Keep getting CUDA index error from apply_edge function.
/opt/conda/conda-bld/pytorch_1573049305765/work/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [415,0,0], thread: [95,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1573049305765/work/aten/src/THC/THCReduceAll.cuh line=327 error=59 : device-side assert triggered
g.apply_edges(u_dot_ve('k_', 'r_', 'q_', 'qrk_'))
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/heterograph.py", line 2704, in apply_edges
Runtime.run(prog)
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/runtime/runtime.py", line 11, in run
exe.run()
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/runtime/ir/executor.py", line 204, in run
udf_ret = fn_data(src_data, edge_data, dst_data)
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/runtime/scheduler.py", line 972, in _mfunc_wrapper
return mfunc(ebatch)
File "/home/keyit/code/alan-framework/modules/semantic_parser/model/graph_encoders.py", line 840, in func
u = es.src[src_field]
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/utils.py", line 285, in __getitem__
return self._fn(key)
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/frame.py", line 655, in <lambda>
return utils.LazyDict(lambda key: self._frame[key][rows], keys=self.keys())
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/frame.py", line 96, in __getitem__
user_idx = idx.tousertensor(F.context(self.data))
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/utils.py", line 105, in tousertensor
self._user_tensor_data[ctx] = F.copy_to(data, ctx)
File "/home/keyit/anaconda3/envs/alan/lib/python3.6/site-packages/dgl/backend/pytorch/tensor.py", line 95, in copy_to
return input.cuda()
RuntimeError: CUDA error: device-side assert triggered
Function To Apply
def u_dot_ve(src_field, edg_field, dst_field, out_field):
def func(es):
v = es.dst[dst_field]
e = es.data[edg_field]
u = es.src[src_field]
ve = v + e
ve_u = ve * u
ve_u_sum = torch.sum(ve_u, dim=-1, keepdim=True)
return {out_field: ve_u_sum}
return func
Call apply_edge
g = dgl.to_homo(hg)
num_nodes = g.number_of_nodes()
num_edges = g.number_of_edges()
k = self.k_w(x).view(num_nodes, self.num_heads, -1).contiguous()
q = self.q_w(x).view(num_nodes, self.num_heads, -1).contiguous()
v = self.v_w(x).view(num_nodes, self.num_heads, -1).contiguous()
r = r.view(num_edges, self.num_heads, -1).contiguous()
g.ndata['k_'] = k
g.ndata['q_'] = q
g.ndata['v_'] = v
g.edata['r_'] = r
# --- Compute attention score ---
# k * (q + r) --> qkr (es, hs, 1)
g.apply_edges(u_dot_ve('k_', 'r_', 'q_', 'qrk_'))
Please help me debug on this error. I checked k, r, and q. Which looks good to me.