when i run demo graphsage with 0.5.2 distribute training
there is a error about ‘Found no NVIDIA driver on your system’
but a have gpu and test as follow
Python 3.6.8 (default, Aug 7 2019, 17:28:10)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] on linux
Type “help”, “copyright”, “credits” or “license” for more information.
import torch
torch.cuda.is_available()
Truex = torch.Tensor(5, 3)
x.cuda()
tensor([[-8.7259e+05, 4.5745e-41, -8.7259e+05],
[ 4.5745e-41, 1.4013e-45, 0.0000e+00],
[ 0.0000e+00, 0.0000e+00, 0.0000e+00],
[ 0.0000e+00, -1.2633e-23, 4.5745e-41],
[ 0.0000e+00, 0.0000e+00, 0.0000e+00]], device=‘cuda:0’)torch.version
‘1.6.0’x.to(0)
tensor([[-8.7259e+05, 4.5745e-41, -8.7259e+05],
[ 4.5745e-41, 1.4013e-45, 0.0000e+00],
[ 0.0000e+00, 0.0000e+00, 0.0000e+00],
[ 0.0000e+00, -1.2633e-23, 4.5745e-41],
[ 0.0000e+00, 0.0000e+00, 0.0000e+00]], device=‘cuda:0’)
DGL log as follow
------------------------------trace log as follow ----------------------------
Traceback (most recent call last):
Traceback (most recent call last):
File “train_dist_trainer.py”, line 337, in
File “train_dist_trainer.py”, line 337, in
main(args)
File “train_dist_trainer.py”, line 288, in main
main(args)
File “train_dist_trainer.py”, line 288, in main
run(args, device, data)
File “train_dist_trainer.py”, line 174, in run
model = model.to(device)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 607, in to
run(args, device, data)
File “train_dist_trainer.py”, line 174, in run
return self._apply(convert)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 376, in _apply
model = model.to(device)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 607, in to
param_applied = fn(param)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 605, in convert
return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
File “/usr/local/lib64/python3.6/site-packages/torch/cuda/init.py”, line 186, in _lazy_init
return self._apply(convert)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
_check_driver()
File “/usr/local/lib64/python3.6/site-packages/torch/cuda/init.py”, line 68, in _check_driver
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
http://www.nvidia.com/Download/index.aspx""")
AssertionError:
Found no NVIDIA driver on your system. Please check that you
have an NVIDIA GPU and installed a driver from
http://www.nvidia.com/Download/index.aspx
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 354, in _apply
module._apply(fn)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 376, in _apply
param_applied = fn(param)
File “/usr/local/lib64/python3.6/site-packages/torch/nn/modules/module.py”, line 605, in convert
return t.to(device, dtype if t.is_floating_point() else None, non_blocking)
File “/usr/local/lib64/python3.6/site-packages/torch/cuda/init.py”, line 186, in _lazy_init
_check_driver()
File “/usr/local/lib64/python3.6/site-packages/torch/cuda/init.py”, line 68, in _check_driver
http://www.nvidia.com/Download/index.aspx""")
AssertionError:
Found no NVIDIA driver on your system. Please check that you
have an NVIDIA GPU and installed a driver from
http://www.nvidia.com/Download/index.aspx
Exception ignored in: <bound method DistDataLoader.del of <dgl.distributed.dist_dataloader.DistDataLoader object at 0x7f65d9a9f9e8>>
Traceback (most recent call last):
File “/usr/local/lib64/python3.6/site-packages/dgl/distributed/dist_dataloader.py”, line 154, in del
results.append(self.pool.apply_async(cleanup_fn, args=(self.barrier, self.name,)))
File “/usr/lib64/python3.6/multiprocessing/pool.py”, line 355, in apply_async
raise ValueError(“Pool not running”)
ValueError: Pool not running
Exception ignored in: <bound method DistDataLoader.del of <dgl.distributed.dist_dataloader.DistDataLoader object at 0x7f12e1d799b0>>
Traceback (most recent call last):
File “/usr/local/lib64/python3.6/site-packages/dgl/distributed/dist_dataloader.py”, line 154, in del
results.append(self.pool.apply_async(cleanup_fn, args=(self.barrier, self.name,)))
File “/usr/lib64/python3.6/multiprocessing/pool.py”, line 355, in apply_async
raise ValueError(“Pool not running”)
ValueError: Pool not running
Traceback (most recent call last):
File “/usr/lib64/python3.6/runpy.py”, line 193, in _run_module_as_main
“main”, mod_spec)
File “/usr/lib64/python3.6/runpy.py”, line 85, in _run_code
exec(code, run_globals)
File “/usr/local/lib64/python3.6/site-packages/torch/distributed/launch.py”, line 261, in
main()
File “/usr/local/lib64/python3.6/site-packages/torch/distributed/launch.py”, line 257, in main
cmd=cmd)
subprocess.CalledProcessError: Command ‘[’/usr/bin/python’, ‘-u’, ‘train_dist_trainer.py’, ‘–local_rank=1’]’ returned non-zero exit status 1.