Hi,
I created my own gpu based server (nvidia 1060 6gb) and I keep running into a problem whenever I execute vgg.predict(imgs, True).
The error is:
In file included from mod.cu:4:0:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:17:0: warning: “PyString_Check” redefined
#define PyString_Check PyUnicode_Check
^
In file included from /home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:11:0,
from mod.cu:4:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include/numpy/npy_3kcompat.h:63:0: note: this is the location of the previous definition
#define PyString_Check PyBytes_Check
^
In file included from mod.cu:4:0:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:18:0: warning: “PyString_FromString” redefined
#define PyString_FromString PyUnicode_FromString
^
In file included from /home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:11:0,
from mod.cu:4:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include/numpy/npy_3kcompat.h:65:0: note: this is the location of the previous definition
#define PyString_FromString PyBytes_FromString
^
In file included from mod.cu:4:0:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:19:0: warning: “PyString_AsString” redefined
#define PyString_AsString PyUnicode_AsUTF8
^
In file included from /home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:11:0,
from mod.cu:4:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include/numpy/npy_3kcompat.h:72:0: note: this is the location of the previous definition
#define PyString_AsString PyBytes_AsString
^
In file included from mod.cu:4:0:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:20:0: warning: “PyString_FromStringAndSize” redefined
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
^
In file included from /home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:11:0,
from mod.cu:4:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include/numpy/npy_3kcompat.h:66:0: note: this is the location of the previous definition
#define PyString_FromStringAndSize PyBytes_FromStringAndSize
^
In file included from mod.cu:4:0:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:21:0: warning: “PyString_Size” redefined
#define PyString_Size PyUnicode_GET_SIZE
^
In file included from /home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/cuda_ndarray.cuh:11:0,
from mod.cu:4:
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include/numpy/npy_3kcompat.h:74:0: note: this is the location of the previous definition
#define PyString_Size PyBytes_Size
^
mod.cu(78): error: identifier “cudnnSetFilterNdDescriptor_v4” is undefined
1 error detected in the compilation of “/tmp/tmpxft_000068de_00000000-9_mod.cpp1.ii”.
[‘nvcc’, ‘-shared’, ‘-O3’, ‘-Xlinker’, ‘-rpath,/usr/local/cuda/lib64’, ‘-use_fast_math’, ‘-arch=sm_61’, ‘-m64’, ‘-Xcompiler’, ‘-fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=mc72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden’, ‘-Xlinker’, ‘-rpath,/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray’, ‘-I/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray’, ‘-I/usr/local/cuda/include’, ‘-I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda’, ‘-I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include’, ‘-I/home/rahul/anaconda2/envs/fastai34/include/python3.4m’, ‘-I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof’, ‘-o’, ‘/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/tmppl0r18xy/m1b4b87b9831a7c1022a8899fd1be603f.so’, ‘mod.cu’, ‘-L/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray’, ‘-L/home/rahul/anaconda2/envs/fastai34/lib’, ‘-lcudart’, ‘-lcublas’, ‘-lcuda_ndarray’, ‘-lcudnn’, ‘-lpython3.4m’]
Exception Traceback (most recent call last)
in ()
----> 1 vgg.predict(imgs, True)
/home/rahul/courses/deeplearning1/nbs/vgg16.py in predict(self, imgs, details)
74 “”"
75 # predict probability of each class for each image
—> 76 all_preds = self.model.predict(imgs)
77 # for each image get the index of the class with max probability
78 idxs = np.argmax(all_preds, axis=1)
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/keras/models.py in predict(self, x, batch_size, verbose)
676 if self.model is None:
677 self.build()
–> 678 return self.model.predict(x, batch_size=batch_size, verbose=verbose)
679
680 def predict_on_batch(self, x):
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/keras/engine/training.py in predict(self, x, batch_size, verbose)
1192 else:
1193 ins = x
-> 1194 self._make_predict_function()
1195 f = self.predict_function
1196 return self._predict_loop(f, ins,
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/keras/engine/training.py in _make_predict_function(self)
750 self.outputs,
751 updates=self.state_updates,
–> 752 **kwargs)
753
754 def _fit_loop(self, f, ins, out_labels=[], batch_size=32,
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/keras/backend/theano_backend.py in function(inputs, outputs, updates, **kwargs)
800 msg = “Invalid argument ‘%s’ passed to K.function” % key
801 raise ValueError(msg)
–> 802 return Function(inputs, outputs, updates=updates, **kwargs)
803
804
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/keras/backend/theano_backend.py in init(self, inputs, outputs, updates, **kwargs)
786 allow_input_downcast=True,
787 on_unused_input=‘ignore’,
–> 788 **kwargs)
789
790 def call(self, inputs):
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/compile/function.py in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
318 on_unused_input=on_unused_input,
319 profile=profile,
–> 320 output_keys=output_keys)
321 # We need to add the flag check_aliased inputs if we have any mutable or
322 # borrowed used defined inputs
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/compile/pfunc.py in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)
477 accept_inplace=accept_inplace, name=name,
478 profile=profile, on_unused_input=on_unused_input,
–> 479 output_keys=output_keys)
480
481
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/compile/function_module.py in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys)
1775 on_unused_input=on_unused_input,
1776 output_keys=output_keys).create(
-> 1777 defaults)
1778
1779 t2 = time.time()
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/compile/function_module.py in create(self, input_storage, trustme, storage_map)
1639 theano.config.traceback.limit = 0
1640 _fn, _i, _o = self.linker.make_thunk(
-> 1641 input_storage=input_storage_lists, storage_map=storage_map)
1642 finally:
1643 theano.config.traceback.limit = limit_orig
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/link.py in make_thunk(self, input_storage, output_storage, storage_map)
688 return self.make_all(input_storage=input_storage,
689 output_storage=output_storage,
–> 690 storage_map=storage_map)[:3]
691
692 def make_all(self, input_storage, output_storage):
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/vm.py in make_all(self, profiler, input_storage, output_storage, storage_map)
1001 storage_map,
1002 compute_map,
-> 1003 no_recycling))
1004 if not hasattr(thunks[-1], ‘lazy’):
1005 # We don’t want all ops maker to think about lazy Ops.
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/init.py in make_thunk(self, node, storage_map, compute_map, no_recycling)
254 enable_cuda=False)
255 return super(GpuOp, self).make_thunk(node, storage_map,
–> 256 compute_map, no_recycling)
257
258 theano.compile.debugmode.default_make_thunk.append(
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/op.py in make_thunk(self, node, storage_map, compute_map, no_recycling)
968 try:
969 return self.make_c_thunk(node, storage_map, compute_map,
–> 970 no_recycling)
971 except (NotImplementedError, utils.MethodNotDefined):
972 logger.debug(‘Falling back on perform’)
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/op.py in make_c_thunk(self, node, storage_map, compute_map, no_recycling)
877 logger.debug(‘Trying CLinker.make_thunk’)
878 outputs = cl.make_thunk(input_storage=node_input_storage,
–> 879 output_storage=node_output_storage)
880 fill_storage, node_input_filters, node_output_filters = outputs
881
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/cc.py in make_thunk(self, input_storage, output_storage, storage_map, keep_lock)
1198 cthunk, in_storage, out_storage, error_storage = self.compile(
1199 input_storage, output_storage, storage_map,
-> 1200 keep_lock=keep_lock)
1201
1202 res = _CThunk(cthunk, init_tasks, tasks, error_storage)
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/cc.py in compile(self, input_storage, output_storage, storage_map, keep_lock)
1141 output_storage,
1142 storage_map,
-> 1143 keep_lock=keep_lock)
1144 return (thunk,
1145 [link.Container(input, storage) for input, storage in
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/cc.py in cthunk_factory(self, error_storage, in_storage, out_storage, storage_map, keep_lock)
1593 else:
1594 module = get_module_cache().module_from_key(
-> 1595 key=key, lnk=self, keep_lock=keep_lock)
1596
1597 vars = self.inputs + self.outputs + self.orphans
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/cmodule.py in module_from_key(self, key, lnk, keep_lock)
1140 try:
1141 location = dlimport_workdir(self.dirname)
-> 1142 module = lnk.compile_cmodule(location)
1143 name = module.file
1144 assert name.startswith(location)
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof/cc.py in compile_cmodule(self, location)
1504 lib_dirs=self.lib_dirs(),
1505 libs=libs,
-> 1506 preargs=preargs)
1507 except Exception as e:
1508 e.args += (str(self.fgraph),)
/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda/nvcc_compiler.py in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, rpaths, py_module, hide_symbols)
397 print(cmd)
398 raise Exception(‘nvcc return status’, p.returncode,
–> 399 ‘for cmd’, ’ '.join(cmd))
400 elif config.cmodule.compilation_warning and nvcc_stdout:
401 print(nvcc_stdout)
Exception: (‘The following error happened while compiling the node’, GpuDnnConv{algo=‘small’, inplace=True}(GpuContiguous.0, GpuContiguous.0, GpuAllocEmpty.0, GpuDnnConvDesc{border_mode=‘valid’, subsample=(1, 1), conv_mode=‘conv’, precision=‘float32’}.0, Constant{1.0}, Constant{0.0}), ‘\n’, ‘nvcc return status’, 2, ‘for cmd’, ‘nvcc -shared -O3 -Xlinker -rpath,/usr/local/cuda/lib64 -use_fast_math -arch=sm_61 -m64 -Xcompiler -fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=mc72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden -Xlinker -rpath,/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray -I/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray -I/usr/local/cuda/include -I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/sandbox/cuda -I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/numpy/core/include -I/home/rahul/anaconda2/envs/fastai34/include/python3.4m -I/home/rahul/anaconda2/envs/fastai34/lib/python3.4/site-packages/theano/gof -o /home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/tmppl0r18xy/m1b4b87b9831a7c1022a8899fd1be603f.so mod.cu -L/home/rahul/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-3.4.5-64/cuda_ndarray -L/home/rahul/anaconda2/envs/fastai34/lib -lcudart -lcublas -lcuda_ndarray -lcudnn -lpython3.4m’, “[GpuDnnConv{algo=‘small’, inplace=True}(<CudaNdarrayType(float32, 4D)>, <CudaNdarrayType(float32, 4D)>, <CudaNdarrayType(float32, 4D)>, <CDataType{cudnnConvolutionDescriptor_t}>, Constant{1.0}, Constant{0.0})]”)
Could someone help me resolve this error?