After updating the fastai lib, I’m getting a runtime error from code that ran previously. I’m assuming there’s been a change somewhere that I wasn’t ready for… I hit the error when trying to run the learning rate finder (this is in a notebook):
RuntimeError: cuDNN error: CUDNN_STATUS_BAD_PARAM
I guess that doesn’t say much. The full backtrace is:
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-11-454ad220aa1f> in <module>
----> 1 learner.lr_find(loss_fn, wd=1e-4)
~/src/fastai/fastai/train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
30 cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
31 a = int(np.ceil(num_it/len(learn.data.train_dl)))
---> 32 learn.fit(a, start_lr, callbacks=[cb], wd=wd)
33
34 def to_fp16(learn:Learner, loss_scale:float=512., flat_master:bool=False)->Learner:
~/src/fastai/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
176 callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
177 fit(epochs, self.model, self.loss_func, opt=self.opt, data=self.data, metrics=self.metrics,
--> 178 callbacks=self.callbacks+callbacks)
179
180 def create_opt(self, lr:Floats, wd:Floats=0.)->None:
~/src/fastai/fastai/utils/mem.py in wrapper(*args, **kwargs)
101
102 try:
--> 103 return func(*args, **kwargs)
104 except Exception as e:
105 if ("CUDA out of memory" in str(e) or
~/src/fastai/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
88 for xb,yb in progress_bar(data.train_dl, parent=pbar):
89 xb, yb = cb_handler.on_batch_begin(xb, yb)
---> 90 loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
91 if cb_handler.on_batch_end(loss): break
92
~/src/fastai/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
18 if not is_listy(xb): xb = [xb]
19 if not is_listy(yb): yb = [yb]
---> 20 out = model(*xb)
21 out = cb_handler.on_loss_begin(out)
22
~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
487 result = self._slow_forward(*input, **kwargs)
488 else:
--> 489 result = self.forward(*input, **kwargs)
490 for hook in self._forward_hooks.values():
491 hook_result = hook(self, input, result)
<ipython-input-8-d629ed2a4cfe> in forward(self, recon_x, x)
57
58 def forward(self, recon_x, x):
---> 59 z, mu, logvar = self.encode(x)
60 z = self.decode(z)
61 return z, mu, logvar
<ipython-input-8-d629ed2a4cfe> in encode(self, x)
47
48 def encode(self, x):
---> 49 h = self.encoder(x)
50 z, mu, logvar = self.bottleneck(h)
51 return z, mu, logvar
~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
487 result = self._slow_forward(*input, **kwargs)
488 else:
--> 489 result = self.forward(*input, **kwargs)
490 for hook in self._forward_hooks.values():
491 hook_result = hook(self, input, result)
~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
90 def forward(self, input):
91 for module in self._modules.values():
---> 92 input = module(input)
93 return input
94
~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
487 result = self._slow_forward(*input, **kwargs)
488 else:
--> 489 result = self.forward(*input, **kwargs)
490 for hook in self._forward_hooks.values():
491 hook_result = hook(self, input, result)
~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
318 def forward(self, input):
319 return F.conv2d(input, self.weight, self.bias, self.stride,
--> 320 self.padding, self.dilation, self.groups)
321
322
RuntimeError: cuDNN error: CUDNN_STATUS_BAD_PARAM```