Cuda errors after update

After updating the fastai lib, I’m getting a runtime error from code that ran previously. I’m assuming there’s been a change somewhere that I wasn’t ready for… I hit the error when trying to run the learning rate finder (this is in a notebook):

RuntimeError: cuDNN error: CUDNN_STATUS_BAD_PARAM

I guess that doesn’t say much. The full backtrace is:

LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-11-454ad220aa1f> in <module>
----> 1 learner.lr_find(loss_fn, wd=1e-4)

~/src/fastai/fastai/train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
     30     cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
     31     a = int(np.ceil(num_it/len(learn.data.train_dl)))
---> 32     learn.fit(a, start_lr, callbacks=[cb], wd=wd)
     33 
     34 def to_fp16(learn:Learner, loss_scale:float=512., flat_master:bool=False)->Learner:

~/src/fastai/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    176         callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
    177         fit(epochs, self.model, self.loss_func, opt=self.opt, data=self.data, metrics=self.metrics,
--> 178             callbacks=self.callbacks+callbacks)
    179 
    180     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

~/src/fastai/fastai/utils/mem.py in wrapper(*args, **kwargs)
    101 
    102         try:
--> 103             return func(*args, **kwargs)
    104         except Exception as e:
    105             if ("CUDA out of memory" in str(e) or

~/src/fastai/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
     88             for xb,yb in progress_bar(data.train_dl, parent=pbar):
     89                 xb, yb = cb_handler.on_batch_begin(xb, yb)
---> 90                 loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     91                 if cb_handler.on_batch_end(loss): break
     92 

~/src/fastai/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     18     if not is_listy(xb): xb = [xb]
     19     if not is_listy(yb): yb = [yb]
---> 20     out = model(*xb)
     21     out = cb_handler.on_loss_begin(out)
     22 

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-8-d629ed2a4cfe> in forward(self, recon_x, x)
     57 
     58     def forward(self, recon_x, x):
---> 59         z, mu, logvar = self.encode(x)
     60         z = self.decode(z)
     61         return z, mu, logvar

<ipython-input-8-d629ed2a4cfe> in encode(self, x)
     47 
     48     def encode(self, x):
---> 49         h = self.encoder(x)
     50         z, mu, logvar = self.bottleneck(h)
     51         return z, mu, logvar

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     90     def forward(self, input):
     91         for module in self._modules.values():
---> 92             input = module(input)
     93         return input
     94 

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    318     def forward(self, input):
    319         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 320                         self.padding, self.dilation, self.groups)
    321 
    322 

RuntimeError: cuDNN error: CUDNN_STATUS_BAD_PARAM```

I think they changed the default embedding size.

Thanks, but I have to confess that I’m not sure how to apply this information… The model is just a simple VAE, and it seems like the problem is in the encoder, but I’m not sure what I should be changing. Any further thoughts appreciated.

I actually guessed from the CUDNN error, which mostly occurs due to bad input size. So maybe changing the input embedding size can help you. What is your current input embedding size for the encoder?