Segmentation error: device-side assert triggered

Hello, I am trying to adapt the code from this notebook from this notebook to my own dataset, (first transform from 256x1600 to 256x400). However, when I run this code:

learn.lr_find()
learn.recorder.plot()

I got “device-side assert triggered”. I am running this code on Colab.

The trace is here:


RuntimeError Traceback (most recent call last)
in ()
----> 1 learn.lr_find()
2 learn.recorder.plot()

11 frames
/usr/local/lib/python3.6/dist-packages/fastai/train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
39 cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
40 epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
—> 41 learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
42
43 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
198 else: self.opt.lr,self.opt.wd = lr,wd
199 callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
–> 200 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
201
202 def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
89 cb_handler = CallbackHandler(callbacks, metrics)
90 pbar = master_bar(range(epochs))
—> 91 cb_handler.on_train_begin(epochs, pbar=pbar, metrics=metrics)
92
93 exception=False

/usr/local/lib/python3.6/dist-packages/fastai/callback.py in on_train_begin(self, epochs, pbar, metrics)
263 self.state_dict.update(dict(n_epochs=epochs, pbar=pbar, metrics=metrics))
264 names = [(met.name if hasattr(met, ‘name’) else camel2snake(met.class.name)) for met in self.metrics]
–> 265 self(‘train_begin’, metrics_names=names)
266 if self.state_dict[‘epoch’] != 0:
267 self.state_dict[‘pbar’].first_bar.total -= self.state_dict[‘epoch’]

/usr/local/lib/python3.6/dist-packages/fastai/callback.py in call(self, cb_name, call_mets, **kwargs)
249 if call_mets:
250 for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
–> 251 for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
252
253 def set_dl(self, dl:DataLoader):

/usr/local/lib/python3.6/dist-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
239 def call_and_update(self, cb, cb_name, **kwargs)->None:
240 “Call cb_name on cb and update the inner state.”
–> 241 new = ifnone(getattr(cb, f’on
{cb_name}’)(**self.state_dict, **kwargs), dict())
242 for k,v in new.items():
243 if k not in self.state_dict:

/usr/local/lib/python3.6/dist-packages/fastai/callbacks/lr_finder.py in on_train_begin(self, pbar, **kwargs)
17 “Initialize optimizer and learner hyperparameters.”
18 setattr(pbar, ‘clean_on_interrupt’, True)
—> 19 self.learn.save(‘tmp’)
20 self.opt = self.learn.opt
21 self.opt.lr = self.sched.start

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in save(self, file, return_path, with_opt)
251 if not with_opt: state = get_model(self.model).state_dict()
252 else: state = {‘model’: get_model(self.model).state_dict(), ‘opt’:self.opt.state_dict()}
–> 253 torch.save(state, target)
254 if return_path: return target
255

/usr/local/lib/python3.6/dist-packages/torch/serialization.py in save(obj, f, pickle_module, pickle_protocol)
258 >>> torch.save(x, buffer)
259 “”"
–> 260 return _with_file_like(f, “wb”, lambda f: _save(obj, f, pickle_module, pickle_protocol))
261
262

/usr/local/lib/python3.6/dist-packages/torch/serialization.py in _with_file_like(f, mode, body)
183 f = open(f, mode)
184 try:
–> 185 return body(f)
186 finally:
187 if new_fd:

/usr/local/lib/python3.6/dist-packages/torch/serialization.py in (f)
258 >>> torch.save(x, buffer)
259 “”"
–> 260 return _with_file_like(f, “wb”, lambda f: _save(obj, f, pickle_module, pickle_protocol))
261
262

/usr/local/lib/python3.6/dist-packages/torch/serialization.py in _save(obj, f, pickle_module, pickle_protocol)
336 f.flush()
337 for key in serialized_storage_keys:
–> 338 serialized_storages[key]._write_file(f, _should_read_directly(f))
339
340

RuntimeError: cuda runtime error (59) : device-side assert triggered at /pytorch/torch/csrc/generic/serialization.cpp:23

Does anyone have hints to solve this issue?