Hello, I am trying to adapt the code from this notebook from this notebook to my own dataset, (first transform from 256x1600 to 256x400). However, when I run this code:
learn.lr_find()
learn.recorder.plot()
I got “device-side assert triggered”. I am running this code on Colab.
The trace is here:
RuntimeError Traceback (most recent call last)
in ()
----> 1 learn.lr_find()
2 learn.recorder.plot()11 frames
/usr/local/lib/python3.6/dist-packages/fastai/train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
39 cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
40 epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
—> 41 learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
42
43 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
198 else: self.opt.lr,self.opt.wd = lr,wd
199 callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
→ 200 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
201
202 def create_opt(self, lr:Floats, wd:Floats=0.)->None:/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
89 cb_handler = CallbackHandler(callbacks, metrics)
90 pbar = master_bar(range(epochs))
—> 91 cb_handler.on_train_begin(epochs, pbar=pbar, metrics=metrics)
92
93 exception=False/usr/local/lib/python3.6/dist-packages/fastai/callback.py in on_train_begin(self, epochs, pbar, metrics)
263 self.state_dict.update(dict(n_epochs=epochs, pbar=pbar, metrics=metrics))
264 names = [(met.name if hasattr(met, ‘name’) else camel2snake(met.class.name)) for met in self.metrics]
→ 265 self(‘train_begin’, metrics_names=names)
266 if self.state_dict[‘epoch’] != 0:
267 self.state_dict[‘pbar’].first_bar.total -= self.state_dict[‘epoch’]/usr/local/lib/python3.6/dist-packages/fastai/callback.py in call(self, cb_name, call_mets, **kwargs)
249 if call_mets:
250 for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
→ 251 for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
252
253 def set_dl(self, dl:DataLoader):/usr/local/lib/python3.6/dist-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
239 def call_and_update(self, cb, cb_name, **kwargs)->None:
240 “Callcb_nameoncband update the inner state.”
→ 241 new = ifnone(getattr(cb, f’on{cb_name}')(**self.state_dict, **kwargs), dict())
242 for k,v in new.items():
243 if k not in self.state_dict:/usr/local/lib/python3.6/dist-packages/fastai/callbacks/lr_finder.py in on_train_begin(self, pbar, **kwargs)
17 “Initialize optimizer and learner hyperparameters.”
18 setattr(pbar, ‘clean_on_interrupt’, True)
—> 19 self.learn.save(‘tmp’)
20 self.opt = self.learn.opt
21 self.opt.lr = self.sched.start/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in save(self, file, return_path, with_opt)
251 if not with_opt: state = get_model(self.model).state_dict()
252 else: state = {‘model’: get_model(self.model).state_dict(), ‘opt’:self.opt.state_dict()}
→ 253 torch.save(state, target)
254 if return_path: return target
255/usr/local/lib/python3.6/dist-packages/torch/serialization.py in save(obj, f, pickle_module, pickle_protocol)
258 >>> torch.save(x, buffer)
259 “”"
→ 260 return _with_file_like(f, “wb”, lambda f: _save(obj, f, pickle_module, pickle_protocol))
261
262/usr/local/lib/python3.6/dist-packages/torch/serialization.py in _with_file_like(f, mode, body)
183 f = open(f, mode)
184 try:
→ 185 return body(f)
186 finally:
187 if new_fd:/usr/local/lib/python3.6/dist-packages/torch/serialization.py in (f)
258 >>> torch.save(x, buffer)
259 “”"
→ 260 return _with_file_like(f, “wb”, lambda f: _save(obj, f, pickle_module, pickle_protocol))
261
262/usr/local/lib/python3.6/dist-packages/torch/serialization.py in _save(obj, f, pickle_module, pickle_protocol)
336 f.flush()
337 for key in serialized_storage_keys:
→ 338 serialized_storages[key]._write_file(f, _should_read_directly(f))
339
340RuntimeError: cuda runtime error (59) : device-side assert triggered at /pytorch/torch/csrc/generic/serialization.cpp:23
Does anyone have hints to solve this issue?