Image segmentation with 2 Image inputs
RuntimeError: CUDA error: device-side assert triggered
Here you can find a notebook that reproduces an issue I’m struggling with for some time. The scenario is the following:
-
The URLs.CAMVID_TINY dataset
-
A datablock with 2 Image inputs: DataBlock(blocks=(ImageBlock, ImageBlock, MaskBlock), …)
-
A custom class that handles 2 image inputs (in this example it simply ignores the second image):
class CustomSequentialEx(SequentialEx):
def forward(self, x, x2):
… -
A CustomDynamicUnet that is identic to the original DynamicUnet but has CustomSequentialEx as a base class.
-
A Custom_unet_learner that is identic to the original unet_learner but initialises a CustomDynamicUnet instead.
and … errors with learn.lr_find / learn.fit_one_cycle.
Can someone help me with this one. I’m out of ideas at the moment.
https://colab.research.google.com/drive/1WRcKjBMkQFMZfF6JGSWqOs4qwaIoAS79
The error is the following:
<ipython-input-15-2ea9996b6c00> in <module>
----> 1 learn.fit_one_cycle(10, slice(lr), pct_start=0.9, wd=1e-2)
~/workspace/fastai2/fastai2/callback/schedule.py in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
110 scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
111 'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 112 self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
113
114 # Cell
~/workspace/fastai2/fastai2/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
189 try:
190 self.epoch=epoch; self('begin_epoch')
--> 191 self._do_epoch_train()
192 self._do_epoch_validate()
193 except CancelEpochException: self('after_cancel_epoch')
~/workspace/fastai2/fastai2/learner.py in _do_epoch_train(self)
162 try:
163 self.dl = self.dls.train; self('begin_train')
--> 164 self.all_batches()
165 except CancelTrainException: self('after_cancel_train')
166 finally: self('after_train')
~/workspace/fastai2/fastai2/learner.py in all_batches(self)
140 def all_batches(self):
141 self.n_iter = len(self.dl)
--> 142 for o in enumerate(self.dl): self.one_batch(*o)
143
144 def one_batch(self, i, b):
~/workspace/fastai2/fastai2/learner.py in one_batch(self, i, b)
154 self.opt.zero_grad()
155 except CancelBatchException: self('after_cancel_batch')
--> 156 finally: self('after_batch')
157
158 def _do_begin_fit(self, n_epoch):
~/workspace/fastai2/fastai2/learner.py in __call__(self, event_name)
121 def ordered_cbs(self, cb_func): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, cb_func)]
122
--> 123 def __call__(self, event_name): L(event_name).map(self._call_one)
124 def _call_one(self, event_name):
125 assert hasattr(event, event_name)
~/workspace/fastcore/fastcore/foundation.py in map(self, f, *args, **kwargs)
360 else f.format if isinstance(f,str)
361 else f.__getitem__)
--> 362 return self._new(map(g, self))
363
364 def filter(self, f, negate=False, **kwargs):
~/workspace/fastcore/fastcore/foundation.py in _new(self, items, *args, **kwargs)
313 @property
314 def _xtra(self): return None
--> 315 def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
316 def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
317 def copy(self): return self._new(self.items.copy())
~/workspace/fastcore/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
39 return x
40
---> 41 res = super().__call__(*((x,) + args), **kwargs)
42 res._newchk = 0
43 return res
~/workspace/fastcore/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
304 if items is None: items = []
305 if (use_list is not None) or not _is_array(items):
--> 306 items = list(items) if use_list else _listify(items)
307 if match is not None:
308 if is_coll(match): match = len(match)
~/workspace/fastcore/fastcore/foundation.py in _listify(o)
240 if isinstance(o, list): return o
241 if isinstance(o, str) or _is_array(o): return [o]
--> 242 if is_iter(o): return list(o)
243 return [o]
244
~/workspace/fastcore/fastcore/foundation.py in __call__(self, *args, **kwargs)
206 if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
207 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 208 return self.fn(*fargs, **kwargs)
209
210 # Cell
~/workspace/fastai2/fastai2/learner.py in _call_one(self, event_name)
124 def _call_one(self, event_name):
125 assert hasattr(event, event_name)
--> 126 [cb(event_name) for cb in sort_by_run(self.cbs)]
127
128 def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)
~/workspace/fastai2/fastai2/learner.py in <listcomp>(.0)
124 def _call_one(self, event_name):
125 assert hasattr(event, event_name)
--> 126 [cb(event_name) for cb in sort_by_run(self.cbs)]
127
128 def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)
~/workspace/fastai2/fastai2/callback/core.py in __call__(self, event_name)
21 _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
22 (self.run_valid and not getattr(self, 'training', False)))
---> 23 if self.run and _run: getattr(self, event_name, noop)()
24 if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
25
~/workspace/fastai2/fastai2/learner.py in after_batch(self)
414 if len(self.yb) == 0: return
415 mets = self._train_mets if self.training else self._valid_mets
--> 416 for met in mets: met.accumulate(self.learn)
417 if not self.training: return
418 self.lrs.append(self.opt.hypers[-1]['lr'])
~/workspace/fastai2/fastai2/learner.py in accumulate(self, learn)
366 def accumulate(self, learn):
367 self.count += 1
--> 368 self.val = torch.lerp(to_detach(learn.loss.mean(), gather=False), self.val, self.beta)
369 @property
370 def value(self): return self.val/(1-self.beta**self.count)