Unet for a single class throws CUDA error

I have tried to provide a single class to segmentation task but it works well with two classes

get_y_fn = lambda x: path_msk/f'{x.stem}{x.suffix}'

src = (ImageFileList.from_folder(path_img)
       .label_from_func(get_y_fn)
       .random_split_by_pct(0.2))

data = (src.datasets(SegmentationDataset, classes=codes, div=True)
        .transform(get_transforms(), size=128, tfm_y=True)
        .databunch(bs=32, num_workers=0)
        .normalize(imagenet_stats))       


learn = Learner.create_unet(data, models.resnet18)

learn.fit_one_cycle(1, slice(1e-2/2))

And the error is:

RuntimeError                              Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
     93         exception = e
---> 94         raise e
     95     finally: cb_handler.on_train_end(exception)

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
     83                 xb, yb = cb_handler.on_batch_begin(xb, yb)
---> 84                 loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     85                 if cb_handler.on_batch_end(loss): break

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     24     if opt is not None:
---> 25         loss = cb_handler.on_backward_begin(loss)
     26         loss.backward()

/usr/local/lib/python3.6/dist-packages/fastai/callback.py in on_backward_begin(self, loss)
    223         for cb in self.callbacks:
--> 224             a = cb.on_backward_begin(**self.state_dict)
    225             if a is not None: self.state_dict['last_loss'] = a

/usr/local/lib/python3.6/dist-packages/fastai/basic_train.py in on_backward_begin(self, smooth_loss, **kwargs)
    266         if self.pbar is not None and hasattr(self.pbar,'child'):
--> 267             self.pbar.child.comment = f'{smooth_loss:.4f}'
    268 

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in __format__(self, format_spec)
    377         if self.dim() == 0:
--> 378             return self.item().__format__(format_spec)
    379         return object.__format__(self, format_spec)

RuntimeError: CUDA error: device-side assert triggered

I’m trying to remove extra demotion by providing single class. In case two classes I have prediction in form of (32, 2, 128,128) as I noticed the 2 comes from number of classes.