Issues with Multi-Class Image Segmentation on Custom Data

fastai
1

I get an error when trying to create a SegmentationDataLoaders object using labels I created myself with PixelAnnotationTool (I also had to adjust the pixels from 255 to somewhere within my category range). It’s probably also worth stating that I can run the example in the fastai NB1 for image segmentation on the CAMVID dataset with GPU enabled.

path = Path('segmentation_test_1')
dls = SegmentationDataLoaders.from_label_func(
    path, bs=2, fnames = get_image_files(path/"train"),
    label_func = lambda o: path/'labels'/f'{o.stem}_watershed_mask.png',
    codes = np.loadtxt(path/'codes.txt', dtype=str)
)

Immediately upon creating it, I get the error message, Could not do one pass in your dataloader, there is something wrong in it. This error message isn’t very helpful, so I drilled into the code to see what SegmentationDataLoaders was doing and tried to create the datablock myself.

dblock = DataBlock(blocks=(ImageBlock, MaskBlock(codes=codes)),
                           splitter=RandomSplitter(0.2, seed=42),
                           get_items = get_image_files,
                           get_y=get_y
   dblock.summary(path/'train', bs=2, show_batch=False)
                  )

This returns the following output and stacktrace:

Setting-up type transforms pipelines
Collecting items from segmentation_test_1/train
Found 8 items
2 datasets of sizes 7,1
Setting up Pipeline: PILBase.create
Setting up Pipeline: <lambda> -> PILBase.create

Building one sample
  Pipeline: PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying PILBase.create gives
      PILImage mode=RGB size=170x128
  Pipeline: <lambda> -> PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying <lambda> gives
      PILMask mode=L size=170x128
    applying PILBase.create gives
      PILMask mode=L size=170x128

Final sample: (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)


Setting up after_item: Pipeline: AddMaskCodes -> ToTensor
Setting up before_batch: Pipeline: 
Setting up after_batch: Pipeline: IntToFloatTensor -- {'div': 255.0, 'div_mask': 1}
Could not do one pass in your dataloader, there is something wrong in it

Building one batch
Applying item_tfms to the first sample:
  Pipeline: AddMaskCodes -> ToTensor
    starting from
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying AddMaskCodes gives
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying ToTensor gives
      (TensorImage of size 3x128x170, TensorMask of size 128x170)

Adding the next 1 samples

No before_batch transform to apply

Collating items in a batch

Applying batch_tfms to the batch built
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-339-8a5a537c7402> in <module>
      7 # dls = dblock.dataloaders(path/'train', bs=2)
      8 # dls.show_batch()
----> 9 dblock.summary(path/'train', bs=2, show_batch=False)
     10 # res = SegmentationDataLoaders.from_dblock(dblock, fnames, path)
     11 # res.show_batch()

/opt/conda/lib/python3.7/site-packages/fastai/data/block.py in summary(self, source, bs, show_batch, **kwargs)
    192     if len([f for f in dls.train.after_batch.fs if f.name != 'noop'])!=0:
    193         print("\nApplying batch_tfms to the batch built")
--> 194         b = to_device(b, dls.device)
    195         b = _apply_pipeline(dls.train.after_batch, b)
    196     else: print("\nNo batch_tfms to apply")

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in to_device(b, device)
    215     elif device is None: device=default_device()
    216     def _inner(o): return o.to(device, non_blocking=True) if isinstance(o,Tensor) else o.to_device(device) if hasattr(o, "to_device") else o
--> 217     return apply(_inner, b)
    218 
    219 # Cell

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in apply(func, x, *args, **kwargs)
    163 def apply(func, x, *args, **kwargs):
    164     "Apply `func` recursively to `x`, passing on args"
--> 165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
    167     res = func(x, *args, **kwargs)

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in <listcomp>(.0)
    163 def apply(func, x, *args, **kwargs):
    164     "Apply `func` recursively to `x`, passing on args"
--> 165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
    167     res = func(x, *args, **kwargs)

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in apply(func, x, *args, **kwargs)
    165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
--> 167     res = func(x, *args, **kwargs)
    168     return res if x is None else retain_type(res, x)
    169 

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in _inner(o)
    214     if defaults.use_cuda==False: device='cpu'
    215     elif device is None: device=default_device()
--> 216     def _inner(o): return o.to(device, non_blocking=True) if isinstance(o,Tensor) else o.to_device(device) if hasattr(o, "to_device") else o
    217     return apply(_inner, b)
    218 

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in _f(self, *args, **kwargs)
    298         def _f(self, *args, **kwargs):
    299             cls = self.__class__
--> 300             res = getattr(super(TensorBase, self), fn)(*args, **kwargs)
    301             return retain_type(res, self, copy_meta=True)
    302         return _f

RuntimeError: CUDA error: device-side assert triggered

Because fastai is imported with the *, I have access to
defaults.use_cuda, so I set it to false and create my dls using either approach just fine.

defaults.use_cuda = False

BTW, I’m running all of this in a docker container created with the image: fastdotai/fastai-course and with the --gpu flag. Maybe that has something to do with it. Here’s my nvidia-smi output, just in case that maybe matters.

nvidia-smi
Fri Sep 18 13:47:18 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.33.01    Driver Version: 440.33.01    CUDA Version: 10.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 108...  Off  | 00000000:01:00.0 Off |                  N/A |
|  0%   55C    P2    63W / 250W |   3554MiB / 11178MiB |     23%      Default |
+-------------------------------+----------------------+----------------------+
|   1  GeForce GTX 108...  Off  | 00000000:02:00.0 Off |                  N/A |
|  0%   28C    P8    10W / 275W |     12MiB / 11178MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      1624      G   /usr/lib/xorg/Xorg                           547MiB |
|    0      4784      G   compiz                                       250MiB |
|    0      5457      G   ...quest-channel-token=3761568144799598588   324MiB |
|    0      7929      C   /opt/conda/bin/python                       1815MiB |
|    0      8819      C   /opt/conda/bin/python                        569MiB |
|    0      9427      G   ...quest-channel-token=1381726256334931566    41MiB |
+-----------------------------------------------------------------------------+
2

Could you show the output of dblock.summary() in a CPU-bound environment?

3

Yep!

Setting-up type transforms pipelines
Collecting items from segmentation_test_1/train
Found 8 items
2 datasets of sizes 7,1
Setting up Pipeline: PILBase.create
Setting up Pipeline: <lambda> -> PILBase.create

Building one sample
  Pipeline: PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying PILBase.create gives
      PILImage mode=RGB size=170x128
  Pipeline: <lambda> -> PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying <lambda> gives
      PILMask mode=L size=170x128
    applying PILBase.create gives
      PILMask mode=L size=170x128

Final sample: (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)


Setting up after_item: Pipeline: AddMaskCodes -> ToTensor
Setting up before_batch: Pipeline: 
Setting up after_batch: Pipeline: IntToFloatTensor -- {'div': 255.0, 'div_mask': 1}

Building one batch
Applying item_tfms to the first sample:
  Pipeline: AddMaskCodes -> ToTensor
    starting from
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying AddMaskCodes gives
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying ToTensor gives
      (TensorImage of size 3x128x170, TensorMask of size 128x170)

Adding the next 1 samples

No before_batch transform to apply

Collating items in a batch

Applying batch_tfms to the batch built
  Pipeline: IntToFloatTensor -- {'div': 255.0, 'div_mask': 1}
    starting from
      (TensorImage of size 2x3x128x170, TensorMask of size 2x128x170)
    applying IntToFloatTensor -- {'div': 255.0, 'div_mask': 1} gives
      (TensorImage of size 2x3x128x170, TensorMask of size 2x128x170)
4

Try removing the codes in your call to MaskBlock

5 
defaults.use_cuda = True
dblock = DataBlock(blocks=(ImageBlock, MaskBlock()),
                           splitter=RandomSplitter(0.2, seed=42),
                           get_items = get_image_files,
                           get_y=get_y
                  )
dblock.summary(path/'train', bs=2, show_batch=False)

results in

Setting-up type transforms pipelines
Collecting items from segmentation_test_1/train
Found 8 items
2 datasets of sizes 7,1
Setting up Pipeline: PILBase.create
Setting up Pipeline: <lambda> -> PILBase.create

Building one sample
  Pipeline: PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying PILBase.create gives
      PILImage mode=RGB size=170x128
  Pipeline: <lambda> -> PILBase.create
    starting from
      segmentation_test_1/train/00020.jpg
    applying <lambda> gives
      PILMask mode=L size=170x128
    applying PILBase.create gives
      PILMask mode=L size=170x128

Final sample: (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)


Setting up after_item: Pipeline: AddMaskCodes -> ToTensor
Setting up before_batch: Pipeline: 
Setting up after_batch: Pipeline: IntToFloatTensor -- {'div': 255.0, 'div_mask': 1}
Could not do one pass in your dataloader, there is something wrong in it

Building one batch
Applying item_tfms to the first sample:
  Pipeline: AddMaskCodes -> ToTensor
    starting from
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying AddMaskCodes gives
      (PILImage mode=RGB size=170x128, PILMask mode=L size=170x128)
    applying ToTensor gives
      (TensorImage of size 3x128x170, TensorMask of size 128x170)

Adding the next 1 samples

No before_batch transform to apply

Collating items in a batch

Applying batch_tfms to the batch built
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-344-e05b279e209c> in <module>
      5                            get_y=get_y
      6                   )
----> 7 dblock.summary(path/'train', bs=2, show_batch=False)

/opt/conda/lib/python3.7/site-packages/fastai/data/block.py in summary(self, source, bs, show_batch, **kwargs)
    192     if len([f for f in dls.train.after_batch.fs if f.name != 'noop'])!=0:
    193         print("\nApplying batch_tfms to the batch built")
--> 194         b = to_device(b, dls.device)
    195         b = _apply_pipeline(dls.train.after_batch, b)
    196     else: print("\nNo batch_tfms to apply")

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in to_device(b, device)
    215     elif device is None: device=default_device()
    216     def _inner(o): return o.to(device, non_blocking=True) if isinstance(o,Tensor) else o.to_device(device) if hasattr(o, "to_device") else o
--> 217     return apply(_inner, b)
    218 
    219 # Cell

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in apply(func, x, *args, **kwargs)
    163 def apply(func, x, *args, **kwargs):
    164     "Apply `func` recursively to `x`, passing on args"
--> 165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
    167     res = func(x, *args, **kwargs)

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in <listcomp>(.0)
    163 def apply(func, x, *args, **kwargs):
    164     "Apply `func` recursively to `x`, passing on args"
--> 165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
    167     res = func(x, *args, **kwargs)

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in apply(func, x, *args, **kwargs)
    165     if is_listy(x): return type(x)([apply(func, o, *args, **kwargs) for o in x])
    166     if isinstance(x,dict):  return {k: apply(func, v, *args, **kwargs) for k,v in x.items()}
--> 167     res = func(x, *args, **kwargs)
    168     return res if x is None else retain_type(res, x)
    169 

/opt/conda/lib/python3.7/site-packages/fastai/torch_core.py in _inner(o)
    214     if defaults.use_cuda==False: device='cpu'
    215     elif device is None: device=default_device()
--> 216     def _inner(o): return o.to(device, non_blocking=True) if isinstance(o,Tensor) else o.to_device(device) if hasattr(o, "to_device") else o
    217     return apply(_inner, b)
    218 

RuntimeError: CUDA error: device-side assert triggered
1 Like
6

Okay, and I just tried writing it in one big python file and just executing it outside of a jupyter notebook. It works with the GPU.

from fastai.vision.all import *

path = Path('segmentation_test_1')
fnames = get_image_files(path/'train')
label_path = (path/'labels')
label_func = lambda o: label_path/f'{o.stem}_watershed_mask.png'
label_names = [label_func(n) for n in fnames]
codes = np.loadtxt(path/'codes.txt', dtype=str)


def n_codes(fnames, is_partial=True):
  "Gather the codes from a list of `fnames`"
  vals = set()
  if is_partial:
    random.shuffle(fnames)
    fnames = fnames[:10]
  for fname in fnames:
    msk = np.array(PILMask.create(fname))
    for val in np.unique(msk):
      if val not in vals:
        vals.add(val)
  vals = list(vals)
  p2c = dict()
  for i,val in enumerate(vals):
    p2c[i] = vals[i]
  return p2c

def get_msk(fn, pix2class):
  "Grab a mask from a `filename` and adjust the pixels based on `pix2class`"
  label_path = (path/'labels')
  fn = label_path/f'{fn.stem}_watershed_mask.png'
  msk = np.array(PILMask.create(fn))
  mx = np.max(msk)
  for i, val in enumerate(p2c):
    msk[msk==p2c[i]] = val
  return PILMask.create(msk)

p2c = n_codes(label_names)
get_y = lambda o: get_msk(o, p2c)
dblock = DataBlock(blocks=(ImageBlock, MaskBlock()),
                           splitter=RandomSplitter(0.2, seed=42),
                           get_items = get_image_files,
                           get_y=get_y
                  )

dblock.summary(path/'train', bs=2, show_batch=False)
1 Like
7

And I just shut down all of my other notebooks and tried running the original code in Jupyter, and that worked as well… shrug Hooray! It works now! :smiley:

1 Like