Loading RAW image files into ImageBlock

Hey guys,

Problem: How to load image files of *.RAW format into ImageBlock?

Suggested solution:

Looking at: (Data.py)

# Cell
def ImageBlock(cls=PILImage):
    "A `TransformBlock` for images of `cls`"
    return TransformBlock(type_tfms=cls.create, batch_tfms=IntToFloatTensor)

What if I can create another cls that is based on rawpy API?

So far it looks like that PIL API supports mostly *.JPG format.

This issue is followed by this issue:

Defining your own image class would be the right solution.
The following suggestion could work:


def load_raw_image(fn):
    "Open and load a raw image with rawpy"
    # your function to load a raw image from fn/Path here
    # return a numpy array `arr` 
    return arr


class RawpyImage(PILBase):
    @classmethod
    def create(cls, fn:(Path,str), **kwargs)->None:
        "Open a raw image from path `fn`"
        arr = load_raw_image(fn)
        return cls(Image.fromarray(arr))

Make sure the format of the np.array generated in load_raw_image is either uint8 (integer values between 0 and 255) or float32 (float values between 0 and 1).

1 Like

Thanks man! Sounds great.

Make sure the format of the np.array generated in load_raw_image is either uint8 (integer values between 0 and 255) or float32 (float values between 0 and 1).

JPG has about 8 bits for each pixel, so there are at most 256 different values. But RAW files have about 16 bits for each pixel, so there are at most 65536 different values. So would I need uint16, isn’t it?

In theory, yes, but IIRC, uint16 cannot be transformed to a PILImage instance. I had those problems when reading DICOM files, which are also 16 bit. Going with float values, rescaled to values between 0 and 1 might be easier.

So if I get it right, I should unpack the image file into a matrix, then divide every pixel by 65536, then convert it into np.array of the datatype of float32?

Yes. This should work.

1 Like

Hi !
I want to load 16 bits png images. Does fastai do this normalization for me?

Hi BresNet !
How does this fit in a simple image classification script? Where I call this class?

Maybe here:

ds = DataBlock(blocks = (ImageBlock(cls=RawpyImage),CategoryBlock),
                  get_items = get_image_files,
                  splitter = GrandparentSplitter(),
                  get_y = parent_label)

:thinking:

I’ve tried this:

def load_16bits_image(fn):
    img = Image.open(fn)
    arr = np.array(img) / 65365

    return arr


class Grayscale16bitsImage(PILBase):
    @classmethod
    def create(cls, fn : (Path, str), **kwargs)->None:
        "Open a raw image from path `fn`"
        arr = load_16bits_image(fn)
        return cls(Image.fromarray(arr))

path = Path('data')

item_tfms = [Resize(224)]

batch_tfms = aug_transforms(
    mult=2, do_flip=False, max_rotate=15
)

dblock = DataBlock(
    blocks=(ImageBlock(cls=Grayscale16bitsImage), CategoryBlock),
    get_items=get_image_files,
    splitter=GrandparentSplitter(train_name='train', valid_name='valid'),
    get_y=parent_label,
    item_tfms=item_tfms,
    batch_tfms=batch_tfms)

But I get the following error:

AttributeError: _tensor_cls
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-56-10885883025c> in <module>
     32 print(classes)
     33 
---> 34 dls_patches_one.show_batch()
     35 

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\data\core.py in show_batch(self, b, max_n, ctxs, show, unique, **kwargs)
     98             old_get_idxs = self.get_idxs
     99             self.get_idxs = lambda: Inf.zeros
--> 100         if b is None: b = self.one_batch()
    101         if not show: return self._pre_show_batch(b, max_n=max_n)
    102         show_batch(*self._pre_show_batch(b, max_n=max_n), ctxs=ctxs, max_n=max_n, **kwargs)

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\data\load.py in one_batch(self)
    146     def one_batch(self):
    147         if self.n is not None and len(self)==0: raise ValueError(f'This DataLoader does not contain any batches')
--> 148         with self.fake_l.no_multiproc(): res = first(self)
    149         if hasattr(self, 'it'): delattr(self, 'it')
    150         return res

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\basics.py in first(x, f, negate, **kwargs)
    545     x = iter(x)
    546     if f: x = filter_ex(x, f=f, negate=negate, gen=True, **kwargs)
--> 547     return next(x, None)
    548 
    549 # Cell

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\data\load.py in __iter__(self)
    107         self.before_iter()
    108         self.__idxs=self.get_idxs() # called in context of main process (not workers/subprocesses)
--> 109         for b in _loaders[self.fake_l.num_workers==0](self.fake_l):
    110             if self.device is not None: b = to_device(b, self.device)
    111             yield self.after_batch(b)

~\Anaconda3\envs\dlenv\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    519             if self._sampler_iter is None:
    520                 self._reset()
--> 521             data = self._next_data()
    522             self._num_yielded += 1
    523             if self._dataset_kind == _DatasetKind.Iterable and \

~\Anaconda3\envs\dlenv\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
    559     def _next_data(self):
    560         index = self._next_index()  # may raise StopIteration
--> 561         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    562         if self._pin_memory:
    563             data = _utils.pin_memory.pin_memory(data)

~\Anaconda3\envs\dlenv\lib\site-packages\torch\utils\data\_utils\fetch.py in fetch(self, possibly_batched_index)
     32                 raise StopIteration
     33         else:
---> 34             data = next(self.dataset_iter)
     35         return self.collate_fn(data)
     36 

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\data\load.py in create_batches(self, samps)
    116         if self.dataset is not None: self.it = iter(self.dataset)
    117         res = filter(lambda o:o is not None, map(self.do_item, samps))
--> 118         yield from map(self.do_batch, self.chunkify(res))
    119 
    120     def new(self, dataset=None, cls=None, **kwargs):

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\basics.py in chunked(it, chunk_sz, drop_last, n_chunks)
    214     if not isinstance(it, Iterator): it = iter(it)
    215     while True:
--> 216         res = list(itertools.islice(it, chunk_sz))
    217         if res and (len(res)==chunk_sz or not drop_last): yield res
    218         if len(res)<chunk_sz: return

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\data\load.py in do_item(self, s)
    131     def prebatched(self): return self.bs is None
    132     def do_item(self, s):
--> 133         try: return self.after_item(self.create_item(s))
    134         except SkipItemException: return None
    135     def chunkify(self, b): return b if self.prebatched else chunked(b, self.bs, self.drop_last)

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in __call__(self, o)
    198         self.fs = self.fs.sorted(key='order')
    199 
--> 200     def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
    201     def __repr__(self): return f"Pipeline: {' -> '.join([f.name for f in self.fs if f.name != 'noop'])}"
    202     def __getitem__(self,i): return self.fs[i]

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in compose_tfms(x, tfms, is_enc, reverse, **kwargs)
    148     for f in tfms:
    149         if not is_enc: f = f.decode
--> 150         x = f(x, **kwargs)
    151     return x
    152 

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in __call__(self, x, **kwargs)
     71     @property
     72     def name(self): return getattr(self, '_name', _get_name(self))
---> 73     def __call__(self, x, **kwargs): return self._call('encodes', x, **kwargs)
     74     def decode  (self, x, **kwargs): return self._call('decodes', x, **kwargs)
     75     def __repr__(self): return f'{self.name}:\nencodes: {self.encodes}decodes: {self.decodes}'

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in _call(self, fn, x, split_idx, **kwargs)
     81     def _call(self, fn, x, split_idx=None, **kwargs):
     82         if split_idx!=self.split_idx and self.split_idx is not None: return x
---> 83         return self._do_call(getattr(self, fn), x, **kwargs)
     84 
     85     def _do_call(self, f, x, **kwargs):

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in _do_call(self, f, x, **kwargs)
     88             ret = f.returns(x) if hasattr(f,'returns') else None
     89             return retain_type(f(x, **kwargs), x, ret)
---> 90         res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
     91         return retain_type(res, x)
     92 

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in <genexpr>(.0)
     88             ret = f.returns(x) if hasattr(f,'returns') else None
     89             return retain_type(f(x, **kwargs), x, ret)
---> 90         res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
     91         return retain_type(res, x)
     92 

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\transform.py in _do_call(self, f, x, **kwargs)
     87             if f is None: return x
     88             ret = f.returns(x) if hasattr(f,'returns') else None
---> 89             return retain_type(f(x, **kwargs), x, ret)
     90         res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
     91         return retain_type(res, x)

~\Anaconda3\envs\dlenv\lib\site-packages\fastcore\dispatch.py in __call__(self, *args, **kwargs)
    116         elif self.inst is not None: f = MethodType(f, self.inst)
    117         elif self.owner is not None: f = MethodType(f, self.owner)
--> 118         return f(*args, **kwargs)
    119 
    120     def __get__(self, inst, owner):

~\Anaconda3\envs\dlenv\lib\site-packages\fastai\vision\core.py in encodes(self, o)
    221 # Cell
    222 @ToTensor
--> 223 def encodes(self, o:PILBase): return o._tensor_cls(image2tensor(o))
    224 @ToTensor
    225 def encodes(self, o:PILMask): return o._tensor_cls(image2tensor(o)[0])

~\Anaconda3\envs\dlenv\lib\site-packages\PIL\Image.py in __getattr__(self, name)
    544             )
    545             return self._category
--> 546         raise AttributeError(name)
    547 
    548     @property

AttributeError: _tensor_cls

Maybe load_16bits_image(fn) isn’t loading correctly the images here img = Image.open(fn). How can I see the values of fn?

My guess would be, too, that the error is in load_16bits_image_().
Try loading a file with the function only and check if you get a valid numpy array.

arr = load_16bits_image_(`path/to/file`)
arr.shape, arr.max(), arr.min()

If that works, maybe the error is in your getters. Does get_image_files return a valid file name? you could use print debugging in load_16bits_image_() function. This way, you’ll know what the values of fn are. It should be a valid FileName.