Issue with loading and displaying bounding boxes

Hey, first post here, so let me know if I’m omitting necessary information.

=== Software ===
python version : 3.7.0
fastai version : 1.0.25.dev0
torch version  : 1.0.0.dev20181116
torch cuda ver
torch cuda is  : **Not available**

=== Hardware ===
No GPUs available

=== Environment ===
platform       : Linux-4.4.0-53-generic-x86_64-with-debian-stretch-sid
distro         : #74-Ubuntu SMP Fri Dec 2 15:59:10 UTC 2016
conda env      : fastai-cpu
python         : /home/username/anaconda3/envs/fastai-cpu/bin/python
sys.path       :
/home/username/anaconda3/envs/fastai-cpu/lib/python37.zip
/home/username/anaconda3/envs/fastai-cpu/lib/python3.7
/home/username/anaconda3/envs/fastai-cpu/lib/python3.7/lib-dynload
/home/username/anaconda3/envs/fastai-cpu/lib/python3.7/site-packages
/home/username/anaconda3/envs/fastai-cpu/lib/python3.7/site-packages/IPython/extensions
no supported gpus found on this system

I’m trying to do object detection on my own dataset, but I have so far been unable to load the dataset correctly. I have the following code and when I run it dataset.show_batch will fail for certain images.

import fastai
from fastai.vision import (data as Data,
learner as Learner,
models as Models,
transform as Transform)
from torch import nn
from pathlib import Path
import matplotlib.pyplot as plt
import pandas as pd
import json
import collections
import numpy as np

PATH = Path(’/home/username/Dropbox/workspace/fastai/data/animal’)
train_json = json.load((PATH / ‘dataset.json’).open())

IMAGES,ANNOTATIONS,CATEGORIES = [‘images’, ‘annotations’, ‘categories’]
FILE_NAME,ID,IMG_ID,CAT_ID,BBOX = ‘file_name’,‘id’,‘image_id’,‘category_id’,‘bbox’

cats = dict((o[ID], o[‘name’]) for o in train_json[CATEGORIES])
train_filenames = dict((o[ID], o[FILE_NAME]) for o in train_json[IMAGES])
train_ids = [o[ID] for o in train_json[IMAGES]]

IMG_PATH = Path(’/home/username/Dropbox/workspace/fastai/data/animal’)

id2cat = list(cats.values())

def get_train_anno_by_filename(train_json):
train_anno = collections.defaultdict(lambda:[])
for o in train_json[ANNOTATIONS]:
if not o[‘ignore’]:
bb = o[BBOX]
bb = list([bb[1], bb[0], bb[3]+bb[1]-1, bb[2]+bb[0]-1])
fn = train_filenames[o[IMG_ID]]
cat = id2cat[o[CAT_ID]]
train_anno[fn].append((bb, cat))
fn2anno = {}
for fn, targets in train_anno.items():
fn2anno[fn] = [list(e) for e in zip(*targets)]
return fn2anno

fn2anno = get_train_anno_by_filename(train_json)

df = pd.DataFrame(
{‘fn’: [train_filenames[i] for i in train_ids]},
columns=[‘fn’]
)

tfms = Transform.get_transforms(
xtra_tfms=Transform.rand_resize_crop(224)
)

def get_y_func(o):
fn = Path(o).name
res = fn2anno[fn]
return res

dataset = (Data.ObjectItemList.from_df(
df_reduced,
IMG_PATH
)
.random_split_by_pct()
.label_from_func(get_y_func)
.transform(tfms, tfm_y=True)
.databunch(bs=4, collate_fn=Data.bb_pad_collate))

dataset.show_batch(rows=2, ds_type=Data.DatasetType.Valid, figsize=(6,6))

I get the following error


TypeError Traceback (most recent call last)
in
----> 1 dataset.show_batch(rows=2, ds_type=Data.DatasetType.Valid, figsize=(6,6))

~/Dropbox/workspace/fastai/fastai/basic_data.py in show_batch(self, rows, ds_type, **kwargs)
119 if rows is None: rows = int(math.sqrt(len(b_idx)))
120 ds = dl.dataset
–> 121 ds[0][0].show_batch(b_idx, rows, ds, **kwargs)
122
123 @property

~/Dropbox/workspace/fastai/fastai/data_block.py in getitem(self, idxs)
343 def getitem(self,idxs:Union[int,np.ndarray])->‘LabelList’:
344 if isinstance(try_int(idxs), int):
–> 345 if self.item is None: x,y = self.x[idxs],self.y[idxs]
346 else: x,y = self.item ,0
347 if self.tfms:

~/Dropbox/workspace/fastai/fastai/data_block.py in getitem(self, idxs)
77
78 def getitem(self,idxs:int)->Any:
—> 79 if isinstance(try_int(idxs), int): return self.get(idxs)
80 else: return self.new(self.items[idxs], xtra=index_row(self.xtra, idxs))
81

~/Dropbox/workspace/fastai/fastai/vision/data.py in get(self, i)
305
306 def get(self, i):
–> 307 return ImageBBox.create(*self.x.sizes[i], *self.items[i])
308
309 class ObjectItemList(ImageItemList):

~/Dropbox/workspace/fastai/fastai/vision/image.py in create(cls, h, w, bboxes, labels, c2i, pad_idx)
–> 390 bboxes = tensor(bboxes).float()
391 tr_corners = torch.cat([bboxes[:,0][:,None], bboxes[:,3][:,None]], 1)
392 bl_corners = bboxes[:,1:3].flip(1)

~/Dropbox/workspace/fastai/fastai/torch_core.py in tensor(x, *rest)
66 “Like torch.as_tensor, but handle lists too, and can pass multiple vector elements directly”
67 if len(rest): x = (x,)+rest
—> 68 return torch.tensor(x) if is_listy(x) else as_tensor(x)
69
70 def np_address(x:np.ndarray)->int:

TypeError: can’t convert np.ndarray of type numpy.object_. The only supported types are: double, float, float16, int64, int32, and uint8.

When I print bboxes before the program fails I consistently get [list(x, y, z, c)] when it fails vs [[x, y, z, c]]. So I can fix this problem by doing bboxes = [list(bbox) for bbox in bboxes], but I am very confused as to why this is an issue in the first place. All bounding boxes are made the same way for all images. Additionally, this only happens for the valid set, never for the train set.

Further more, if I do show_batch for the train set, the bounding boxes become displaced because of the transformations. This seems to be because of the rand_resize_crop transform. Is it a known issue that it does not work or am I doing something wrong?

Thanks:)

1 Like

@locus Hi, did you make any progress on solving this problem? I was trying to run something similar but the code doesn’t behave consistently…For example, if I set fn2anno like this:

train_anno = {
    'file1': [(list([1, 2, 3, 4]), 0), (list([3, 4, 5, 6]), 1)]
}

then the dataset loading fails with this error

Traceback (most recent call last):
  File "/kdang/IDDetection/try_image_datalist.py", line 46, in <module>
    .label_from_func(get_label) \
  File "/usr/local/lib/python3.6/dist-packages/fastai/data_block.py", line 371, in _inner
    self.train = ft(*args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/fastai/data_block.py", line 231, in label_from_func
    return self.label_from_list([func(o) for o in self.items], **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/fastai/data_block.py", line 210, in label_from_list
    labels = array(labels, dtype=object)
  File "/usr/local/lib/python3.6/dist-packages/fastai/core.py", line 246, in array
    return np.array(a, *args, **kwargs)
ValueError: could not broadcast input array from shape (2,4) into shape (2)

Could you share how you started the code and how you know the label format should be in this format? Thanks

Note that ObjectItemList is our default to go with the function get_annotations, so you have to make sure your inputs are exactly the same way if you don’t use it. Maybe work around the example in the data block API docs?
We won’t go back to object detection until January at least, so there isn’t going to be any other example before that.

Hi @locus,
I get exact the same error. I use this code to generate the databunch.

get_y_func = lambda o:[[[0.,0.,100.,100.],[200.,200.,100.,100.]],[‘text’,‘text’]]
data = (ObjectItemList.from_df(img_df, path, folder=’’)
.random_split_by_pct()
.label_from_func(get_y_func)
.transform(get_transforms(), size=300, tfm_y=True)
.databunch(bs=20, collate_fn=bb_pad_collate))

When I change the else part in the tensor function to

as_tensor([x[0],x[1]]) (works for 2 bounding boxes)

This works and the bounding boxes seem to be displayed correctly.

Where did you add this code? bboxes = [list(bbox) for bbox in bboxes]
In the fastai code or in your notebook?

In this code (inside the array function) the fastai library is changing it to an numpy object

def label_from_list(self, labels:Iterator, **kwargs)->‘LabelList’:
"Label self.items with labels using label_cls"
labels = array(labels, dtype=object)
label_cls = self.get_label_cls(labels, **kwargs)
y = label_cls(labels, path=self.path, **kwargs)
res = self._label_list(x=self, y=y)
return res

I’m continuing debugging. Let me know if you have progress!

PS: I tried to exactly reproduce the output of the get_annotations function … but probably I oversee something there.

I will post something when I have something that works:)

1 Like

Do you get the exact same stack trace as me? Go to the ImageBBox class’s create() function. Put bboxes = [list(bbox) for bbox in bboxes] on the first line there.

Thanks.

Yes I do get the exact trace. The thing is that one dataset is loading correctly and another is not. The input which get_y_func generates looks exact the same in jupyter notebook, same types and same formats.

The only difference is the way jupyter is displaying the output:

The working input is displayed in a jupyter notebook like:
53

The not working input is displayed in a jupyter notebook like
11

so to be exact, you had to change fastai’s code to make it work? Because I tried multiple variations of the input it still doesn’t work

Yes I do get the exact trace. The thing is that one dataset is loading correctly and another is not. The input which get_y_func generates looks exact the same in jupyter notebook, same types and same formats.

Yes, I noticed that displaying Train works, but valid doesn’t (or the other way aorund, without the fix). But I haven’t had time to look into why there’s a discrepancy.

The only difference is the way jupyter is displaying the output:

Which one of those do not work? looks like both should work. Does one belong to the valid dataset and the other not?

so to be exact, you had to change fastai’s code to make it work?

That is correct.

Which one of those do not work? looks like both should work. Does one belong to the valid dataset and the other not?

The first one does work, the second one gives an error (like your stack trace). It is working correctly with your update. I have no clue why the first is working without your correction and the second fails…

Yesterday eve, I started again from the beginning, using the get_annotations function. It now works, but I’m still figuring out why :-). I will post the working version these days.

See attached my notebook (exported as pdf) Oxford synth text.pdf (911.2 KB)

It seems to load the dataset and annotations correctly by ‘data.show_batch()’

On training I encounter the next issue. Which probably has to do with incorrect loss function:


ValueError Traceback (most recent call last)
in
----> 1 learn.fit_one_cycle(2)

/opt/anaconda3/lib/python3.6/site-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, wd, callbacks, **kwargs)
18 callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor,
19 pct_start=pct_start, **kwargs))
—> 20 learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
21
22 def lr_find(learn:Learner, start_lr:Floats=1e-7, end_lr:Floats=10, num_it:int=100, stop_div:bool=True, **kwargs:Any):

/opt/anaconda3/lib/python3.6/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
160 callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
161 fit(epochs, self.model, self.loss_func, opt=self.opt, data=self.data, metrics=self.metrics,
–> 162 callbacks=self.callbacks+callbacks)
163
164 def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/opt/anaconda3/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
92 except Exception as e:
93 exception = e
—> 94 raise e
95 finally: cb_handler.on_train_end(exception)
96

/opt/anaconda3/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, model, loss_func, opt, data, callbacks, metrics)
82 for xb,yb in progress_bar(data.train_dl, parent=pbar):
83 xb, yb = cb_handler.on_batch_begin(xb, yb)
—> 84 loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
85 if cb_handler.on_batch_end(loss): break
86

/opt/anaconda3/lib/python3.6/site-packages/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
20
21 if not loss_func: return to_detach(out), yb[0].detach()
—> 22 loss = loss_func(out, *yb)
23
24 if opt is not None:

/opt/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py in binary_cross_entropy_with_logits(input, target, weight, size_average, reduce, reduction, pos_weight)
1768
1769 if not (target.size() == input.size()):
-> 1770 raise ValueError(“Target size ({}) must be the same as input size ({})”.format(target.size(), input.size()))
1771
1772 return torch.binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction)

ValueError: Target size (torch.Size([20, 11, 4])) must be the same as input size (torch.Size([20, 2]))

Ramon what you are doing does not make much sense. Resnet does not do object localization and you have not specified a custom head/loss function. Look at this, it might clear up your confusion.

Thanks for your reply. It was indeed a quick, not well though line of code :smiley:
I indeed planned to have a relook at the lesson 8 of the previous fastai lib, thanks for sharing the link!