Lesson1 with apples and oranges - please help with errors


(Anant) #1

HI all,

As suggested, I’m trying to use a custom image dataset for Lesson 1. I downloaded some using google-image-downloader mentioned here, and got 77 images for each label.

I used a 60-20-20 for train-valid-test, and created similar dir struct as catsdogs.

When I ran learn.fit, this is the error I’m seeing:

---------------------------------------------------------------------
error                               Traceback (most recent call last)
~/fastai/courses/dl1/fastai/dataset.py in open_image(fn)
    134         try:
--> 135             return cv2.cvtColor(cv2.imread(fn, flags), cv2.COLOR_BGR2RGB).astype(np.float32)/255
    136         except Exception as e:

error: /io/opencv/modules/imgproc/src/color.cpp:11079: error: (-215) scn == 3 || scn == 4 in function cvtColor


The above exception was the direct cause of the following exception:

OSError                             Traceback (most recent call last)
<ipython-input-81-e6c87b20ce86> in <module>()
      1 arch=resnet34
      2 data = ImageClassifierData.from_paths(PATH, tfms=tfms_from_model(arch, sz))
----> 3 learn = ConvLearner.pretrained(arch, data, precompute=True)
      4 learn.fit(0.01, 2)

~/fastai/courses/dl1/fastai/conv_learner.py in pretrained(cls, f, data, ps, xtra_fc, xtra_cut, **kwargs)
     96     def pretrained(cls, f, data, ps=None, xtra_fc=None, xtra_cut=0, **kwargs):
     97         models = ConvnetBuilder(f, data.c, data.is_multi, data.is_reg, ps=ps, xtra_fc=xtra_fc, xtra_cut=xtra_cut)
---> 98         return cls(data, models, **kwargs)
     99 
    100     @property

~/fastai/courses/dl1/fastai/conv_learner.py in __init__(self, data, models, precompute, **kwargs)
     89         elif self.metrics is None:
     90             self.metrics = [accuracy_thresh(0.5)] if self.data.is_multi else [accuracy]
---> 91         if precompute: self.save_fc1()
     92         self.freeze()
     93         self.precompute = precompute

~/fastai/courses/dl1/fastai/conv_learner.py in save_fc1(self)
    141         m=self.models.top_model
    142         if len(self.activations[0])!=len(self.data.trn_ds):
--> 143             predict_to_bcolz(m, self.data.fix_dl, act)
    144         if len(self.activations[1])!=len(self.data.val_ds):
    145             predict_to_bcolz(m, self.data.val_dl, val_act)

~/fastai/courses/dl1/fastai/model.py in predict_to_bcolz(m, gen, arr, workers)
     11     lock=threading.Lock()
     12     m.eval()
---> 13     for x,*_ in tqdm(gen):
     14         y = to_np(m(VV(x)).data)
     15         with lock:

~/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py in __iter__(self)
    951 """, fp_write=getattr(self.fp, 'write', sys.stderr.write))
    952 
--> 953             for obj in iterable:
    954                 yield obj
    955                 # Update and possibly print the progressbar.

~/fastai/courses/dl1/fastai/dataset.py in __next__(self)
    241         if self.i>=len(self.dl): raise StopIteration
    242         self.i+=1
--> 243         return next(self.it)
    244 
    245     @property

~/fastai/courses/dl1/fastai/dataloader.py in __iter__(self)
     73     def __iter__(self):
     74         with ThreadPoolExecutor(max_workers=self.num_workers) as e:
---> 75             for batch in e.map(self.get_batch, iter(self.batch_sampler)):
     76                 yield get_tensor(batch, self.pin_memory)
     77 

~/anaconda3/envs/fastai/lib/python3.6/concurrent/futures/_base.py in result_iterator()
    584                     # Careful not to keep a reference to the popped future
    585                     if timeout is None:
--> 586                         yield fs.pop().result()
    587                     else:
    588                         yield fs.pop().result(end_time - time.time())

~/anaconda3/envs/fastai/lib/python3.6/concurrent/futures/_base.py in result(self, timeout)
    423                 raise CancelledError()
    424             elif self._state == FINISHED:
--> 425                 return self.__get_result()
    426 
    427             self._condition.wait(timeout)

~/anaconda3/envs/fastai/lib/python3.6/concurrent/futures/_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

~/anaconda3/envs/fastai/lib/python3.6/concurrent/futures/thread.py in run(self)
     54 
     55         try:
---> 56             result = self.fn(*self.args, **self.kwargs)
     57         except BaseException as exc:
     58             self.future.set_exception(exc)

~/fastai/courses/dl1/fastai/dataloader.py in get_batch(self, indices)
     66 
     67     def get_batch(self, indices):
---> 68         res = self.collate_fn([self.dataset[i] for i in indices], self.pad_idx)
     69         if not self.transpose: return res
     70         res[0] = res[0].T

~/fastai/courses/dl1/fastai/dataloader.py in <listcomp>(.0)
     66 
     67     def get_batch(self, indices):
---> 68         res = self.collate_fn([self.dataset[i] for i in indices], self.pad_idx)
     69         if not self.transpose: return res
     70         res[0] = res[0].T

~/fastai/courses/dl1/fastai/dataset.py in __getitem__(self, idx)
     94 
     95     def __getitem__(self, idx):
---> 96         x,y = self.get_x(idx),self.get_y(idx)
     97         return self.get(self.transform, x, y)
     98 

~/fastai/courses/dl1/fastai/dataset.py in get_x(self, i)
    143     def get_n(self): return len(self.y)
    144     def get_sz(self): return self.transform.sz
--> 145     def get_x(self, i): return open_image(os.path.join(self.path, self.fnames[i]))
    146 
    147     def resize_imgs(self, targ, new_path):

~/fastai/courses/dl1/fastai/dataset.py in open_image(fn)
    135             return cv2.cvtColor(cv2.imread(fn, flags), cv2.COLOR_BGR2RGB).astype(np.float32)/255
    136         except Exception as e:
--> 137             raise OSError('Error handling image at: {}'.format(fn)) from e
    138 
    139 class FilesDataset(BaseDataset):

OSError: Error handling image at: data/applesoranges/train/oranges/30811-oranges.jpg

Could someone help?

Thanks,
Anant


(Anant) #2

OK, I installed imagemagick with sudo apt-get, and it does not identify this file as a jpeg. removed the file, and my trainer completed.

Lesson learned: Verify data for proper jpg using imagemagick (probably a script in the making)

Anyways, now I see some horrible accuracy. Fun to be had…


#3

Hi, I had the same problem. You can just type ‘file’ and then the file name to see if the file is an image file.


(Santosh) #4

Hi @shimsan

Were you able to resolve this issue?

I am seeing the following issue -
im = open_image(IMG_PATH/im0_d[FILE_NAME])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
in ()
1 import numpy
----> 2 im = open_image(IMG_PATH/im0_d[FILE_NAME])

/usr/local/lib/python3.6/dist-packages/fastai/dataset.py in open_image(fn)
117 def open_image(fn):
118 flags = cv2.IMREAD_UNCHANGED+cv2.IMREAD_ANYDEPTH+cv2.IMREAD_ANYCOLOR
–> 119 return cv2.cvtColor(cv2.imread(fn, flags), cv2.COLOR_BGR2RGB).astype(np.float32)/255
120
121 class FilesDataset(BaseDataset):

TypeError: bad argument type for built-in operation