Problems with MultiCategoryBlock

I am trying to perform MultiCategory classification with some images in folders named ['LS', 'LP', 'LC', 'FS', 'FP', 'FC'].

I used Datablock as follows:

dblock = DataBlock(blocks    = (ImageBlock, MultiCategoryBlock,
                   get_items = get_image_files,
                   get_y     = parent_label , 
                   splitter  = RandomSplitter())

If I use dblock.summary(path) everything goes nicely and dls = dblock.dataloaders(path, bs=16, num_workers=8) too. dls.vocab gives (#5) ['C','F','L','P','S'] as expected.

However, when I run:

learn = cnn_learner(dls, resnet34, metrics=[accuracy])
learn.fit_one_cycle(1)

Got an AssertionError

epoch	train_loss	valid_loss	accuracy	time
0	0.350248	0.294326	None	03:11
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-130-4dfb24161c57> in <module>
----> 1 learn.fit_one_cycle(1)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/callback/schedule.py in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
    110     scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
    111               'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 112     self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
    113 
    114 # Cell

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
    191                         self.epoch=epoch;          self('begin_epoch')
    192                         self._do_epoch_train()
--> 193                         self._do_epoch_validate()
    194                     except CancelEpochException:   self('after_cancel_epoch')
    195                     finally:                       self('after_epoch')

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    173             dl,old,has = change_attrs(dl, names, [False,False])
    174             self.dl = dl;                                    self('begin_validate')
--> 175             with torch.no_grad(): self.all_batches()
    176         except CancelValidException:                         self('after_cancel_validate')
    177         finally:

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in all_batches(self)
    141     def all_batches(self):
    142         self.n_iter = len(self.dl)
--> 143         for o in enumerate(self.dl): self.one_batch(*o)
    144 
    145     def one_batch(self, i, b):

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in one_batch(self, i, b)
    155             self.opt.zero_grad()
    156         except CancelBatchException:                         self('after_cancel_batch')
--> 157         finally:                                             self('after_batch')
    158 
    159     def _do_begin_fit(self, n_epoch):

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in __call__(self, event_name)
    122     def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
    123 
--> 124     def __call__(self, event_name): L(event_name).map(self._call_one)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, *args, **kwargs)
    370              else f.format if isinstance(f,str)
    371              else f.__getitem__)
--> 372         return self._new(map(g, self))
    373 
    374     def filter(self, f, negate=False, **kwargs):

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    321     @property
    322     def _xtra(self): return None
--> 323     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    324     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    325     def copy(self): return self._new(self.items.copy())

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     39             return x
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0
     43         return res

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    312         if items is None: items = []
    313         if (use_list is not None) or not _is_array(items):
--> 314             items = list(items) if use_list else _listify(items)
    315         if match is not None:
    316             if is_coll(match): match = len(match)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in _listify(o)
    248     if isinstance(o, list): return o
    249     if isinstance(o, str) or _is_array(o): return [o]
--> 250     if is_iter(o): return list(o)
    251     return [o]
    252 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __call__(self, *args, **kwargs)
    214             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    215         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 216         return self.fn(*fargs, **kwargs)
    217 
    218 # Cell

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in _call_one(self, event_name)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)
--> 127         [cb(event_name) for cb in sort_by_run(self.cbs)]
    128 
    129     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in <listcomp>(.0)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)
--> 127         [cb(event_name) for cb in sort_by_run(self.cbs)]
    128 
    129     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/callback/core.py in __call__(self, event_name)
     22         _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
     23                (self.run_valid and not getattr(self, 'training', False)))
---> 24         if self.run and _run: getattr(self, event_name, noop)()
     25         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     26 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in after_batch(self)
    410         if len(self.yb) == 0: return
    411         mets = self._train_mets if self.training else self._valid_mets
--> 412         for met in mets: met.accumulate(self.learn)
    413         if not self.training: return
    414         self.lrs.append(self.opt.hypers[-1]['lr'])

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in accumulate(self, learn)
    335     def accumulate(self, learn):
    336         bs = find_bs(learn.yb)
--> 337         self.total += to_detach(self.func(learn.pred, *learn.yb))*bs
    338         self.count += bs
    339     @property

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/metrics.py in accuracy(inp, targ, axis)
     72 def accuracy(inp, targ, axis=-1):
     73     "Compute accuracy with `targ` when `pred` is bs * n_classes"
---> 74     pred,targ = flatten_check(inp.argmax(dim=axis), targ)
     75     return (pred == targ).float().mean()
     76 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/torch_core.py in flatten_check(inp, targ)
    770     "Check that `out` and `targ` have the same number of elements and flatten them."
    771     inp,targ = inp.contiguous().view(-1),targ.contiguous().view(-1)
--> 772     test_eq(len(inp), len(targ))
    773     return inp,targ

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/test.py in test_eq(a, b)
     30 def test_eq(a,b):
     31     "`test` that `a==b`"
---> 32     test(a,b,equals, '==')
     33 
     34 # Cell

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/test.py in test(a, b, cmp, cname)
     20     "`assert` that `cmp(a,b)`; display inputs and `cname or cmp.__name__` if it fails"
     21     if cname is None: cname=cmp.__name__
---> 22     assert cmp(a,b),f"{cname}:\n{a}\n{b}"
     23 
     24 # Cell

AssertionError: ==:
16
80

It seems to throw the error when validation step starts. I checked dls.train.vocab and dls.valid.vocab and both have the same labels. Also I tried to increase RandomSplitter to 0.5 but the error still persists.

I guess it’s something related with the labels but I am not quite sure where the problem is. I am using the last version of fastai (0.0.16) and fastcore (0.1.16)

Thanks

1 Like

Accuracy is for regular classification. You should use accuracy_thresh instead.

3 Likes

Accuracy_thresh or accuracy_multi? Is multi just for text?

Thank for this helpful thread!:wink:
I’m having the same issue in this Multi-Label Classification competition.
Like Joan, I should use metrics = accuracy_multi instead of metrics = accuracy in cnn_learner().

1 Like