Multiple target for tabular model

fastai
1

Hi,

I can’t make multi label classification to train on tabular data.
I have 5 categorical targets for each row of data.
The y targets have dtype of “CategoricalDtype”
When using lr_find with the following code, I get “ValueError: Expected input batch_size (64) to match target batch_size (320).”

This is the code I used:

to = TabularPandas(df, procs=[Categorify, Normalize],
                   cat_names=cat_cols,
                   cont_names=cont_cols,
                   y_names=y_names,
                   splits= splits)
dls = to.dataloaders(bs=64)
learn = tabular_learner(dls, metrics=[accuracy])
learn.lr_find()

this is the traceback with the error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [345], in <cell line: 1>()
----> 1 learn.lr_find()

File /opt/conda/lib/python3.9/site-packages/fastai/callback/schedule.py:289, in lr_find(self, start_lr, end_lr, num_it, stop_div, show_plot, suggest_funcs)
    287 n_epoch = num_it//len(self.dls.train) + 1
    288 cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)
--> 289 with self.no_logging(): self.fit(n_epoch, cbs=cb)
    290 if suggest_funcs is not None:
    291     lrs, losses = tensor(self.recorder.lrs[num_it//10:-5]), tensor(self.recorder.losses[num_it//10:-5])

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:222, in Learner.fit(self, n_epoch, lr, wd, cbs, reset_opt)
    220 self.opt.set_hypers(lr=self.lr if lr is None else lr)
    221 self.n_epoch = n_epoch
--> 222 self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:164, in Learner._with_events(self, f, event_type, ex, final)
    163 def _with_events(self, f, event_type, ex, final=noop):
--> 164     try: self(f'before_{event_type}');  f()
    165     except ex: self(f'after_cancel_{event_type}')
    166     self(f'after_{event_type}');  final()

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:213, in Learner._do_fit(self)
    211 for epoch in range(self.n_epoch):
    212     self.epoch=epoch
--> 213     self._with_events(self._do_epoch, 'epoch', CancelEpochException)

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:164, in Learner._with_events(self, f, event_type, ex, final)
    163 def _with_events(self, f, event_type, ex, final=noop):
--> 164     try: self(f'before_{event_type}');  f()
    165     except ex: self(f'after_cancel_{event_type}')
    166     self(f'after_{event_type}');  final()

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:207, in Learner._do_epoch(self)
    206 def _do_epoch(self):
--> 207     self._do_epoch_train()
    208     self._do_epoch_validate()

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:199, in Learner._do_epoch_train(self)
    197 def _do_epoch_train(self):
    198     self.dl = self.dls.train
--> 199     self._with_events(self.all_batches, 'train', CancelTrainException)

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:164, in Learner._with_events(self, f, event_type, ex, final)
    163 def _with_events(self, f, event_type, ex, final=noop):
--> 164     try: self(f'before_{event_type}');  f()
    165     except ex: self(f'after_cancel_{event_type}')
    166     self(f'after_{event_type}');  final()

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:170, in Learner.all_batches(self)
    168 def all_batches(self):
    169     self.n_iter = len(self.dl)
--> 170     for o in enumerate(self.dl): self.one_batch(*o)

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:195, in Learner.one_batch(self, i, b)
    193 b = self._set_device(b)
    194 self._split(b)
--> 195 self._with_events(self._do_one_batch, 'batch', CancelBatchException)

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:164, in Learner._with_events(self, f, event_type, ex, final)
    163 def _with_events(self, f, event_type, ex, final=noop):
--> 164     try: self(f'before_{event_type}');  f()
    165     except ex: self(f'after_cancel_{event_type}')
    166     self(f'after_{event_type}');  final()

File /opt/conda/lib/python3.9/site-packages/fastai/learner.py:176, in Learner._do_one_batch(self)
    174 self('after_pred')
    175 if len(self.yb):
--> 176     self.loss_grad = self.loss_func(self.pred, *self.yb)
    177     self.loss = self.loss_grad.clone()
    178 self('after_loss')

File /opt/conda/lib/python3.9/site-packages/fastai/losses.py:35, in BaseLoss.__call__(self, inp, targ, **kwargs)
     33 if targ.dtype in [torch.int8, torch.int16, torch.int32]: targ = targ.long()
     34 if self.flatten: inp = inp.view(-1,inp.shape[-1]) if self.is_2d else inp.view(-1)
---> 35 return self.func.__call__(inp, targ.view(-1) if self.flatten else targ, **kwargs)

File /opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py:1110, in Module._call_impl(self, *input, **kwargs)
   1106 # If we don't have any hooks, we want to skip the rest of the logic in
   1107 # this function, and just call forward.
   1108 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110     return forward_call(*input, **kwargs)
   1111 # Do not call functions when jit is used
   1112 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.9/site-packages/torch/nn/modules/loss.py:1163, in CrossEntropyLoss.forward(self, input, target)
   1162 def forward(self, input: Tensor, target: Tensor) -> Tensor:
-> 1163     return F.cross_entropy(input, target, weight=self.weight,
   1164                            ignore_index=self.ignore_index, reduction=self.reduction,
   1165                            label_smoothing=self.label_smoothing)

File /opt/conda/lib/python3.9/site-packages/torch/nn/functional.py:2982, in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
   2916 r"""This criterion computes the cross entropy loss between input and target.
   2917 
   2918 See :class:`~torch.nn.CrossEntropyLoss` for details.
   (...)
   2979     >>> loss.backward()
   2980 """
   2981 if has_torch_function_variadic(input, target, weight):
-> 2982     return handle_torch_function(
   2983         cross_entropy,
   2984         (input, target, weight),
   2985         input,
   2986         target,
   2987         weight=weight,
   2988         size_average=size_average,
   2989         ignore_index=ignore_index,
   2990         reduce=reduce,
   2991         reduction=reduction,
   2992         label_smoothing=label_smoothing,
   2993     )
   2994 if size_average is not None or reduce is not None:
   2995     reduction = _Reduction.legacy_get_string(size_average, reduce)

File /opt/conda/lib/python3.9/site-packages/torch/overrides.py:1394, in handle_torch_function(public_api, relevant_args, *args, **kwargs)
   1388     warnings.warn("Defining your `__torch_function__ as a plain method is deprecated and "
   1389                   "will be an error in PyTorch 1.11, please define it as a classmethod.",
   1390                   DeprecationWarning)
   1392 # Use `public_api` instead of `implementation` so __torch_function__
   1393 # implementations can do equality/identity comparisons.
-> 1394 result = torch_func_method(public_api, types, args, kwargs)
   1396 if result is not NotImplemented:
   1397     return result

File /opt/conda/lib/python3.9/site-packages/fastai/torch_core.py:341, in TensorBase.__torch_function__(self, func, types, args, kwargs)
    339 convert=False
    340 if _torch_handled(args, self._opt, func): convert,types = type(self),(torch.Tensor,)
--> 341 res = super().__torch_function__(func, types, args=args, kwargs=kwargs)
    342 if convert: res = convert(res)
    343 if isinstance(res, TensorBase): res.set_meta(self, as_copy=True)

File /opt/conda/lib/python3.9/site-packages/torch/_tensor.py:1142, in Tensor.__torch_function__(cls, func, types, args, kwargs)
   1139     return NotImplemented
   1141 with _C.DisableTorchFunction():
-> 1142     ret = func(*args, **kwargs)
   1143     if func in get_default_nowrap_functions():
   1144         return ret

File /opt/conda/lib/python3.9/site-packages/torch/nn/functional.py:2996, in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
   2994 if size_average is not None or reduce is not None:
   2995     reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2996 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)

ValueError: Expected input batch_size (64) to match target batch_size (320).

Thanks in advance!

2

Hi @OFire

You will have to make a couple of changes to your code

  1. Since you’re doing multi-label classification, you cannot pass accuracy as a metric, you’ll need to change it to accuracy_multi
  2. Also, you should specify an explicit loss function in the case where you don’t want to do a standard classification i.e. BCEWithLogitsLossFlat in this case.

So your learner definition will change to something like this

# Specify the loss function and appropriate metric
learn = tabular_learner(dls, metrics=[accuracy_multi], loss_func = BCEWithLogitsLossFlat())
learn.lr_find()

image

This should make your code work. I have tried to replicate a simple multilabel tabular classification problem in the below gist for demonstrating the same.
Refer here for the example

Hope this helps.

Thanks! :slight_smile:

2 Likes
3

Hi @ElisonSherton ,
First of all, I really do appreciate your answer and the effort of makeing a gist for demonstration.

I’ve tried to apply the changes to metrics and loss_func, but still can’t make it run.
I guess I’ve missed the right term for what I’m trying to do.
I have 5 targets, for each target there are 25 classes. (in total, there are 125 different classes)
I guess it’s not multi label classification but something like multi class classification.

I’ve googled the term and found a towardsdatascience post by you from Aug’21 which does something similar, but for images.
This is the post:
https://towardsdatascience.com/multi-label-classification-in-fastai-using-spreadsheets-25ae570c8ff9

I’ve tried to set y_block=MultiCategoryBlock()
But when trying to get a dataloader, I get TypeError, which means I can’t have my targets as “CategoricalDtype”

to = TabularPandas(a, procs=[Categorify, Normalize],
                   cat_names=cat_cols,
                   cont_names=cont_cols,
                   y_names=y_names,
                   splits= splits, 
                   y_block = MultiCategoryBlock())
dls = to.dataloaders(bs=64)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [112], in <cell line: 1>()
----> 1 dls = to.dataloaders(bs=64)

File /opt/conda/lib/python3.9/site-packages/fastai/data/core.py:236, in FilteredBase.dataloaders(self, bs, shuffle_train, shuffle, val_shuffle, n, path, dl_type, dl_kwargs, device, drop_last, val_bs, **kwargs)
    234 dl = dl_type(self.subset(0), **merge(kwargs,def_kwargs, dl_kwargs[0]))
    235 def_kwargs = {'bs':bs if val_bs is None else val_bs,'shuffle':val_shuffle,'n':None,'drop_last':False}
--> 236 dls = [dl] + [dl.new(self.subset(i), **merge(kwargs,def_kwargs,val_kwargs,dl_kwargs[i]))
    237               for i in range(1, self.n_subsets)]
    238 return self._dbunch_type(*dls, path=path, device=device)

File /opt/conda/lib/python3.9/site-packages/fastai/data/core.py:236, in <listcomp>(.0)
    234 dl = dl_type(self.subset(0), **merge(kwargs,def_kwargs, dl_kwargs[0]))
    235 def_kwargs = {'bs':bs if val_bs is None else val_bs,'shuffle':val_shuffle,'n':None,'drop_last':False}
--> 236 dls = [dl] + [dl.new(self.subset(i), **merge(kwargs,def_kwargs,val_kwargs,dl_kwargs[i]))
    237               for i in range(1, self.n_subsets)]
    238 return self._dbunch_type(*dls, path=path, device=device)

File /opt/conda/lib/python3.9/site-packages/fastai/data/core.py:66, in TfmdDL.new(self, dataset, cls, **kwargs)
     64 if not hasattr(self, '_n_inp') or not hasattr(self, '_types'):
     65     try:
---> 66         self._one_pass()
     67         res._n_inp,res._types = self._n_inp,self._types
     68     except Exception as e:

File /opt/conda/lib/python3.9/site-packages/fastai/data/core.py:53, in TfmdDL._one_pass(self)
     51 b = self.do_batch([self.do_item(None)])
     52 if self.device is not None: b = to_device(b, self.device)
---> 53 its = self.after_batch(b)
     54 self._n_inp = 1 if not isinstance(its, (list,tuple)) or len(its)==1 else len(its)-1
     55 self._types = explode_types(its)

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:200, in Pipeline.__call__(self, o)
--> 200 def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:150, in compose_tfms(x, tfms, is_enc, reverse, **kwargs)
    148 for f in tfms:
    149     if not is_enc: f = f.decode
--> 150     x = f(x, **kwargs)
    151 return x

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:113, in ItemTransform.__call__(self, x, **kwargs)
--> 113 def __call__(self, x, **kwargs): return self._call1(x, '__call__', **kwargs)

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:116, in ItemTransform._call1(self, x, name, **kwargs)
    115 def _call1(self, x, name, **kwargs):
--> 116     if not _is_tuple(x): return getattr(super(), name)(x, **kwargs)
    117     y = getattr(super(), name)(list(x), **kwargs)
    118     if not self._retain: return y

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:73, in Transform.__call__(self, x, **kwargs)
---> 73 def __call__(self, x, **kwargs): return self._call('encodes', x, **kwargs)

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:83, in Transform._call(self, fn, x, split_idx, **kwargs)
     81 def _call(self, fn, x, split_idx=None, **kwargs):
     82     if split_idx!=self.split_idx and self.split_idx is not None: return x
---> 83     return self._do_call(getattr(self, fn), x, **kwargs)

File /opt/conda/lib/python3.9/site-packages/fastcore/transform.py:89, in Transform._do_call(self, f, x, **kwargs)
     87     if f is None: return x
     88     ret = f.returns(x) if hasattr(f,'returns') else None
---> 89     return retain_type(f(x, **kwargs), x, ret)
     90 res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
     91 return retain_type(res, x)

File /opt/conda/lib/python3.9/site-packages/fastcore/dispatch.py:123, in TypeDispatch.__call__(self, *args, **kwargs)
    121 elif self.inst is not None: f = MethodType(f, self.inst)
    122 elif self.owner is not None: f = MethodType(f, self.owner)
--> 123 return f(*args, **kwargs)

File /opt/conda/lib/python3.9/site-packages/fastai/tabular/core.py:327, in ReadTabBatch.encodes(self, to)
    325 else: res = (tensor(to.cats).long(),tensor(to.conts).float())
    326 ys = [n for n in to.y_names if n in to.items.columns]
--> 327 if len(ys) == len(to.y_names): res = res + (tensor(to.targ),)
    328 if to.device is not None: res = to_device(res, to.device)
    329 return res

File /opt/conda/lib/python3.9/site-packages/fastai/torch_core.py:134, in tensor(x, *rest, **kwargs)
    128     if len(rest): x = (x,)+rest
    129     # There was a Pytorch bug in dataloader using num_workers>0. Haven't confirmed if fixed
    130     # if isinstance(x, (tuple,list)) and len(x)==0: return tensor(0)
    131     res = (x if isinstance(x, Tensor)
    132            else torch.tensor(x, **kwargs) if isinstance(x, (tuple,list))
    133            else _array2tensor(x) if isinstance(x, ndarray)
--> 134            else as_tensor(x.values, **kwargs) if isinstance(x, (pd.Series, pd.DataFrame))
    135 #            else as_tensor(array(x, **kwargs)) if hasattr(x, '__array__') or is_iter(x)
    136            else _array2tensor(array(x), **kwargs))
    137     if res.dtype is torch.float64: return res.float()
    138     return res

TypeError: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, complex64, complex128, int64, int32, int16, int8, uint8, and bool.

I’ve tried to set each target to have his code number, now instead of being CategoricalDtype they are int8 dtype.

I’ve successfully being able to train it as regression for each target, but it’s not really what I’m aiming for.

1 Like
4

Oh, I get it now, thanks for adding that context.

So, the problem you are facing could be addressed in the following way I presume

  1. For each of the five targets, you one-hot encode them so that you get a 125 length vector.
  2. Right now, I think each of the 5 targets you have is a numpy array.
  3. To elaborate, I am proposing you to have 125 target columns in your dataframe instead of 5 columns and each column again being a list/numpy array of those targets which I think is the case now. So your ynames will be 125 columns now.

This should enable you to train the model using BCEWithLogitsLossFlat as a proper multi-label classification problem instead of using MSELoss and looking at it like a regression problem.

If you could share a couple of rows of your data as a csv gist, I could give it a shot as well.

Thanks! :slight_smile:

2 Likes