Tabular learner confusion matrix error

Part 1 (2020)
1

Hi All. I am using tabular module to do some classification task. Please see my code so far

# Get the weights of classes
class_count_df = df_HR_train.groupby('Attrition_Voluntary').count()
class_count_df

n_0, n_1 = class_count_df.iloc[0, 0], class_count_df.iloc[1, 0]

w_0 = (n_0 + n_1) / (2.0 * n_0)
w_1 = (n_0 + n_1) / (2.0 * n_1)

w_0, w_1

# Convert to category column
df_HR_train['Attrition_Voluntary'] = df_HR_train['Attrition_Voluntary'].astype('category')
df_HR_train.info()

dls = TabularDataLoaders.from_df(df_HR_train, y_names='Attrition_Voluntary', y_block=CategoryBlock,
                                 cat_names=CAT_NAMES,
                                 cont_names=CONT_NAMES,
                                 procs=[Categorify, FillMissing, Normalize],
                                 splits=TrainTestSplitter(test_size = 0.2, stratify=df_HR_train['Attrition_Voluntary'], 
                                                          random_state = 12))


%%time
weights = [w_0, w_1]
class_weights=torch.FloatTensor(weights).cuda()
loss_func = FocalLossFlat(weight=class_weights)
prec = Precision()


learn = tabular_learner(dls, 
                        layers=[500, 250],    
                        loss_func=loss_func, 
                        metrics=prec)


learn.fit_one_cycle(3)

For some reason this part throws an error.

interp = ClassificationInterpretation.from_learner(learn)

interp.plot_confusion_matrix()

RuntimeError Traceback (most recent call last)
in
----> 1 interp = ClassificationInterpretation.from_learner(learn)
2 interp.plot_confusion_matrix()

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/interpret.py in from_learner(cls, learn, ds_idx, dl, act)
27 “Construct interpretation object from a learner”
28 if dl is None: dl = learn.dls[ds_idx].new(shuffled=False, drop_last=False)
—> 29 return cls(dl, *learn.get_preds(dl=dl, with_input=True, with_loss=True, with_decoded=True, act=None))
30
31 def top_losses(self, k=None, largest=True):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, reorder, cbs, **kwargs)
251 if with_loss: ctx_mgrs.append(self.loss_not_reduced())
252 with ContextManagers(ctx_mgrs):
→ 253 self._do_epoch_validate(dl=dl)
254 if act is None: act = getattr(self.loss_func, ‘activation’, noop)
255 res = cb.all_tensors()

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in _do_epoch_validate(self, ds_idx, dl)
201 if dl is None: dl = self.dls[ds_idx]
202 self.dl = dl
→ 203 with torch.no_grad(): self._with_events(self.all_batches, ‘validate’, CancelValidException)
204
205 def _do_epoch(self):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in with_events(self, f, event_type, ex, final)
161
162 def with_events(self, f, event_type, ex, final=noop):
→ 163 try: self(f’before{event_type}’); f()
164 except ex: self(f’after_cancel{event_type}’)
165 self(f’after_{event_type}’); final()

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in all_batches(self)
167 def all_batches(self):
168 self.n_iter = len(self.dl)
→ 169 for o in enumerate(self.dl): self.one_batch(*o)
170
171 def _do_one_batch(self):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in one_batch(self, i, b)
192 b = self._set_device(b)
193 self._split(b)
→ 194 self._with_events(self._do_one_batch, ‘batch’, CancelBatchException)
195
196 def _do_epoch_train(self):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in with_events(self, f, event_type, ex, final)
163 try: self(f’before{event_type}’); f()
164 except ex: self(f’after_cancel_{event_type}’)
→ 165 self(f’after_{event_type}’); final()
166
167 def all_batches(self):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in call(self, event_name)
139
140 def ordered_cbs(self, event): return [cb for cb in self.cbs.sorted(‘order’) if hasattr(cb, event)]
→ 141 def call(self, event_name): L(event_name).map(self._call_one)
142
143 def _call_one(self, event_name):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastcore/foundation.py in map(self, f, gen, *args, **kwargs)
152 def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
153
→ 154 def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
155 def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
156 def filter(self, f=noop, negate=False, gen=False, **kwargs):

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastcore/basics.py in map_ex(iterable, f, gen, *args, **kwargs)
664 res = map(g, iterable)
665 if gen: return res
→ 666 return list(res)
667
668 # Cell

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastcore/basics.py in call(self, *args, **kwargs)
649 if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
650 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
→ 651 return self.func(*fargs, **kwargs)
652
653 # Cell

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/learner.py in _call_one(self, event_name)
143 def _call_one(self, event_name):
144 if not hasattr(event, event_name): raise Exception(f’missing {event_name}’)
→ 145 for cb in self.cbs.sorted(‘order’): cb(event_name)
146
147 def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/callback/core.py in call(self, event_name)
43 (self.run_valid and not getattr(self, ‘training’, False)))
44 res = None
—> 45 if self.run and _run: res = getattr(self, event_name, noop)()
46 if event_name==‘after_fit’: self.run=True #Reset self.run to True at each end of fit
47 return res

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/callback/core.py in after_batch(self)
129 if self.with_loss:
130 bs = find_bs(self.yb)
→ 131 loss = self.loss if self.loss.numel() == bs else self.loss.view(bs,-1).mean(1)
132 self.losses.append(self.learn.to_detach(loss))
133

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/fastai/torch_core.py in torch_function(self, func, types, args, kwargs)
338 convert=False
339 if _torch_handled(args, self._opt, func): convert,types = type(self),(torch.Tensor,)
→ 340 res = super().torch_function(func, types, args=args, kwargs=kwargs)
341 if convert: res = convert(res)
342 if isinstance(res, TensorBase): res.set_meta(self, as_copy=True)

/anaconda/envs/azureml_py38/lib/python3.8/site-packages/torch/_tensor.py in torch_function(cls, func, types, args, kwargs)
1021
1022 with _C.DisableTorchFunction():
→ 1023 ret = func(*args, **kwargs)
1024 return _convert(ret, cls)
1025

RuntimeError: shape ‘[64, -1]’ is invalid for input of size 1

What is the reason that I cannot print confusion matrix?

1 Like
2

Hello,
I don’t know the answer, but I have some further info while I’m digging into it.
I also tried running learn.get_preds(with_loss=True), or rather equivalent in my code, and this is where the error starts. You can see in the trace in the Interpreter.from_learner() that it calls learn.get_preds in line 29. This also only happens (to me) when using FocalLossFlat(), like you do. In my case, I’m working with an image classifier. Beyond that I am not sure why this loss function causes a problem, aside from it may not have the same shape as the default loss function? For your issue, I would (And in my cases did) try not passing in the loss_func to the learner, it selects one by default (Not sure which). This at least should get you unstuck if having a focal loss is not important to your project.

3

Ran into this same issue while using FocalLossFlat with cnn_learner.

Agree with @jhunter that the problem with ClassificationInterpretation arises in the call to learn.get_preds(with_loss=True).

Furthermore, I think the issue may trace to the definition of FocalLossFlat, which is a subclass of CrossEntropyLossFlat.

My inspection of the following code may hint at the root of the problem:

1   learn.loss_func
>>> Flattened loss of CrossEntropyLoss
2   learn.loss_func.func
>>> CrossEntropyLoss

What I suspect is happening is that there is some call to CrossEntropyLoss that occurs as a result of calling learn.get_preds(with_loss=True), which may conflict with how FocalLossFlat is implemented.

I’m going to experiment with alternate definitions for FocalLossFlat and will post again if I find any solutions.

Update:

Using the following custom loss functions solved the issue for me:

class FocalLoss(Module):
    y_int=True
    def __init__(self, alpha: Optional[float] = 0.25, gamma: Optional[float] = 2.0,
                 reduction: str = 'mean') -> None:
        store_attr()
    
    def forward(self, inp: torch.Tensor, targ: torch.Tensor):
        ce_loss = F.cross_entropy(inp, targ, reduction="none")
        p_t = torch.exp(-ce_loss)
        loss = (1 - p_t)**self.gamma * ce_loss
        if self.alpha >= 0:
            alpha_t = self.alpha * targ + (1 - self.alpha) * (1 - targ)
            loss = alpha_t * loss
        if self.reduction == "mean":
            loss = loss.mean()
        elif self.reduction == "sum":
            loss = loss.sum()
        return loss


class FocalLossFlatten(BaseLoss):
    y_int = True
    @use_kwargs_dict(keep=True, reduction='mean')
    def __init__(self, *args, alpha=-1, gamma=2.0, axis=-1, **kwargs):
        super().__init__(FocalLoss, *args, alpha=alpha, gamma=gamma, axis=axis, **kwargs)
        
    def decodes(self, x): return x.argmax(dim=self.axis)
    def activation(self, x): return F.softmax(x, dim=self.axis)
1 Like
4

This is very helpful. Thank you for making the fix. You should send a pr to the fastai repo.

1 Like
5

PR submitted this evening. Thanks for the suggestion/reminder!