[SOLVED] Mixup on Image Regression

Hello Guys

@sgugger I am marking you here if you can kindly help me in solving this issue.

I am working on an Image Regression task where I created ImageDataBunch in following way:

tfms = get_transforms(max_zoom=1., max_lighting=0.20, 
                  max_warp=0., xtra_tfms=[crop_pad(size=600, padding_mode='border', row_pct=0.,col_pct=0.), symmetric_warp(magnitude=(-0.1,0.1)) ])

path = Path('../input/wiki-face-data/wiki_crop/wiki_crop/')

data = ImageList.from_df(df_age, path, cols=['full_path'], folder = '.').split_by_rand_pct(0.2).label_from_df(label_cls=FloatList).transform(tfms, resize_method=ResizeMethod.CROP, padding_mode='border', size=224*2).databunch(bs=64//2).normalize(imagenet_stats)

For this, I have made a custom flattened loss function:

class L1LossFlat(nn.L1Loss):
    def forward(self, input:Tensor, target:Tensor) -> Rank0Tensor:
        return super().forward(input.view(-1), target.view(-1))

and to tackle with the regression problem, I used following module:

class AgeModel(nn.Module):
    def __init__(self):
        super().__init__()
        layers = list(resnet50(pretrained=True).children())[:-2]
        layers += [AdaptiveConcatPool2d(),Flatten()]
        layers += [nn.BatchNorm1d(4096, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        layers += [nn.Dropout(p=0.75)]
        layers += [nn.Linear(4096, 1024, bias=True), nn.ReLU(inplace=True)]
        #layers += [nn.BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        #layers += [nn.Dropout(p=0.60)]
        #layers += [nn.Linear(2048, 1024, bias=True), nn.ReLU(inplace=True)]
        #layers += [nn.BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        #layers += [nn.Dropout(p=0.75)]
        #layers += [nn.Linear(1024, 256, bias=True), nn.ReLU(inplace=True)]
        #layers += [nn.BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        #layers += [nn.Dropout(p=0.50)]
        #layers += [nn.Linear(512,256 , bias=True), nn.ReLU(inplace=True)]
        layers += [nn.BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        layers += [nn.Dropout(p=0.60)]
        layers += [nn.Linear(1024, 256, bias=True), nn.ReLU(inplace=True)]
        layers += [nn.BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)]
        layers += [nn.Dropout(p=0.40)]
        layers += [nn.Linear(256, 16, bias=True), nn.ReLU(inplace=True)]
        layers += [nn.Linear(16,1)]
        self.agemodel = nn.Sequential(*layers)
    def forward(self, x):
        return self.agemodel(x).squeeze(-1)
          # could add 116*torch.sigmoid

Now in below code, while creating learner function, I want to use mixup in following manner:

learn = Learner(data, model,  model_dir = "/temp/model/",
               callback_fns=[ShowGraph]).mixup(stack_y=False)
learn.loss_func = L1LossFlat()

learn.split([model.agemodel[4],model.agemodel[6], model.agemodel[8]])

But when I am fitting one cycle, I am getting following error:

learn.freeze_to(-1)
learn.fit_one_cycle(4, wd=1e-4)

Error:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-38-43fd45a4db16> in <module>()
      1 import fastai
      2 learn.freeze_to(-1)
----> 3 learn.fit_one_cycle(1, wd=1e-4)

/opt/conda/lib/python3.6/site-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)
     20     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,
     21                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))
---> 22     learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
     23 
     24 def lr_find(learn:Learner, start_lr:Floats=1e-7, end_lr:Floats=10, num_it:int=100, stop_div:bool=True, wd:float=None):

/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    194         callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
    195         if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
--> 196         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    197 
    198     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
    109         exception = e
    110         raise
--> 111     finally: cb_handler.on_train_end(exception)
    112 
    113 loss_func_name2activ = {'cross_entropy_loss': F.softmax, 'nll_loss': torch.exp, 'poisson_nll_loss': torch.exp,

/opt/conda/lib/python3.6/site-packages/fastai/callback.py in on_train_end(self, exception)
    320     def on_train_end(self, exception:Union[bool,Exception])->None:
    321         "Handle end of training, `exception` is an `Exception` or False if no exceptions during training."
--> 322         self('train_end', exception=exception)
    323 
    324 class AverageMetric(Callback):

/opt/conda/lib/python3.6/site-packages/fastai/callback.py in __call__(self, cb_name, call_mets, **kwargs)
    248         if call_mets:
    249             for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
--> 250         for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
    251 
    252     def set_dl(self, dl:DataLoader):

/opt/conda/lib/python3.6/site-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
    238     def _call_and_update(self, cb, cb_name, **kwargs)->None:
    239         "Call `cb_name` on `cb` and update the inner state."
--> 240         new = ifnone(getattr(cb, f'on_{cb_name}')(**self.state_dict, **kwargs), dict())
    241         for k,v in new.items():
    242             if k not in self.state_dict:

/opt/conda/lib/python3.6/site-packages/fastai/callbacks/mixup.py in on_train_end(self, **kwargs)
     34 
     35     def on_train_end(self, **kwargs):
---> 36         self.learn.loss_func = self.learn.loss_func.get_old()
     37 
     38 

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
    533                 return modules[name]
    534         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 535             type(self).__name__, name))
    536 
    537     def __setattr__(self, name, value):

AttributeError: 'L1LossFlat' object has no attribute 'get_old'

How to solve this issue?

Thanks
Abhik

2 Likes

It looks like this particular bug has been fixed in master (the source code is different from what I saw in your stack trace). You should do an editable install or wait for the next release.

Hi @sgugger

I will wait for the next release then. Any idea by when it will happen? By the way, I checked fastai version ‘1.0.51’. Is there any newer version than this?

Thanks
AJ

Yes, the latest is 1.0.52

1 Like

cool, it worked on 1.0.52 :slight_smile: