I can't save via add_metrics in custom callback


I’m trying to use my own callback. It’s being called, but its output seems not to be recorded or writed to history.csv

from fastai.callbacks import LearnerCallback
class MyCallback(LearnerCallback):

def __init__(self, learn:Learner):

def on_train_begin(self, **kwargs):
    self.learn.recorder.add_metric_names(['fold', 'cycle'])
def on_epoch_end(self, last_metrics, **kwargs):
    f=globals().get('fold_id', -1)
    c=-1 if not 'checkpoints_saver' in globals() else get_int_parameter_from_string('cyc',checkpoints_saver.name)

    print("last_metrics, [f,c]",last_metrics, [f,c])
    return add_metrics(last_metrics, [f,c])

learn = cnn_learner(…)
callbacks=[SaveModelCallback(learn, every=‘epoch’, monitor=‘accuracy’, name=checkpoints_save_name),
CSVLogger(learn, filename=CSV_LOGGER_FILENAME.replace(".csv",""),append=True)]

learn.fit_one_cycle(…, callbacks=callbacks)

What am I doing wrong?


You have to make the _order attribute of your back negative (so that it runs before the Recorder). I usually put -20 for things that add new metrics.

1 Like

Thanks @sgugger !

Now it’s working

Thanks for posting thiis info. I hit this error even though the callback was using -20.
I changed to -50 and now working like a champ, so just wanted to add that you might have to go lower than -20.

ugh, even setting it to -1000 doesn’t reliably work now.

anaconda3/lib/python3.7/site-packages/fastai/callback.py in on_epoch_end(self, val_loss)
315         "Epoch is done, process `val_loss`."
316         self.state_dict['last_metrics'] = [val_loss] if val_loss is not None else [None]

–> 317 self(‘epoch_end’, call_mets = val_loss is not None)
318 self.state_dict[‘epoch’] += 1
319 return self.state_dict[‘stop_training’]

~/anaconda3/lib/python3.7/site-packages/fastai/callback.py in call(self, cb_name, call_mets, **kwargs)
249 if call_mets:
250 for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
–> 251 for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
253 def set_dl(self, dl:DataLoader):

~/anaconda3/lib/python3.7/site-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
239 def call_and_update(self, cb, cb_name, **kwargs)->None:
240 “Call cb_name on cb and update the inner state.”
–> 241 new = ifnone(getattr(cb, f’on
{cb_name}’)(**self.state_dict, **kwargs), dict())
242 for k,v in new.items():
243 if k not in self.state_dict:

~/course-v3/nbs/dl1/selprop.py in on_epoch_end(self, epoch, last_metrics, **kwargs)
47 loss_perc = self.sel_losses_sum / self.losses_sum if epoch > 0 else 1.
48 sample_perc = self.sel_samples / self.samples if epoch > 0 else 1.
—> 49 return add_metrics(last_metrics, [loss_perc, sample_perc])
51 def on_train_end(self, **kwargs):

NameError: name ‘add_metrics’ is not defined