Class based metrics don't work?

using the example TraceMallocMetric from the docs
This is a fully contained example that should throw the same error:

from fastai import *
from fastai.text import *

path = untar_data(URLs.IMDB_SAMPLE)
data = TextClasDataBunch.from_csv(path, 'texts.csv')

import tracemalloc
class TraceMallocMetric(Callback):
    def __init__(self):
        super().__init__()
        self.name = "peak RAM"

    def on_epoch_begin(self, **kwargs):
        tracemalloc.start()
        
    def on_epoch_end(self, last_metrics, **kwargs):
        current, peak =  tracemalloc.get_traced_memory()
        tracemalloc.stop()
        return add_metrics(last_metrics, torch.tensor(peak))
    
    
learn_kwargs = {'metrics':TraceMallocMetric()}
learn = text_classifier_learner(data, AWD_LSTM, drop_mult=0.5,**learn_kwargs)
learn.fit(1)

and get this error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-2-e0133d6518b6> in <module>
     23 learn = text_classifier_learner(data, AWD_LSTM, drop_mult=0.5,**learn_kwargs)
     24 
---> 25 learn.fit(1)

~/fast_ai/fastai-fork/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    194         callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
    195         if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
--> 196         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    197 
    198     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

~/fast_ai/fastai-fork/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
     86     assert len(learn.data.train_dl) != 0, f"""Your training dataloader is empty, can't train a model.
     87         Use a smaller batch size (batch size={data.train_dl.batch_size} for {len(data.train_dl.dataset)} elements)."""
---> 88     cb_handler = CallbackHandler(callbacks, metrics)
     89     pbar = master_bar(range(epochs))
     90     cb_handler.on_train_begin(epochs, pbar=pbar, metrics=metrics)

<string> in __init__(self, callbacks, metrics, beta)

~/fast_ai/fastai-fork/fastai/callback.py in __post_init__(self)
    231         self.callbacks = ifnone(self.callbacks, [])
    232         self.metrics = ifnone(self.metrics, [])
--> 233         self.metrics = [(met if isinstance(met, Callback) else AverageMetric(met)) for met in self.metrics]
    234         self.callbacks = sorted(self.callbacks, key=lambda o: getattr(o, '_order', 0))
    235         self.smoothener = SmoothenValue(self.beta)

TypeError: 'TraceMallocMetric' object is not iterable

I am on the latest dev version of fastai. Did I set this up wrong?

Ah, text_classifier_learner doesn’t lisitfy metrics. Working on a fix, in the meantime pass your TraceMallocMetric() in a list.

1 Like

Easy change. Thank you so much!