IndexError: list index out of range when using lr_find() with SaveModelCallback()

fastai fastai dev
1

I get the following error when calling learn.lr_find() on a learner with SaveModelCallback() in the lesson1-pets.ipynb notebook. That is with

learn = cnn_learner(dbunch, resnet34, metrics=error_rate,cbs=SaveModelCallback()).to_fp16()

calling

learn.lr_find()

results in

`IndexError: list index out of range` f:\dev\repos\fastai\fastai_dev\fastai2\callback\tracker.py in after_epoch(self) 83 if self.every_epoch: self._save(f'{self.fname}_{self.epoch}') 84 else: #every improvement ---> 85 super().after_epoch() 86 if self.new_best: self._save(f'{self.fname}') 87

f:\dev\repos\fastai\fastai_dev\fastai2\callback\tracker.py in after_epoch(self)
45 def after_epoch(self):
46 “Compare the last value to the best up to know”
—> 47 val = self.recorder.values[-1][self.idx]
48 if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True
49 else: self.new_best = False

f:\dev\repos\fastai\fastai_dev\fastai2\core\foundation.py in getitem(self, idx)
315 def _xtra(self): return None
316 def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
–> 317 def getitem(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
318 def copy(self): return self._new(self.items.copy())
319

f:\dev\repos\fastai\fastai_dev\fastai2\core\foundation.py in _get(self, i)
319
320 def _get(self, i):
–> 321 if is_indexer(i) or isinstance(i,slice): return getattr(self.items,‘iloc’,self.items)[i]
322 i = mask2idxs(i)
323 return (self.items.iloc[list(i)] if hasattr(self.items,‘iloc’)

IndexError: list index out of range

It looks to be a result of SaveModelCallback() running during lr_find(). Because lr_find() replaces the contents of learn.recorder.values the error is thrown if TrackerCallback() is monitoring a metric other than train_loss. Should SaveModelCallBack() run during lr_find()?

2

No, SaveModelCallback shouldn’t be run during an lr finder. I’ll add some checks for that.

3

Thanks for that, working perfectly now.

1 Like
4

@sgugger Sorry to be a pain but I am getting the same error when using learn.get_preds() and learn.get_preds(dl=test_dl_) where

test_dl_ = test_dl(dbunch_val, test_items)

5

Added checks for that too. Note that in general, if you want a callback to run only during one call to fit, you should pass it in that call, not to the learner constructor.

2 Likes
6

Noted, cheers.

7

the same issue seems to be for ShowGraphCallback()

when I run lr_find with ShowGraphCallback() it alse gives IndexError: list index out of range

8

I would guess this is not a priority fix as this is the recommended advice.

9

Added the check for ShowGraph too.

10

I have this problem while using learn.tta() too. Not sure if it’s related.

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-14-63aae3d2fce3> in <module>
----> 1 learn.tta()

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in tta(self, ds_idx, dl, n, item_tfms, batch_tfms, beta, use_max)
    522         for i in self.progress.mbar if hasattr(self,'progress') else range(n):
    523             self.epoch = i #To keep track of progress on mbar since the progress callback will use self.epoch
--> 524             aug_preds.append(self.get_preds(dl=dl, inner=True)[0][None])
    525     aug_preds = torch.cat(aug_preds)
    526     aug_preds = aug_preds.max(0)[0] if use_max else aug_preds.mean(0)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, **kwargs)
    218             self(event.begin_epoch if inner else _before_epoch)
    219             self._do_epoch_validate(dl=dl)
--> 220             self(event.after_epoch if inner else _after_epoch)
    221             if act is None: act = getattr(self.loss_func, 'activation', noop)
    222             res = cb.all_tensors()

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in __call__(self, event_name)
    122     def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
    123 
--> 124     def __call__(self, event_name): L(event_name).map(self._call_one)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, *args, **kwargs)
    370              else f.format if isinstance(f,str)
    371              else f.__getitem__)
--> 372         return self._new(map(g, self))
    373 
    374     def filter(self, f, negate=False, **kwargs):

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    321     @property
    322     def _xtra(self): return None
--> 323     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    324     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    325     def copy(self): return self._new(self.items.copy())

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     39             return x
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0
     43         return res

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    312         if items is None: items = []
    313         if (use_list is not None) or not _is_array(items):
--> 314             items = list(items) if use_list else _listify(items)
    315         if match is not None:
    316             if is_coll(match): match = len(match)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in _listify(o)
    248     if isinstance(o, list): return o
    249     if isinstance(o, str) or _is_array(o): return [o]
--> 250     if is_iter(o): return list(o)
    251     return [o]
    252 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __call__(self, *args, **kwargs)
    214             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    215         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 216         return self.fn(*fargs, **kwargs)
    217 
    218 # Cell

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in _call_one(self, event_name)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)
--> 127         [cb(event_name) for cb in sort_by_run(self.cbs)]
    128 
    129     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/learner.py in <listcomp>(.0)
    125     def _call_one(self, event_name):
    126         assert hasattr(event, event_name)
--> 127         [cb(event_name) for cb in sort_by_run(self.cbs)]
    128 
    129     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/callback/core.py in __call__(self, event_name)
     22         _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
     23                (self.run_valid and not getattr(self, 'training', False)))
---> 24         if self.run and _run: getattr(self, event_name, noop)()
     25         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     26 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/callback/tracker.py in after_epoch(self)
     77         if self.every_epoch: self._save(f'{self.fname}_{self.epoch}')
     78         else: #every improvement
---> 79             super().after_epoch()
     80             if self.new_best: self._save(f'{self.fname}')
     81 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastai2/callback/tracker.py in after_epoch(self)
     37     def after_epoch(self):
     38         "Compare the last value to the best up to know"
---> 39         val = self.recorder.values[-1][self.idx]
     40         if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True
     41         else: self.new_best = False

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in __getitem__(self, idx)
    322     def _xtra(self): return None
    323     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
--> 324     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    325     def copy(self): return self._new(self.items.copy())
    326 

~/anaconda3/envs/fastai2/lib/python3.7/site-packages/fastcore/foundation.py in _get(self, i)
    326 
    327     def _get(self, i):
--> 328         if is_indexer(i) or isinstance(i,slice): return getattr(self.items,'iloc',self.items)[i]
    329         i = mask2idxs(i)
    330         return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')

IndexError: list index out of range
2 Likes