SaveModelCallback error

Hi all. I’m trying to use SaveModelCallback but it keeps throwing when I change what to monitor from valid_loss to accuracy and try to validate the model. Here is what I’ve done so far with the stack trace

def addCallback(learner):
#changing monitor from `valid_loss` to `accuracy` throws an error
    learner.add_cbs([SaveModelCallback(monitor='accuracy', fname='classifier-model-saved-by-callback')])

learn = cnn_learner(dls, resnet152, loss_func=CrossEntropyLossFlat(), metrics=accuracy).to_fp16()

addCallback(learn)

learn.validate(1)
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-70-631604a2e07b> in <module>()
----> 1 learn.validate()

14 frames
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in validate(self, ds_idx, dl, cbs)
    211             self(_before_epoch)
    212             self._do_epoch_validate(ds_idx, dl)
--> 213             self(_after_epoch)
    214         return getattr(self, 'final_record', None)
    215 

/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in __call__(self, event_name)
    132     def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
    133 
--> 134     def __call__(self, event_name): L(event_name).map(self._call_one)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in map(self, f, *args, **kwargs)
    375              else f.format if isinstance(f,str)
    376              else f.__getitem__)
--> 377         return self._new(map(g, self))
    378 
    379     def filter(self, f, negate=False, **kwargs):

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    325     @property
    326     def _xtra(self): return None
--> 327     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    328     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    329     def copy(self): return self._new(self.items.copy())

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     45             return x
     46 
---> 47         res = super().__call__(*((x,) + args), **kwargs)
     48         res._newchk = 0
     49         return res

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    316         if items is None: items = []
    317         if (use_list is not None) or not _is_array(items):
--> 318             items = list(items) if use_list else _listify(items)
    319         if match is not None:
    320             if is_coll(match): match = len(match)

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _listify(o)
    252     if isinstance(o, list): return o
    253     if isinstance(o, str) or _is_array(o): return [o]
--> 254     if is_iter(o): return list(o)
    255     return [o]
    256 

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in __call__(self, *args, **kwargs)
    218             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    219         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 220         return self.fn(*fargs, **kwargs)
    221 
    222 # Cell

/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in _call_one(self, event_name)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    138 
    139     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in <listcomp>(.0)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    138 
    139     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

/usr/local/lib/python3.6/dist-packages/fastai2/callback/core.py in __call__(self, event_name)
     22         _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
     23                (self.run_valid and not getattr(self, 'training', False)))
---> 24         if self.run and _run: getattr(self, event_name, noop)()
     25         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     26 

/usr/local/lib/python3.6/dist-packages/fastai2/callback/tracker.py in after_epoch(self)
     79         if self.every_epoch: self._save(f'{self.fname}_{self.epoch}')
     80         else: #every improvement
---> 81             super().after_epoch()
     82             if self.new_best: self._save(f'{self.fname}')
     83 

/usr/local/lib/python3.6/dist-packages/fastai2/callback/tracker.py in after_epoch(self)
     37     def after_epoch(self):
     38         "Compare the last value to the best up to know"
---> 39         val = self.recorder.values[-1][self.idx]
     40         if self.comp(val - self.min_delta, self.best): self.best,self.new_best = val,True
     41         else: self.new_best = False

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in __getitem__(self, idx)
    326     def _xtra(self): return None
    327     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
--> 328     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    329     def copy(self): return self._new(self.items.copy())
    330 

/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _get(self, i)
    330 
    331     def _get(self, i):
--> 332         if is_indexer(i) or isinstance(i,slice): return getattr(self.items,'iloc',self.items)[i]
    333         i = mask2idxs(i)
    334         return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')

IndexError: list index out of range
  1. SaveModelCallback is used when training, not validating, as you are here
  2. In my experience, it works best when you attach it to the fit function. For your use case, this should work:
learn = cnn_learner(dls, resnet152, loss_func=CrossEntropyLossFlat(), metrics=accuracy).to_fp16()
learn.fit_one_cycle(num_epochs, max_lr, cbs=[SaveModelCallback(monitor='accuray')])
4 Likes

i get this error even using with fit.
/opt/conda/lib/python3.7/site-packages/fastai/learner.py:54: UserWarning: Saved filed doesn’t contain an optimizer state.
elif with_opt: warn(“Saved filed doesn’t contain an optimizer state.”)
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in with_events(self, f, event_type, ex, final)
153 def with_events(self, f, event_type, ex, final=noop):
–> 154 try: self(f’before
{event_type}’) ;f()
155 except ex: self(f’after_cancel
{event_type}’)
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in call(self, event_name)
131
–> 132 def call(self, event_name): L(event_name).map(self._call_one)
133
/opt/conda/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, gen, *args, **kwargs)
178
–> 179 def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
180 def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
/opt/conda/lib/python3.7/site-packages/fastcore/basics.py in map_ex(iterable, f, gen, *args, **kwargs)
606 if gen: return res
–> 607 return list(res)
608
/opt/conda/lib/python3.7/site-packages/fastcore/basics.py in call(self, *args, **kwargs)
596 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
–> 597 return self.func(*fargs, **kwargs)
598
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in _call_one(self, event_name)
135 assert hasattr(event, event_name), event_name
–> 136 [cb(event_name) for cb in sort_by_run(self.cbs)]
137
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in (.0)
135 assert hasattr(event, event_name), event_name
–> 136 [cb(event_name) for cb in sort_by_run(self.cbs)]
137
/opt/conda/lib/python3.7/site-packages/fastai/callback/core.py in call(self, event_name)
43 res = None
—> 44 if self.run and run: res = getattr(self, event_name, noop)()
45 if event_name==‘after_fit’: self.run=True #Reset self.run to True at each end of fit
/opt/conda/lib/python3.7/site-packages/fastai/callback/fp16.py in before_fit(self)
84 def before_fit(self):
—> 85 assert self.dls.device.type == ‘cuda’, "Mixed-precision training requires a GPU, remove the call to_fp16"
86 if self.learn.opt is None: self.learn.create_opt()
AssertionError: Mixed-precision training requires a GPU, remove the call to_fp16
During handling of the above exception, another exception occurred:
FileNotFoundError Traceback (most recent call last)
in
8 #MixUp1(),
9 ReduceLROnPlateau(monitor=‘accuracy_multi’,factor=5,patience=2)
—> 10 ,SaveModelCallback(monitor=‘accuracy_multi’,fname=f’best_model_dense_ext
{fold}’) ] )
/opt/conda/lib/python3.7/site-packages/fastai/callback/schedule.py in fit_sgdr(self, n_cycles, cycle_len, lr_max, cycle_mult, cbs, reset_opt, wd)
146 scheds = [SchedCos(lr_max, 0) for _ in range(n_cycles)]
147 scheds = {‘lr’: combine_scheds(pcts, scheds)}
–> 148 self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
149
150 # Cell
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
203 self.opt.set_hypers(lr=self.lr if lr is None else lr)
204 self.n_epoch = n_epoch
–> 205 self._with_events(self._do_fit, ‘fit’, CancelFitException, self.end_cleanup)
206
207 def end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in with_events(self, f, event_type, ex, final)
154 try: self(f’before
{event_type}’) ;f()
155 except ex: self(f’after_cancel
{event_type}’)
–> 156 finally: self(f’after
{event_type}’) ;final()
157
158 def all_batches(self):
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in call(self, event_name)
130 def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
131
–> 132 def call(self, event_name): L(event_name).map(self._call_one)
133
134 def _call_one(self, event_name):
/opt/conda/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, gen, *args, **kwargs)
177 def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
178
–> 179 def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
180 def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
181 def filter(self, f=noop, negate=False, gen=False, **kwargs):
/opt/conda/lib/python3.7/site-packages/fastcore/basics.py in map_ex(iterable, f, gen, *args, **kwargs)
605 res = map(g, iterable)
606 if gen: return res
–> 607 return list(res)
608
609 # Cell
/opt/conda/lib/python3.7/site-packages/fastcore/basics.py in call(self, *args, **kwargs)
595 if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
596 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
–> 597 return self.func(*fargs, **kwargs)
598
599 # Cell
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in _call_one(self, event_name)
134 def _call_one(self, event_name):
135 assert hasattr(event, event_name), event_name
–> 136 [cb(event_name) for cb in sort_by_run(self.cbs)]
137
138 def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in (.0)
134 def _call_one(self, event_name):
135 assert hasattr(event, event_name), event_name
–> 136 [cb(event_name) for cb in sort_by_run(self.cbs)]
137
138 def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)
/opt/conda/lib/python3.7/site-packages/fastai/callback/core.py in call(self, event_name)
42 (self.run_valid and not getattr(self, ‘training’, False)))
43 res = None
—> 44 if self.run and _run: res = getattr(self, event_name, noop)()
45 if event_name==‘after_fit’: self.run=True #Reset self.run to True at each end of fit
46 return res
/opt/conda/lib/python3.7/site-packages/fastai/callback/tracker.py in after_fit(self, **kwargs)
85 def after_fit(self, **kwargs):
86 “Load the best model.”
—> 87 if not self.every_epoch: self.learn.load(f’{self.fname}’, with_opt=self.with_opt)
88
89 # Cell
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in load(self, file, with_opt, device, **kwargs)
291 if self.opt is None: self.create_opt()
292 file = join_path_file(file, self.path/self.model_dir, ext=’.pth’)
–> 293 load_model(file, self.model, self.opt, with_opt=with_opt, device=device, **kwargs)
294 return self
295
/opt/conda/lib/python3.7/site-packages/fastai/learner.py in load_model(file, model, opt, with_opt, device, strict)
44 if isinstance(device, int): device = torch.device(‘cuda’, device)
45 elif device is None: device = ‘cpu’
—> 46 state = torch.load(file, map_location=device)
47 hasopt = set(state)=={‘model’, ‘opt’}
48 model_state = state[‘model’] if hasopt else state
/opt/conda/lib/python3.7/site-packages/torch/serialization.py in load(f, map_location, pickle_module, **pickle_load_args)
579 pickle_load_args[‘encoding’] = ‘utf-8’
580
–> 581 with _open_file_like(f, ‘rb’) as opened_file:
582 if _is_zipfile(opened_file):
583 # The zipfile reader is going to advance the current file position.
/opt/conda/lib/python3.7/site-packages/torch/serialization.py in _open_file_like(name_or_buffer, mode)
228 def _open_file_like(name_or_buffer, mode):
229 if _is_path(name_or_buffer):
–> 230 return _open_file(name_or_buffer, mode)
231 else:
232 if ‘w’ in mode:
/opt/conda/lib/python3.7/site-packages/torch/serialization.py in init(self, name, mode)
209 class _open_file(_opener):
210 def init(self, name, mode):
–> 211 super(_open_file, self).init(open(name, mode))
212
213 def exit(self, *args):
FileNotFoundError: [Errno 2] No such file or directory: ‘models/best_model_dense_ext_3.pth’

The error suggests an issue with accessing a list index, possibly due to an undefined metric ‘accuracy’. Check your model and dataset setup.