UserWarning: Your generator is empty ... when running "learn.predict" on trained text classifier

fastai fastai dev
1 
inf_learn = load_learner(SENTIMENT_PATH/f'{m_pre}export_clas{m_suf}.pkl')
inf_learn.predict('I feel very uncomfortable talking to my supervisor. Help!!!')`

/lib/python3.7/site-packages/fastprogress/fastprogress.py:74: UserWarning: Your generator is empty.
  warn("Your generator is empty.")

Am I calling this wrong? Seems like something with fastprogress (btw, I have done editable installs of the latest for both fastcore and fastai2)

Full stack trace:

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    172             dl,old,has = change_attrs(dl, names, [False,False])
--> 173             self.dl = dl;                                    self('begin_validate')
    174             with torch.no_grad(): self.all_batches()

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in __call__(self, event_name)
    122 
--> 123     def __call__(self, event_name): L(event_name).map(self._call_one)
    124     def _call_one(self, event_name):

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in map(self, f, *args, **kwargs)
    361              else f.__getitem__)
--> 362         return self._new(map(g, self))
    363 

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    314     def _xtra(self): return None
--> 315     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    316     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    305         if (use_list is not None) or not _is_array(items):
--> 306             items = list(items) if use_list else _listify(items)
    307         if match is not None:

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _listify(o)
    241     if isinstance(o, str) or _is_array(o): return [o]
--> 242     if is_iter(o): return list(o)
    243     return [o]

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(self, *args, **kwargs)
    207         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 208         return self.fn(*fargs, **kwargs)
    209 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _call_one(self, event_name)
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in <listcomp>(.0)
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/callback/core.py in __call__(self, event_name)
     22                (self.run_valid and not getattr(self, 'training', False)))
---> 23         if self.run and _run: getattr(self, event_name, noop)()
     24         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit

~/development/_training/ml/nlp-playground/tritonlytics/callbacks.py in begin_validate(self, **kwargs)
     37     def begin_validate(self, **kwargs):
---> 38         self.opt_thresh = self._opt_th()
     39         if (not self.update_metric_thresh): return

~/development/_training/ml/nlp-playground/tritonlytics/callbacks.py in _opt_th(self)
     62             # (see https://dev.fast.ai/learner#Learner.get_preds)
---> 63             probs, targs = learn.get_preds(inner=True)
     64             probs, targs = to_np(probs), to_np(targs)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, **kwargs)
    217             self(event.begin_epoch if inner else _before_epoch)
--> 218             self._do_epoch_validate(dl=dl)
    219             self(event.after_epoch if inner else _after_epoch)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    176         finally:
--> 177             dl,*_ = change_attrs(dl, names, old, has);       self('after_validate')
    178 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in __call__(self, event_name)
    122 
--> 123     def __call__(self, event_name): L(event_name).map(self._call_one)
    124     def _call_one(self, event_name):

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in map(self, f, *args, **kwargs)
    361              else f.__getitem__)
--> 362         return self._new(map(g, self))
    363 

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    314     def _xtra(self): return None
--> 315     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    316     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    305         if (use_list is not None) or not _is_array(items):
--> 306             items = list(items) if use_list else _listify(items)
    307         if match is not None:

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _listify(o)
    241     if isinstance(o, str) or _is_array(o): return [o]
--> 242     if is_iter(o): return list(o)
    243     return [o]

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(self, *args, **kwargs)
    207         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 208         return self.fn(*fargs, **kwargs)
    209 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _call_one(self, event_name)
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in <listcomp>(.0)
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/callback/core.py in __call__(self, event_name)
     22                (self.run_valid and not getattr(self, 'training', False)))
---> 23         if self.run and _run: getattr(self, event_name, noop)()
     24         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/callback/core.py in after_validate(self)
     93         "Concatenate all recorded tensors"
---> 94         if self.with_input:     self.inputs  = detuplify(to_concat(self.inputs, dim=self.concat_dim))
     95         if not self.save_preds: self.preds   = detuplify(to_concat(self.preds, dim=self.concat_dim))

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/torch_core.py in to_concat(xs, dim)
    212     "Concat the element in `xs` (recursively if they are tuples/lists of tensors)"
--> 213     if is_listy(xs[0]): return type(xs[0])([to_concat([x[i] for x in xs], dim=dim) for i in range_of(xs[0])])
    214     if isinstance(xs[0],dict):  return {k: to_concat([x[k] for x in xs], dim=dim) for k in xs.keys()}

IndexError: list index out of range

During handling of the above exception, another exception occurred:

IndexError                                Traceback (most recent call last)
<ipython-input-80-ef38ad4132d8> in <module>
----> 1 inf_learn.predict('I feel very uncomfortable talking to my supervisor even though she loves dogs. Help!!!')

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in predict(self, item, rm_type_tfms, with_input)
    228     def predict(self, item, rm_type_tfms=None, with_input=False):
    229         dl = self.dls.test_dl([item], rm_type_tfms=rm_type_tfms)
--> 230         inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
    231         dec = self.dls.decode_batch((*tuplify(inp),*tuplify(dec_preds)))[0]
    232         i = getattr(self.dls, 'n_inp', -1)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, **kwargs)
    216             for mgr in ctx_mgrs: stack.enter_context(mgr)
    217             self(event.begin_epoch if inner else _before_epoch)
--> 218             self._do_epoch_validate(dl=dl)
    219             self(event.after_epoch if inner else _after_epoch)
    220             if act is None: act = getattr(self.loss_func, 'activation', noop)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    175         except CancelValidException:                         self('after_cancel_validate')
    176         finally:
--> 177             dl,*_ = change_attrs(dl, names, old, has);       self('after_validate')
    178 
    179     def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in __call__(self, event_name)
    121     def ordered_cbs(self, cb_func): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, cb_func)]
    122 
--> 123     def __call__(self, event_name): L(event_name).map(self._call_one)
    124     def _call_one(self, event_name):
    125         assert hasattr(event, event_name)

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in map(self, f, *args, **kwargs)
    360              else f.format if isinstance(f,str)
    361              else f.__getitem__)
--> 362         return self._new(map(g, self))
    363 
    364     def filter(self, f, negate=False, **kwargs):

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    313     @property
    314     def _xtra(self): return None
--> 315     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    316     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    317     def copy(self): return self._new(self.items.copy())

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     39             return x
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0
     43         return res

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    304         if items is None: items = []
    305         if (use_list is not None) or not _is_array(items):
--> 306             items = list(items) if use_list else _listify(items)
    307         if match is not None:
    308             if is_coll(match): match = len(match)

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in _listify(o)
    240     if isinstance(o, list): return o
    241     if isinstance(o, str) or _is_array(o): return [o]
--> 242     if is_iter(o): return list(o)
    243     return [o]
    244 

~/development/_training/ml/nlp-playground/_libs/fastcore/fastcore/foundation.py in __call__(self, *args, **kwargs)
    206             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    207         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 208         return self.fn(*fargs, **kwargs)
    209 
    210 # Cell

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in _call_one(self, event_name)
    124     def _call_one(self, event_name):
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 
    128     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/learner.py in <listcomp>(.0)
    124     def _call_one(self, event_name):
    125         assert hasattr(event, event_name)
--> 126         [cb(event_name) for cb in sort_by_run(self.cbs)]
    127 
    128     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/callback/core.py in __call__(self, event_name)
     21         _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
     22                (self.run_valid and not getattr(self, 'training', False)))
---> 23         if self.run and _run: getattr(self, event_name, noop)()
     24         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     25 

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/callback/core.py in after_validate(self)
     92     def after_validate(self):
     93         "Concatenate all recorded tensors"
---> 94         if self.with_input:     self.inputs  = detuplify(to_concat(self.inputs, dim=self.concat_dim))
     95         if not self.save_preds: self.preds   = detuplify(to_concat(self.preds, dim=self.concat_dim))
     96         if not self.save_targs: self.targets = detuplify(to_concat(self.targets, dim=self.concat_dim))

~/development/_training/ml/nlp-playground/_libs/fastai2/fastai2/torch_core.py in to_concat(xs, dim)
    211 def to_concat(xs, dim=0):
    212     "Concat the element in `xs` (recursively if they are tuples/lists of tensors)"
--> 213     if is_listy(xs[0]): return type(xs[0])([to_concat([x[i] for x in xs], dim=dim) for i in range_of(xs[0])])
    214     if isinstance(xs[0],dict):  return {k: to_concat([x[k] for x in xs], dim=dim) for k in xs.keys()}
    215     #We may receives xs that are not concatenatable (inputs of a text classifier for instance),

IndexError: list index out of range
2

That’s the second error once you had your failure, so not the real cause. The real cause is up in the stack trace (see why I ask for it :wink: ) and comes from your input being empty, or not all of the same size:

    212     "Concat the element in `xs` (recursively if they are tuples/lists of tensors)"
--> 213     if is_listy(xs[0]): return type(xs[0])([to_concat([x[i] for x in xs], dim=dim) for i in range_of(xs[0])])
    214     if isinstance(xs[0],dict):  return {k: to_concat([x[k] for x in xs], dim=dim) for k in xs.keys()}

IndexError: list index out of range

will see if I can reproduce tomorrow.

3

That’s why I added it back in as an edit … I didn’t want to face Sylvain’s wrath a second time, haha.

Thanks for looking at it. I’m wondering if it is because of a callback I’m using (I threw in the new one I proposed for the framework on optimizing FBeta thresholds for multi-label). I’ll try again without it let ya know.

1 Like
4

Well, guess what, it IS my custom callback.

Is there a way to remove it when exporting? It’s really there just for training (e.g., to find the optimal FBeta threshold and adjust the thresh argument/attribute of metrics to use it). Or is there something else recommended?

5

If it’s just for training, pass it in your call to fit. Or otherwise, you have learn.remove_cb to remove a callback.

4 Likes
6

I’m bookmarking this and liking it!

I never realized that the cbs supplied to fit were not associated to the Learner and thus not saved/exported when those calls were made! Really changes how I’ll use callbacks going forward (up till now I’ve always associated them to Learner when instantiated).

Thanks again!

2 Likes
7

Np. We use this for schedulers for instance (like LRFinder, 1cycle or sgdr).