[Solved] Problem with tabular_learner.predict() on a single row

In fastbook/09_tabular.ipynb, while tabular_learner()'s get_preds() works fine on a batch of inputs, I run into problems when using predict() on a single row, one input.

df_final_nn is the Dataframe used to create the TabularPandas, which in turn is used to create the DataLoaders of the learner object.

Wonder if anyone has seen this, and what I might have missed/done wrong here … :thinking: Would appreciate any idea…

  1. df_nn_final.iloc[0] is:
YearMade                                                  2004
Coupler_System                                             NaN
ProductSize                                                NaN
fiProductClassDesc    Wheel Loader - 110.0 to 120.0 Horsepower
ModelID                                                   3157
fiSecondaryDesc                                              D
saleElapsed                                         1163635200
Enclosure                                           EROPS w AC
Hydraulics_Flow                                            NaN
fiModelDesc                                               521D
fiModelDescriptor                                          NaN
ProductGroup                                                WL
Drive_System                                               NaN
Hydraulics                                             2 Valve
Tire_Size                                  None or Unspecified
SalePrice                                              11.0974
Name: 0, dtype: object
  1. The error when invoking in a notebook cell: learn.predict(df_nn_final.iloc[0]):
TypeError                                 Traceback (most recent call last)
~/Dropbox/fastai2/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    182             self.dl = dl;                                    self('begin_validate')
--> 183             with torch.no_grad(): self.all_batches()
    184         except CancelValidException:                         self('after_cancel_validate')

~/Dropbox/fastai2/fastai2/learner.py in all_batches(self)
    152         self.n_iter = len(self.dl)
--> 153         for o in enumerate(self.dl): self.one_batch(*o)

~/Dropbox/fastai2/fastai2/data/load.py in __iter__(self)
     99             if self.device is not None: b = to_device(b, self.device)
--> 100             yield self.after_batch(b)
    101         self.after_iter()

~/Dropbox/fastcore/fastcore/transform.py in __call__(self, o)
--> 187     def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
    188     def __repr__(self): return f"Pipeline: {' -> '.join([f.name for f in self.fs if f.name != 'noop'])}"

~/Dropbox/fastcore/fastcore/transform.py in compose_tfms(x, tfms, is_enc, reverse, **kwargs)
    139         if not is_enc: f = f.decode
--> 140         x = f(x, **kwargs)
    141     return x

~/Dropbox/fastcore/fastcore/transform.py in __call__(self, x, **kwargs)
    102     _retain = True
--> 103     def __call__(self, x, **kwargs): return self._call1(x, '__call__', **kwargs)
    104     def decode(self, x, **kwargs):   return self._call1(x, 'decode', **kwargs)

~/Dropbox/fastcore/fastcore/transform.py in _call1(self, x, name, **kwargs)
    105     def _call1(self, x, name, **kwargs):
--> 106         if not _is_tuple(x): return getattr(super(), name)(x, **kwargs)
    107         y = getattr(super(), name)(list(x), **kwargs)

~/Dropbox/fastcore/fastcore/transform.py in __call__(self, x, **kwargs)
     71     def name(self): return getattr(self, '_name', _get_name(self))
---> 72     def __call__(self, x, **kwargs): return self._call('encodes', x, **kwargs)
     73     def decode  (self, x, **kwargs): return self._call('decodes', x, **kwargs)

~/Dropbox/fastcore/fastcore/transform.py in _call(self, fn, x, split_idx, **kwargs)
     81         if split_idx!=self.split_idx and self.split_idx is not None: return x
---> 82         return self._do_call(getattr(self, fn), x, **kwargs)

~/Dropbox/fastcore/fastcore/transform.py in _do_call(self, f, x, **kwargs)
     85         if not _is_tuple(x):
---> 86             return x if f is None else retain_type(f(x, **kwargs), x, f.returns_none(x))
     87         res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)

~/Dropbox/fastcore/fastcore/dispatch.py in __call__(self, *args, **kwargs)
     97         if self.inst is not None: f = MethodType(f, self.inst)
---> 98         return f(*args, **kwargs)

~/Dropbox/fastai2/fastai2/tabular/core.py in encodes(self, to)
    288         ys = [n for n in to.y_names if n in to.items.columns]
--> 289         if len(ys) == len(to.y_names): res = res + (tensor(to.targ),)
    290         if to.device is not None: res = to_device(res, to.device)

~/Dropbox/fastai2/fastai2/torch_core.py in tensor(x, *rest, **kwargs)
    111            else _array2tensor(x) if isinstance(x, ndarray)
--> 112            else as_tensor(x.values, **kwargs) if isinstance(x, (pd.Series, pd.DataFrame))
    113            else as_tensor(x, **kwargs) if hasattr(x, '__array__') or is_iter(x)

TypeError: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, int64, int32, int16, int8, uint8, and bool.

During handling of the above exception, another exception occurred:

IndexError                                Traceback (most recent call last)
<ipython-input-109-71ce0638ebf9> in <module>
----> 1 learn.predict(df_nn_final.iloc[0])

~/Dropbox/fastai2/fastai2/tabular/learner.py in predict(self, row)
     17         tst_to.conts = tst_to.conts.astype(np.float32)
     18         dl = self.dls.valid.new(tst_to)
---> 19         inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
     20         i = getattr(self.dls, 'n_inp', -1)
     21         b = (*tuplify(inp),*tuplify(dec_preds))

~/Dropbox/fastai2/fastai2/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, reorder, **kwargs)
    227             for mgr in ctx_mgrs: stack.enter_context(mgr)
    228             self(event.begin_epoch if inner else _before_epoch)
--> 229             self._do_epoch_validate(dl=dl)
    230             self(event.after_epoch if inner else _after_epoch)
    231             if act is None: act = getattr(self.loss_func, 'activation', noop)

~/Dropbox/fastai2/fastai2/learner.py in _do_epoch_validate(self, ds_idx, dl)
    183             with torch.no_grad(): self.all_batches()
    184         except CancelValidException:                         self('after_cancel_validate')
--> 185         finally:                                             self('after_validate')
    187     @log_args(but='cbs')

~/Dropbox/fastai2/fastai2/learner.py in __call__(self, event_name)
    132     def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
--> 134     def __call__(self, event_name): L(event_name).map(self._call_one)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)

~/Dropbox/fastcore/fastcore/foundation.py in map(self, f, *args, **kwargs)
    373              else f.format if isinstance(f,str)
    374              else f.__getitem__)
--> 375         return self._new(map(g, self))
    377     def filter(self, f, negate=False, **kwargs):

~/Dropbox/fastcore/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    324     @property
    325     def _xtra(self): return None
--> 326     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    327     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    328     def copy(self): return self._new(self.items.copy())

~/Dropbox/fastcore/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     45             return x
---> 47         res = super().__call__(*((x,) + args), **kwargs)
     48         res._newchk = 0
     49         return res

~/Dropbox/fastcore/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    315         if items is None: items = []
    316         if (use_list is not None) or not _is_array(items):
--> 317             items = list(items) if use_list else _listify(items)
    318         if match is not None:
    319             if is_coll(match): match = len(match)

~/Dropbox/fastcore/fastcore/foundation.py in _listify(o)
    251     if isinstance(o, list): return o
    252     if isinstance(o, str) or _is_array(o): return [o]
--> 253     if is_iter(o): return list(o)
    254     return [o]

~/Dropbox/fastcore/fastcore/foundation.py in __call__(self, *args, **kwargs)
    217             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    218         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 219         return self.fn(*fargs, **kwargs)
    221 # Cell

~/Dropbox/fastai2/fastai2/learner.py in _call_one(self, event_name)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    139     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/Dropbox/fastai2/fastai2/learner.py in <listcomp>(.0)
    135     def _call_one(self, event_name):
    136         assert hasattr(event, event_name)
--> 137         [cb(event_name) for cb in sort_by_run(self.cbs)]
    139     def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)

~/Dropbox/fastai2/fastai2/callback/core.py in __call__(self, event_name)
     22         _run = (event_name not in _inner_loop or (self.run_train and getattr(self, 'training', True)) or
     23                (self.run_valid and not getattr(self, 'training', False)))
---> 24         if self.run and _run: getattr(self, event_name, noop)()
     25         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit

~/Dropbox/fastai2/fastai2/callback/core.py in after_validate(self)
     93     def after_validate(self):
     94         "Concatenate all recorded tensors"
---> 95         if self.with_input:     self.inputs  = detuplify(to_concat(self.inputs, dim=self.concat_dim))
     96         if not self.save_preds: self.preds   = detuplify(to_concat(self.preds, dim=self.concat_dim))
     97         if not self.save_targs: self.targets = detuplify(to_concat(self.targets, dim=self.concat_dim))

~/Dropbox/fastai2/fastai2/torch_core.py in to_concat(xs, dim)
    211 def to_concat(xs, dim=0):
    212     "Concat the element in `xs` (recursively if they are tuples/lists of tensors)"
--> 213     if is_listy(xs[0]): return type(xs[0])([to_concat([x[i] for x in xs], dim=dim) for i in range_of(xs[0])])
    214     if isinstance(xs[0],dict):  return {k: to_concat([x[k] for x in xs], dim=dim) for k in xs[0].keys()}
    215     #We may receives xs that are not concatenatable (inputs of a text classifier for instance),

IndexError: list index out of range

Solved. Input to predict(row) must be processed, numeric data, not the raw, alpha-numeric data in df_final_nn above.

Thus learn.predict(learn.dls.train.all_cols.iloc[0]) works, coz’ the fields are all numeric:

YearMade                 1.000000
Coupler_System           0.000000
ProductSize              3.000000
fiProductClassDesc      48.000000
ModelID                893.000000
fiSecondaryDesc         43.000000
Enclosure                1.000000
Hydraulics_Flow          0.000000
fiModelDesc           2293.000000
ProductGroup             5.000000
Drive_System             0.000000
Hydraulics               1.000000
Tire_Size                0.000000
saleElapsed              1.225075
SalePrice               10.043249

p.s. learn.predict(learn.dls.train.iloc[0]) doesn’t work, coz’ it’s a TabularPandas, not a panda.series.Series