How to do batch inference?

daleevans · February 25, 2019, 6:06pm

I’ve read the inference tutorial, and I can load a trained learner and do a single prediction with .predict() but I don’t see how to easily do batch inference. I can construct a batch tensor and run it directly through learner.model(data) but it seem like there’s probably a smarter way to do that. Can anyone point me in the right direction?

yeldarb · February 25, 2019, 7:02pm

Try .pred_batch(), it’s what .predict() is actually calling behind the scenes (with a batch containing a single item).

github.com

fastai/fastai/blob/master/fastai/basic_train.py#L336


def backward(self, item):
    "Pass `item` through the model and computes the gradient. Useful if `backward_hooks` are attached."
    xb,yb = self.data.one_item(item)
    loss = loss_batch(self.model.eval(), xb, yb, self.loss_func, opt=FakeOptimizer(),
                      cb_handler=CallbackHandler(self.callbacks))
    return loss


def predict(self, item:ItemBase, **kwargs):
    "Return predicted class, label and probabilities for `item`."
    batch = self.data.one_item(item)
    res = self.pred_batch(batch=batch)
    pred,x = res[0],batch[0]
    norm = getattr(self.data,'norm',False)
    if norm:
        x = self.data.denorm(x)
        if norm.keywords.get('do_y',False): pred = self.data.denorm(pred)
    ds = self.data.single_ds
    pred = ds.y.analyze_pred(pred, **kwargs)
    out = ds.y.reconstruct(pred, ds.x.reconstruct(x[0])) if has_arg(ds.y.reconstruct, 'x') else ds.y.reconstruct(pred)
    return out, pred, res[0]

github.com

fastai/fastai/blob/master/fastai/basic_train.py#L310


    gc.collect()
    return self


def get_preds(self, ds_type:DatasetType=DatasetType.Valid, with_loss:bool=False, n_batch:Optional[int]=None,
              pbar:Optional[PBar]=None) -> List[Tensor]:
    "Return predictions and targets on `ds_type` dataset."
    lf = self.loss_func if with_loss else None
    return get_preds(self.model, self.dl(ds_type), cb_handler=CallbackHandler(self.callbacks),
                     activ=_loss_func2activ(self.loss_func), loss_func=lf, n_batch=n_batch, pbar=pbar)


def pred_batch(self, ds_type:DatasetType=DatasetType.Valid, batch:Tuple=None, reconstruct:bool=False) -> List[Tensor]:
    "Return output of the model on one batch from `ds_type` dataset."
    if batch is not None: xb,yb = batch
    else: xb,yb = self.data.one_batch(ds_type, detach=False, denorm=False)
    cb_handler = CallbackHandler(self.callbacks)
    xb,yb = cb_handler.on_batch_begin(xb,yb, train=False)
    preds = loss_batch(self.model.eval(), xb, yb, cb_handler=cb_handler)
    res = _loss_func2activ(self.loss_func)(preds[0])
    if not reconstruct: return res
    res = res.detach().cpu()
    ds = self.dl(ds_type).dataset

austinmw · March 22, 2019, 7:30pm

@yeldarb Hi, I seem to be getting the same average inference speed using a loop of predict versus a single pred_batch. Is that normal? I thought batch prediction would be faster.

noamholz · July 17, 2019, 2:32pm

Same experience here. No speed differences.

EpiphanyMania · July 28, 2019, 5:54am

Predict internally makes your image as “a batch of size one”. Hence same average inference speed.