V 1.0.50: Lr_find Expected input batch_size (102512) to match target batch_size (16)

Part 1 (2019)
1

I’m following https://github.com/fastai/course-v3/blob/master/nbs/dl1/lesson3-imdb.ipynb. I’m using fastai 1.0.50.post1, Python 3.7.1. I got batch size error when I run lr_find.

ValueError: Expected input batch_size (102512) to match target batch_size (16).

Below is the related my scripts:

bs = 16
data_lm = TextLMDataBunch.from_csv(path, ‘all_texts.csv’, text_cols=1, label_cols=0, bs=bs)
data_lm.save(‘data_lm_export.pkl’)
data_lm = load_data(path, fname=‘data_clas_export.pkl’, bs = bs)
data_lm.vocab.itos[:20]
data_lm.train_ds[0][0]
learner = language_model_learner(data_lm, arch=AWD_LSTM, drop_mult=0.5)
learner.lr_find()

Then I got error

LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.

ValueError Traceback (most recent call last)
in
----> 1 learner.lr_find()

/home/application/anaconda/lib/python3.7/site-packages/fastai/train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
30 cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
31 epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
—> 32 learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
33
34 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=False, clip:float=None,

/home/application/anaconda/lib/python3.7/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
194 callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
195 if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
–> 196 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
197
198 def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/home/application/anaconda/lib/python3.7/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
98 for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
99 xb, yb = cb_handler.on_batch_begin(xb, yb)
–> 100 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)
101 if cb_handler.on_batch_end(loss): break
102

/home/application/anaconda/lib/python3.7/site-packages/fastai/basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
27
28 if not loss_func: return to_detach(out), yb[0].detach()
—> 29 loss = loss_func(out, *yb)
30
31 if opt is not None:

/home/application/anaconda/lib/python3.7/site-packages/fastai/layers.py in call(self, input, target, **kwargs)
243 if self.floatify: target = target.float()
244 input = input.view(-1,input.shape[-1]) if self.is_2d else input.view(-1)
–> 245 return self.func.call(input, target.view(-1), **kwargs)
246
247 def CrossEntropyFlat(*args, axis:int=-1, **kwargs):

/home/application/anaconda/lib/python3.7/site-packages/torch/nn/modules/module.py in call(self, *input, **kwargs)
487 result = self._slow_forward(*input, **kwargs)
488 else:
–> 489 result = self.forward(*input, **kwargs)
490 for hook in self._forward_hooks.values():
491 hook_result = hook(self, input, result)

/home/application/anaconda/lib/python3.7/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
902 def forward(self, input, target):
903 return F.cross_entropy(input, target, weight=self.weight,
–> 904 ignore_index=self.ignore_index, reduction=self.reduction)
905
906

/home/application/anaconda/lib/python3.7/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
1968 if size_average is not None or reduce is not None:
1969 reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 1970 return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
1971
1972

/home/application/anaconda/lib/python3.7/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
1786 if input.size(0) != target.size(0):
1787 raise ValueError(‘Expected input batch_size ({}) to match target batch_size ({}).’
-> 1788 .format(input.size(0), target.size(0)))
1789 if dim == 2:
1790 ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (102512) to match target batch_size (16).

2

Discard it. It turns out I should not set bs size when I save language model. data_lm = TextLMDataBunch.from_csv(path, ‘all_texts.csv’). Now I don’t have this error anymore

3

I save the TextLMDataBunch.from_csv without set bs size, but when I load_data, set bs size, and run lr_find, this error happens. Have you solved it?

4

Hey everyone,

I’m currently classifying text with the ULMFiT approach.

When I use the data block api to prepare data_lm and data_clas, I get the same error.
However - if I use TextLMDataBunch and TextClasDataBunch it works fine.

I tried to figure out the error, but without success.

Here is my code:

    data_lm = (TextList.from_df(df=df, path=PATH, cols='TEXT')
               .split_subsets(train_size=0.4, valid_size=0.1)
               .label_from_df('LABEL')
               .databunch())

    data_clas = (TextList.from_df(df=df, path=PATH, cols='TEXT', vocab=data_lm.vocab)
                 .split_subsets(train_size=0.4, valid_size=0.1)
                 .label_from_df('LABEL')
                 .databunch())

data_lm.save('data_lm_export.pkl')
data_clas.save('data_clas_export.pkl')

data_lm = load_data(PATH, 'data_lm_export.pkl')
data_clas = load_data(PATH, 'data_clas_export.pkl', bs=64)

weights = 'path/to/weights/30k-pre-ger'
vocs = 'path/to/30k-pre-ger-itos'

learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5, pretrained_fnames=[weights, vocs])

learn.lr_find()

Then the following error occurs:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-30-d81c6bd29d71> in <module>
----> 1 learn.lr_find()

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\fastai\train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
     30     cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
     31     epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
---> 32     learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
     33 
     34 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\fastai\basic_train.py in fit(self, epochs, lr, wd, callbacks)
    194         callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
    195         if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
--> 196         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    197 
    198     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\fastai\basic_train.py in fit(epochs, learn, callbacks, metrics)
     98             for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
     99                 xb, yb = cb_handler.on_batch_begin(xb, yb)
--> 100                 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)
    101                 if cb_handler.on_batch_end(loss): break
    102 

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\fastai\basic_train.py in loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     27 
     28     if not loss_func: return to_detach(out), yb[0].detach()
---> 29     loss = loss_func(out, *yb)
     30 
     31     if opt is not None:

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\fastai\layers.py in __call__(self, input, target, **kwargs)
    265         if self.floatify: target = target.float()
    266         input = input.view(-1,input.shape[-1]) if self.is_2d else input.view(-1)
--> 267         return self.func.__call__(input, target.view(-1), **kwargs)
    268 
    269 def CrossEntropyFlat(*args, axis:int=-1, **kwargs):

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
    902     def forward(self, input, target):
    903         return F.cross_entropy(input, target, weight=self.weight,
--> 904                                ignore_index=self.ignore_index, reduction=self.reduction)
    905 
    906 

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\torch\nn\functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   1968     if size_average is not None or reduce is not None:
   1969         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 1970     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   1971 
   1972 

~\AppData\Local\Continuum\anaconda3\envs\env\lib\site-packages\torch\nn\functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1786     if input.size(0) != target.size(0):
   1787         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'
-> 1788                          .format(input.size(0), target.size(0)))
   1789     if dim == 2:
   1790         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (7872) to match target batch_size (64).

Does anyone have an idea?

Thanks in advance

Edit:
I’m using fastai V 1.0.51 and pytorch V 1.0.1