Bayesian Optimization for Hyperparameter Tuning

Hi all,

I am attempting to apply Bayesian Optimization to discover the best combination of hyperparameters for my image segmentation model. For now, this is only the learning rate (lr) and weight decay (wd). Please would someone kindly troubleshoot my error message. Note for legacy reasons, I am using Fastai1.

> RuntimeError: The size of tensor a (3744) must match the size of tensor b (101376) at non-singleton dimension 1

CODE:

# source: https://github.com/muellerzr/BaysianOptimizationFastAI/blob/master/Baysian_Optimization_in_FastAI.ipynb

from fastai.vision import *
from fastai.vision.interpret import *
from fastai.callbacks.hooks import *

# split image-mask pairs into train and valid sets
get_y_fn = lambda x: path_rtk_gt_masks/f'{x.stem}{x.suffix}' # func: gets image's corresponding mask
src = (SegmentationItemList.from_folder(path_rtk_gt_images)
            .split_by_fname_file('../valid.txt')
            .label_from_func(get_y_fn, classes=codes))

# create databunch
src_size = get_dimensions(rtk_gt_images[0])
data = (src.transform(get_transforms(), size=src_size, tfm_y=True)
            .databunch(bs=8)
            .normalize(imagenet_stats))

# custom fit parameters
def fit_with(lr, wd):
  ''' create and fit learner '''
  
  # create learner
  learn = unet_learner(data, models.resnet34, metrics=accuracy, wd=wd) #  main purpose of Learner is to train model using Learner.fit. learn is a trainer for model using data to minimize loss_func with optimizer opt_func

  # train model at specified learning rate
  with progress_disabled_ctx(learn) as learn:
      learn.fit_one_cycle(3, max_lr=lr)
    
  # save, print and return model's overall accuracy
  acc = float(learn.validate(learn.data.valid_dl)[1])
  
  return acc

# call 'fit_with' func and return accuracy
pbounds = {'lr': (1e-5, 1e-2), 'wd': (4e-4, 0.4)} # specify parameter limits
optimizer = BayesianOptimization( 
    f = fit_with,     # use custom fit function    
    pbounds=pbounds,  # use specified limits
    verbose = 2,      # 1 prints a maximum only when it is observed, 0 is completely silent
    random_state=1)

optimizer.maximize()
print(optimizer.max)

DATABUNCH:

Train: LabelList (561 items)
x: SegmentationItemList
Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352)
y: SegmentationLabelList
ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352)
Path: <...>;

Valid: LabelList (140 items)
x: SegmentationItemList
Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352)
y: SegmentationLabelList
ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352)
Path: <...>;

ERROR:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/bayes_opt/target_space.py in probe(self, params)
    190         try:
--> 191             target = self._cache[_hashable(x)]
    192         except KeyError:

KeyError: (0.004176049826978714, 0.2882416675794864)

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
13 frames
<ipython-input-74-de2a7cddfda6> in <module>
     62     random_state=1)
     63 
---> 64 optimizer.maximize()
     65 print(optimizer.max)

/usr/local/lib/python3.7/dist-packages/bayes_opt/bayesian_optimization.py in maximize(self, init_points, n_iter, acq, kappa, kappa_decay, kappa_decay_delay, xi, **gp_params)
    183                 iteration += 1
    184 
--> 185             self.probe(x_probe, lazy=False)
    186 
    187             if self._bounds_transformer:

/usr/local/lib/python3.7/dist-packages/bayes_opt/bayesian_optimization.py in probe(self, params, lazy)
    114             self._queue.add(params)
    115         else:
--> 116             self._space.probe(params)
    117             self.dispatch(Events.OPTIMIZATION_STEP)
    118 

/usr/local/lib/python3.7/dist-packages/bayes_opt/target_space.py in probe(self, params)
    192         except KeyError:
    193             params = dict(zip(self._keys, x))
--> 194             target = self.target_func(**params)
    195             self.register(x, target)
    196         return target

<ipython-input-74-de2a7cddfda6> in fit_with(lr, wd)
     47   # train model at specified learning rate
     48   with progress_disabled_ctx(learn) as learn:
---> 49       learn.fit_one_cycle(3, max_lr=lr)
     50 
     51   # save, print and return model's overall accuracy

/usr/local/lib/python3.7/dist-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)
     21     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,
     22                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))
---> 23     learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
     24 
     25 def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,

/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    198         else: self.opt.lr,self.opt.wd = lr,wd
    199         callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
--> 200         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    201 
    202     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
    104             if not cb_handler.skip_validate and not learn.data.empty_val:
    105                 val_loss = validate(learn.model, learn.data.valid_dl, loss_func=learn.loss_func,
--> 106                                        cb_handler=cb_handler, pbar=pbar)
    107             else: val_loss=None
    108             if cb_handler.on_epoch_end(val_loss): break

/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in validate(model, dl, loss_func, cb_handler, pbar, average, n_batch)
     61             if not is_listy(yb): yb = [yb]
     62             nums.append(first_el(yb).shape[0])
---> 63             if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
     64             if n_batch and (len(nums)>=n_batch): break
     65         nums = np.array(nums, dtype=np.float32)

/usr/local/lib/python3.7/dist-packages/fastai/callback.py in on_batch_end(self, loss)
    306         "Handle end of processing one batch with `loss`."
    307         self.state_dict['last_loss'] = loss
--> 308         self('batch_end', call_mets = not self.state_dict['train'])
    309         if self.state_dict['train']:
    310             self.state_dict['iteration'] += 1

/usr/local/lib/python3.7/dist-packages/fastai/callback.py in __call__(self, cb_name, call_mets, **kwargs)
    248         "Call through to all of the `CallbakHandler` functions."
    249         if call_mets:
--> 250             for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
    251         for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
    252 

/usr/local/lib/python3.7/dist-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
    239     def _call_and_update(self, cb, cb_name, **kwargs)->None:
    240         "Call `cb_name` on `cb` and update the inner state."
--> 241         new = ifnone(getattr(cb, f'on_{cb_name}')(**self.state_dict, **kwargs), dict())
    242         for k,v in new.items():
    243             if k not in self.state_dict:

/usr/local/lib/python3.7/dist-packages/fastai/callback.py in on_batch_end(self, last_output, last_target, **kwargs)
    342         if not is_listy(last_target): last_target=[last_target]
    343         self.count += first_el(last_target).size(0)
--> 344         val = self.func(last_output, *last_target)
    345         if self.world:
    346             val = val.clone()

/usr/local/lib/python3.7/dist-packages/fastai/metrics.py in accuracy(input, targs)
     28     input = input.argmax(dim=-1).view(n,-1)
     29     targs = targs.view(n,-1)
---> 30     return (input==targs).float().mean()
     31 
     32 def accuracy_thresh(y_pred:Tensor, y_true:Tensor, thresh:float=0.5, sigmoid:bool=True)->Rank0Tensor:

RuntimeError: The size of tensor a (3744) must match the size of tensor b (101376) at non-singleton dimension 1

Hi @taylort139,

I think the issue here is that the target is not getting transformed.

Your predictions are 3744 in number which I guess is the product of the size tuple obtained from src_size and the predictions are still 288*352=101376 in number. Hence the comparison is not happening properly and accuracy figure cannot be calculated.

To validate this hypothesis, could you simple try keeping the src_size = (288, 352) and check? Also, I don’t think accuracy is the best metric to evaluate the model, you could try IoU or Dice co-efficient to get a fair performance measure…

Thanks,
Vinayak.

Try weights and biases Sweeps. It’s a very simple tool combined with a fastai callback. We actually used it with Jeremy to do the resnet18 arch explore. It has Bayesian optimisation built in.

PD: I would encourage you to move out of fastai v1. I have done a bunch of Semantic Seg using fastai, if you need any help.

Check @scottire recent tweet: https://twitter.com/_ScottCondron/status/1575419805309575168

1 Like