Hi all,
I was fine-tuning BART from huggingface to perform conditionalgeneration as part of a summarisation task. I instantiated a Learner object with the dataloader, model, etc without any issues:
learn = Learner(
dls,
FastaiWrapper(),
loss_func=SummarisationLoss(),
opt_func=ranger,
splitter=bart_splitter
).to_fp16()
However, when I tried to call lr_find on the learner object, it threw me the following error:
AttributeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in fit(self=<fastai2.learner.Learner object>, n_epoch=1, lr=None, wd=None, cbs=LRFinder, reset_opt=False)
195 try:
–> 196 self._do_begin_fit(n_epoch)
self._do_begin_fit = <bound method Learner._do_begin_fit of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
n_epoch = 1
197 for epoch in range(n_epoch):
27 frames
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in _do_begin_fit(self=<fastai2.learner.Learner object>, n_epoch=1)
169 def _do_begin_fit(self, n_epoch):
–> 170 self.n_epoch,self.loss = n_epoch,tensor(0.); self(‘begin_fit’)
self.n_epoch = 1
self.loss = tensor(0.)
n_epoch = 1
global tensor = <function tensor at 0x7f7e04df8f28>
self = <fastai2.learner.Learner object at 0x7f7d4843dc88>
171
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in call(self=<fastai2.learner.Learner object>, event_name=‘begin_fit’)
133
–> 134 def call(self, event_name): L(event_name).map(self._call_one)
global call = undefined
self = <fastai2.learner.Learner object at 0x7f7d4843dc88>
event_name = ‘begin_fit’
global L = <class ‘fastcore.foundation.L’>
event_name.map = undefined
self._call_one = <bound method Learner._call_one of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
135 def _call_one(self, event_name):
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in map(self=(#1) [‘begin_fit’], f=<bound method Learner._call_one of <fastai2.learner.Learner object>>, *args=(), **kwargs={})
375 else f.getitem)
–> 376 return self._new(map(g, self))
self._new = <bound method L._new of (#1) [‘begin_fit’]>
global map = undefined
g = <fastcore.foundation.bind object at 0x7f7d46b2c2e8>
self = (#1) [‘begin_fit’]
377
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _new(self=(#1) [‘begin_fit’], items=, *args=(), **kwargs={})
326 def _xtra(self): return None
–> 327 def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
global _new = undefined
self = (#1) [‘begin_fit’]
items = <map object at 0x7f7d46b2cf60>
args = ()
kwargs = {}
global type = undefined
global use_list = undefined
328 def getitem(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in call(cls=<class ‘fastcore.foundation.L’>, x=, args=(), **kwargs={‘use_list’: None})
46
—> 47 res = super().call(((x,) + args), **kwargs)
res = undefined
global super.call = undefined
x = <map object at 0x7f7d46b2cf60>
args = ()
kwargs = {‘use_list’: None}
48 res._newchk = 0
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in init(self=<class ‘fastcore.foundation.L’> instance, items=, use_list=None, match=None, *rest=())
317 if (use_list is not None) or not _is_array(items):
–> 318 items = list(items) if use_list else _listify(items)
items = <map object at 0x7f7d46b2cf60>
global list = undefined
use_list = None
global _listify = <function _listify at 0x7f7e05086c80>
319 if match is not None:
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _listify(o=)
253 if isinstance(o, str) or _is_array(o): return [o]
–> 254 if is_iter(o): return list(o)
global is_iter = <function is_iter at 0x7f7e0505de18>
o = <map object at 0x7f7d46b2cf60>
global list = undefined
255 return [o]
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in call(self=<fastcore.foundation.bind object>, *args=[‘begin_fit’], **kwargs={})
219 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
–> 220 return self.fn(*fargs, **kwargs)
self.fn = <bound method Learner._call_one of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
fargs = [‘begin_fit’]
kwargs = {}
221
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in _call_one(self=<fastai2.learner.Learner object>, event_name=‘begin_fit’)
136 assert hasattr(event, event_name)
–> 137 [cb(event_name) for cb in sort_by_run(self.cbs)]
global cb = undefined
event_name = ‘begin_fit’
global sort_by_run = <function sort_by_run at 0x7f7e04de4ae8>
self.cbs = (#5) [TrainEvalCallback,Recorder,ProgressCallback,ModelToHalf,MixedPrecision]
138
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in (.0=<list_iterator object>)
136 assert hasattr(event, event_name)
–> 137 [cb(event_name) for cb in sort_by_run(self.cbs)]
cb = MixedPrecision
global event_name = undefined
global sort_by_run = <function sort_by_run at 0x7f7e04de4ae8>
global self.cbs = undefined
138
/usr/local/lib/python3.6/dist-packages/fastai2/callback/core.py in call(self=MixedPrecision, event_name=‘begin_fit’)
23 (self.run_valid and not getattr(self, ‘training’, False)))
—> 24 if self.run and _run: getattr(self, event_name, noop)()
self.run = True
_run = True
global getattr = undefined
self = MixedPrecision
event_name = ‘begin_fit’
global noop = <function noop at 0x7f7e04df8158>
25 if event_name==‘after_fit’: self.run=True #Reset self.run to True at each end of fit
/usr/local/lib/python3.6/dist-packages/fastai2/callback/fp16.py in begin_fit(self=MixedPrecision)
84 def begin_fit(self):
—> 85 assert self.dls.device.type == ‘cuda’, "Mixed-precision training requires a GPU, remove the call to_fp16"
self.dls.device.type = undefined
86 if self.learn.opt is None: self.learn.create_opt()
AttributeError: ‘str’ object has no attribute ‘type’
During handling of the above exception, another exception occurred:
AttributeError Traceback (most recent call last)
in ()
----> 1 learn.lr_find(start_lr=1e-5, end_lr=20)
global learn.lr_find = <bound method Learner.lr_find of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
global start_lr = undefined
global end_lr = undefined
/usr/local/lib/python3.6/dist-packages/fastai2/callback/schedule.py in lr_find(self=<fastai2.learner.Learner object>, start_lr=1e-05, end_lr=20, num_it=100, stop_div=True, show_plot=True, suggestions=True)
226 n_epoch = num_it//len(self.dls.train) + 1
227 cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)
–> 228 with self.no_logging(): self.fit(n_epoch, cbs=cb)
self.no_logging = <bound method Learner.no_logging of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
self.fit = <bound method Learner.fit of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
n_epoch = 1
global cbs = undefined
cb = LRFinder
229 if show_plot: self.recorder.plot_lr_find()
230 if suggestions:
/usr/local/lib/python3.6/dist-packages/fastcore/utils.py in _f(*args=(<fastai2.learner.Learner object>, 1), **kwargs={‘cbs’: LRFinder})
429 init_args.update(log)
430 setattr(inst, ‘init_args’, init_args)
–> 431 return inst if to_return else f(*args, **kwargs)
inst = <fastai2.learner.Learner object at 0x7f7d4843dc88>
global to_return = undefined
global f = undefined
args = (<fastai2.learner.Learner object at 0x7f7d4843dc88>, 1)
kwargs = {‘cbs’: LRFinder}
432 return _f
433
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in fit(self=<fastai2.learner.Learner object>, n_epoch=1, lr=None, wd=None, cbs=LRFinder, reset_opt=False)
204
205 except CancelFitException: self(‘after_cancel_fit’)
–> 206 finally: self(‘after_fit’)
self = <fastai2.learner.Learner object at 0x7f7d4843dc88>
207
208 def validate(self, ds_idx=1, dl=None, cbs=None):
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in call(self=<fastai2.learner.Learner object>, event_name=‘after_fit’)
132 def ordered_cbs(self, event): return [cb for cb in sort_by_run(self.cbs) if hasattr(cb, event)]
133
–> 134 def call(self, event_name): L(event_name).map(self._call_one)
global call = undefined
self = <fastai2.learner.Learner object at 0x7f7d4843dc88>
event_name = ‘after_fit’
global L = <class ‘fastcore.foundation.L’>
event_name.map = undefined
self._call_one = <bound method Learner._call_one of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
135 def _call_one(self, event_name):
136 assert hasattr(event, event_name)
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in map(self=(#1) [‘after_fit’], f=<bound method Learner._call_one of <fastai2.learner.Learner object>>, *args=(), **kwargs={})
374 else f.format if isinstance(f,str)
375 else f.getitem)
–> 376 return self._new(map(g, self))
self._new = <bound method L._new of (#1) [‘after_fit’]>
global map = undefined
g = <fastcore.foundation.bind object at 0x7f7d46d42d68>
self = (#1) [‘after_fit’]
377
378 def filter(self, f, negate=False, **kwargs):
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _new(self=(#1) [‘after_fit’], items=, *args=(), **kwargs={})
325 @property
326 def _xtra(self): return None
–> 327 def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
global _new = undefined
self = (#1) [‘after_fit’]
items = <map object at 0x7f7d46d42ef0>
args = ()
kwargs = {}
global type = undefined
global use_list = undefined
328 def getitem(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
329 def copy(self): return self._new(self.items.copy())
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in call(cls=<class ‘fastcore.foundation.L’>, x=, args=(), **kwargs={‘use_list’: None})
45 return x
46
—> 47 res = super().call(((x,) + args), **kwargs)
res = undefined
global super.call = undefined
x = <map object at 0x7f7d46d42ef0>
args = ()
kwargs = {‘use_list’: None}
48 res._newchk = 0
49 return res
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in init(self=<class ‘fastcore.foundation.L’> instance, items=, use_list=None, match=None, *rest=())
316 if items is None: items = []
317 if (use_list is not None) or not _is_array(items):
–> 318 items = list(items) if use_list else _listify(items)
items = <map object at 0x7f7d46d42ef0>
global list = undefined
use_list = None
global _listify = <function _listify at 0x7f7e05086c80>
319 if match is not None:
320 if is_coll(match): match = len(match)
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in _listify(o=)
252 if isinstance(o, list): return o
253 if isinstance(o, str) or _is_array(o): return [o]
–> 254 if is_iter(o): return list(o)
global is_iter = <function is_iter at 0x7f7e0505de18>
o = <map object at 0x7f7d46d42ef0>
global list = undefined
255 return [o]
256
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in call(self=<fastcore.foundation.bind object>, *args=[‘after_fit’], **kwargs={})
218 if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
219 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
–> 220 return self.fn(*fargs, **kwargs)
self.fn = <bound method Learner._call_one of <fastai2.learner.Learner object at 0x7f7d4843dc88>>
fargs = [‘after_fit’]
kwargs = {}
221
222 # Cell
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in _call_one(self=<fastai2.learner.Learner object>, event_name=‘after_fit’)
135 def _call_one(self, event_name):
136 assert hasattr(event, event_name)
–> 137 [cb(event_name) for cb in sort_by_run(self.cbs)]
global cb = undefined
event_name = ‘after_fit’
global sort_by_run = <function sort_by_run at 0x7f7e04de4ae8>
self.cbs = (#5) [TrainEvalCallback,Recorder,ProgressCallback,ModelToHalf,MixedPrecision]
138
139 def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)
/usr/local/lib/python3.6/dist-packages/fastai2/learner.py in (.0=<list_iterator object>)
135 def _call_one(self, event_name):
136 assert hasattr(event, event_name)
–> 137 [cb(event_name) for cb in sort_by_run(self.cbs)]
cb = MixedPrecision
global event_name = undefined
global sort_by_run = <function sort_by_run at 0x7f7e04de4ae8>
global self.cbs = undefined
138
139 def _bn_bias_state(self, with_bias): return bn_bias_params(self.model, with_bias).map(self.opt.state)
/usr/local/lib/python3.6/dist-packages/fastai2/callback/core.py in call(self=MixedPrecision, event_name=‘after_fit’)
22 _run = (event_name not in _inner_loop or (self.run_train and getattr(self, ‘training’, True)) or
23 (self.run_valid and not getattr(self, ‘training’, False)))
—> 24 if self.run and _run: getattr(self, event_name, noop)()
self.run = True
_run = True
global getattr = undefined
self = MixedPrecision
event_name = ‘after_fit’
global noop = <function noop at 0x7f7e04df8158>
25 if event_name==‘after_fit’: self.run=True #Reset self.run to True at each end of fit
26
/usr/local/lib/python3.6/dist-packages/fastai2/callback/fp16.py in after_fit(self=MixedPrecision)
122
123 def after_fit(self):
–> 124 _copy_state(self.learn.opt, self.master_pgs, self.model_pgs)
global _copy_state = <function _copy_state at 0x7f7df0056268>
self.learn.opt = <fastai2.optimizer.Lookahead object at 0x7f7d477f0438>
self.master_pgs = undefined
self.model_pgs = undefined
125 self.learn.opt.param_lists = self.old_pgs
126 delattr(self, “master_pgs”)
/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py in getattr(self=MixedPrecision, k=‘master_pgs’)
232 if self._component_attr_filter(k):
233 attr = getattr(self,self._default,None)
–> 234 if attr is not None: return getattr(attr,k)
attr = <fastai2.learner.Learner object at 0x7f7d4843dc88>
global getattr = undefined
k = ‘master_pgs’
235 raise AttributeError(k)
236 def dir(self): return custom_dir(self,self._dir())
AttributeError: ‘Learner’ object has no attribute ‘master_pgs’
I managed to identify the error as being the to_fp16 callback, and I could run lr_find after removing it. However, as transformer models require a lot of GPU RAM, I’d want to attempt incorporating mixed precision into my fine-tuning. Is there any way to do so?