How to validate test set? - fastai v2

Hello everyone,

I’m trying to validate a test set on a previously trained model.

The error I have is that when the function “learn.validate(dl=test_dls)” is executed the following error is returned: TypeError: object of type ‘DataLoaders’ has no len().

The test dataloader (test_dls) is build correctly because if I print the length of test_dls.train.items and test_dls.valid.items I get the expected results.

My dataset structure is:

RepoGit_ref

test
train_val

train
valid

I tried to find a reference example on the fastai forums and searching on the Internet but I didn’t find any.

I hope someone can help me solve this issue.

At the end of this post I show the key parts of my code and the full stack trace of the error:

Thank you.

Load trained model

data_train = DataBlock(
    blocks=(ImageBlock, CategoryBlock), # CategoryBlock = label
    get_items=get_image_files,
    get_y= parent_label, # parent_label = the folders names of the images are their labels (normal/effusion)
    splitter= GrandparentSplitter(), # split training/validation
    # validation images will be the ones that are stored inside "valid" folder of the dataset
    batch_tfms= [*aug_transforms(do_flip=False, size=(120,160)), Normalize.from_stats(*imagenet_stats)] 
)

dls = data_train.dataloaders(path_dl, path=path_dl, bs = 64) # bs: how many samples per batch to load 

learn = cnn_learner(dls, resnet50, metrics=accuracy)

learn.load('learned_effusion_classif_cossos_resnet50')

Inference

data_test = DataBlock(
    blocks=(ImageBlock, CategoryBlock), # CategoryBlock = label
    get_items=get_image_files,
    get_y= parent_label, # parent_label = the folders names of the images are their labels (normal/effusion)
    splitter= GrandparentSplitter(train_name='train', valid_name='test'), # split training/validation
    # validation images will be the ones that are stored inside "valid" folder of the dataset
    batch_tfms= [*aug_transforms(do_flip=False, size=(120,160)), Normalize.from_stats(*imagenet_stats)] )

test_ds_path = '../../../../Datasets/repoGit_ref/'

test_ds_path_dl = Path(test_ds_path)
Path.BASE_PATH = test_ds_path_dl
test_ds_path_dl.ls().sorted()

test_dls = data_test.dataloaders(test_ds_path_dl, path=test_ds_path_dl, bs = 64) # bs: how many samples per batch to load 

learn.validate(dl=test_dls)

Stack trace error

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_15424\1554754936.py in <module>
----> 1 learn.validate(dl=test_dls)

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\learner.py in validate(self, ds_idx, dl, cbs)
    233     def validate(self, ds_idx=1, dl=None, cbs=None):
    234         if dl is None: dl = self.dls[ds_idx]
--> 235         with self.validation_context(cbs=cbs): self._do_epoch_validate(ds_idx, dl)
    236         return getattr(self, 'final_record', None)
    237 

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\learner.py in _do_epoch_validate(self, ds_idx, dl)
    201         if dl is None: dl = self.dls[ds_idx]
    202         self.dl = dl
--> 203         with torch.no_grad(): self._with_events(self.all_batches, 'validate', CancelValidException)
    204 
    205     def _do_epoch(self):

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\learner.py in _with_events(self, f, event_type, ex, final)
    161 
    162     def _with_events(self, f, event_type, ex, final=noop):
--> 163         try: self(f'before_{event_type}');  f()
    164         except ex: self(f'after_cancel_{event_type}')
    165         self(f'after_{event_type}');  final()

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\learner.py in __call__(self, event_name)
    139 
    140     def ordered_cbs(self, event): return [cb for cb in self.cbs.sorted('order') if hasattr(cb, event)]
--> 141     def __call__(self, event_name): L(event_name).map(self._call_one)
    142 
    143     def _call_one(self, event_name):

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastcore\foundation.py in map(self, f, gen, *args, **kwargs)
    153     def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
    154 
--> 155     def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
    156     def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
    157     def argfirst(self, f, negate=False): return first(i for i,o in self.enumerate() if f(o))

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastcore\basics.py in map_ex(iterable, f, gen, *args, **kwargs)
    696     res = map(g, iterable)
    697     if gen: return res
--> 698     return list(res)
    699 
    700 # Cell

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastcore\basics.py in __call__(self, *args, **kwargs)
    681             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    682         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 683         return self.func(*fargs, **kwargs)
    684 
    685 # Cell

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\learner.py in _call_one(self, event_name)
    143     def _call_one(self, event_name):
    144         if not hasattr(event, event_name): raise Exception(f'missing {event_name}')
--> 145         for cb in self.cbs.sorted('order'): cb(event_name)
    146 
    147     def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\callback\core.py in __call__(self, event_name)
     43                (self.run_valid and not getattr(self, 'training', False)))
     44         res = None
---> 45         if self.run and _run: res = getattr(self, event_name, noop)()
     46         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
     47         return res

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\callback\progress.py in before_validate(self)
     24 
     25     def before_train(self):    self._launch_pbar()
---> 26     def before_validate(self): self._launch_pbar()
     27     def after_train(self):     self.pbar.on_iter_end()
     28     def after_validate(self):  self.pbar.on_iter_end()

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastai\callback\progress.py in _launch_pbar(self)
     32 
     33     def _launch_pbar(self):
---> 34         self.pbar = progress_bar(self.dl, parent=getattr(self, 'mbar', None), leave=False)
     35         self.pbar.update(0)
     36 

c:\Users\marcp\AppData\Local\Programs\Python\Python37\lib\site-packages\fastprogress\fastprogress.py in __init__(self, gen, total, display, leave, parent, master, comment)
     17     def __init__(self, gen, total=None, display=True, leave=True, parent=None, master=None, comment=''):
     18         self.gen,self.parent,self.master,self.comment = gen,parent,master,comment
---> 19         self.total = len(gen) if total is None else total
     20         self.last_v = 0
     21         if parent is None: self.leave,self.display = leave,display

TypeError: object of type 'DataLoaders' has no len()