Hello,
I want to create a test dataloader for inference.
Until now I created a whole new DataBlock and from that Block a new dataloader.
I recently saw a code snippet in the fastai library, where a test dataloader was created the following way:
dl = learner.dls.test_dl(test_items=items)
Unfortunately when I call
learner.get_preds(dl=dl, with_loss=True)
I get the following error:
IndexError Traceback (most recent call last)
<ipython-input-121-175f385d0cf4> in <module>
----> 1 preds = learner.get_preds(dl=dl_valid_exp, with_loss=True)
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in get_preds(self, ds_idx, dl, with_input, with_decoded, with_loss, act, inner, reorder, cbs, **kwargs)
241 if with_loss: ctx_mgrs.append(self.loss_not_reduced())
242 with ContextManagers(ctx_mgrs):
--> 243 self._do_epoch_validate(dl=dl)
244 if act is None: act = getattr(self.loss_func, 'activation', noop)
245 res = cb.all_tensors()
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in _do_epoch_validate(self, ds_idx, dl)
191 if dl is None: dl = self.dls[ds_idx]
192 self.dl = dl
--> 193 with torch.no_grad(): self._with_events(self.all_batches, 'validate', CancelValidException)
194
195 def _do_epoch(self):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
158
159 def _with_events(self, f, event_type, ex, final=noop):
--> 160 try: self(f'before_{event_type}'); f()
161 except ex: self(f'after_cancel_{event_type}')
162 self(f'after_{event_type}'); final()
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in all_batches(self)
164 def all_batches(self):
165 self.n_iter = len(self.dl)
--> 166 for o in enumerate(self.dl): self.one_batch(*o)
167
168 def _do_one_batch(self):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in one_batch(self, i, b)
182 self.iter = i
183 self._split(b)
--> 184 self._with_events(self._do_one_batch, 'batch', CancelBatchException)
185
186 def _do_epoch_train(self):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
160 try: self(f'before_{event_type}'); f()
161 except ex: self(f'after_cancel_{event_type}')
--> 162 self(f'after_{event_type}'); final()
163
164 def all_batches(self):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in __call__(self, event_name)
139
140 def ordered_cbs(self, event): return [cb for cb in self.cbs.sorted('order') if hasattr(cb, event)]
--> 141 def __call__(self, event_name): L(event_name).map(self._call_one)
142
143 def _call_one(self, event_name):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, gen, *args, **kwargs)
152 def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
153
--> 154 def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
155 def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
156 def filter(self, f=noop, negate=False, gen=False, **kwargs):
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastcore/basics.py in map_ex(iterable, f, gen, *args, **kwargs)
664 res = map(g, iterable)
665 if gen: return res
--> 666 return list(res)
667
668 # Cell
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastcore/basics.py in __call__(self, *args, **kwargs)
649 if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
650 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 651 return self.func(*fargs, **kwargs)
652
653 # Cell
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/learner.py in _call_one(self, event_name)
143 def _call_one(self, event_name):
144 if not hasattr(event, event_name): raise Exception(f'missing {event_name}')
--> 145 for cb in self.cbs.sorted('order'): cb(event_name)
146
147 def _bn_bias_state(self, with_bias): return norm_bias_params(self.model, with_bias).map(self.opt.state)
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/callback/core.py in __call__(self, event_name)
42 (self.run_valid and not getattr(self, 'training', False)))
43 res = None
---> 44 if self.run and _run: res = getattr(self, event_name, noop)()
45 if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit
46 return res
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/callback/core.py in after_batch(self)
125 else: (self.save_targs/str(self.iter)).save_array(targs[0])
126 if self.with_loss:
--> 127 bs = find_bs(self.yb)
128 loss = self.loss if self.loss.numel() == bs else self.loss.view(bs,-1).mean(1)
129 self.losses.append(self.learn.to_detach(loss))
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/torch_core.py in find_bs(b)
550 def find_bs(b):
551 "Recursively search the batch size of `b`."
--> 552 return item_find(b).shape[0]
553
554 # Cell
/opt/conda/envs/DLM_Py3/lib/python3.7/site-packages/fastai/torch_core.py in item_find(x, idx)
536 def item_find(x, idx=0):
537 "Recursively takes the `idx`-th element of `x`"
--> 538 if is_listy(x): return item_find(x[idx])
539 if isinstance(x,dict):
540 key = list(x.keys())[idx] if isinstance(idx, int) else idx
IndexError: tuple index out of range
I am not quite sure what this means. Can someon give me a hint in the right direction?
My data block and dataloader creation look like this:
final_size = 512
bs = 35
data = fastai.data.block.DataBlock(
blocks=(TileImageBlock, fastai.data.block.MultiCategoryBlock),
get_x=lambda x: x,
get_y=lambda x: x.get_labels(),
splitter=fastai.data.transforms.FuncSplitter(lambda x: x.get_dataset_type() == shared.enums.DatasetType.validation),
item_tfms=fastai.vision.augment.Resize(size=final_size, method = 'squish'),
batch_tfms=fastai.vision.augment.aug_transforms(flip_vert=True))
dls = data.dataloaders(patient_manager.get_tiles(dataset_type=DatasetType.train)\
+patient_manager.get_tiles(dataset_type=DatasetType.validation),
bs=bs,
verbose=False)
(TileImageBlock and the object type “Tile” which is returned by patient_manager.get_tiles are custom objects that represent a Tile of a whole-slide image and make it possible to extract png images on the fly from whole-slide images)
Thanks in advance,
Christoph