I suspect I’m missing something super obvious, but I’m having some problems using the PartialDL / partial_dataloaders
[source] callback. At the moment I’m just trying to get it working on the Pets data.
The example/test in the docs:
n = 160
dsets = Datasets(torch.arange(n).float())
dls = dsets.partial_dataloaders(partial_n=32, bs=16)
assert len(dls[0])==2
for batch in dls[0]:
assert len(batch[0])==16
seems to work okay. But when I try with the pets data:
dls.dataset.partial_dataloaders(partial_n=32, bs=8)
I get an error:
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
[<ipython-input-36-fa775b2a0b09>](https://localhost:8080/#) in <module>()
----> 1 dls.dataset.partial_dataloaders(partial_n=32, bs=8)
9 frames
[/usr/local/lib/python3.6/dist-packages/fastai/callback/data.py](https://localhost:8080/#) in partial_dataloaders(self, partial_n, bs, **kwargs)
**62** "Create a partial dataloader `PartialDL` for the training set"
**63** xtra_kwargs = [{}] * (self.n_subsets-1)
---> 64 return self.dataloaders(bs=bs, dl_type=PartialDL, dl_kwargs=({'partial_n':partial_n}, *xtra_kwargs), **kwargs)
[/usr/local/lib/python3.6/dist-packages/fastai/data/core.py](https://localhost:8080/#) in dataloaders(self, bs, val_bs, shuffle_train, n, path, dl_type, dl_kwargs, device, **kwargs)
**211** if dl_type is None: dl_type = self._dl_type
**212** drop_last = kwargs.pop('drop_last', shuffle_train)
--> 213 dl = dl_type(self.subset(0), bs=bs, shuffle=shuffle_train, drop_last=drop_last, n=n, device=device,
**214** **merge(kwargs, dl_kwargs[0]))
**215** dls = [dl] + [dl.new(self.subset(i), bs=(bs if val_bs is None else val_bs), shuffle=False, drop_last=False,
[/usr/local/lib/python3.6/dist-packages/fastai/data/core.py](https://localhost:8080/#) in subset(self, i)
**321** def __repr__(self): return coll_repr(self)
**322** def decode(self, o, full=True): return tuple(tl.decode(o_, full=full) for o_,tl in zip(o,tuplify(self.tls, match=o)))
--> 323 def subset(self, i): return type(self)(tls=L(tl.subset(i) for tl in self.tls), n_inp=self.n_inp)
**324** def _new(self, items, *args, **kwargs): return super()._new(items, tfms=self.tfms, do_setup=False, **kwargs)
**325** def overlapping_splits(self): return self.tls[0].overlapping_splits()
[/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py](https://localhost:8080/#) in __call__(cls, x, *args, **kwargs)
**95** def __call__(cls, x=None, *args, **kwargs):
**96** if not args and not kwargs and x is not None and isinstance(x,cls): return x
---> 97 return super().__call__(x, *args, **kwargs)
**98**
**99** # Cell
[/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py](https://localhost:8080/#) in __init__(self, items, use_list, match, *rest)
**103** def __init__(self, items=None, *rest, use_list=False, match=None):
**104** if (use_list is not None) or not is_array(items):
--> 105 items = listify(items, *rest, use_list=use_list, match=match)
**106** super().__init__(items)
**107**
[/usr/local/lib/python3.6/dist-packages/fastcore/basics.py](https://localhost:8080/#) in listify(o, use_list, match, *rest)
**54** elif isinstance(o, list): res = o
**55** elif isinstance(o, str) or is_array(o): res = [o]
---> 56 elif is_iter(o): res = list(o)
**57** else: res = [o]
**58** if match is not None:
[/usr/local/lib/python3.6/dist-packages/fastai/data/core.py](https://localhost:8080/#) in <genexpr>(.0)
**321** def __repr__(self): return coll_repr(self)
**322** def decode(self, o, full=True): return tuple(tl.decode(o_, full=full) for o_,tl in zip(o,tuplify(self.tls, match=o)))
--> 323 def subset(self, i): return type(self)(tls=L(tl.subset(i) for tl in self.tls), n_inp=self.n_inp)
**324** def _new(self, items, *args, **kwargs): return super()._new(items, tfms=self.tfms, do_setup=False, **kwargs)
**325** def overlapping_splits(self): return self.tls[0].overlapping_splits()
[/usr/local/lib/python3.6/dist-packages/fastai/data/core.py](https://localhost:8080/#) in subset(self, i)
**239** split_idx = ifnone(split_idx,self.split_idx)
**240** return super()._new(items, tfms=self.tfms, do_setup=False, types=self.types, split_idx=split_idx, **kwargs)
--> 241 def subset(self, i): return self._new(self._get(self.splits[i]), split_idx=i)
**242** def _after_item(self, o): return self.tfms(o)
**243** def __repr__(self): return f"{self.__class__.__name__}: {self.items}\ntfms - {self.tfms.fs}"
[/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py](https://localhost:8080/#) in _get(self, i)
**117** return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')
**118** else self.items.__array__()[(i,)] if hasattr(self.items,'__array__')
--> 119 else [self.items[i_] for i_ in i])
**120**
**121** def __setitem__(self, idx, o):
[/usr/local/lib/python3.6/dist-packages/fastcore/foundation.py](https://localhost:8080/#) in <listcomp>(.0)
**117** return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')
**118** else self.items.__array__()[(i,)] if hasattr(self.items,'__array__')
--> 119 else [self.items[i_] for i_ in i])
**120**
**121** def __setitem__(self, idx, o):
IndexError: list index out of range
Again, I suspect I’m missing something super obvious but I’d love to get this working
Example Colab notebook: https://colab.research.google.com/drive/1yzR6GVEzlpEufzAfJYXZgl0LgVeKMwwq?usp=sharing