Alright, I managed to adjust the “get_files” function a little so it will also check my condition before it adds the desired photo to the list.
def get_files2(path, extensions=image_extensions, recurse=True, folders=None, followlinks=True,modulus=10,x_m=2,y_m=3):
"Get all the files in `path` with optional `extensions`, optionally with `recurse`, only in `folders`, if specified."
path = Path(path)
folders=L(folders)
extensions = setify(extensions)
extensions = {e.lower() for e in extensions}
if recurse:
res = []
for i,(p,d,f) in enumerate(os.walk(path, followlinks=followlinks)): # returns (dirpath, dirnames, filenames)
if len(folders) !=0 and i==0: d[:] = [o for o in d if o in folders]
else: d[:] = [o for o in d if not o.startswith('.')]
if len(folders) !=0 and i==0 and '.' not in folders: continue
res += _get_files(p, f, extensions)
**elif x_m==False and y_m==False:**
** f = [o.name for o in os.scandir(path) if o.is_file()]**
** res = _get_files(path, f, extensions)**
** else:**
** f = [o.name for o in os.scandir(path) if (o.is_file() and (int(re.findall('\d+',o.name)[0]))%modulus in (x_m,y_m))]**
** res = _get_files(path, f, extensions)**
return L(res)
I mostly added this: (with import re, as well)
(o.is_file() and (int(re.findall(’\d+’,o.name)[0]))%modulus in (x_m,y_m))]
So right afterward, in the main folder, the items that were appended into the list were only the ones that followed the condition that I added with the modulus.
But when I run the “summary()” command, it couldn’t really work out. It kept adding other photos in the folder to the list of objects (of which it marked as “None”).
This is what I get by running “one_batch”
/usr/local/lib/python3.7/dist-packages/fastai/data/load.py in one_batch(self)
146 def one_batch(self):
147 if self.n is not None and len(self)==0: raise ValueError(f'This DataLoader does not contain any batches')
--> 148 with self.fake_l.no_multiproc(): res = first(self)
149 if hasattr(self, 'it'): delattr(self, 'it')
150 return res
/usr/local/lib/python3.7/dist-packages/fastcore/basics.py in first(x, f, negate, **kwargs)
553 x = iter(x)
554 if f: x = filter_ex(x, f=f, negate=negate, gen=True, **kwargs)
--> 555 return next(x, None)
556
557 # Cell
/usr/local/lib/python3.7/dist-packages/fastai/data/load.py in __iter__(self)
107 self.before_iter()
108 self.__idxs=self.get_idxs() # called in context of main process (not workers/subprocesses)
--> 109 for b in _loaders[self.fake_l.num_workers==0](self.fake_l):
110 if self.device is not None: b = to_device(b, self.device)
111 yield self.after_batch(b)
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py in __next__(self)
519 if self._sampler_iter is None:
520 self._reset()
--> 521 data = self._next_data()
522 self._num_yielded += 1
523 if self._dataset_kind == _DatasetKind.Iterable and \
/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py in _next_data(self)
559 def _next_data(self):
560 index = self._next_index() # may raise StopIteration
--> 561 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
562 if self._pin_memory:
563 data = _utils.pin_memory.pin_memory(data)
/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
37 raise StopIteration
38 else:
---> 39 data = next(self.dataset_iter)
40 return self.collate_fn(data)
41
/usr/local/lib/python3.7/dist-packages/fastai/data/load.py in create_batches(self, samps)
116 if self.dataset is not None: self.it = iter(self.dataset)
117 res = filter(lambda o:o is not None, map(self.do_item, samps))
--> 118 yield from map(self.do_batch, self.chunkify(res))
119
120 def new(self, dataset=None, cls=None, **kwargs):
/usr/local/lib/python3.7/dist-packages/fastcore/basics.py in chunked(it, chunk_sz, drop_last, n_chunks)
215 if not isinstance(it, Iterator): it = iter(it)
216 while True:
--> 217 res = list(itertools.islice(it, chunk_sz))
218 if res and (len(res)==chunk_sz or not drop_last): yield res
219 if len(res)<chunk_sz: return
/usr/local/lib/python3.7/dist-packages/fastai/data/load.py in do_item(self, s)
131 def prebatched(self): return self.bs is None
132 def do_item(self, s):
--> 133 try: return self.after_item(self.create_item(s))
134 except SkipItemException: return None
135 def chunkify(self, b): return b if self.prebatched else chunked(b, self.bs, self.drop_last)
/usr/local/lib/python3.7/dist-packages/fastcore/transform.py in __call__(self, o)
198 self.fs = self.fs.sorted(key='order')
199
--> 200 def __call__(self, o): return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
201 def __repr__(self): return f"Pipeline: {' -> '.join([f.name for f in self.fs if f.name != 'noop'])}"
202 def __getitem__(self,i): return self.fs[i]
/usr/local/lib/python3.7/dist-packages/fastcore/transform.py in compose_tfms(x, tfms, is_enc, reverse, **kwargs)
148 for f in tfms:
149 if not is_enc: f = f.decode
--> 150 x = f(x, **kwargs)
151 return x
152
/usr/local/lib/python3.7/dist-packages/fastai/vision/augment.py in __call__(self, b, split_idx, **kwargs)
32
33 def __call__(self, b, split_idx=None, **kwargs):
---> 34 self.before_call(b, split_idx=split_idx)
35 return super().__call__(b, split_idx=split_idx, **kwargs) if self.do else b
36
/usr/local/lib/python3.7/dist-packages/fastai/vision/augment.py in before_call(self, b, split_idx)
241
242 def before_call(self, b, split_idx):
--> 243 w,h = self.orig_sz = _get_sz(b)
244 if split_idx:
245 xtra = math.ceil(max(*self.size[:2])*self.val_xtra/8)*8
/usr/local/lib/python3.7/dist-packages/fastai/vision/augment.py in _get_sz(x)
142 def _get_sz(x):
143 if isinstance(x, tuple): x = x[0]
--> 144 if not isinstance(x, Tensor): return fastuple(x.size)
145 return fastuple(getattr(x, 'img_size', getattr(x, 'sz', (x.shape[-1], x.shape[-2]))))
146
AttributeError: 'NoneType' object has no attribute 'size'
Out of 59 photos in a folder, the get_files successfully reads only the relevant 24 files.
But when it goes through summary() method, it tries once again to read all the 59 photos, which leaves many (35 objects) as “NoneType”.
What shall I change then?