Batch normalization normalizes activations and not the input data.
Yes it is absolutely necessary. Without it, the range of pixel values will be between 0-255 and running pretrained networks with these values will give very bad results as these networks were trained on normalized images.
If ur training ur own network without transfer learning(i.e.randomly initialized weights), then this is not a mandatory thing to do as batch norm layers will take care of it in later stages but still a good thing to have to help smoother optimization
1 Like
i dint look at the quickdoodle competition data but combining the multiple CSVs to get a single csv is an option
I have written a brief blog post based on my notes for lesson 1.
Please have a look.
4 Likes
He was using AWS SageMaker during the session.
You might want to get permission before sharing link to the course site
4 Likes
It was asked, that you do not share the course website outside of this forum. (See the first post in this topic.)
I will edit it. Thnaks for poiniting
Yeah, oftentimes it’s not in the first linear layer
Has anyone encountered issues running cell [9]?
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-9-5299576508f3> in <module>
1 data = ImageDataBunch.from_name_re(path_img, fnames, pat, ds_tfms=get_transforms(), size=224)
----> 2 data.normalize(imagenet_stats)
~/git/fastai/fastai/vision/data.py in normalize(self, stats)
340 def normalize(self, stats:Collection[Tensor]=None)->None:
341 "Add normalize transform using `stats` (defaults to `DataBunch.batch_stats`)"
--> 342 stats = ifnone(stats, self.batch_stats())
343 if getattr(self,'norm',False): raise Exception('Can not call normalize twice')
344 self.norm,self.denorm = normalize_funcs(*stats)
~/git/fastai/fastai/vision/data.py in batch_stats(self, funcs)
335 "Grab a batch of data and call reduction function `func` per channel"
336 funcs = ifnone(funcs, [torch.mean,torch.std])
--> 337 x = self.valid_dl.one_batch()[0].cpu()
338 return [func(channel_view(x), 1) for func in funcs]
339
~/git/fastai/fastai/basic_data.py in one_batch(self)
57 self.num_workers = 0
58 it = iter(self)
---> 59 try: return next(it)
60 finally: self.num_workers = w
61
~/git/fastai/fastai/basic_data.py in __iter__(self)
50 def __iter__(self):
51 "Process and returns items from `DataLoader`."
---> 52 for b in self.dl: yield self.proc_batch(b)
53
54 def one_batch(self)->Collection[Tensor]:
~/.pyenv/versions/3.6.6/envs/fastai/lib/python3.6/site-packages/torch/utils/data/dataloader.py in __next__(self)
312 if self.num_workers == 0: # same-process loading
313 indices = next(self.sample_iter) # may raise StopIteration
--> 314 batch = self.collate_fn([self.dataset[i] for i in indices])
315 if self.pin_memory:
316 batch = pin_memory_batch(batch)
~/.pyenv/versions/3.6.6/envs/fastai/lib/python3.6/site-packages/torch/utils/data/dataloader.py in <listcomp>(.0)
312 if self.num_workers == 0: # same-process loading
313 indices = next(self.sample_iter) # may raise StopIteration
--> 314 batch = self.collate_fn([self.dataset[i] for i in indices])
315 if self.pin_memory:
316 batch = pin_memory_batch(batch)
~/git/fastai/fastai/vision/data.py in __getitem__(self, idx)
196 "Return tfms(x),y."
197 x,y = self.ds[idx]
--> 198 x = apply_tfms(self.tfms, x, **self.kwargs)
199 if self.tfm_y: y = apply_tfms(self.tfms, y, **self.y_kwargs)
200 return x, y
~/git/fastai/fastai/vision/image.py in apply_tfms(tfms, x, do_resolve, xtra, size, mult, do_crop, padding_mode, **kwargs)
549 for tfm in tfms:
550 if tfm.tfm in xtra: x = tfm(x, **xtra[tfm.tfm])
--> 551 elif tfm in size_tfms: x = tfm(x, size=size, padding_mode=padding_mode)
552 else: x = tfm(x)
553 return x
~/git/fastai/fastai/vision/image.py in __call__(self, x, *args, **kwargs)
456 def __call__(self, x:Image, *args, **kwargs)->Image:
457 "Randomly execute our tfm on `x`."
--> 458 return self.tfm(x, *args, **{**self.resolved, **kwargs}) if self.do_run else x
459
460 def _resolve_tfms(tfms:TfmList):
~/git/fastai/fastai/vision/image.py in __call__(self, p, is_random, *args, **kwargs)
400 def __call__(self, *args:Any, p:float=1., is_random:bool=True, **kwargs:Any)->Image:
401 "Calc now if `args` passed; else create a transform called prob `p` if `random`."
--> 402 if args: return self.calc(*args, **kwargs)
403 else: return RandTransform(self, kwargs=kwargs, is_random=is_random, p=p)
404
~/git/fastai/fastai/vision/image.py in calc(self, x, *args, **kwargs)
405 def calc(self, x:Image, *args:Any, **kwargs:Any)->Image:
406 "Apply to image `x`, wrapping it if necessary."
--> 407 if self._wrap: return getattr(x, self._wrap)(self.func, *args, **kwargs)
408 else: return self.func(x, *args, **kwargs)
409
~/git/fastai/fastai/vision/image.py in pixel(self, func, *args, **kwargs)
154 def pixel(self, func:PixelFunc, *args, **kwargs)->'Image':
155 "Equivalent to `image.px = func(image.px)`."
--> 156 self.px = func(self.px, *args, **kwargs)
157 return self
158
~/git/fastai/fastai/vision/image.py in px(self)
127 def px(self)->TensorImage:
128 "Get the tensor pixel buffer."
--> 129 self.refresh()
130 return self._px
131 @px.setter
~/git/fastai/fastai/vision/image.py in refresh(self)
119 self._logit_px = None
120 if self._affine_mat is not None or self._flow is not None:
--> 121 self._px = _grid_sample(self._px, self.flow, **self.sample_kwargs)
122 self.sample_kwargs = {}
123 self._flow = None
~/git/fastai/fastai/vision/image.py in _grid_sample(x, coords, mode, padding_mode, **kwargs)
465 "Grab pixels in `coords` from `input` sampling by `mode`. `paddding_mode` is reflection, border or zeros."
466 coords = coords.flow.permute(0, 3, 1, 2).contiguous().permute(0, 2, 3, 1) # optimize layout for grid_sample
--> 467 return F.grid_sample(x[None], coords, mode=mode, padding_mode=padding_mode)[0]
468
469 def _affine_grid(size:TensorImageSize)->FlowField:
~/.pyenv/versions/3.6.6/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py in grid_sample(input, grid, mode, padding_mode)
2090 padding_mode = GRID_SAMPLE_MODE_BORDER
2091 else:
-> 2092 raise ValueError("padding_mode needs to be 'zeros' or 'border', but got {}".format(padding_mode))
2093 return torch.grid_sampler(input, grid, padding_mode)
2094
ValueError: padding_mode needs to be 'zeros' or 'border', but got reflection
I’ve installed python 3.6.6, fastai 1.0.12.dev0 and :
torch==0.4.1
torchvision==0.2.1
torchvision-nightly==0.2.1
2 Likes
I think you need to install pytorch 1.0.x
conda install -c pytorch -c fastai fastai pytorch-nightly cuda92 (or whichever cuda you have)
2 Likes
That’s simply done with require_grads=False in pytorch. Note that this advanced stuff so you should ask it on the dedicated subforum 
3 Likes
what is that?
Could anyone share the link to the GPU forum post that @jeremy mentions in the slides?
Or just press Shift-R on your keyboard! 
Press ? to see a full list of keyboard shortcuts.
11 Likes
I have always wondered why batch sizes are …34,64,128,… Is that something to do with GPU memory? If I set 100 as batch size will it run out of memory without really using all memory because it is not dividable with 100?
NO.It has more to do with efficiency and convention. As long as batch size can fit in the GPU memory, you’re good to go. But to ensure accuracy and save time, you fit in the max batch size that you can.
2 Likes
Its just a habit that coders size up in powers of two.
Have a bs=800 because that was the maximum i could fit in the gpu
1 Like
He mentions it at around 57 minutes in the video.
I guess this is what you are looking for http://course-v3.fast.ai/#using-a-gpu