01_intro, cuda runtime error (801) : operation not supported at ..\torch/csrc/generic/StorageSharing.cpp:247

Hey,

I’m trying to do the fastbook and I get an error message right away in the first chapter.

cuda runtime error (801) : operation not supported at ..\torch/csrc/generic/StorageSharing.cpp:247

The Error occurs here:
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
in
23
24 learn = cnn_learner(dls, resnet34, metrics=error_rate)
—> 25 learn.fine_tune(1)
26 if dim == 2:
27 target = target.long() #new input

~\Anaconda3\lib\site-packages\fastai\callback\schedule.py in fine_tune(self, epochs, base_lr, freeze_epochs, lr_mult, pct_start, div, **kwargs)
    155     "Fine tune with `freeze` for `freeze_epochs` then with `unfreeze` from `epochs` using discriminative LR"
    156     self.freeze()
--> 157     self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)
    158     base_lr /= 2
    159     self.unfreeze()

~\Anaconda3\lib\site-packages\fastai\callback\schedule.py in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
    110     scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
    111               'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 112     self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
    113 
    114 # Cell

~\Anaconda3\lib\site-packages\fastai\learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
    203             self.opt.set_hypers(lr=self.lr if lr is None else lr)
    204             self.n_epoch = n_epoch
--> 205             self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)
    206 
    207     def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None

~\Anaconda3\lib\site-packages\fastai\learner.py in _with_events(self, f, event_type, ex, final)
    152 
    153     def _with_events(self, f, event_type, ex, final=noop):
--> 154         try:       self(f'before_{event_type}')       ;f()
    155         except ex: self(f'after_cancel_{event_type}')
    156         finally:   self(f'after_{event_type}')        ;final()

~\Anaconda3\lib\site-packages\fastai\learner.py in _do_fit(self)
    194         for epoch in range(self.n_epoch):
    195             self.epoch=epoch
--> 196             self._with_events(self._do_epoch, 'epoch', CancelEpochException)
    197 
    198     def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):

~\Anaconda3\lib\site-packages\fastai\learner.py in _with_events(self, f, event_type, ex, final)
    152 
    153     def _with_events(self, f, event_type, ex, final=noop):
--> 154         try:       self(f'before_{event_type}')       ;f()
    155         except ex: self(f'after_cancel_{event_type}')
    156         finally:   self(f'after_{event_type}')        ;final()

~\Anaconda3\lib\site-packages\fastai\learner.py in _do_epoch(self)
    188 
    189     def _do_epoch(self):
--> 190         self._do_epoch_train()
    191         self._do_epoch_validate()
    192 

~\Anaconda3\lib\site-packages\fastai\learner.py in _do_epoch_train(self)
    180     def _do_epoch_train(self):
    181         self.dl = self.dls.train
--> 182         self._with_events(self.all_batches, 'train', CancelTrainException)
    183 
    184     def _do_epoch_validate(self, ds_idx=1, dl=None):

~\Anaconda3\lib\site-packages\fastai\learner.py in _with_events(self, f, event_type, ex, final)
    152 
    153     def _with_events(self, f, event_type, ex, final=noop):
--> 154         try:       self(f'before_{event_type}')       ;f()
    155         except ex: self(f'after_cancel_{event_type}')
    156         finally:   self(f'after_{event_type}')        ;final()

~\Anaconda3\lib\site-packages\fastai\learner.py in all_batches(self)
    158     def all_batches(self):
    159         self.n_iter = len(self.dl)
--> 160         for o in enumerate(self.dl): self.one_batch(*o)
    161 
    162     def _do_one_batch(self):

~\Anaconda3\lib\site-packages\fastai\data\load.py in __iter__(self)
     99         self.before_iter()
    100         self.__idxs=self.get_idxs() # called in context of main process (not workers/subprocesses)
--> 101         for b in _loaders[self.fake_l.num_workers==0](self.fake_l):
    102             if self.device is not None: b = to_device(b, self.device)
    103             yield self.after_batch(b)

~\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __init__(self, loader)
    799             #     before it starts, and __del__ tries to join but will get:
    800             #     AssertionError: can only join a started process.
--> 801             w.start()
    802             self._index_queues.append(index_queue)
    803             self._workers.append(w)

~\Anaconda3\lib\multiprocessing\process.py in start(self)
    110                'daemonic processes are not allowed to have children'
    111         _cleanup()
--> 112         self._popen = self._Popen(self)
    113         self._sentinel = self._popen.sentinel
    114         # Avoid a refcycle if the target function holds an indirect

~\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
    221     @staticmethod
    222     def _Popen(process_obj):
--> 223         return _default_context.get_context().Process._Popen(process_obj)
    224 
    225 class DefaultContext(BaseContext):

~\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
    320         def _Popen(process_obj):
    321             from .popen_spawn_win32 import Popen
--> 322             return Popen(process_obj)
    323 
    324     class SpawnContext(BaseContext):

~\Anaconda3\lib\multiprocessing\popen_spawn_win32.py in __init__(self, process_obj)
     87             try:
     88                 reduction.dump(prep_data, to_child)
---> 89                 reduction.dump(process_obj, to_child)
     90             finally:
     91                 set_spawning_popen(None)

~\Anaconda3\lib\multiprocessing\reduction.py in dump(obj, file, protocol)
     58 def dump(obj, file, protocol=None):
     59     '''Replacement for pickle.dump() using ForkingPickler.'''
---> 60     ForkingPickler(file, protocol).dump(obj)
     61 
     62 #

~\Anaconda3\lib\site-packages\torch\multiprocessing\reductions.py in reduce_tensor(tensor)
    238          ref_counter_offset,
    239          event_handle,
--> 240          event_sync_required) = storage._share_cuda_()
    241         tensor_offset = tensor.storage_offset()
    242         shared_cache[handle] = StorageWeakRef(storage)

I think I installed everything correctly.

Best Regards
David

I’m running it in JupytherLab on Windows.

Hi. Searching that error message on Google gave this. Try that solution (setting num_workers=0) and see how you get on?

Yijin

Hey utkb,

thanks for your reply.

Where should I put (setting num_workers=0)?

Just running in in JupyterLab?

Best regards
David

I figured it out. This works:

from fastai.vision.all import *
path = untar_data(URLs.PETS)/'images'

def is_cat(x): return x[0].isupper()
dls = ImageDataLoaders.from_name_func(
    path, get_image_files(path), valid_pct=0.2, seed=42,
    label_func=is_cat, item_tfms=Resize(224),num_workers=0,bs=6)

learn = cnn_learner(dls, resnet34, metrics=error_rate)
learn.fine_tune(1)

I had to set num_workers=0 and bs=6 and that works for me.
Note: I’m running this on my notebook.

Best regards
David