Hello everyone: I am facing this issue for few weeks but I am unable to solve this.
I am working on medical images where I am diagnosing disease. I am creating a databunch by using some custom data loader using fastai. The code which I am using is below:
But this code is creating only train and valid split. The valid split includes the test set as combine I don’t know how to separately load the test set. My sets are already defined in a data frame into 80%, 10%, 10% train, valid, and test respectively.
I know I am doing something wrong in the code but I am a newbie so can’t make new changes if I do then I am facing many other problems.
My Dataloader code:
def get_chestxray8(path:PathOrStr, bs:int, img_sz:int, valid_only_bbx:bool=False, tfms:bool=True, convert_mode:str='RGB',
normalize:bool=True, norm_stats:Tuple[Floats, Floats]=imagenet_stats, processor:Optional[Callable]=None,
**kwargs:Any)->DataBunch:
'''
TODO
'''
path = Path(path)
df = pd.read_pickle(path / 'full_ds_bbx.pkl')
df['is_valid'] = df.set!='Train'
if valid_only_bbx: df = df[(df.set=='Train') | df.bbx]
if processor is not None: df = processor(df)
lbl_dict = df[['file','label']].set_index('file')['label'].to_dict()
def bbox_label_func(fn:str)->list: return lbl_dict[Path(fn).name]
lbls = ['No finding', 'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Infiltration',
'Lung Opacity', 'Mass', 'Pleural effusion', 'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis']
src = (CustomObjectItemList.from_df(df, path / 'images', cols='file', convert_mode=convert_mode)
.split_from_df('is_valid')
.label_from_func(bbox_label_func, classes=lbls))
if tfms: src = src.transform(get_transforms(**kwargs), size=img_sz, tfm_y=True)
data = src.databunch(bs=bs, collate_fn=multiclass_bb_pad_collate)
if normalize: data = data.normalize(stats=norm_stats)
return data
Please help me thanks.
@much_learner @muellerzr