#default_exp vision.data
#export
from local.test import *
from local.torch_basics import *
from local.data.all import *
from local.vision.core import *
from local.notebook.showdoc import *
Helper functions to get data in a
DataBunch
un the vision applicaiton and higher classImageDataBunch
#export
class ImageDataBunch(DataBunch):
@classmethod
@delegates(DataBunch.from_dblock)
def from_folder(cls, path, train='train', valid='valid', valid_pct=None, seed=None, vocab=None, **kwargs):
"Create from imagenet style dataset in `path` with `train`,`valid`,`test` subfolders (or provide `valid_pct`)."
splitter = GrandparentSplitter(train_name=train, valid_name=valid) if valid_pct is None else RandomSplitter(valid_pct, seed=seed)
dblock = DataBlock(blocks=(ImageBlock, CategoryBlock(vocab=vocab)),
get_items=get_image_files,
splitter=splitter,
get_y=parent_label)
return cls.from_dblock(dblock, path, path=path, **kwargs)
@classmethod
@delegates(DataBunch.from_dblock)
def from_name_func(cls, path, fnames, label_func, valid_pct=0.2, seed=None, **kwargs):
"Create from list of `fnames` in `path`s with `label_func`."
dblock = DataBlock(blocks=(ImageBlock, CategoryBlock),
splitter=RandomSplitter(valid_pct, seed=seed),
get_y=label_func)
return cls.from_dblock(dblock, fnames, path=path, **kwargs)
@classmethod
@delegates(DataBunch.from_dblock)
def from_name_re(cls, path, fnames, pat, **kwargs):
"Create from list of `fnames` in `path`s with re expression `pat`."
return cls.from_name_func(path, fnames, RegexLabeller(pat), **kwargs)
@classmethod
@delegates(DataBunch.from_dblock)
def from_df(cls, df, path='.', valid_pct=0.2, seed=None, fn_col=0, folder=None, suff='', label_col=1, label_delim=None, y_block=None, **kwargs):
pref = f'{Path(path) if folder is None else Path(path)/folder}{os.path.sep}'
if y_block is None: y_block = MultiCategoryBlock if is_listy(label_col) and len(label_col) > 1 else CategoryBlock
dblock = DataBlock(blocks=(ImageBlock, y_block),
get_x=ColReader(fn_col, pref=pref, suff=suff),
get_y=ColReader(label_col, label_delim=label_delim),
splitter=RandomSplitter(valid_pct, seed=seed))
return cls.from_dblock(dblock, df, path=path, **kwargs)
@classmethod
def from_csv(cls, path, csv_fname='labels.csv', header='infer', delimiter=None, **kwargs):
df = pd.read_csv(Path(path)/csv_fname, header=header, delimiter=delimiter)
return cls.from_df(df, path=path, **kwargs)
@classmethod
@delegates(DataBunch.from_dblock)
def from_lists(cls, path, fnames, labels, valid_pct=0.2, seed:int=None, y_block=None, **kwargs):
"Create from list of `fnames` in `path`."
if y_block is None:
y_block = MultiCategoryBlock if is_listy(labels[0]) and len(labels[0]) > 1 else (TransformBlock if isinstance(labels[0], float) else CategoryBlock)
dblock = DataBlock(blocks=(ImageBlock, y_block),
splitter=RandomSplitter(valid_pct, seed=seed))
return cls.from_dblock(dblock, (fnames, labels), path=path, **kwargs)
ImageDataBunch.from_csv = delegates(to=ImageDataBunch.from_df)(ImageDataBunch.from_csv)
ImageDataBunch.from_name_re = delegates(to=ImageDataBunch.from_name_func)(ImageDataBunch.from_name_re)
show_doc(ImageDataBunch.from_folder)
ImageDataBunch.from_folder
[source]
ImageDataBunch.from_folder
(path
,train
='train'
,valid
='valid'
,valid_pct
=None
,seed
=None
,vocab
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
Create from imagenet style dataset in path
with train
,valid
,test
subfolders (or provide valid_pct
).
show_doc(ImageDataBunch.from_name_func)
ImageDataBunch.from_name_func
[source]
ImageDataBunch.from_name_func
(path
,fnames
,label_func
,valid_pct
=0.2
,seed
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
Create from list of fnames
in path
s with label_func
.
show_doc(ImageDataBunch.from_name_re)
ImageDataBunch.from_name_re
[source]
ImageDataBunch.from_name_re
(path
,fnames
,pat
,valid_pct
=0.2
,seed
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
Create from list of fnames
in path
s with re expression pat
.
show_doc(ImageDataBunch.from_df)
ImageDataBunch.from_df
[source]
ImageDataBunch.from_df
(df
,path
='.'
,valid_pct
=0.2
,seed
=None
,fn_col
=0
,pref
=''
,suff
=''
,label_col
=1
,label_delim
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
show_doc(ImageDataBunch.from_csv)
ImageDataBunch.from_csv
[source]
ImageDataBunch.from_csv
(path
,csv_fname
='labels.csv'
,header
='infer'
,delimiter
=None
,valid_pct
=0.2
,seed
=None
,fn_col
=0
,pref
=''
,suff
=''
,label_col
=1
,label_delim
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
show_doc(ImageDataBunch.from_lists)
ImageDataBunch.from_lists
[source]
ImageDataBunch.from_lists
(path
,fnames
,labels
,valid_pct
=0.2
,seed
:int
=None
,y_block
=None
,type_tfms
=None
,item_tfms
=None
,batch_tfms
=None
,bs
=16
,shuffle
=False
,num_workers
=None
,pin_memory
=False
,timeout
=0
,drop_last
=False
,indexed
=None
,n
=None
,wif
=None
,before_iter
=None
,create_batches
=None
,create_item
=None
,after_item
=None
,before_batch
=None
,create_batch
=None
,retain
=None
,after_batch
=None
,after_iter
=None
)
Create from list of fnames
in path
.
#export
def get_grid(n, rows=None, cols=None, add_vert=0, figsize=None, double=False, title=None, return_fig=False):
rows = rows or int(np.ceil(math.sqrt(n)))
cols = cols or int(np.ceil(n/rows))
if double: cols*=2 ; n*=2
figsize = (cols*3, rows*3+add_vert) if figsize is None else figsize
fig,axs = subplots(rows, cols, figsize=figsize)
axs = axs.flatten()
for ax in axs[n:]: ax.set_axis_off()
if title is not None: fig.suptitle(title, weight='bold', size=14)
return (fig,axs) if return_fig else axs
#export
@typedispatch
def show_batch(x:TensorImage, y, samples, ctxs=None, max_n=10, rows=None, cols=None, figsize=None, **kwargs):
if ctxs is None: ctxs = get_grid(min(len(samples), max_n), rows=rows, cols=cols, figsize=figsize)
ctxs = show_batch[object](x, y, samples, ctxs=ctxs, max_n=max_n, **kwargs)
return ctxs
# export
def clip_remove_empty(bbox, label):
"Clip bounding boxes with image border and label background the empty ones."
bbox = torch.clamp(bbox, -1, 1)
empty = ((bbox[...,2] - bbox[...,0])*(bbox[...,3] - bbox[...,1]) < 0.)
return (bbox[~empty], label[~empty])
bb = tensor([[-2,-0.5,0.5,1.5], [-0.5,-0.5,0.5,0.5], [1,0.5,0.5,0.75], [-0.5,-0.5,0.5,0.5]])
bb,lbl = clip_remove_empty(bb, tensor([1,2,3,2]))
test_eq(bb, tensor([[-1,-0.5,0.5,1.], [-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]]))
test_eq(lbl, tensor([1,2,2]))
#export
def bb_pad(samples, pad_idx=0):
"Function that collect `samples` of labelled bboxes and adds padding with `pad_idx`."
samples = [(s[0], *clip_remove_empty(*s[1:])) for s in samples]
max_len = max([len(s[2]) for s in samples])
def _f(img,bbox,lbl):
bbox = torch.cat([bbox,bbox.new_zeros(max_len-bbox.shape[0], 4)])
lbl = torch.cat([lbl, lbl .new_zeros(max_len-lbl .shape[0])+pad_idx])
return img,bbox,lbl
return [_f(*s) for s in samples]
img1,img2 = TensorImage(torch.randn(16,16,3)),TensorImage(torch.randn(16,16,3))
bb1 = tensor([[-2,-0.5,0.5,1.5], [-0.5,-0.5,0.5,0.5], [1,0.5,0.5,0.75], [-0.5,-0.5,0.5,0.5]])
lbl1 = tensor([1, 2, 3, 2])
bb2 = tensor([[-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]])
lbl2 = tensor([2, 2])
samples = [(img1, bb1, lbl1), (img2, bb2, lbl2)]
res = bb_pad(samples)
non_empty = tensor([True,True,False,True])
test_eq(res[0][0], img1)
test_eq(res[0][1], tensor([[-1,-0.5,0.5,1.], [-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5]]))
test_eq(res[0][2], tensor([1,2,2]))
test_eq(res[1][0], img2)
test_eq(res[1][1], tensor([[-0.5,-0.5,0.5,0.5], [-0.5,-0.5,0.5,0.5], [0,0,0,0]]))
test_eq(res[1][2], tensor([2,2,0]))
TransformBlock
s for vision¶#export
def ImageBlock(cls=PILImage): return TransformBlock(type_tfms=cls.create, batch_tfms=IntToFloatTensor)
#export
MaskBlock = TransformBlock(type_tfms=PILMask.create, batch_tfms=IntToFloatTensor)
#export
PointBlock = TransformBlock(type_tfms=TensorPoint.create, item_tfms=PointScaler)
BBoxBlock = TransformBlock(type_tfms=TensorBBox.create, item_tfms=PointScaler, dbunch_kwargs = {'before_batch': bb_pad})
#export
BBoxBlock = TransformBlock(type_tfms=TensorBBox.create, item_tfms=PointScaler, dbunch_kwargs = {'before_batch': bb_pad})
#export
def BBoxLblBlock(vocab=None, add_na=True):
return TransformBlock(type_tfms=MultiCategorize(vocab=vocab, add_na=add_na), item_tfms=BBoxLabeler)
#hide
from local.notebook.export import notebook2script
notebook2script(all_fs=True)
Converted 00_test.ipynb. Converted 01_core_foundation.ipynb. Converted 01a_core_utils.ipynb. Converted 01b_core_dispatch.ipynb. Converted 01c_core_transform.ipynb. Converted 02_core_script.ipynb. Converted 03_torchcore.ipynb. Converted 03a_layers.ipynb. Converted 04_data_load.ipynb. Converted 05_data_core.ipynb. Converted 06_data_transforms.ipynb. Converted 07_data_block.ipynb. Converted 08_vision_core.ipynb. Converted 09_vision_augment.ipynb. Converted 09a_vision_data.ipynb. Converted 10_pets_tutorial.ipynb. Converted 11_vision_models_xresnet.ipynb. Converted 12_optimizer.ipynb. Converted 13_learner.ipynb. Converted 13a_metrics.ipynb. Converted 14_callback_schedule.ipynb. Converted 14a_callback_data.ipynb. Converted 15_callback_hook.ipynb. Converted 15a_vision_models_unet.ipynb. Converted 16_callback_progress.ipynb. Converted 17_callback_tracker.ipynb. Converted 18_callback_fp16.ipynb. Converted 19_callback_mixup.ipynb. Converted 20_interpret.ipynb. Converted 20a_distributed.ipynb. Converted 21_vision_learner.ipynb. Converted 22_tutorial_imagenette.ipynb. Converted 23_tutorial_transfer_learning.ipynb. Converted 30_text_core.ipynb. Converted 31_text_data.ipynb. Converted 32_text_models_awdlstm.ipynb. Converted 33_text_models_core.ipynb. Converted 34_callback_rnn.ipynb. Converted 35_tutorial_wikitext.ipynb. Converted 36_text_models_qrnn.ipynb. Converted 37_text_learner.ipynb. Converted 38_tutorial_ulmfit.ipynb. Converted 40_tabular_core.ipynb. Converted 41_tabular_model.ipynb. Converted 42_tabular_rapids.ipynb. Converted 50_data_block_examples.ipynb. Converted 60_medical_imaging.ipynb. Converted 65_medical_text.ipynb. Converted 70_callback_wandb.ipynb. Converted 71_callback_tensorboard.ipynb. Converted 90_notebook_core.ipynb. Converted 91_notebook_export.ipynb. Converted 92_notebook_showdoc.ipynb. Converted 93_notebook_export2html.ipynb. Converted 94_notebook_test.ipynb. Converted 95_index.ipynb. Converted 96_data_external.ipynb. Converted 97_utils_test.ipynb. Converted notebook2jekyll.ipynb. Converted xse_resnext.ipynb.