from local.test import *
from local.data.all import *
from local.notebook.showdoc import show_doc
Examples for computer vision, NLP, and tabular
from local.vision.all import *
mnist = DataBlock(blocks=(ImageBlock(cls=PILImageBW), CategoryBlock),
get_items=get_image_files,
splitter=GrandparentSplitter(),
get_y=parent_label)
dbunch = mnist.databunch(untar_data(URLs.MNIST_TINY))
dbunch.show_batch(max_n=9, figsize=(4,4))
pets = DataBlock(blocks=(ImageBlock, CategoryBlock),
get_items=get_image_files,
splitter=RandomSplitter(),
get_y=RegexLabeller(pat = r'/([^/]+)_\d+.jpg$'))
dbunch = pets.databunch(untar_data(URLs.PETS)/"images", item_tfms=Resize(128),
batch_tfms=aug_transforms())
dbunch.show_batch(max_n=9)
planet_source = untar_data(URLs.PLANET_TINY)
df = pd.read_csv(planet_source/"labels.csv")
planet = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),
get_x=ColReader(0, pref=planet_source/"train", suff='.jpg'),
splitter=RandomSplitter(),
get_y=ColReader(1, label_delim=' '))
planet = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),
get_x=lambda x:planet_source/"train"/f'{x[0]}.jpg',
splitter=RandomSplitter(),
get_y=lambda x:x[1].split(' '))
dbunch = planet.databunch(df.values,
batch_tfms=aug_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.))
dbunch.show_batch(max_n=9, figsize=(12,9))
def _planet_items(x): return (
f'{planet_source}/train/'+x.image_name+'.jpg', x.tags.str.split())
planet = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),
get_items = _planet_items,
splitter = RandomSplitter())
dbunch = planet.databunch(df, batch_tfms=aug_transforms(
flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.))
dbunch.show_batch(max_n=9, figsize=(12,9))
class PlanetDataBlock(DataBlock):
blocks = ImageBlock,MultiCategoryBlock
splitter = staticmethod(RandomSplitter())
def get_items(self, x): return (
f'{planet_source}/train/' + x.image_name + '.jpg', x.tags.str.split())
planet = PlanetDataBlock()
dbunch = planet.databunch(df, batch_tfms=aug_transforms(flip_vert=True))
dbunch.show_batch(max_n=9, figsize=(12,9))
planet = DataBlock(blocks=(ImageBlock, MultiCategoryBlock),
get_x = lambda o:f'{planet_source}/train/'+o.image_name+'.jpg',
get_y = lambda o:o.tags.split(),
splitter = RandomSplitter())
dbunch = planet.databunch(df, batch_tfms=aug_transforms(
flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.))
dbunch.show_batch(max_n=9, figsize=(12,9))
camvid = DataBlock(blocks=(ImageBlock, ImageBlock(cls=PILMask)),
get_items=get_image_files,
splitter=RandomSplitter(),
get_y=lambda o: untar_data(URLs.CAMVID_TINY)/'labels'/f'{o.stem}_P{o.suffix}')
dbunch = camvid.databunch(untar_data(URLs.CAMVID_TINY)/"images",
batch_tfms=aug_transforms())
dbunch.show_batch(max_n=9, vmin=1, vmax=30)
biwi_source = untar_data(URLs.BIWI_SAMPLE)
fn2ctr = (biwi_source/'centers.pkl').load()
biwi = DataBlock(blocks=(ImageBlock, PointBlock),
get_items=get_image_files,
splitter=RandomSplitter(),
get_y=lambda o:fn2ctr[o.name].flip(0))
dbunch = biwi.databunch(biwi_source, batch_tfms=aug_transforms())
dbunch.show_batch(max_n=9)
coco_source = untar_data(URLs.COCO_TINY)
images, lbl_bbox = get_annotations(coco_source/'train.json')
img2bbox = dict(zip(images, lbl_bbox))
coco = DataBlock(blocks=(ImageBlock, BBoxBlock, BBoxLblBlock),
get_items=get_image_files,
splitter=RandomSplitter(),
getters=[noop, lambda o: img2bbox[o.name][0], lambda o: img2bbox[o.name][1]], n_inp=1)
dbunch = coco.databunch(coco_source, item_tfms=Resize(128),
batch_tfms=aug_transforms())
dbunch.show_batch(max_n=9)
from local.text.all import *
path = untar_data(URLs.IMDB_SAMPLE)
df = pd.read_csv(path/'texts.csv')
df_tok,count = tokenize_df(df, 'text')
imdb_lm = DataBlock(blocks=(TextBlock(make_vocab(count), is_lm=True),),
get_x=attrgetter('text'),
splitter=RandomSplitter())
dbunch = imdb_lm.databunch(df_tok, bs=64, seq_len=72)
dbunch.show_batch(max_n=6)
text | text_ | |
---|---|---|
0 | xxbos xxmaj hello . this is my first review for any movie i have seen . i went through the trouble of doing this to tell everyone that this is quite literally , the most disgusting movie i have ever seen . i feel like the movie was xxunk made , which i will give some understanding due to budget xxunk on making it . i felt like i was watching a | xxmaj hello . this is my first review for any movie i have seen . i went through the trouble of doing this to tell everyone that this is quite literally , the most disgusting movie i have ever seen . i feel like the movie was xxunk made , which i will give some understanding due to budget xxunk on making it . i felt like i was watching a very |
1 | - time , we focus on three sets of children : one , the lonely son of a rich couple who wants nothing more for xxmaj christmas than their company ( projected as a wish - xxunk fantasy where the boy finds his parents wrapped in extra - large xxunk ! ) , a girl from a poor family who xxunk to own a doll of her own ( the xxunk one | time , we focus on three sets of children : one , the lonely son of a rich couple who wants nothing more for xxmaj christmas than their company ( projected as a wish - xxunk fantasy where the boy finds his parents wrapped in extra - large xxunk ! ) , a girl from a poor family who xxunk to own a doll of her own ( the xxunk one first |
2 | camera - work , xxunk on the print , flickering lights … i had to rub my eyes when i realised it was made in 2001 , and not 1971 . xxmaj even the clothes and fashioned look about three decades out of date ! \n\n xxmaj if you think xxmaj i 'm not qualified to do a review of xxmaj chronicles having not seen the whole film , then go ahead | - work , xxunk on the print , flickering lights … i had to rub my eyes when i realised it was made in 2001 , and not 1971 . xxmaj even the clothes and fashioned look about three decades out of date ! \n\n xxmaj if you think xxmaj i 'm not qualified to do a review of xxmaj chronicles having not seen the whole film , then go ahead . |
3 | bank , and the doctor , xxmaj xxunk , and xxmaj xxunk at night but not in the evening getting off their boat . ) xxmaj the best acting in the movie was probably from the sheriff , xxmaj xxunk ( although , there 's a xxunk of character when the pulse xxunk * whatever that thing is when people die , it xxunk * shows xxmaj xxunk has died , he | , and the doctor , xxmaj xxunk , and xxmaj xxunk at night but not in the evening getting off their boat . ) xxmaj the best acting in the movie was probably from the sheriff , xxmaj xxunk ( although , there 's a xxunk of character when the pulse xxunk * whatever that thing is when people die , it xxunk * shows xxmaj xxunk has died , he still |
4 | of the story work - displaying a xxunk frustration mixed with xxunk determination . xxmaj director xxmaj parks , who was already known for his coverage of controversial subjects in his photography , does not shy away from the xxunk of the story . xxmaj rather , the movie is xxunk in portrayal of the xxunk of the world of police and streets criminals that these two men xxunk . xxmaj adding | the story work - displaying a xxunk frustration mixed with xxunk determination . xxmaj director xxmaj parks , who was already known for his coverage of controversial subjects in his photography , does not shy away from the xxunk of the story . xxmaj rather , the movie is xxunk in portrayal of the xxunk of the world of police and streets criminals that these two men xxunk . xxmaj adding to |
5 | xxunk of plot xxunk that borrow from nearly every xxunk and dagger government conspiracy cliché that has ever been written . xxmaj the film stars xxmaj nicholas xxmaj cage as xxmaj benjamin xxmaj xxunk xxmaj xxunk ( how precious is that , i ask you ? ) ; a seemingly normal fellow who , for no other reason than being of a xxunk of like - minded misguided fortune hunters , decides | of plot xxunk that borrow from nearly every xxunk and dagger government conspiracy cliché that has ever been written . xxmaj the film stars xxmaj nicholas xxmaj cage as xxmaj benjamin xxmaj xxunk xxmaj xxunk ( how precious is that , i ask you ? ) ; a seemingly normal fellow who , for no other reason than being of a xxunk of like - minded misguided fortune hunters , decides to |
imdb_clas = DataBlock(blocks=(TextBlock(make_vocab(count)), CategoryBlock),
get_x=attrgetter('text'),
get_y=attrgetter('label'),
splitter=RandomSplitter())
dbunch = imdb_clas.databunch(df_tok, bs=64, seq_len=72)
dbunch.show_batch(max_n=2)
text | category | |
---|---|---|
0 | xxbos xxmaj now , i like sci - fi cartoons . xxmaj however , when " xxunk " appeared in xxmaj xxunk in late 2006 , i watched the premiere and was inevitably xxunk . xxmaj the characters are generic and stereotypical ( do they xxup really need to make an african - american man wear xxunk - xxunk print clothing and speak in a xxmaj xxunk accent ? xxup why are all the xxmaj asian characters xxunk yellow and xxunk ? xxmaj does the mother xxup have to have big xxunk and chest and constantly complain ? ) to the point where things become unrealistic , predictable , gross and sometimes disturbing ... | negative |
1 | xxbos xxmaj we bought the xxup dvd set of " xxunk war xxunk xxunk xxmaj xxunk " ( german ) / " once xxmaj upon a xxmaj time … xxmaj life " ( english ) for our xxunk kids because everyone loved the " xxunk war xxunk der xxmaj xxunk " ( german ) / " once xxmaj upon a xxmaj time … xxmaj man " ( english ) series ( us parents had seen it as kids ) and it has xxunk even high expectations ! xxmaj the series is very well made , does not show its age , and our kids at various ages really like to watch it . xxmaj at the same time , they learn things us parents did n't know until way , way later . xx... | positive |
from local.tabular.core import *
adult_source = untar_data(URLs.ADULT_SAMPLE)
df = pd.read_csv(adult_source/'adult.csv')
cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']
cont_names = ['age', 'fnlwgt', 'education-num']
procs = [Categorify, FillMissing, Normalize]
splits = RandomSplitter()(range_of(df))
to = TabularPandas(df, procs, cat_names, cont_names, y_names="salary", splits=splits)
dbch = to.databunch()
dbch.show_batch()
age | fnlwgt | education-num | workclass | education | marital-status | occupation | relationship | race | age_na | fnlwgt_na | education-num_na | salary | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 33.000000 | 258498.001394 | 10.0 | Private | Some-college | Married-civ-spouse | Craft-repair | Wife | White | False | False | False | <50k |
1 | 45.000000 | 25648.999148 | 10.0 | Private | Some-college | Married-civ-spouse | Transport-moving | Husband | White | False | False | False | >=50k |
2 | 30.000000 | 271710.000650 | 13.0 | Private | Bachelors | Married-civ-spouse | Sales | Husband | White | False | False | False | >=50k |
3 | 53.000000 | 143822.000988 | 14.0 | State-gov | Masters | Married-civ-spouse | Prof-specialty | Husband | White | False | False | False | >=50k |
4 | 35.000000 | 166235.000640 | 10.0 | Private | Some-college | Never-married | Machine-op-inspct | Not-in-family | Black | False | False | False | <50k |
5 | 28.000000 | 197932.000295 | 10.0 | Local-gov | Some-college | Never-married | Adm-clerical | Own-child | White | False | False | False | <50k |
6 | 33.000000 | 192001.999933 | 14.0 | Private | Masters | Married-civ-spouse | Exec-managerial | Husband | White | False | False | False | >=50k |
7 | 59.000001 | 116441.997963 | 10.0 | Private | Some-college | Married-civ-spouse | Machine-op-inspct | Husband | White | False | False | False | <50k |
8 | 55.000000 | 158641.000089 | 9.0 | Private | HS-grad | Widowed | Adm-clerical | Not-in-family | White | False | False | False | <50k |
9 | 55.000000 | 141727.000403 | 13.0 | Private | Bachelors | Married-civ-spouse | Adm-clerical | Wife | White | False | False | False | <50k |