# !apt-get update
# !apt-get install wget
# !wget 'YOUR_URL_HERE' -O 'flowers.zip'
from pathlib import Path
path = Path('../data/flowers/').resolve()
# from zipfile import ZipFile
# with ZipFile('flowers.zip', 'r') as zip_ref:
# zip_ref.extractall(path)
Note: after cloning the tfrecord
repository you need to go into tfrecord/__init__.py
and change:
tf.python_io
to tf.io
tf.python_io
to tf.compat.v1.io
# !git clone https://github.com/pgmmpk/tfrecord.git
# !conda install tensorflow -y
# !pip install gpustat
from fastai2.vision.all import *
import tensorflow as tf
from tfrecord.tfrecord import *
def unpack_sample(feats):
return {
'class' : unpack_int64_list(feats['class']),
'image' : unpack_bytes_list(feats['image'])
}
train_fnames = get_files(path/'train', extensions='.tfrec')
test_fnames = get_files(path/'val', extensions='.tfrec')
data = []
for name in train_fnames+test_fnames:
r = Reader(str(name), unpack_sample)
for sample in r:
data.append([sample['image'][0], sample['class'][0]])
WARNING:tensorflow:From /notebooks/FastGarden/nbs/tfrecord/tfrecord/__init__.py:81: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version. Instructions for updating: Use eager execution and: `tf.data.TFRecordDataset(path)`
len(data)
16465
get_x = lambda o: PILImage.create(io.BytesIO(o[0]))
get_y = lambda o: o[1]
def get_items(noop): return data
splitter = IndexSplitter(range(12753, len(data)))
item_tfms = [RandomResizedCrop(224, min_scale=0.7)]
batch_tfms = [*aug_transforms()]
block = DataBlock(
blocks=(ImageBlock, CategoryBlock),
get_items=get_items,
get_x=get_x,
get_y=get_y,
splitter=splitter,
item_tfms=item_tfms,
batch_tfms=batch_tfms
)
dls = block.dataloaders('', bs=64)
dls.show_batch()
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.lr_find()
(0.010000000149011612, 0.00363078061491251)
fit_flat_cos(5, lr=3e-3)
: 0.7056 ± 0.0055¶# 1/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.768008 | 2.363963 | 0.391972 | 02:34 |
1 | 2.146107 | 2.075266 | 0.451239 | 02:33 |
2 | 1.799903 | 1.973233 | 0.487877 | 02:33 |
3 | 1.520557 | 1.615816 | 0.569235 | 02:33 |
4 | 1.063737 | 1.121813 | 0.701509 | 02:33 |
# 2/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.746529 | 2.462022 | 0.358836 | 02:33 |
1 | 2.161063 | 1.997275 | 0.464978 | 02:33 |
2 | 1.775223 | 1.774945 | 0.529364 | 02:33 |
3 | 1.511754 | 1.796667 | 0.533136 | 02:33 |
4 | 1.067415 | 1.085076 | 0.713362 | 02:32 |
# 3/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.786595 | 2.523889 | 0.352371 | 02:32 |
1 | 2.179846 | 1.986795 | 0.471444 | 02:32 |
2 | 1.806607 | 1.840603 | 0.521013 | 02:32 |
3 | 1.528591 | 1.909116 | 0.501616 | 02:32 |
4 | 1.045687 | 1.115731 | 0.702047 | 02:32 |
results = [
0.701509, # trial 1
0.713362, # trial 2
0.702047, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')
Avg: 0.7056 Std: 0.0055
fit_flat_cos(5, lr=6e-3)
: 0.7170 ± 0.0037¶# 1/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.789303 | 2.535100 | 0.342672 | 02:30 |
1 | 2.253267 | 2.219538 | 0.416218 | 02:32 |
2 | 1.847394 | 2.218594 | 0.432651 | 02:31 |
3 | 1.629426 | 1.591986 | 0.571390 | 02:32 |
4 | 1.086009 | 1.082340 | 0.712823 | 02:32 |
# 2/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.723681 | 2.540765 | 0.369881 | 02:32 |
1 | 2.177144 | 2.206023 | 0.426994 | 02:32 |
2 | 1.825692 | 1.986223 | 0.483297 | 02:32 |
3 | 1.552183 | 1.755216 | 0.523438 | 02:32 |
4 | 1.065394 | 1.075901 | 0.716325 | 02:32 |
# 3/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.804643 | 2.740090 | 0.310614 | 02:32 |
1 | 2.225041 | 2.061276 | 0.465787 | 02:32 |
2 | 1.850433 | 2.436078 | 0.418103 | 02:32 |
3 | 1.572415 | 1.700506 | 0.543912 | 02:32 |
4 | 1.060845 | 1.070454 | 0.721713 | 02:32 |
results = [
0.712823, # trial 1
0.716325, # trial 2
0.721713, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')
Avg: 0.7170 Std: 0.0037
fit_one_cycle(5, lr=8e-3, wd=1e-3)
: 0.6711 ± 0.0149¶# 1/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.258210 | 3.183470 | 0.228179 | 02:32 |
1 | 2.566364 | 2.634067 | 0.310884 | 02:31 |
2 | 2.032837 | 1.950492 | 0.478448 | 02:31 |
3 | 1.489237 | 1.434236 | 0.620420 | 02:31 |
4 | 1.174728 | 1.269128 | 0.660022 | 02:31 |
# 2/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.234499 | 3.018919 | 0.273976 | 02:31 |
1 | 2.509192 | 2.515717 | 0.340248 | 02:31 |
2 | 1.911371 | 1.857579 | 0.509160 | 02:31 |
3 | 1.410966 | 1.312312 | 0.646552 | 02:31 |
4 | 1.058215 | 1.161696 | 0.692080 | 02:30 |
# 3/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.266664 | 2.975242 | 0.257004 | 02:31 |
1 | 2.529052 | 2.491700 | 0.354526 | 02:31 |
2 | 2.040608 | 2.057193 | 0.446659 | 02:31 |
3 | 1.507227 | 1.425681 | 0.615032 | 02:31 |
4 | 1.190011 | 1.268649 | 0.661099 | 02:31 |
results = [
0.660022, # trial 1
0.692080, # trial 2
0.661099, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')
Avg: 0.6711 Std: 0.0149
fit_one_cycle(5, lr=8e-3, wd=1e-3, pct_start=0.7)
: 0.6785 ± 0.0037¶# 1/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3, pct_start=0.7)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.307375 | 2.897245 | 0.263200 | 02:30 |
1 | 2.579807 | 2.805717 | 0.285560 | 02:32 |
2 | 2.352891 | 2.744128 | 0.300108 | 02:32 |
3 | 1.958257 | 2.084558 | 0.450162 | 02:32 |
4 | 1.245100 | 1.171975 | 0.681304 | 02:32 |
# 2/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3, pct_start=0.7)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.339589 | 3.054797 | 0.257812 | 02:32 |
1 | 2.634054 | 2.761052 | 0.324892 | 02:32 |
2 | 2.354805 | 2.928357 | 0.321121 | 02:32 |
3 | 1.989341 | 2.127400 | 0.443157 | 02:32 |
4 | 1.282349 | 1.216697 | 0.673222 | 02:32 |
# 3/3
net = xresnet50(pretrained=False, sa=True, act_cls=Mish)
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_one_cycle(5, 8e-3, wd=1e-3, pct_start=0.7)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 3.308072 | 2.880099 | 0.280981 | 02:32 |
1 | 2.651013 | 2.931513 | 0.276401 | 02:32 |
2 | 2.328457 | 8.501055 | 0.109914 | 02:32 |
3 | 1.976508 | 2.296954 | 0.406250 | 02:32 |
4 | 1.261625 | 1.191077 | 0.681035 | 02:32 |
results = [
0.681304, # trial 1
0.673222, # trial 2
0.681035, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')
Avg: 0.6785 Std: 0.0037
xresnet50
+ MaxBlurPool: 0.7079 ± 0.0057¶import kornia
# see, e.g., https://github.com/ducha-aiki/imagewoofv2-fastv2-maxpoolblur/blob/master/fastai2-imagenette-train-maxblurpool.ipynb
def convert_MP_to_blurMP(model, layer_type_old=nn.MaxPool2d):
for name, module in reversed(model._modules.items()):
if len(list(module.children())) > 0:
# recurse
model._modules[name] = convert_MP_to_blurMP(module, layer_type_old)
if type(module) == layer_type_old:
layer_old = module
layer_new = kornia.contrib.MaxBlurPool2d(3, True)
model._modules[name] = layer_new
return model
# 1/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.828057 | 2.455099 | 0.355603 | 02:55 |
1 | 2.243564 | 2.129641 | 0.444504 | 02:55 |
2 | 1.873171 | 2.273885 | 0.444504 | 02:55 |
3 | 1.593557 | 1.904136 | 0.512931 | 02:55 |
4 | 1.046666 | 1.085186 | 0.714709 | 02:55 |
# 2/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.763338 | 2.566091 | 0.317619 | 02:55 |
1 | 2.257158 | 2.182703 | 0.432112 | 02:55 |
2 | 1.940125 | 2.020298 | 0.476293 | 02:55 |
3 | 1.613202 | 1.650954 | 0.554957 | 02:55 |
4 | 1.144398 | 1.128705 | 0.700700 | 02:56 |
# 3/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.807843 | 2.502607 | 0.349138 | 02:55 |
1 | 2.251971 | 3.100409 | 0.329203 | 02:55 |
2 | 1.893489 | 2.334129 | 0.407866 | 02:55 |
3 | 1.570825 | 1.650394 | 0.558728 | 02:55 |
4 | 1.096632 | 1.099856 | 0.708244 | 02:55 |
results = [
0.714709, # trial 1
0.700700, # trial 2
0.708244, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')
Avg: 0.7079 Std: 0.0057
xresnet50
+ MaxBlurPool, learn.fit_flat_cos(5, 3e-3)
:¶!gpustat
71419a76a9f5 Wed Mar 25 02:15:06 2020 418.67 [0] Quadro M4000 | 36'C, 0 % | 12 / 8126 MB |
dls = block.dataloaders('', bs=32) # sometimes I get a Quadro M4000 GPU
# 1/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.685957 | 2.550505 | 0.333513 | 06:59 |
1 | 2.178324 | 2.213792 | 0.412177 | 06:59 |
2 | 1.783730 | 1.727573 | 0.538254 | 06:59 |
3 | 1.496804 | 1.432747 | 0.611530 | 06:59 |
4 | 1.038597 | 1.052808 | 0.727101 | 06:59 |
# 2/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.739494 | 2.577640 | 0.335938 | 06:59 |
1 | 2.216475 | 2.113397 | 0.448815 | 06:59 |
2 | 1.788160 | 1.740925 | 0.551994 | 06:59 |
3 | 1.564232 | 1.524614 | 0.598599 | 06:59 |
4 | 1.053690 | 1.072532 | 0.721983 | 06:59 |
# 3/3
net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish))
learn = Learner(dls, net, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 3e-3)
epoch | train_loss | valid_loss | accuracy | time |
---|---|---|---|---|
0 | 2.711620 | 06:01 |
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-19-8abd8a004746> in <module> 2 net = convert_MP_to_blurMP(xresnet50(pretrained=False, sa=True, act_cls=Mish)) 3 learn = Learner(dls, net, opt_func=ranger, metrics=accuracy) ----> 4 learn.fit_flat_cos(5, 3e-3) /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/callback/schedule.py in fit_flat_cos(self, n_epoch, lr, div_final, pct_start, wd, cbs, reset_opt) 133 lr = np.array([h['lr'] for h in self.opt.hypers]) 134 scheds = {'lr': combined_cos(pct_start, lr, lr, lr/div_final)} --> 135 self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd) 136 137 # Cell /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt) 189 try: 190 self.epoch=epoch; self('begin_epoch') --> 191 self._do_epoch_train() 192 self._do_epoch_validate() 193 except CancelEpochException: self('after_cancel_epoch') /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/learner.py in _do_epoch_train(self) 162 try: 163 self.dl = self.dls.train; self('begin_train') --> 164 self.all_batches() 165 except CancelTrainException: self('after_cancel_train') 166 finally: self('after_train') /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/learner.py in all_batches(self) 140 def all_batches(self): 141 self.n_iter = len(self.dl) --> 142 for o in enumerate(self.dl): self.one_batch(*o) 143 144 def one_batch(self, i, b): /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/learner.py in one_batch(self, i, b) 151 if not self.training: return 152 self.loss.backward(); self('after_backward') --> 153 self.opt.step(); self('after_step') 154 self.opt.zero_grad() 155 except CancelBatchException: self('after_cancel_batch') /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/optimizer.py in step(self) 275 def step(self): 276 if self.slow_weights is None: self._copy_weights() --> 277 self.opt.step() 278 self.count += 1 279 if self.count%self.k != 0: return /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/optimizer.py in step(self) 71 def step(self): 72 for p,pg,state,hyper in self.all_params(with_grad=True): ---> 73 for cb in self.cbs: state = _update(state, cb(p, **{**state, **hyper})) 74 self.state[p] = state 75 /opt/conda/envs/fastai/lib/python3.7/site-packages/fastai2/optimizer.py in average_grad(p, mom, dampening, grad_avg, **kwargs) 111 if grad_avg is None: grad_avg = torch.zeros_like(p.grad.data) 112 damp = 1-mom if dampening else 1. --> 113 grad_avg.mul_(mom).add_(damp, p.grad.data) 114 return {'grad_avg': grad_avg} 115 KeyboardInterrupt:
results = [
0.727101, # trial 1
0.721983, # trial 2
, # trial 3
]
print(f' Avg: {np.mean(results):.4f}' '\n Std: ' f'{np.std(results):.4f}')