Here we compare the different validation techniques on a pretrained resnet50 model
%reload_ext autoreload
%autoreload 2
from validation_utils import sort_ar, chunks, map_idx2ar, ValDataset, SequentialIndexSampler, RectangularCropTfm, validate
import sys, os, shutil, time, warnings
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
import urllib.request
import pandas as pd
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.utils.data.sampler import Sampler
cudnn.benchmark = True
data = Path.home()/'data/imagenet'
workers = 7
valdir = data/'validation'
batch_size = 64
fp16 = True
idx_ar_sorted = sort_ar(data, valdir)
OR just download precomputed imagenet sizes
idx2ar_path = data/'sorted_idxar.p'
url = 'https://s3-us-west-2.amazonaws.com/ashaw-fastai-imagenet/sorted_idxar.p'
if not idx2ar_path.exists(): urllib.request.urlretrieve(url, idx2ar_path)
idx_ar_sorted = sort_ar(data, valdir)
import resnet
model = resnet.resnet50(pretrained=True)
model = model.cuda()
criterion = nn.CrossEntropyLoss().cuda()
if fp16: model = model.half()
val_bs = 64
target_size = 288
idx_sorted, _ = zip(*idx_ar_sorted)
idx2ar, ar_means = map_idx2ar(idx_ar_sorted, val_bs)
val_sampler_ar = SequentialIndexSampler(idx_sorted)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
tensor_tfm = [transforms.ToTensor(), normalize]
This was the validation technique used in fast.ai's original DAWNBenchmark
Resize Image 1.14x -> Crop to target size (288)
val_tfms = [transforms.Resize(int(target_size*1.14)), transforms.CenterCrop(target_size)] + tensor_tfm
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms))
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
orig_prec5 = validate(val_loader, model, criterion)
Perform validation with rectangular images!
val_ar_tfms = [transforms.Resize(int(target_size*1.14)), RectangularCropTfm(idx2ar, target_size)]
val_dataset_rect = ValDataset(valdir, val_ar_tfms+tensor_tfm)
val_loader = torch.utils.data.DataLoader(
val_dataset_rect, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
rect_prec5 = validate(val_loader, model, criterion)
def batch_mean(array, size=10): return [np.array(c).mean() for c in chunks(array, 100)]
batch_means = batch_mean(ar_means)
rect_prec5_mean = batch_mean(rect_prec5)
orig_prec5_mean = batch_mean(orig_prec5)
d = {'OriginalValidation': orig_prec5_mean,
'RectangularValidation': rect_prec5_mean,
'AR Mean': batch_means,
'Difference': np.array(rect_prec5_mean)-np.array(orig_prec5_mean)}
df = pd.DataFrame(data=d); df
You can see that rectangular validation outperforms the original when the aspect ratio is farther away from 1 (square crop)
Take 4 random crops + original validation image and averages the predictions together
min_scale = 0.5
trn_tfms = [
transforms.RandomResizedCrop(target_size, scale=(min_scale, 1.0)),
transforms.RandomHorizontalFlip(),
] + tensor_tfm
aug_dataset = datasets.ImageFolder(valdir, transforms.Compose(trn_tfms))
val_tfms = [transforms.Resize(int(target_size*1.14)), transforms.CenterCrop(target_size)] + tensor_tfm
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(val_tfms))
aug_loader = torch.utils.data.DataLoader(
aug_dataset, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
tta_prec5 = validate(val_loader, model, criterion, aug_loader=aug_loader, num_augmentations=4)
Take 4 random crops + recangular validation image and averages the predictions together
min_scale = 0.5
trn_tfms = [
transforms.RandomResizedCrop(target_size, scale=(min_scale, 1.0)),
transforms.RandomHorizontalFlip(),
] + tensor_tfm
aug_dataset = datasets.ImageFolder(valdir, transforms.Compose(trn_tfms))
aug_loader = torch.utils.data.DataLoader(
aug_dataset, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
val_ar_tfms = [transforms.Resize(int(target_size*1.14)), RectangularCropTfm(idx2ar, target_size)]
val_dataset_rect = ValDataset(valdir, val_ar_tfms+tensor_tfm)
val_loader = torch.utils.data.DataLoader(
val_dataset_rect, batch_size=val_bs, shuffle=False,
num_workers=workers, pin_memory=True, sampler=val_sampler_ar)
tta_rect_prec5 = validate(val_loader, model, criterion, aug_loader=aug_loader, num_augmentations=4)
def batch_mean(array, size=10): return [np.array(c).mean() for c in chunks(array, 100)]
batch_means = batch_mean(ar_means)
rect_prec5_mean = batch_mean(rect_prec5)
orig_prec5_mean = batch_mean(orig_prec5)
tta_prec5_mean = batch_mean(tta_prec5)
tta_rect_prec5_mean = batch_mean(tta_rect_prec5)
d = {'Original Validation': orig_prec5_mean,
'Rectangular Validation': rect_prec5_mean,
'TTA Validation': tta_prec5_mean,
'TTA + Rectangular Validation': tta_rect_prec5_mean,
'AR Mean': batch_means}
df = pd.DataFrame(data=d); df