# boilerplate code
import tensorflow as tf
print(tf.__version__)
2.0.0-alpha0
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Conv2D, Layer, MaxPool2D, \
Dropout, Dense, Input, concatenate, \
GlobalAveragePooling2D, AveragePooling2D,\
Flatten, BatchNormalization
import cv2 #python -m pip install opencv-python
import numpy as np
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.utils import to_categorical
from tensorflow import keras
import math
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.callbacks import LearningRateScheduler
num_classes = 10
def load_cifar10_data(img_rows, img_cols):
# Load cifar10 training and test sets
(X_train, Y_train), (X_test, Y_test) = cifar10.load_data()
# Resize training images
X_train = np.array([cv2.resize(img, (img_rows, img_cols)) for img in X_train[:, :, :, :]])
X_test = np.array([cv2.resize(img, (img_rows, img_cols)) for img in X_test[:, :, :, :]])
X_train = X_train.astype('float16') / 255.0
X_test = X_test.astype('float16') / 255.0
# Transform targets to keras compatible format
Y_train = to_categorical(Y_train, num_classes)
Y_test = to_categorical(Y_test, num_classes)
print("X_train: {0}".format(X_train.shape))
print("Y_train: {0}".format(Y_train.shape))
print("X_test: {0}".format(X_test.shape))
print("Y_test: {0}".format(Y_test.shape))
return X_train, Y_train, X_test, Y_test
X_train, y_train, X_test, y_test = load_cifar10_data(224, 224)
X_train: (50000, 224, 224, 3) Y_train: (50000, 10) X_test: (10000, 224, 224, 3) Y_test: (10000, 10)
kernel_init = tf.keras.initializers.glorot_uniform()
bias_init = tf.keras.initializers.Constant(value=0.2)
from functools import partial
DefaultConv2D = partial(
Conv2D,
kernel_size=(3, 3),
strides=(1, 1),
padding="SAME",
kernel_initializer=kernel_init,
bias_initializer=bias_init
)
class ResidualUnit(Layer):
def __init__(self, filters, strides=1, activation="relu", **kwargs):
super().__init__(**kwargs)
self.activation = keras.activations.get(activation)
self.main_layers = [
DefaultConv2D(filters, strides=strides),
keras.layers.BatchNormalization(),
self.activation,
DefaultConv2D(filters),
keras.layers.BatchNormalization()
]
if strides > 1:
self.skip_layers = [
DefaultConv2D(filters, kernel_size=1, strides=strides),
keras.layers.BatchNormalization()
]
else:
self.skip_layers = []
def call(self, inputs):
Z = inputs
for layer in self.main_layers:
Z = layer(Z)
skip_Z = inputs
for layer in self.skip_layers:
skip_Z = layer(skip_Z)
return self.activation(Z + skip_Z)
model = keras.models.Sequential(name='resnet_34')
model.add(DefaultConv2D(
64, kernel_size=7, strides=2, input_shape=[224, 224, 3]
))
model.add(keras.layers.BatchNormalization())
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.MaxPool2D(pool_size=3, strides=2, padding="SAME"))
prev_filters = 64
for filters in [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3:
strides = 1 if filters == prev_filters else 2
model.add(ResidualUnit(filters, strides=strides))
prev_filters = filters
model.add(keras.layers.GlobalAvgPool2D())
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(10, activation="softmax"))
model.summary()
Model: "resnet_34" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 112, 112, 64) 9472 _________________________________________________________________ batch_normalization_v2 (Batc (None, 112, 112, 64) 256 _________________________________________________________________ activation (Activation) (None, 112, 112, 64) 0 _________________________________________________________________ max_pooling2d (MaxPooling2D) (None, 56, 56, 64) 0 _________________________________________________________________ residual_unit (ResidualUnit) (None, 56, 56, 64) 74368 _________________________________________________________________ residual_unit_1 (ResidualUni (None, 56, 56, 64) 74368 _________________________________________________________________ residual_unit_2 (ResidualUni (None, 56, 56, 64) 74368 _________________________________________________________________ residual_unit_3 (ResidualUni (None, 28, 28, 128) 231296 _________________________________________________________________ residual_unit_4 (ResidualUni (None, 28, 28, 128) 296192 _________________________________________________________________ residual_unit_5 (ResidualUni (None, 28, 28, 128) 296192 _________________________________________________________________ residual_unit_6 (ResidualUni (None, 28, 28, 128) 296192 _________________________________________________________________ residual_unit_7 (ResidualUni (None, 14, 14, 256) 921344 _________________________________________________________________ residual_unit_8 (ResidualUni (None, 14, 14, 256) 1182208 _________________________________________________________________ residual_unit_9 (ResidualUni (None, 14, 14, 256) 1182208 _________________________________________________________________ residual_unit_10 (ResidualUn (None, 14, 14, 256) 1182208 _________________________________________________________________ residual_unit_11 (ResidualUn (None, 14, 14, 256) 1182208 _________________________________________________________________ residual_unit_12 (ResidualUn (None, 14, 14, 256) 1182208 _________________________________________________________________ residual_unit_13 (ResidualUn (None, 7, 7, 512) 3677696 _________________________________________________________________ residual_unit_14 (ResidualUn (None, 7, 7, 512) 4723712 _________________________________________________________________ residual_unit_15 (ResidualUn (None, 7, 7, 512) 4723712 _________________________________________________________________ global_average_pooling2d (Gl (None, 512) 0 _________________________________________________________________ flatten (Flatten) (None, 512) 0 _________________________________________________________________ dense (Dense) (None, 10) 5130 ================================================================= Total params: 21,315,338 Trainable params: 21,298,314 Non-trainable params: 17,024 _________________________________________________________________
initial_lrate = 0.01
def decay(epoch, steps=100):
drop = 0.96
epochs_drop = 8
lrate = initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop))
return lrate
lr_sc = LearningRateScheduler(decay, verbose=1)
sgd = SGD(lr=initial_lrate, momentum=0.9, nesterov=True)
model.compile(
loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy']
)
epochs = 35
history = model.fit(
x=X_train,
y=y_train,
validation_data=(X_test, y_test),
epochs=epochs, batch_size=256, callbacks=[lr_sc]
)
Train on 50000 samples, validate on 10000 samples Epoch 00001: LearningRateScheduler reducing learning rate to 0.01. Epoch 1/35 50000/50000 [==============================] - 205s 4ms/sample - loss: 1.4258 - accuracy: 0.4903 - val_loss: 1.9112 - val_accuracy: 0.3711 Epoch 00002: LearningRateScheduler reducing learning rate to 0.01. Epoch 2/35 50000/50000 [==============================] - 193s 4ms/sample - loss: 0.8504 - accuracy: 0.6988 - val_loss: 1.1408 - val_accuracy: 0.6201 Epoch 00003: LearningRateScheduler reducing learning rate to 0.01. Epoch 3/35 50000/50000 [==============================] - 192s 4ms/sample - loss: 0.5902 - accuracy: 0.7951 - val_loss: 1.1549 - val_accuracy: 0.6433 Epoch 00004: LearningRateScheduler reducing learning rate to 0.01. Epoch 4/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.4049 - accuracy: 0.8586 - val_loss: 1.1226 - val_accuracy: 0.6340 Epoch 00005: LearningRateScheduler reducing learning rate to 0.01. Epoch 5/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.2547 - accuracy: 0.9108 - val_loss: 1.6556 - val_accuracy: 0.6001 Epoch 00006: LearningRateScheduler reducing learning rate to 0.01. Epoch 6/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.1696 - accuracy: 0.9421 - val_loss: 1.8149 - val_accuracy: 0.6156 Epoch 00007: LearningRateScheduler reducing learning rate to 0.01. Epoch 7/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 0.1107 - accuracy: 0.9640 - val_loss: 1.5841 - val_accuracy: 0.6699 Epoch 00008: LearningRateScheduler reducing learning rate to 0.0096. Epoch 8/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0691 - accuracy: 0.9771 - val_loss: 2.2737 - val_accuracy: 0.6329 Epoch 00009: LearningRateScheduler reducing learning rate to 0.0096. Epoch 9/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0437 - accuracy: 0.9857 - val_loss: 2.2252 - val_accuracy: 0.6192 Epoch 00010: LearningRateScheduler reducing learning rate to 0.0096. Epoch 10/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0300 - accuracy: 0.9906 - val_loss: 1.1013 - val_accuracy: 0.7675 Epoch 00011: LearningRateScheduler reducing learning rate to 0.0096. Epoch 11/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0163 - accuracy: 0.9953 - val_loss: 1.4369 - val_accuracy: 0.7379 Epoch 00012: LearningRateScheduler reducing learning rate to 0.0096. Epoch 12/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0141 - accuracy: 0.9957 - val_loss: 1.4038 - val_accuracy: 0.7509 Epoch 00013: LearningRateScheduler reducing learning rate to 0.0096. Epoch 13/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0129 - accuracy: 0.9964 - val_loss: 1.4215 - val_accuracy: 0.7438 Epoch 00014: LearningRateScheduler reducing learning rate to 0.0096. Epoch 14/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0067 - accuracy: 0.9983 - val_loss: 1.2960 - val_accuracy: 0.7695 Epoch 00015: LearningRateScheduler reducing learning rate to 0.0096. Epoch 15/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0014 - accuracy: 0.9999 - val_loss: 0.9393 - val_accuracy: 0.8182 Epoch 00016: LearningRateScheduler reducing learning rate to 0.009216. Epoch 16/35 50000/50000 [==============================] - 191s 4ms/sample - loss: 3.6020e-04 - accuracy: 1.0000 - val_loss: 0.9105 - val_accuracy: 0.8231 Epoch 00017: LearningRateScheduler reducing learning rate to 0.009216. Epoch 17/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 2.1701e-04 - accuracy: 1.0000 - val_loss: 0.9101 - val_accuracy: 0.8233 Epoch 00018: LearningRateScheduler reducing learning rate to 0.009216. Epoch 18/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 1.7623e-04 - accuracy: 1.0000 - val_loss: 0.9136 - val_accuracy: 0.8238 Epoch 00019: LearningRateScheduler reducing learning rate to 0.009216. Epoch 19/35 50000/50000 [==============================] - 191s 4ms/sample - loss: 1.5298e-04 - accuracy: 1.0000 - val_loss: 0.9170 - val_accuracy: 0.8248 Epoch 00020: LearningRateScheduler reducing learning rate to 0.009216. Epoch 20/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 1.3303e-04 - accuracy: 1.0000 - val_loss: 0.9178 - val_accuracy: 0.8265 Epoch 00021: LearningRateScheduler reducing learning rate to 0.009216. Epoch 21/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 1.2130e-04 - accuracy: 1.0000 - val_loss: 0.9188 - val_accuracy: 0.8253 Epoch 00022: LearningRateScheduler reducing learning rate to 0.009216. Epoch 22/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 1.0718e-04 - accuracy: 1.0000 - val_loss: 0.9209 - val_accuracy: 0.8254 Epoch 00023: LearningRateScheduler reducing learning rate to 0.009216. Epoch 23/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 9.9417e-05 - accuracy: 1.0000 - val_loss: 0.9238 - val_accuracy: 0.8266 Epoch 00024: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 24/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 9.1316e-05 - accuracy: 1.0000 - val_loss: 0.9236 - val_accuracy: 0.8266 Epoch 00025: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 25/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 8.8492e-05 - accuracy: 1.0000 - val_loss: 0.9260 - val_accuracy: 0.8270 Epoch 00026: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 26/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 8.4004e-05 - accuracy: 1.0000 - val_loss: 0.9270 - val_accuracy: 0.8264 Epoch 00027: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 27/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 7.6821e-05 - accuracy: 1.0000 - val_loss: 0.9286 - val_accuracy: 0.8264 Epoch 00028: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 28/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 7.1725e-05 - accuracy: 1.0000 - val_loss: 0.9297 - val_accuracy: 0.8265 Epoch 00029: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 29/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 6.9910e-05 - accuracy: 1.0000 - val_loss: 0.9298 - val_accuracy: 0.8270 Epoch 00030: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 30/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 6.7411e-05 - accuracy: 1.0000 - val_loss: 0.9316 - val_accuracy: 0.8272 Epoch 00031: LearningRateScheduler reducing learning rate to 0.008847359999999999. Epoch 31/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 6.2929e-05 - accuracy: 1.0000 - val_loss: 0.9318 - val_accuracy: 0.8275 Epoch 00032: LearningRateScheduler reducing learning rate to 0.008493465599999998. Epoch 32/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 5.9789e-05 - accuracy: 1.0000 - val_loss: 0.9338 - val_accuracy: 0.8272 Epoch 00033: LearningRateScheduler reducing learning rate to 0.008493465599999998. Epoch 33/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 5.8887e-05 - accuracy: 1.0000 - val_loss: 0.9339 - val_accuracy: 0.8267 Epoch 00034: LearningRateScheduler reducing learning rate to 0.008493465599999998. Epoch 34/35 50000/50000 [==============================] - 189s 4ms/sample - loss: 5.7854e-05 - accuracy: 1.0000 - val_loss: 0.9351 - val_accuracy: 0.8264 Epoch 00035: LearningRateScheduler reducing learning rate to 0.008493465599999998. Epoch 35/35 50000/50000 [==============================] - 190s 4ms/sample - loss: 5.6941e-05 - accuracy: 1.0000 - val_loss: 0.9361 - val_accuracy: 0.8266