import cv2
import numpy as np
import pandas as pd
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard
from keras.models import Model
from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Activation, UpSampling2D, BatchNormalization
from keras.optimizers import RMSprop
from keras.losses import binary_crossentropy
import keras.backend as K
from sklearn.model_selection import train_test_split
Using TensorFlow backend.
import random
import gzip
import pickle
import matplotlib.pyplot as plt
%matplotlib inline
def dice_coeff(y_true, y_pred):
smooth = 1.
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
return score
def dice_loss(y_true, y_pred):
loss = 1 - dice_coeff(y_true, y_pred)
return loss
def bce_dice_loss(y_true, y_pred):
loss = binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
return loss
def unet_down_one_block(inputs, num_filters):
x = Conv2D(num_filters, (3, 3), padding='same')(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(num_filters, (3, 3), padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
def unet_max_pool(inputs):
x = MaxPooling2D((2, 2), strides=(2, 2))(inputs)
return x
def unet_up_one_block(up_input, down_input, num_filters):
x = UpSampling2D((2,2))(up_input)
x = concatenate([down_input, x], axis=3)
x = Conv2D(num_filters, (3,3), padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(num_filters, (3,3), padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(num_filters, (3,3), padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
def get_unet(input_shape = (256, 256, 3),
num_classes = 1,
initial_filters = 32,
central_filters = 1024):
num_filters = initial_filters
out_list = [Input(shape=input_shape)]
down_interim_list = []
while num_filters <= central_filters/2:
x = unet_down_one_block(out_list[-1], num_filters)
down_interim_list.append(x)
num_filters = num_filters * 2
y = unet_max_pool(x)
out_list.append(y)
x = unet_down_one_block(out_list[-1], num_filters)
out_list.append(x)
num_filters = int(num_filters / 2)
while num_filters >= initial_filters:
x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)
out_list.append(x)
num_filters = int(num_filters / 2)
classify = Conv2D(num_classes, (1,1), activation = 'sigmoid')(out_list[-1])
model = Model(inputs=out_list[0], outputs=classify)
model.compile(optimizer=RMSprop(lr=0.0001),
loss=bce_dice_loss,
metrics=[dice_coeff])
return model
model_512 = get_unet(input_shape=(512,512,3), initial_filters=16)
model_512.load_weights('./weights/best_weights_512_2.hdf5')
input_size=512
from os import listdir
ids_list = [x.split('.')[0] for x in listdir('data/train')]
all_imgs=[]
for id in ids_list:
img = cv2.imread('data/train/{}.jpg'.format(id))
img = cv2.resize(img, (input_size, input_size))
all_imgs.append(img)
preds = model_512.predict(np.array(all_imgs, np.float32)/255, batch_size=16)
preds = np.squeeze(preds, axis=3)
preds.shape
(5088, 512, 512)
for i in range(len(ids_list)):
this_id = ids_list[i]
this_pred = np.copy(preds[i,:,:])
f = gzip.open('./data/preds512/' + this_id+ '.pklz','wb')
pickle.dump(this_pred, f)
f.close()
model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=64)
model_fullres.summary()
____________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ==================================================================================================== input_1 (InputLayer) (None, 1280, 1920, 4) 0 ____________________________________________________________________________________________________ conv2d_1 (Conv2D) (None, 1280, 1920, 8) 296 input_1[0][0] ____________________________________________________________________________________________________ batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_1[0][0] ____________________________________________________________________________________________________ activation_1 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_1[0][0] ____________________________________________________________________________________________________ conv2d_2 (Conv2D) (None, 1280, 1920, 8) 584 activation_1[0][0] ____________________________________________________________________________________________________ batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_2[0][0] ____________________________________________________________________________________________________ activation_2 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_2[0][0] ____________________________________________________________________________________________________ max_pooling2d_1 (MaxPooling2D) (None, 640, 960, 8) 0 activation_2[0][0] ____________________________________________________________________________________________________ conv2d_3 (Conv2D) (None, 640, 960, 16) 1168 max_pooling2d_1[0][0] ____________________________________________________________________________________________________ batch_normalization_3 (BatchNorm (None, 640, 960, 16) 64 conv2d_3[0][0] ____________________________________________________________________________________________________ activation_3 (Activation) (None, 640, 960, 16) 0 batch_normalization_3[0][0] ____________________________________________________________________________________________________ conv2d_4 (Conv2D) (None, 640, 960, 16) 2320 activation_3[0][0] ____________________________________________________________________________________________________ batch_normalization_4 (BatchNorm (None, 640, 960, 16) 64 conv2d_4[0][0] ____________________________________________________________________________________________________ activation_4 (Activation) (None, 640, 960, 16) 0 batch_normalization_4[0][0] ____________________________________________________________________________________________________ max_pooling2d_2 (MaxPooling2D) (None, 320, 480, 16) 0 activation_4[0][0] ____________________________________________________________________________________________________ conv2d_5 (Conv2D) (None, 320, 480, 32) 4640 max_pooling2d_2[0][0] ____________________________________________________________________________________________________ batch_normalization_5 (BatchNorm (None, 320, 480, 32) 128 conv2d_5[0][0] ____________________________________________________________________________________________________ activation_5 (Activation) (None, 320, 480, 32) 0 batch_normalization_5[0][0] ____________________________________________________________________________________________________ conv2d_6 (Conv2D) (None, 320, 480, 32) 9248 activation_5[0][0] ____________________________________________________________________________________________________ batch_normalization_6 (BatchNorm (None, 320, 480, 32) 128 conv2d_6[0][0] ____________________________________________________________________________________________________ activation_6 (Activation) (None, 320, 480, 32) 0 batch_normalization_6[0][0] ____________________________________________________________________________________________________ max_pooling2d_3 (MaxPooling2D) (None, 160, 240, 32) 0 activation_6[0][0] ____________________________________________________________________________________________________ conv2d_7 (Conv2D) (None, 160, 240, 64) 18496 max_pooling2d_3[0][0] ____________________________________________________________________________________________________ batch_normalization_7 (BatchNorm (None, 160, 240, 64) 256 conv2d_7[0][0] ____________________________________________________________________________________________________ activation_7 (Activation) (None, 160, 240, 64) 0 batch_normalization_7[0][0] ____________________________________________________________________________________________________ conv2d_8 (Conv2D) (None, 160, 240, 64) 36928 activation_7[0][0] ____________________________________________________________________________________________________ batch_normalization_8 (BatchNorm (None, 160, 240, 64) 256 conv2d_8[0][0] ____________________________________________________________________________________________________ activation_8 (Activation) (None, 160, 240, 64) 0 batch_normalization_8[0][0] ____________________________________________________________________________________________________ up_sampling2d_1 (UpSampling2D) (None, 320, 480, 64) 0 activation_8[0][0] ____________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, 320, 480, 96) 0 activation_6[0][0] up_sampling2d_1[0][0] ____________________________________________________________________________________________________ conv2d_9 (Conv2D) (None, 320, 480, 32) 27680 concatenate_1[0][0] ____________________________________________________________________________________________________ batch_normalization_9 (BatchNorm (None, 320, 480, 32) 128 conv2d_9[0][0] ____________________________________________________________________________________________________ activation_9 (Activation) (None, 320, 480, 32) 0 batch_normalization_9[0][0] ____________________________________________________________________________________________________ conv2d_10 (Conv2D) (None, 320, 480, 32) 9248 activation_9[0][0] ____________________________________________________________________________________________________ batch_normalization_10 (BatchNor (None, 320, 480, 32) 128 conv2d_10[0][0] ____________________________________________________________________________________________________ activation_10 (Activation) (None, 320, 480, 32) 0 batch_normalization_10[0][0] ____________________________________________________________________________________________________ conv2d_11 (Conv2D) (None, 320, 480, 32) 9248 activation_10[0][0] ____________________________________________________________________________________________________ batch_normalization_11 (BatchNor (None, 320, 480, 32) 128 conv2d_11[0][0] ____________________________________________________________________________________________________ activation_11 (Activation) (None, 320, 480, 32) 0 batch_normalization_11[0][0] ____________________________________________________________________________________________________ up_sampling2d_2 (UpSampling2D) (None, 640, 960, 32) 0 activation_11[0][0] ____________________________________________________________________________________________________ concatenate_2 (Concatenate) (None, 640, 960, 48) 0 activation_4[0][0] up_sampling2d_2[0][0] ____________________________________________________________________________________________________ conv2d_12 (Conv2D) (None, 640, 960, 16) 6928 concatenate_2[0][0] ____________________________________________________________________________________________________ batch_normalization_12 (BatchNor (None, 640, 960, 16) 64 conv2d_12[0][0] ____________________________________________________________________________________________________ activation_12 (Activation) (None, 640, 960, 16) 0 batch_normalization_12[0][0] ____________________________________________________________________________________________________ conv2d_13 (Conv2D) (None, 640, 960, 16) 2320 activation_12[0][0] ____________________________________________________________________________________________________ batch_normalization_13 (BatchNor (None, 640, 960, 16) 64 conv2d_13[0][0] ____________________________________________________________________________________________________ activation_13 (Activation) (None, 640, 960, 16) 0 batch_normalization_13[0][0] ____________________________________________________________________________________________________ conv2d_14 (Conv2D) (None, 640, 960, 16) 2320 activation_13[0][0] ____________________________________________________________________________________________________ batch_normalization_14 (BatchNor (None, 640, 960, 16) 64 conv2d_14[0][0] ____________________________________________________________________________________________________ activation_14 (Activation) (None, 640, 960, 16) 0 batch_normalization_14[0][0] ____________________________________________________________________________________________________ up_sampling2d_3 (UpSampling2D) (None, 1280, 1920, 16 0 activation_14[0][0] ____________________________________________________________________________________________________ concatenate_3 (Concatenate) (None, 1280, 1920, 24 0 activation_2[0][0] up_sampling2d_3[0][0] ____________________________________________________________________________________________________ conv2d_15 (Conv2D) (None, 1280, 1920, 8) 1736 concatenate_3[0][0] ____________________________________________________________________________________________________ batch_normalization_15 (BatchNor (None, 1280, 1920, 8) 32 conv2d_15[0][0] ____________________________________________________________________________________________________ activation_15 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_15[0][0] ____________________________________________________________________________________________________ conv2d_16 (Conv2D) (None, 1280, 1920, 8) 584 activation_15[0][0] ____________________________________________________________________________________________________ batch_normalization_16 (BatchNor (None, 1280, 1920, 8) 32 conv2d_16[0][0] ____________________________________________________________________________________________________ activation_16 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_16[0][0] ____________________________________________________________________________________________________ conv2d_17 (Conv2D) (None, 1280, 1920, 8) 584 activation_16[0][0] ____________________________________________________________________________________________________ batch_normalization_17 (BatchNor (None, 1280, 1920, 8) 32 conv2d_17[0][0] ____________________________________________________________________________________________________ activation_17 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_17[0][0] ____________________________________________________________________________________________________ conv2d_18 (Conv2D) (None, 1280, 1920, 1) 9 activation_17[0][0] ==================================================================================================== Total params: 135,969 Trainable params: 135,153 Non-trainable params: 816 ____________________________________________________________________________________________________
from keras.utils import plot_model
plot_model(model_fullres, to_file='model-fullres-v2.png', show_shapes=True, show_layer_names=True)
from IPython.display import FileLink
FileLink('model-fullres-v2.png')
threshold = 0.5
df_train = pd.read_csv('data/train_masks.csv')
ids_train = df_train['img'].map(lambda s: s.split('.')[0])
ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42)
def randomHueSaturationValue(image, hue_shift_limit=(-180, 180),
sat_shift_limit=(-255, 255),
val_shift_limit=(-255, 255), u=0.5):
if np.random.random() < u:
image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(image)
hue_shift = np.random.uniform(hue_shift_limit[0], hue_shift_limit[1])
h = cv2.add(h, hue_shift)
sat_shift = np.random.uniform(sat_shift_limit[0], sat_shift_limit[1])
s = cv2.add(s, sat_shift)
val_shift = np.random.uniform(val_shift_limit[0], val_shift_limit[1])
v = cv2.add(v, val_shift)
image = cv2.merge((h, s, v))
image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)
return image
def randomShiftScaleRotate(image, mask,
shift_limit=(-0.0625, 0.0625),
scale_limit=(-0.1, 0.1),
rotate_limit=(-45, 45), aspect_limit=(0, 0),
borderMode=cv2.BORDER_CONSTANT, u=0.5):
if np.random.random() < u:
height, width, channel = image.shape
angle = np.random.uniform(rotate_limit[0], rotate_limit[1]) # degree
scale = np.random.uniform(1 + scale_limit[0], 1 + scale_limit[1])
aspect = np.random.uniform(1 + aspect_limit[0], 1 + aspect_limit[1])
sx = scale * aspect / (aspect ** 0.5)
sy = scale / (aspect ** 0.5)
dx = round(np.random.uniform(shift_limit[0], shift_limit[1]) * width)
dy = round(np.random.uniform(shift_limit[0], shift_limit[1]) * height)
cc = np.math.cos(angle / 180 * np.math.pi) * sx
ss = np.math.sin(angle / 180 * np.math.pi) * sy
rotate_matrix = np.array([[cc, -ss], [ss, cc]])
box0 = np.array([[0, 0], [width, 0], [width, height], [0, height], ])
box1 = box0 - np.array([width / 2, height / 2])
box1 = np.dot(box1, rotate_matrix.T) + np.array([width / 2 + dx, height / 2 + dy])
box0 = box0.astype(np.float32)
box1 = box1.astype(np.float32)
mat = cv2.getPerspectiveTransform(box0, box1)
image = cv2.warpPerspective(image, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,
borderValue=(
0, 0,
0,))
mask = cv2.warpPerspective(mask, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,
borderValue=(
0, 0,
0,))
return image, mask
def randomHorizontalFlip(image, mask, u=0.5):
if np.random.random() < u:
image = cv2.flip(image, 1)
mask = cv2.flip(mask, 1)
return image, mask
def m512_stuff(ids_list):
final_512_preds=[]
for this_id in ids_list:
f = gzip.open('./data/preds512/' + this_id + '.pklz','rb')
mask = pickle.load(f)
f.close()
resized_mask = cv2.resize(mask, (1920, 1280))
resized_mask = np.expand_dims(resized_mask, axis=3)*255
final_512_preds.append(resized_mask)
return final_512_preds
def train_generator(train_batch_size):
while True:
this_ids_train_split = random.sample(list(ids_train_split), len(list(ids_train_split)))
for start in range(0, len(ids_train_split), train_batch_size):
x_batch = []
y_batch = []
start=0
end = min(start + train_batch_size, len(ids_train_split))
ids_train_batch = this_ids_train_split[start:end]
all_imgs = []
for id in ids_train_batch:
img = cv2.imread('data/train/{}.jpg'.format(id))
all_imgs.append(img)
final_512_preds = m512_stuff(ids_train_batch)
for i in range(len(all_imgs)):
img = cv2.resize(all_imgs[i], (1920,1280))
img = randomHueSaturationValue(img,
hue_shift_limit=(-50, 50),
sat_shift_limit=(-5, 5),
val_shift_limit=(-15, 15))
img = np.concatenate([img, final_512_preds[i]], axis=2)
mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_train_batch[i]), cv2.IMREAD_GRAYSCALE)
mask = cv2.resize(mask, (1920,1280))
img, mask = randomShiftScaleRotate(img, mask,
shift_limit=(-0.0625, 0.0625),
scale_limit=(-0.1, 0.1),
rotate_limit=(-0, 0))
img, mask = randomHorizontalFlip(img, mask)
mask = np.expand_dims(mask, axis=2)
x_batch.append(img)
y_batch.append(mask)
x_batch = np.array(x_batch, np.float32) / 255
y_batch = np.array(y_batch, np.float32) / 255
yield x_batch, y_batch
def valid_generator(val_batch_size):
while True:
for start in range(0, len(ids_valid_split), val_batch_size):
x_batch = []
y_batch = []
end = min(start + val_batch_size, len(ids_valid_split))
ids_valid_batch = ids_valid_split[start:end]
all_imgs = []
for id in ids_valid_batch.values:
img = cv2.imread('data/train/{}.jpg'.format(id))
all_imgs.append(img)
final_512_preds = m512_stuff(ids_valid_batch.values)
for i in range(len(all_imgs)):
img = cv2.resize(all_imgs[i], (1920,1280))
img = np.concatenate([img, final_512_preds[i]], axis=2)
mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_valid_batch.values[i]), cv2.IMREAD_GRAYSCALE)
mask = cv2.resize(mask, (1920,1280))
mask = np.expand_dims(mask, axis=2)
x_batch.append(img)
y_batch.append(mask)
x_batch = np.array(x_batch, np.float32) / 255
y_batch = np.array(y_batch, np.float32) / 255
yield x_batch, y_batch
max_epochs = 50
train_batch_size = 3
val_batch_size = 8
callbacks = [EarlyStopping(monitor='val_loss',
patience=8,
verbose=1,
min_delta=1e-4),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=4,
verbose=1,
epsilon=1e-4),
ModelCheckpoint(monitor='val_loss',
filepath='weights/best_weights_fullresv2.hdf5',
save_best_only=True,
save_weights_only=True)]
model_fullres.fit_generator(generator=train_generator(train_batch_size),
steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),
epochs=max_epochs,
verbose=2,
callbacks=callbacks,
validation_data=valid_generator(val_batch_size),
validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))
Epoch 1/50 4466s - loss: 0.7622 - dice_coeff: 0.5814 - val_loss: 0.6156 - val_dice_coeff: 0.6557 Epoch 2/50 4459s - loss: 0.4551 - dice_coeff: 0.7366 - val_loss: 0.3884 - val_dice_coeff: 0.7606 Epoch 3/50 4459s - loss: 0.2361 - dice_coeff: 0.8573 - val_loss: 0.2059 - val_dice_coeff: 0.8662 Epoch 4/50 4460s - loss: 0.2182 - dice_coeff: 0.8479 - val_loss: 0.1380 - val_dice_coeff: 0.9107 Epoch 5/50 4460s - loss: 0.0832 - dice_coeff: 0.9425 - val_loss: 0.0594 - val_dice_coeff: 0.9633 Epoch 6/50 4460s - loss: 0.0290 - dice_coeff: 0.9808 - val_loss: 0.0294 - val_dice_coeff: 0.9848 Epoch 7/50 4459s - loss: 0.0129 - dice_coeff: 0.9913 - val_loss: 0.0186 - val_dice_coeff: 0.9920 Epoch 8/50 4458s - loss: 0.0083 - dice_coeff: 0.9943 - val_loss: 0.0174 - val_dice_coeff: 0.9937 Epoch 9/50 4460s - loss: 0.0049 - dice_coeff: 0.9973 - val_loss: 0.0141 - val_dice_coeff: 0.9953 Epoch 10/50 4461s - loss: 0.0045 - dice_coeff: 0.9975 - val_loss: 0.0148 - val_dice_coeff: 0.9953 Epoch 11/50 4461s - loss: 0.0031 - dice_coeff: 0.9984 - val_loss: 0.0141 - val_dice_coeff: 0.9957 Epoch 12/50 4462s - loss: 0.0029 - dice_coeff: 0.9985 - val_loss: 0.0152 - val_dice_coeff: 0.9955 Epoch 13/50 4461s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0144 - val_dice_coeff: 0.9956 Epoch 14/50 Epoch 00013: reducing learning rate to 9.999999747378752e-06. 4461s - loss: 0.0032 - dice_coeff: 0.9983 - val_loss: 0.0155 - val_dice_coeff: 0.9954 Epoch 15/50
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-25-59878f25d0c3> in <module>() 19 callbacks=callbacks, 20 validation_data=valid_generator(val_batch_size), ---> 21 validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size))) /home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs) 85 warnings.warn('Update your `' + object_name + 86 '` call to the Keras 2 API: ' + signature, stacklevel=2) ---> 87 return func(*args, **kwargs) 88 wrapper._original_function = func 89 return wrapper /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch) 2040 outs = self.train_on_batch(x, y, 2041 sample_weight=sample_weight, -> 2042 class_weight=class_weight) 2043 2044 if not isinstance(outs, list): /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight) 1760 ins = x + y + sample_weights 1761 self._make_train_function() -> 1762 outputs = self.train_function(ins) 1763 if len(outputs) == 1: 1764 return outputs[0] /home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs) 2271 updated = session.run(self.outputs + [self.updates_op], 2272 feed_dict=feed_dict, -> 2273 **self.session_kwargs) 2274 return updated[:len(self.outputs)] 2275 /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata) 893 try: 894 result = self._run(None, fetches, feed_dict, options_ptr, --> 895 run_metadata_ptr) 896 if run_metadata: 897 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata) 1122 if final_fetches or final_targets or (handle and feed_dict_tensor): 1123 results = self._do_run(handle, final_targets, final_fetches, -> 1124 feed_dict_tensor, options, run_metadata) 1125 else: 1126 results = [] /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata) 1319 if handle is None: 1320 return self._do_call(_run_fn, self._session, feeds, fetches, targets, -> 1321 options, run_metadata) 1322 else: 1323 return self._do_call(_prun_fn, self._session, handle, feeds, fetches) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args) 1325 def _do_call(self, fn, *args): 1326 try: -> 1327 return fn(*args) 1328 except errors.OpError as e: 1329 message = compat.as_text(e.message) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata) 1304 return tf_session.TF_Run(session, options, 1305 feed_dict, fetch_list, target_list, -> 1306 status, run_metadata) 1307 1308 def _prun_fn(session, handle, feed_dict, fetch_list): KeyboardInterrupt:
model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)
model_fullres.summary()
____________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ==================================================================================================== input_1 (InputLayer) (None, 1280, 1920, 4) 0 ____________________________________________________________________________________________________ conv2d_1 (Conv2D) (None, 1280, 1920, 8) 296 input_1[0][0] ____________________________________________________________________________________________________ batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_1[0][0] ____________________________________________________________________________________________________ activation_1 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_1[0][0] ____________________________________________________________________________________________________ conv2d_2 (Conv2D) (None, 1280, 1920, 8) 584 activation_1[0][0] ____________________________________________________________________________________________________ batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_2[0][0] ____________________________________________________________________________________________________ activation_2 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_2[0][0] ____________________________________________________________________________________________________ max_pooling2d_1 (MaxPooling2D) (None, 640, 960, 8) 0 activation_2[0][0] ____________________________________________________________________________________________________ conv2d_3 (Conv2D) (None, 640, 960, 16) 1168 max_pooling2d_1[0][0] ____________________________________________________________________________________________________ batch_normalization_3 (BatchNorm (None, 640, 960, 16) 64 conv2d_3[0][0] ____________________________________________________________________________________________________ activation_3 (Activation) (None, 640, 960, 16) 0 batch_normalization_3[0][0] ____________________________________________________________________________________________________ conv2d_4 (Conv2D) (None, 640, 960, 16) 2320 activation_3[0][0] ____________________________________________________________________________________________________ batch_normalization_4 (BatchNorm (None, 640, 960, 16) 64 conv2d_4[0][0] ____________________________________________________________________________________________________ activation_4 (Activation) (None, 640, 960, 16) 0 batch_normalization_4[0][0] ____________________________________________________________________________________________________ max_pooling2d_2 (MaxPooling2D) (None, 320, 480, 16) 0 activation_4[0][0] ____________________________________________________________________________________________________ conv2d_5 (Conv2D) (None, 320, 480, 32) 4640 max_pooling2d_2[0][0] ____________________________________________________________________________________________________ batch_normalization_5 (BatchNorm (None, 320, 480, 32) 128 conv2d_5[0][0] ____________________________________________________________________________________________________ activation_5 (Activation) (None, 320, 480, 32) 0 batch_normalization_5[0][0] ____________________________________________________________________________________________________ conv2d_6 (Conv2D) (None, 320, 480, 32) 9248 activation_5[0][0] ____________________________________________________________________________________________________ batch_normalization_6 (BatchNorm (None, 320, 480, 32) 128 conv2d_6[0][0] ____________________________________________________________________________________________________ activation_6 (Activation) (None, 320, 480, 32) 0 batch_normalization_6[0][0] ____________________________________________________________________________________________________ max_pooling2d_3 (MaxPooling2D) (None, 160, 240, 32) 0 activation_6[0][0] ____________________________________________________________________________________________________ conv2d_7 (Conv2D) (None, 160, 240, 64) 18496 max_pooling2d_3[0][0] ____________________________________________________________________________________________________ batch_normalization_7 (BatchNorm (None, 160, 240, 64) 256 conv2d_7[0][0] ____________________________________________________________________________________________________ activation_7 (Activation) (None, 160, 240, 64) 0 batch_normalization_7[0][0] ____________________________________________________________________________________________________ conv2d_8 (Conv2D) (None, 160, 240, 64) 36928 activation_7[0][0] ____________________________________________________________________________________________________ batch_normalization_8 (BatchNorm (None, 160, 240, 64) 256 conv2d_8[0][0] ____________________________________________________________________________________________________ activation_8 (Activation) (None, 160, 240, 64) 0 batch_normalization_8[0][0] ____________________________________________________________________________________________________ max_pooling2d_4 (MaxPooling2D) (None, 80, 120, 64) 0 activation_8[0][0] ____________________________________________________________________________________________________ conv2d_9 (Conv2D) (None, 80, 120, 128) 73856 max_pooling2d_4[0][0] ____________________________________________________________________________________________________ batch_normalization_9 (BatchNorm (None, 80, 120, 128) 512 conv2d_9[0][0] ____________________________________________________________________________________________________ activation_9 (Activation) (None, 80, 120, 128) 0 batch_normalization_9[0][0] ____________________________________________________________________________________________________ conv2d_10 (Conv2D) (None, 80, 120, 128) 147584 activation_9[0][0] ____________________________________________________________________________________________________ batch_normalization_10 (BatchNor (None, 80, 120, 128) 512 conv2d_10[0][0] ____________________________________________________________________________________________________ activation_10 (Activation) (None, 80, 120, 128) 0 batch_normalization_10[0][0] ____________________________________________________________________________________________________ max_pooling2d_5 (MaxPooling2D) (None, 40, 60, 128) 0 activation_10[0][0] ____________________________________________________________________________________________________ conv2d_11 (Conv2D) (None, 40, 60, 256) 295168 max_pooling2d_5[0][0] ____________________________________________________________________________________________________ batch_normalization_11 (BatchNor (None, 40, 60, 256) 1024 conv2d_11[0][0] ____________________________________________________________________________________________________ activation_11 (Activation) (None, 40, 60, 256) 0 batch_normalization_11[0][0] ____________________________________________________________________________________________________ conv2d_12 (Conv2D) (None, 40, 60, 256) 590080 activation_11[0][0] ____________________________________________________________________________________________________ batch_normalization_12 (BatchNor (None, 40, 60, 256) 1024 conv2d_12[0][0] ____________________________________________________________________________________________________ activation_12 (Activation) (None, 40, 60, 256) 0 batch_normalization_12[0][0] ____________________________________________________________________________________________________ max_pooling2d_6 (MaxPooling2D) (None, 20, 30, 256) 0 activation_12[0][0] ____________________________________________________________________________________________________ conv2d_13 (Conv2D) (None, 20, 30, 512) 1180160 max_pooling2d_6[0][0] ____________________________________________________________________________________________________ batch_normalization_13 (BatchNor (None, 20, 30, 512) 2048 conv2d_13[0][0] ____________________________________________________________________________________________________ activation_13 (Activation) (None, 20, 30, 512) 0 batch_normalization_13[0][0] ____________________________________________________________________________________________________ conv2d_14 (Conv2D) (None, 20, 30, 512) 2359808 activation_13[0][0] ____________________________________________________________________________________________________ batch_normalization_14 (BatchNor (None, 20, 30, 512) 2048 conv2d_14[0][0] ____________________________________________________________________________________________________ activation_14 (Activation) (None, 20, 30, 512) 0 batch_normalization_14[0][0] ____________________________________________________________________________________________________ max_pooling2d_7 (MaxPooling2D) (None, 10, 15, 512) 0 activation_14[0][0] ____________________________________________________________________________________________________ conv2d_15 (Conv2D) (None, 10, 15, 1024) 4719616 max_pooling2d_7[0][0] ____________________________________________________________________________________________________ batch_normalization_15 (BatchNor (None, 10, 15, 1024) 4096 conv2d_15[0][0] ____________________________________________________________________________________________________ activation_15 (Activation) (None, 10, 15, 1024) 0 batch_normalization_15[0][0] ____________________________________________________________________________________________________ conv2d_16 (Conv2D) (None, 10, 15, 1024) 9438208 activation_15[0][0] ____________________________________________________________________________________________________ batch_normalization_16 (BatchNor (None, 10, 15, 1024) 4096 conv2d_16[0][0] ____________________________________________________________________________________________________ activation_16 (Activation) (None, 10, 15, 1024) 0 batch_normalization_16[0][0] ____________________________________________________________________________________________________ up_sampling2d_1 (UpSampling2D) (None, 20, 30, 1024) 0 activation_16[0][0] ____________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, 20, 30, 1536) 0 activation_14[0][0] up_sampling2d_1[0][0] ____________________________________________________________________________________________________ conv2d_17 (Conv2D) (None, 20, 30, 512) 7078400 concatenate_1[0][0] ____________________________________________________________________________________________________ batch_normalization_17 (BatchNor (None, 20, 30, 512) 2048 conv2d_17[0][0] ____________________________________________________________________________________________________ activation_17 (Activation) (None, 20, 30, 512) 0 batch_normalization_17[0][0] ____________________________________________________________________________________________________ conv2d_18 (Conv2D) (None, 20, 30, 512) 2359808 activation_17[0][0] ____________________________________________________________________________________________________ batch_normalization_18 (BatchNor (None, 20, 30, 512) 2048 conv2d_18[0][0] ____________________________________________________________________________________________________ activation_18 (Activation) (None, 20, 30, 512) 0 batch_normalization_18[0][0] ____________________________________________________________________________________________________ conv2d_19 (Conv2D) (None, 20, 30, 512) 2359808 activation_18[0][0] ____________________________________________________________________________________________________ batch_normalization_19 (BatchNor (None, 20, 30, 512) 2048 conv2d_19[0][0] ____________________________________________________________________________________________________ activation_19 (Activation) (None, 20, 30, 512) 0 batch_normalization_19[0][0] ____________________________________________________________________________________________________ up_sampling2d_2 (UpSampling2D) (None, 40, 60, 512) 0 activation_19[0][0] ____________________________________________________________________________________________________ concatenate_2 (Concatenate) (None, 40, 60, 768) 0 activation_12[0][0] up_sampling2d_2[0][0] ____________________________________________________________________________________________________ conv2d_20 (Conv2D) (None, 40, 60, 256) 1769728 concatenate_2[0][0] ____________________________________________________________________________________________________ batch_normalization_20 (BatchNor (None, 40, 60, 256) 1024 conv2d_20[0][0] ____________________________________________________________________________________________________ activation_20 (Activation) (None, 40, 60, 256) 0 batch_normalization_20[0][0] ____________________________________________________________________________________________________ conv2d_21 (Conv2D) (None, 40, 60, 256) 590080 activation_20[0][0] ____________________________________________________________________________________________________ batch_normalization_21 (BatchNor (None, 40, 60, 256) 1024 conv2d_21[0][0] ____________________________________________________________________________________________________ activation_21 (Activation) (None, 40, 60, 256) 0 batch_normalization_21[0][0] ____________________________________________________________________________________________________ conv2d_22 (Conv2D) (None, 40, 60, 256) 590080 activation_21[0][0] ____________________________________________________________________________________________________ batch_normalization_22 (BatchNor (None, 40, 60, 256) 1024 conv2d_22[0][0] ____________________________________________________________________________________________________ activation_22 (Activation) (None, 40, 60, 256) 0 batch_normalization_22[0][0] ____________________________________________________________________________________________________ up_sampling2d_3 (UpSampling2D) (None, 80, 120, 256) 0 activation_22[0][0] ____________________________________________________________________________________________________ concatenate_3 (Concatenate) (None, 80, 120, 384) 0 activation_10[0][0] up_sampling2d_3[0][0] ____________________________________________________________________________________________________ conv2d_23 (Conv2D) (None, 80, 120, 128) 442496 concatenate_3[0][0] ____________________________________________________________________________________________________ batch_normalization_23 (BatchNor (None, 80, 120, 128) 512 conv2d_23[0][0] ____________________________________________________________________________________________________ activation_23 (Activation) (None, 80, 120, 128) 0 batch_normalization_23[0][0] ____________________________________________________________________________________________________ conv2d_24 (Conv2D) (None, 80, 120, 128) 147584 activation_23[0][0] ____________________________________________________________________________________________________ batch_normalization_24 (BatchNor (None, 80, 120, 128) 512 conv2d_24[0][0] ____________________________________________________________________________________________________ activation_24 (Activation) (None, 80, 120, 128) 0 batch_normalization_24[0][0] ____________________________________________________________________________________________________ conv2d_25 (Conv2D) (None, 80, 120, 128) 147584 activation_24[0][0] ____________________________________________________________________________________________________ batch_normalization_25 (BatchNor (None, 80, 120, 128) 512 conv2d_25[0][0] ____________________________________________________________________________________________________ activation_25 (Activation) (None, 80, 120, 128) 0 batch_normalization_25[0][0] ____________________________________________________________________________________________________ up_sampling2d_4 (UpSampling2D) (None, 160, 240, 128) 0 activation_25[0][0] ____________________________________________________________________________________________________ concatenate_4 (Concatenate) (None, 160, 240, 192) 0 activation_8[0][0] up_sampling2d_4[0][0] ____________________________________________________________________________________________________ conv2d_26 (Conv2D) (None, 160, 240, 64) 110656 concatenate_4[0][0] ____________________________________________________________________________________________________ batch_normalization_26 (BatchNor (None, 160, 240, 64) 256 conv2d_26[0][0] ____________________________________________________________________________________________________ activation_26 (Activation) (None, 160, 240, 64) 0 batch_normalization_26[0][0] ____________________________________________________________________________________________________ conv2d_27 (Conv2D) (None, 160, 240, 64) 36928 activation_26[0][0] ____________________________________________________________________________________________________ batch_normalization_27 (BatchNor (None, 160, 240, 64) 256 conv2d_27[0][0] ____________________________________________________________________________________________________ activation_27 (Activation) (None, 160, 240, 64) 0 batch_normalization_27[0][0] ____________________________________________________________________________________________________ conv2d_28 (Conv2D) (None, 160, 240, 64) 36928 activation_27[0][0] ____________________________________________________________________________________________________ batch_normalization_28 (BatchNor (None, 160, 240, 64) 256 conv2d_28[0][0] ____________________________________________________________________________________________________ activation_28 (Activation) (None, 160, 240, 64) 0 batch_normalization_28[0][0] ____________________________________________________________________________________________________ up_sampling2d_5 (UpSampling2D) (None, 320, 480, 64) 0 activation_28[0][0] ____________________________________________________________________________________________________ concatenate_5 (Concatenate) (None, 320, 480, 96) 0 activation_6[0][0] up_sampling2d_5[0][0] ____________________________________________________________________________________________________ conv2d_29 (Conv2D) (None, 320, 480, 32) 27680 concatenate_5[0][0] ____________________________________________________________________________________________________ batch_normalization_29 (BatchNor (None, 320, 480, 32) 128 conv2d_29[0][0] ____________________________________________________________________________________________________ activation_29 (Activation) (None, 320, 480, 32) 0 batch_normalization_29[0][0] ____________________________________________________________________________________________________ conv2d_30 (Conv2D) (None, 320, 480, 32) 9248 activation_29[0][0] ____________________________________________________________________________________________________ batch_normalization_30 (BatchNor (None, 320, 480, 32) 128 conv2d_30[0][0] ____________________________________________________________________________________________________ activation_30 (Activation) (None, 320, 480, 32) 0 batch_normalization_30[0][0] ____________________________________________________________________________________________________ conv2d_31 (Conv2D) (None, 320, 480, 32) 9248 activation_30[0][0] ____________________________________________________________________________________________________ batch_normalization_31 (BatchNor (None, 320, 480, 32) 128 conv2d_31[0][0] ____________________________________________________________________________________________________ activation_31 (Activation) (None, 320, 480, 32) 0 batch_normalization_31[0][0] ____________________________________________________________________________________________________ up_sampling2d_6 (UpSampling2D) (None, 640, 960, 32) 0 activation_31[0][0] ____________________________________________________________________________________________________ concatenate_6 (Concatenate) (None, 640, 960, 48) 0 activation_4[0][0] up_sampling2d_6[0][0] ____________________________________________________________________________________________________ conv2d_32 (Conv2D) (None, 640, 960, 16) 6928 concatenate_6[0][0] ____________________________________________________________________________________________________ batch_normalization_32 (BatchNor (None, 640, 960, 16) 64 conv2d_32[0][0] ____________________________________________________________________________________________________ activation_32 (Activation) (None, 640, 960, 16) 0 batch_normalization_32[0][0] ____________________________________________________________________________________________________ conv2d_33 (Conv2D) (None, 640, 960, 16) 2320 activation_32[0][0] ____________________________________________________________________________________________________ batch_normalization_33 (BatchNor (None, 640, 960, 16) 64 conv2d_33[0][0] ____________________________________________________________________________________________________ activation_33 (Activation) (None, 640, 960, 16) 0 batch_normalization_33[0][0] ____________________________________________________________________________________________________ conv2d_34 (Conv2D) (None, 640, 960, 16) 2320 activation_33[0][0] ____________________________________________________________________________________________________ batch_normalization_34 (BatchNor (None, 640, 960, 16) 64 conv2d_34[0][0] ____________________________________________________________________________________________________ activation_34 (Activation) (None, 640, 960, 16) 0 batch_normalization_34[0][0] ____________________________________________________________________________________________________ up_sampling2d_7 (UpSampling2D) (None, 1280, 1920, 16 0 activation_34[0][0] ____________________________________________________________________________________________________ concatenate_7 (Concatenate) (None, 1280, 1920, 24 0 activation_2[0][0] up_sampling2d_7[0][0] ____________________________________________________________________________________________________ conv2d_35 (Conv2D) (None, 1280, 1920, 8) 1736 concatenate_7[0][0] ____________________________________________________________________________________________________ batch_normalization_35 (BatchNor (None, 1280, 1920, 8) 32 conv2d_35[0][0] ____________________________________________________________________________________________________ activation_35 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_35[0][0] ____________________________________________________________________________________________________ conv2d_36 (Conv2D) (None, 1280, 1920, 8) 584 activation_35[0][0] ____________________________________________________________________________________________________ batch_normalization_36 (BatchNor (None, 1280, 1920, 8) 32 conv2d_36[0][0] ____________________________________________________________________________________________________ activation_36 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_36[0][0] ____________________________________________________________________________________________________ conv2d_37 (Conv2D) (None, 1280, 1920, 8) 584 activation_36[0][0] ____________________________________________________________________________________________________ batch_normalization_37 (BatchNor (None, 1280, 1920, 8) 32 conv2d_37[0][0] ____________________________________________________________________________________________________ activation_37 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_37[0][0] ____________________________________________________________________________________________________ conv2d_38 (Conv2D) (None, 1280, 1920, 1) 9 activation_37[0][0] ==================================================================================================== Total params: 34,637,409 Trainable params: 34,623,153 Non-trainable params: 14,256 ____________________________________________________________________________________________________
callbacks = [EarlyStopping(monitor='val_loss',
patience=8,
verbose=1,
min_delta=1e-4),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=4,
verbose=1,
epsilon=1e-4),
ModelCheckpoint(monitor='val_loss',
filepath='weights/best_weights_fullresv3.hdf5',
save_best_only=True,
save_weights_only=True)]
model_fullres.fit_generator(generator=train_generator(train_batch_size),
steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),
epochs=max_epochs,
verbose=2,
callbacks=callbacks,
validation_data=valid_generator(val_batch_size),
validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))
Epoch 1/50 5437s - loss: 0.6051 - dice_coeff: 0.6865 - val_loss: 0.5555 - val_dice_coeff: 0.6788 Epoch 2/50 5413s - loss: 0.5022 - dice_coeff: 0.6988 - val_loss: 0.3965 - val_dice_coeff: 0.7570 Epoch 3/50 5411s - loss: 0.1670 - dice_coeff: 0.8868 - val_loss: 0.1190 - val_dice_coeff: 0.9239 Epoch 5/50 5414s - loss: 0.0846 - dice_coeff: 0.9406 - val_loss: 0.0596 - val_dice_coeff: 0.9635 Epoch 6/50 5414s - loss: 0.0291 - dice_coeff: 0.9803 - val_loss: 0.0271 - val_dice_coeff: 0.9856 Epoch 7/50 5415s - loss: 0.0101 - dice_coeff: 0.9936 - val_loss: 0.0176 - val_dice_coeff: 0.9929 Epoch 8/50 5416s - loss: 0.0048 - dice_coeff: 0.9971 - val_loss: 0.0148 - val_dice_coeff: 0.9950 Epoch 9/50 5415s - loss: 0.0036 - dice_coeff: 0.9978 - val_loss: 0.0150 - val_dice_coeff: 0.9954 Epoch 10/50 5416s - loss: 0.0030 - dice_coeff: 0.9986 - val_loss: 0.0139 - val_dice_coeff: 0.9958 Epoch 11/50 5418s - loss: 0.0026 - dice_coeff: 0.9986 - val_loss: 0.0145 - val_dice_coeff: 0.9958 Epoch 12/50 5417s - loss: 0.0023 - dice_coeff: 0.9988 - val_loss: 0.0146 - val_dice_coeff: 0.9958 Epoch 13/50 5417s - loss: 0.0025 - dice_coeff: 0.9986 - val_loss: 0.0148 - val_dice_coeff: 0.9958 Epoch 14/50 5417s - loss: 0.0027 - dice_coeff: 0.9986 - val_loss: 0.0147 - val_dice_coeff: 0.9958 Epoch 15/50
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) <ipython-input-22-db2ff83ed2f4> in <module>() 19 callbacks=callbacks, 20 validation_data=valid_generator(val_batch_size), ---> 21 validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size))) /home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs) 85 warnings.warn('Update your `' + object_name + 86 '` call to the Keras 2 API: ' + signature, stacklevel=2) ---> 87 return func(*args, **kwargs) 88 wrapper._original_function = func 89 return wrapper /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch) 2040 outs = self.train_on_batch(x, y, 2041 sample_weight=sample_weight, -> 2042 class_weight=class_weight) 2043 2044 if not isinstance(outs, list): /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight) 1760 ins = x + y + sample_weights 1761 self._make_train_function() -> 1762 outputs = self.train_function(ins) 1763 if len(outputs) == 1: 1764 return outputs[0] /home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs) 2271 updated = session.run(self.outputs + [self.updates_op], 2272 feed_dict=feed_dict, -> 2273 **self.session_kwargs) 2274 return updated[:len(self.outputs)] 2275 /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata) 893 try: 894 result = self._run(None, fetches, feed_dict, options_ptr, --> 895 run_metadata_ptr) 896 if run_metadata: 897 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata) 1122 if final_fetches or final_targets or (handle and feed_dict_tensor): 1123 results = self._do_run(handle, final_targets, final_fetches, -> 1124 feed_dict_tensor, options, run_metadata) 1125 else: 1126 results = [] /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata) 1319 if handle is None: 1320 return self._do_call(_run_fn, self._session, feeds, fetches, targets, -> 1321 options, run_metadata) 1322 else: 1323 return self._do_call(_prun_fn, self._session, handle, feeds, fetches) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args) 1325 def _do_call(self, fn, *args): 1326 try: -> 1327 return fn(*args) 1328 except errors.OpError as e: 1329 message = compat.as_text(e.message) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata) 1304 return tf_session.TF_Run(session, options, 1305 feed_dict, fetch_list, target_list, -> 1306 status, run_metadata) 1307 1308 def _prun_fn(session, handle, feed_dict, fetch_list): KeyboardInterrupt:
model_fullres.load_weights('./weights/best_weights_fullresv3.hdf5')
model_fullres.compile(optimizer=RMSprop(lr=0.00001),
loss=bce_dice_loss,
metrics=[dice_coeff])
callbacks = [EarlyStopping(monitor='val_loss',
patience=4,
verbose=1,
min_delta=1e-4),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=3,
verbose=1,
epsilon=1e-4),
ModelCheckpoint(monitor='val_loss',
filepath='weights/best_weights_fullresv3.hdf5',
save_best_only=True,
save_weights_only=True)]
model_fullres.fit_generator(generator=train_generator(train_batch_size),
steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),
epochs=max_epochs,
verbose=2,
callbacks=callbacks,
validation_data=valid_generator(val_batch_size),
validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))
Epoch 1/50 5415s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0153 - val_dice_coeff: 0.9956 Epoch 2/50
--------------------------------------------------------------------------- ResourceExhaustedError Traceback (most recent call last) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args) 1326 try: -> 1327 return fn(*args) 1328 except errors.OpError as e: /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata) 1305 feed_dict, fetch_list, target_list, -> 1306 status, run_metadata) 1307 /home/sainath/anaconda3/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback) 88 try: ---> 89 next(self.gen) 90 except StopIteration: /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status() 465 compat.as_text(pywrap_tensorflow.TF_Message(status)), --> 466 pywrap_tensorflow.TF_GetCode(status)) 467 finally: ResourceExhaustedError: OOM when allocating tensor with shape[3,24,1280,1920] [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=["loc:@conv2d_35/convolution"], data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]] During handling of the above exception, another exception occurred: ResourceExhaustedError Traceback (most recent call last) <ipython-input-28-de876fd6dcfd> in <module>() 19 callbacks=callbacks, 20 validation_data=valid_generator(val_batch_size), ---> 21 validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size))) /home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs) 85 warnings.warn('Update your `' + object_name + 86 '` call to the Keras 2 API: ' + signature, stacklevel=2) ---> 87 return func(*args, **kwargs) 88 wrapper._original_function = func 89 return wrapper /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch) 2040 outs = self.train_on_batch(x, y, 2041 sample_weight=sample_weight, -> 2042 class_weight=class_weight) 2043 2044 if not isinstance(outs, list): /home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight) 1760 ins = x + y + sample_weights 1761 self._make_train_function() -> 1762 outputs = self.train_function(ins) 1763 if len(outputs) == 1: 1764 return outputs[0] /home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs) 2271 updated = session.run(self.outputs + [self.updates_op], 2272 feed_dict=feed_dict, -> 2273 **self.session_kwargs) 2274 return updated[:len(self.outputs)] 2275 /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata) 893 try: 894 result = self._run(None, fetches, feed_dict, options_ptr, --> 895 run_metadata_ptr) 896 if run_metadata: 897 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata) 1122 if final_fetches or final_targets or (handle and feed_dict_tensor): 1123 results = self._do_run(handle, final_targets, final_fetches, -> 1124 feed_dict_tensor, options, run_metadata) 1125 else: 1126 results = [] /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata) 1319 if handle is None: 1320 return self._do_call(_run_fn, self._session, feeds, fetches, targets, -> 1321 options, run_metadata) 1322 else: 1323 return self._do_call(_prun_fn, self._session, handle, feeds, fetches) /home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args) 1338 except KeyError: 1339 pass -> 1340 raise type(e)(node_def, op, message) 1341 1342 def _extend_graph(self): ResourceExhaustedError: OOM when allocating tensor with shape[3,24,1280,1920] [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=["loc:@conv2d_35/convolution"], data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]] Caused by op 'training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput', defined at: File "/home/sainath/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "/home/sainath/anaconda3/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module> app.launch_new_instance() File "/home/sainath/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance app.start() File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 477, in start ioloop.IOLoop.instance().start() File "/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py", line 177, in start super(ZMQIOLoop, self).start() File "/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 888, in start handler_func(fd_obj, events) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper return fn(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events self._handle_recv() File "/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv self._run_callback(callback, msg) File "/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback callback(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper return fn(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher return self.dispatch_shell(stream, msg) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell handler(stream, idents, msg) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request user_expressions, allow_stdin) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 196, in do_execute res = shell.run_cell(code, store_history=store_history, silent=silent) File "/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 533, in run_cell return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell interactivity=interactivity, compiler=compiler, result=result) File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2827, in run_ast_nodes if self.run_code(code, result): File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-28-de876fd6dcfd>", line 21, in <module> validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size))) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 87, in wrapper return func(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py", line 1926, in fit_generator self._make_train_function() File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py", line 960, in _make_train_function loss=self.total_loss) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 87, in wrapper return func(*args, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py", line 225, in get_updates grads = self.get_gradients(loss, params) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py", line 73, in get_gradients grads = K.gradients(loss, params) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2310, in gradients return tf.gradients(loss, variables, colocate_gradients_with_ops=True) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 542, in gradients grad_scope, op, func_call, lambda: grad_fn(op, *out_grads)) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 348, in _MaybeCompile return grad_fn() # Exit early File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 542, in <lambda> grad_scope, op, func_call, lambda: grad_fn(op, *out_grads)) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py", line 446, in _Conv2DGrad op.get_attr("data_format")), File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 486, in conv2d_backprop_input data_format=data_format, name=name) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op op_def=op_def) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2630, in create_op original_op=self._default_original_op, op_def=op_def) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1204, in __init__ self._traceback = self._graph._extract_stack() # pylint: disable=protected-access ...which was originally created as op 'conv2d_35/convolution', defined at: File "/home/sainath/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) [elided 16 identical lines from previous traceback] File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell interactivity=interactivity, compiler=compiler, result=result) File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2821, in run_ast_nodes if self.run_code(code, result): File "/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code exec(code_obj, self.user_global_ns, self.user_ns) File "<ipython-input-8-56ff7dc954b0>", line 1, in <module> model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024) File "<ipython-input-7-204678a39f7c>", line 23, in get_unet x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters) File "<ipython-input-6-592812654380>", line 4, in unet_up_one_block x = Conv2D(num_filters, (3,3), padding='same')(x) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py", line 602, in __call__ output = self.call(inputs, **kwargs) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py", line 164, in call dilation_rate=self.dilation_rate) File "/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 3164, in conv2d data_format='NHWC') File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 672, in convolution op=op) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 338, in with_space_to_batch return op(input, num_spatial_dims, padding) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 664, in op name=name) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 131, in _non_atrous_convolution name=name) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 397, in conv2d data_format=data_format, name=name) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op op_def=op_def) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2630, in create_op original_op=self._default_original_op, op_def=op_def) File "/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1204, in __init__ self._traceback = self._graph._extract_stack() # pylint: disable=protected-access ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[3,24,1280,1920] [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=["loc:@conv2d_35/convolution"], data_format="NHWC", padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/gpu:0"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]