This notebook is an assignment of fast.ai lesson8.
Neural Style Transfer is based on this paper.
%matplotlib inline
import importlib
import utils2; importlib.reload(utils2)
from utils2 import *
from scipy.optimize import fmin_l_bfgs_b
from scipy.misc import imsave
from keras import metrics
from vgg16_avg import VGG16_Avg
Using TensorFlow backend. /home/ubuntu/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20. "This module will be removed in 0.20.", DeprecationWarning)
# Tell Tensorflow to use no more GPU RAM than necessary
limit_mem()
path = '/home/ubuntu/cutting-edge-dl-for-coders-part2'
img=Image.open(path+'/data/bridge.jpg')
plt.imshow(np.array(img))
<matplotlib.image.AxesImage at 0x7fea5406df28>
rn_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32)
preproc = lambda x: (x - rn_mean)[:, :, :, ::-1]
deproc = lambda x,s: np.clip(x.reshape(s)[:, :, :, ::-1] + rn_mean, 0, 255)
img_arr = preproc(np.expand_dims(np.array(img), 0))
shp = img_arr.shape
model = VGG16_Avg(include_top=False)
layer = model.get_layer('block5_conv1').output
layer_model = Model(model.input, layer)
targ = K.variable(layer_model.predict(img_arr))
class Evaluator(object):
def __init__(self, f, shp): self.f, self.shp = f, shp
def loss(self, x):
loss_, self.grad_values = self.f([x.reshape(self.shp)])
return loss_.astype(np.float64)
def grads(self, x): return self.grad_values.flatten().astype(np.float64)
loss = metrics.mse(layer, targ)
grads = K.gradients(loss, model.input)
fn = K.function([model.input], [loss]+grads)
evaluator = Evaluator(fn, shp)
def solve_image(eval_obj, niter, x):
for i in range(niter):
x, min_val, info = fmin_l_bfgs_b(eval_obj.loss, x.flatten(),
fprime=eval_obj.grads, maxfun=20)
x = np.clip(x, -127,127)
print('Current loss value:', min_val)
imsave(f'{path}/results/res_at_iteration_{i}.png', deproc(x.copy(), shp)[0])
return x
rand_img = lambda shape: np.random.uniform(-2.5, 2.5, shape)/100
x = rand_img(shp)
plt.imshow(x[0]);
iterations=10
x = solve_image(evaluator, iterations, x)
Current loss value: 48.2291946411 Current loss value: 11.814458847 Current loss value: 6.15314722061 Current loss value: 4.10418081284 Current loss value: 3.15371417999 Current loss value: 2.62931346893 Current loss value: 2.59162425995 Current loss value: 2.59218120575 Current loss value: 2.59218072891 Current loss value: 2.59218072891
Image.open(path + '/results/res_at_iteration_9.png')
from IPython.display import HTML
from matplotlib import animation, rc
fig, ax = plt.subplots()
def animate(i): ax.imshow(Image.open(f'{path}/results/res_at_iteration_{i}.png'))
anim = animation.FuncAnimation(fig, animate, frames=10, interval=200)
HTML(anim.to_html5_video())
def plot_arr(arr): plt.imshow(deproc(arr,arr.shape)[0].astype('uint8'))
style = Image.open(path+'/data/starry_night.jpg')
style_arr = preproc(np.expand_dims(style,0)[:,:,:,:3])
shp = style_arr.shape
model = VGG16_Avg(include_top=False, input_shape=shp[1:])
outputs = {l.name: l.output for l in model.layers}
layers = [outputs['block{}_conv1'.format(o)] for o in range(1,3)]
layers_model = Model(model.input, layers)
targs = [K.variable(o) for o in layers_model.predict(style_arr)]
def gram_matrix(x):
# We want each row to be a channel, and the columns to be flattened x,y locations
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
# The dot product of this with its transpose shows the correlation
# between each pair of channels
return K.dot(features, K.transpose(features)) / x.get_shape().num_elements()
def style_loss(x, targ):
return metrics.mse(gram_matrix(x), gram_matrix(targ))
loss = sum(style_loss(l1[0], l2[0]) for l1,l2 in zip(layers, targs))
grads = K.gradients(loss, model.input)
style_fn = K.function([model.input], [loss]+grads)
evaluator = Evaluator(style_fn, shp)
rand_img = lambda shape: np.random.uniform(-2.5, 2.5, shape)/1
x = rand_img(shp)
x = scipy.ndimage.filters.gaussian_filter(x, [0,2,2,0])
plt.imshow(x[0])
<matplotlib.image.AxesImage at 0x7fea462fb240>
iterations=10
x = rand_img(shp)
x = solve_image(evaluator, iterations, x)
Current loss value: 2065.63867188 Current loss value: 196.682067871 Current loss value: 99.5655059814 Current loss value: 67.0934753418 Current loss value: 41.5264167786 Current loss value: 25.6243247986 Current loss value: 16.4236316681 Current loss value: 10.5170497894 Current loss value: 7.78677654266 Current loss value: 5.92503118515
Image.open(path + '/results/res_at_iteration_9.png')
w,h = style.size
src = img_arr[:,:h,:w]
plot_arr(src)
style_layers = [outputs['block{}_conv2'.format(o)] for o in range(1,6)]
content_name = 'block4_conv2'
content_layer = outputs[content_name]
style_model = Model(model.input, style_layers)
style_targs = [K.variable(o) for o in style_model.predict(style_arr)]
content_model = Model(model.input, content_layer)
content_targ = K.variable(content_model.predict(src))
style_wgts = [0.05,0.2,0.2,0.25,0.3]
loss = sum(style_loss(l1[0], l2[0])*w
for l1,l2,w in zip(style_layers, style_targs, style_wgts))
loss += metrics.mse(content_layer, content_targ)/10
grads = K.gradients(loss, model.input)
transfer_fn = K.function([model.input], [loss]+grads)
evaluator = Evaluator(transfer_fn, shp)
iterations=10
x = rand_img(shp)
x = solve_image(evaluator, iterations, x)
Current loss value: 2612.48999023 Current loss value: 639.54473877 Current loss value: 386.221099854 Current loss value: 292.773681641 Current loss value: 243.464904785 Current loss value: 209.745819092 Current loss value: 183.074325562 Current loss value: 163.345794678 Current loss value: 148.702102661 Current loss value: 136.507385254
Image.open(path + '/results/res_at_iteration_9.png')