####Algorithm needs a good energy measure for the image, and I have tried sobel and prewitt edge detection operators. Sobel operator provides good edge detection and enhancement hence is used throughout my implementation.
####(Read me)
####Functions to be modified wrt input image and resolution - main(),CAOB/CAIR
###Image resize
####Input: Image to be resized and the desired resolution after resize. Call function CAIR with the desired resolution
####Output: Resized image
###Object removal/retaining
####Input: Image with object to be removed/retained
#### Mask of image with black region painted over object to be removed. White region painted over object to be retained.
#### Call function CAOB with the desired resolution to which image can be resized such that object is removed.
####Output: Image with object removed/retained. If object is removed, image is brought back to original size.
###Content Amplification
####Input: Image that is enlarged (resized to greater resolution) using standard resize. Call function CAIR with original resolution of image.
####Output: Image with its original resolution, with its main content amplified.
from scipy.ndimage.filters import generic_gradient_magnitude, sobel
from scipy.misc import imresize
import scipy.ndimage as ndimage
from math import fabs
import sys
import os
import gc
import psutil
from skimage import feature
%pylab inline
from __future__ import print_function
from __future__ import division
Populating the interactive namespace from numpy and matplotlib
WARNING: pylab import has clobbered these variables: ['fabs', 'inf'] `%matplotlib` prevents importing * from pylab and numpy
from PIL import Image
inf = 1e100000
#Function to convert the image to greyscale. Also Used when calculating the gradient of the image.
def grayscale_filter ( img ):
return img.convert("F")
#Function to perform sobel edge detection - Enhanced edges are obtained since the mask gives more weight to the surrounding pixels
def gradient_sobel(img1):
sx = ndimage.sobel(img1, axis=0, mode='constant')
sy = ndimage.sobel(img1, axis=1, mode='constant')
sob = hypot(sx, sy)
gradientsob = Image.new("F", img1.size)
gradientsob.putdata( list( sob.flat ) )
return gradientsob
#Function to perform Prewitt edge detection
def gradient_prewitt(img1):
sx = ndimage.prewitt(img1, axis=0, mode='constant')
sy = ndimage.prewitt(img1, axis=1, mode='constant')
pre = hypot(sx, sy)
gradientpre = Image.new("F", img1.size)
gradientpre.putdata( list( pre.flat ) )
return gradientpre
#By Calling this function, we can compare the results obtained by performing Content aware Image resizing with standard resizing techniques
def standardresize(img,resolution):
resizedc = imresize(img, (resolution[0],resolution[1]), interp='cubic') #Cubic interpolation
subplot(221)
imshow(resizedc)
#bilinear
resizedbl = imresize(img, (resolution[0],resolution[1]), interp='bilinear') #Biliniear interpolation
subplot(222)
imshow(resizedbl)
#bicubic
resizedbc = imresize(img, (resolution[0],resolution[1]), interp='bicubic') #Bicubic interpolation
subplot(223)
imshow(resizedbc)
#Nearest
resized = imresize(img, (resolution[0],resolution[1]), interp='nearest') #Nearest interpolation
subplot(224)
imshow(resized)
#Function to obtain the traspose of the image and store in a new image
def img_transpose(im):
im_width, im_height = im.size
im_arr = numpy.reshape( im.getdata( ), (im_height, im_width) )
im_arr = numpy.transpose(im_arr)
im = Image.new(im.mode, (im_height, im_width) )
im.putdata( list( im_arr.flat) )
return im
#Function to obtain the horizontal seam across the image - Algorithm implemented from the paper
def find_horizontal_seam ( im ):
im_width, im_height = im.size
cost = numpy.zeros( im.size )
#
im_arr = numpy.reshape( im.getdata( ), (im_height, im_width) )
im_arr = numpy.transpose(im_arr)
for y in range(im_height):
cost[0,y] = im_arr[0, y] #First row of the image has same cost as image gradient energy
# Finding the cumulative least cost of all the pixels in the image
for x in range(1, im_width):
for y in range(im_height):
if y == 0:
min_val = min( cost[x-1,y], cost[x-1,y+1] )
elif y < im_height - 2:
min_val = min( cost[x-1,y], cost[x-1,y+1] )
min_val = min( min_val, cost[x-1,y-1] )
else:
min_val = min( cost[x-1,y], cost[x-1,y-1] )
cost[x,y] = im_arr[x,y] + min_val
min_val = inf #Set to be a high value
path = [ ]
#Tracing back the path from the last row in the image
#Initially finding the least cumulative cost from the cost matrix
for y in range(im_height):
if cost[im_width-1,y] < min_val:
min_val = cost[im_width-1,y]
min_ptr = y
pos = (im_width-1,min_ptr)
path.append(pos)
while pos[0] != 0:
val = cost[pos] - im_arr[pos]
x,y = pos
if y == 0:
if val == cost[x-1,y+1]:
pos = (x-1,y+1)
else:
pos = (x-1,y)
elif y < im_height - 2:
if val == cost[x-1,y+1]:
pos = (x-1,y+1)
elif val == cost[x-1,y]:
pos = (x-1,y)
else:
pos = (x-1,y-1)
else:
if val == cost[x-1,y]:
pos = (x-1,y)
else:
pos = (x-1,y-1)
path.append(pos)
return path
# Finding horizontal seam by transposing the image and then finding the horizontal seam.
def find_vertical_seam ( im ):
im = img_transpose(im)
u = find_horizontal_seam(im)
# Reversng the coordinates in the path of the seam from transposed image to match that of the original image
for i in range(len(u)):
temp = list(u[i])
temp.reverse()
u[i] = tuple(temp)
return u
# To obtain a visual representation of the seams that have been obtained by the algorithm
def mark_seam (img, path):
pix = img.load()
path = flatten(path)
if img.mode == "RGB":
for pixel in path:
pix[pixel] = (255,0,0,1) # to mark the seams in red
else:
for pixel in path:
pix[pixel] = 255 #mark the seams in white for grayscale image
return img
def delete_horizontal_seam (img, path):
img_width, img_height = img.size
i = Image.new(img.mode, (img_width, img_height-1)) #Creating a new image with one lesser row
input = img.load()
output = i.load()
path_set = set(path)
seen_set = set()
for y in range(img_height):
for x in range(img_width):
if (x,y) not in path_set and x not in seen_set:
output[x,y] = input[x,y]
elif (x,y) in path_set:
seen_set.add(x)
else:
output[x,y-1] = input[x,y]
return i
def delete_vertical_seam (img, path):
img_width, img_height = img.size
i = Image.new(img.mode, (img_width-1, img_height)) #Creating a new image with one lesser column
input = img.load()
output = i.load()
path_set = set(path)
seen_set = set()
for x in range(img_width):
for y in range(img_height):
if (x,y) not in path_set and y not in seen_set:
output[x,y] = input[x,y]
elif (x,y) in path_set:
seen_set.add(y)
else:
output[x-1,y] = input[x,y]
return i
def add_vertical_seam(img, path):
img_width, img_height = img.size
i = Image.new(img.mode, (img_width + 1, img_height) ) #Creating image with one additional column
input = img.load()
output = i.load()
path_set = set(path)
seen_set = set()
for x in range(img_width):
for y in range(img_height):
if (x,y) not in path_set and y not in seen_set:
output[x,y] = input[x,y]
elif (x,y) in path_set and y not in seen_set:
output[x,y] = input[x,y]
seen_set.add( y )
if x < img_width -1 and x > 0:
output[x+1,y] = vector_avg(input[x-1,y], input[x+1,y])
elif x==img_width-1:
output[x+1,y] = vector_avg(input[x,y], input[x-1,y])
elif x==0:
output[x+1,y] = vector_avg(input[x,y], input[x+1,y])
else:
output[x+1,y] = input[x,y]
return i
def add_horizontal_seam(img, path):
img_width, img_height = img.size
i = Image.new(img.mode, (img_width, img_height+1) ) #Creating image with additional row
input = img.load()
output = i.load()
path_set = set(path)
seen_set = set()
for y in range(img_height):
for x in range(img_width):
if (x,y) not in path_set and x not in seen_set:
output[x,y] = input[x,y]
elif (x,y) in path_set and x not in seen_set:
output[x,y] = input[x,y]
seen_set.add( x )
if y < img_height -1 and y > 0:
output[x,y+1] = vector_avg(input[x,y-1], input[x,y+1])
elif y==img_height -1:
output[x,y+1] = vector_avg(input[x,y], input[x,y-1])
elif y==0:
output[x,y+1] = vector_avg(input[x,y], input[x,y+1])
else:
output[x,y+1] = input[x,y]
return i
#Function to obtain the average value of the new pixel to be added
def vector_avg (u, v):
w = list(u)
for i in range(len(u)):
w[i] = (u[i] + v[i]) / 2
for j in range(len(w)):
w[j]=int(w[j])
return(tuple(w))
def CAIR( resolution):
# function that calls the seam functions until the desired size is reached
input = Image.open('flowerpot.jpg')
intermediate = Image.open('flowerpot.jpg')
(im_width, im_height) = input.size
marked = [ ]
markedd = [ ]
#First adding/removing vertical seams
while im_width > resolution[0]:
u = find_vertical_seam(gradient_sobel(grayscale_filter(input)))
marked.append(u)
input = delete_vertical_seam(input, u)
im_width = input.size[0]
afterverticalremoval=input
check=0
while im_width < resolution[0]:
diff=resolution[0]-im_width #Calculating all seams that have least cost and adding seams only in those paths
path=[]
for i in range(diff):
v = find_vertical_seam(gradient_sobel(grayscale_filter(intermediate)))
marked.append(v)
path.append(v)
intermediate = delete_vertical_seam(intermediate,v)
for i in range(diff):
input = add_vertical_seam(input,path[i])
im_width = input.size[0]
afterverticaladdition=input
check=1
#After vertical addition/removal, adding/removing horizontal seams
while im_height > resolution[1]:
v = find_horizontal_seam(gradient_sobel(grayscale_filter(input)))
markedd.append(v)
input = delete_horizontal_seam(input,v)
im_height = input.size[1]
while im_height < resolution[1]:
diff=resolution[1]-im_height
path1=[]
intermediate=input
for i in range(diff): #Calculating all seams that have least cost and adding seams only in those paths
v = find_horizontal_seam(gradient_sobel(grayscale_filter(intermediate)))
path1.append(v)
markedd.append(v)
intermediate = delete_horizontal_seam(intermediate,v)
for i in range(diff):
input = add_horizontal_seam(input,path1[i])
im_height = input.size[1]
input.show()
print(input.size)
mark_seam(gradient_sobel(grayscale_filter(Image.open('flowerpot.jpg'))), marked).show()
mark_seam(Image.open('flowerpot.jpg'), marked).show()
if (check==1):
mark_seam(afterverticaladdition, markedd).show()
else:
mark_seam(afterverticalremoval, markedd).show()
del(input)
del(marked)
#Function to flatten a list of lists.
def flatten(lst):
for i in lst:
if type(i) == list:
for i in flatten(i):
yield i
else:
yield i
def objremoval(img,im):
#Function to obtain black and white pixels from mask and mark the corresponding pixels with low/high cost in gradient of image.
imgx=img.convert("F")
data = imgx.getdata()
size=imgx.size
width=size[0]
height=size[1]
pixelList = []
for i in range(height):
for j in range(width):
stride = (width*i) + j
pixelList.append((j,i, data[stride]))
whites = []
blacks=[]
for pixel in pixelList:
if pixel[2] <140:
blacks.append(pixel[0:2])
if pixel[2] ==255:
whites.append(pixel[0:2])
gradient=gradient_sobel(grayscale_filter(im))
pix = gradient.load()
for i in range(len(whites)):
pix[whites[i]] = 10000 #Cost for white region (region to be retained)
for i in range(len(blacks)):
pix[blacks[i]] = -10000 #Cost for black region(region to be removed)
return gradient
def CAOB( resolution):
#Function that resizes image such that object is removed/retianed from image. If object is removed ,then bring back to original size.
input = Image.open('dogg.png')
(im_width_orig, im_height_orig) = input.size
intermediate = Image.open('dogg.png')
mask=Image.open('dog.png')
(im_width, im_height) = input.size
marked = [ ]
markedd = [ ]
gradient=objremoval(mask,input)
gradient1=gradient
gradient.show()
#Vertical opertaions are performed first. Manipulations are performed on both the masked gradient and the original input image.
while im_width > resolution[0]:
diff=im_width-resolution[0]
path=[]
for i in range(diff):
v = find_vertical_seam(gradient)
marked.append(v)
path.append(v)
gradient = delete_vertical_seam(gradient,v)
for i in range(diff):
input = delete_vertical_seam(input,path[i])
im_width = input.size[0]
afterverticalremoval=input
check=0
while resolution[0]>im_width:
diff=resolution[0]-im_width
path=[]
for i in range(diff):
v = find_vertical_seam(gradient)
marked.append(v)
path.append(v)
gradient = delete_vertical_seam(gradient,v)
for i in range(diff):
input = add_vertical_seam(input,path[i])
im_width = input.size[0]
afterverticaladdition=input
check=1
#After vertical, perform horizontal manipulations based on resolution required.
while im_height > resolution[1]:
diff=im_height-resolution[1]
path1=[]
for i in range(diff):
v = find_horizontal_seam(gradient)
markedd.append(v)
path1.append(v)
gradient = delete_horizontal_seam(gradient,v)
for i in range(diff):
input = delete_horizontal_seam(input,path1[i])
im_height = input.size[1]
inputt=input
while resolution[1]>im_height :
diff=resolution[1]-im_height
path1=[]
for i in range(diff):
v = find_horizontal_seam(gradient)
markedd.append(v)
path1.append(v)
gradient = delete_horizontal_seam(gradient,v)
for i in range(diff):
input = add_horizontal_seam(input,path1[i])
im_height = input.size[1]
input.show()
print(input.size)
mark_seam(Image.open('dogg.png'), marked).show()
mark_seam(gradient1, marked).show()
if (check==1):
mark_seam(afterverticaladdition, markedd).show()
else:
mark_seam(afterverticalremoval, markedd).show()
#bring back the image to its original size with the object removed
while im_width < im_width_orig:
diff=im_width_orig-resolution[0]
path=[]
for i in range(diff):
v = find_vertical_seam(gradient)
marked.append(v)
path.append(v)
gradient = delete_vertical_seam(gradient,v)
for i in range(diff):
input = add_vertical_seam(input,path[i])
im_width=input.size[0]
afterverticaladdition=input
while im_height < im_height_orig:
diff=im_height_orig-resolution[1]
path1=[]
intermediate=input
for i in range(diff):
v = find_horizontal_seam(gradient)
path1.append(v)
markedd.append(v)
gradient = delete_horizontal_seam(gradient,v)
for i in range(diff):
input = add_horizontal_seam(input,path1[i])
im_height=input.size[1]
print(input.size)
input.show()
imshow(input)
del(input)
del(marked)
gc.collect()
0
def main():
#Main function where the desired resolution of the image is given.
resolution=[]
resolution.append(550)
resolution.append(423)
CAOB(resolution)
#CAIR(resolution)
gc.collect()
if __name__ == "__main__":
main()
(550, 423) (637, 423)
input = Image.open('dogg.png')
imshow(input)
<matplotlib.image.AxesImage at 0x375c3c30>
imshow(gradient_sobel(grayscale_filter(input))) #Enhanced edges
<matplotlib.image.AxesImage at 0x1179cdb0>
imshow(gradient_prewitt(grayscale_filter(input)))
<matplotlib.image.AxesImage at 0x45c6adf0>