This is one of the 100 recipes of the IPython Cookbook, the definitive guide to high-performance scientific computing and data science in Python.
In this recipe, we show how to communicate data in both directions from the notebook to the Python kernel, and conversely. Specifically, we will retrieve the webcam feed from the browser using HTML5's <video>
element, and pass it to Python in real time using the interactive capabilities of the IPython notebook 2.0+. This way, we can process the image in Python with an edge detector (implemented in scikit-image), and display it in the notebook in real time.
Most of the code for this recipe comes from Jason Grout's example.
from IPython.html.widgets import DOMWidget
from IPython.utils.traitlets import Unicode, Bytes, Instance
from IPython.display import display
from skimage import io, filter, color
import urllib
import base64
from PIL import Image
from io import BytesIO
import numpy as np
from numpy import array, ndarray
import matplotlib.pyplot as plt
def to_b64(img):
imgdata = BytesIO()
pil = Image.fromarray(img)
pil.save(imgdata, format='PNG')
imgdata.seek(0)
return urllib.parse.quote(base64.b64encode(imgdata.getvalue()))
def from_b64(b64):
im = Image.open(BytesIO(base64.b64decode(b64)))
return array(im)
roberts()
function in scikit-image.def process_image(image):
img = filter.roberts(image[:,:,0]/255.)
return (255-img*255).astype(np.uint8)
class Camera(DOMWidget):
_view_name = Unicode('CameraView', sync=True)
# This string contains the base64-encoded raw
# webcam image (browser -> Python).
imageurl = Unicode('', sync=True)
# This string contains the base64-encoded processed
# webcam image(Python -> browser).
imageurl2 = Unicode('', sync=True)
# This function is called whenever the raw webcam
# image is changed.
def _imageurl_changed(self, name, new):
head, data = new.split(',', 1)
if not data:
return
# We convert the base64-encoded string
# to a NumPy array.
image = from_b64(data)
# We process the image.
image = process_image(image)
# We convert the processed image
# to a base64-encoded string.
b64 = to_b64(image)
self.imageurl2 = 'data:image/png;base64,' + b64
%%javascript
var video = $('<video>')[0];
var canvas = $('<canvas>')[0];
var canvas2 = $('<img>')[0];
var width = 320;
var height = 0;
require(["widgets/js/widget"], function(WidgetManager){
var CameraView = IPython.DOMWidgetView.extend({
render: function(){
var that = this;
// We append the HTML elements.
setTimeout(function() {
that.$el.append(video).
append(canvas).
append(canvas2);}, 200);
// We initialize the webcam.
var streaming = false;
navigator.getMedia = ( navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia);
navigator.getMedia({video: true, audio: false},
function(stream) {
if (navigator.mozGetUserMedia) {
video.mozSrcObject = stream;
} else {
var vendorURL = (window.URL ||
window.webkitURL);
video.src = vendorURL.createObjectURL(
stream);
}
video.controls = true;
video.play();
},
function(err) {
console.log("An error occured! " + err);
}
);
// We initialize the size of the canvas.
video.addEventListener('canplay', function(ev){
if (!streaming) {
height = video.videoHeight / (
video.videoWidth/width);
video.setAttribute('width', width);
video.setAttribute('height', height);
canvas.setAttribute('width', width);
canvas.setAttribute('height', height);
canvas2.setAttribute('width', width);
canvas2.setAttribute('height', height);
streaming = true;
}
}, false);
// Play/Pause functionality.
var interval;
video.addEventListener('play', function(ev){
// We get the picture every 100ms.
interval = setInterval(takepicture, 100);
})
video.addEventListener('pause', function(ev){
clearInterval(interval);
})
// This function is called at each time step.
// It takes a picture and sends it to the model.
function takepicture() {
canvas.width = width; canvas.height = height;
canvas2.width = width; canvas2.height = height;
video.style.display = 'none';
canvas.style.display = 'none';
// We take a screenshot from the webcam feed and
// we put the image in the first canvas.
canvas.getContext('2d').drawImage(video,
0, 0, width, height);
// We export the canvas image to the model.
that.model.set('imageurl',
canvas.toDataURL('image/png'));
that.touch();
}
},
update: function(){
// This function is called whenever Python modifies
// the second (processed) image. We retrieve it and
// we display it in the second canvas.
var img = this.model.get('imageurl2');
canvas2.src = img;
return CameraView.__super__.update.apply(this);
}
});
// Register the view with the widget manager.
WidgetManager.register_widget_view('CameraView',
CameraView);
});
c = Camera()
display(c)
You'll find all the explanations, figures, references, and much more in the book (to be released later this summer).
IPython Cookbook, by Cyrille Rossant, Packt Publishing, 2014 (500 pages).