%erp5_url http://10.0.2.15:20001/erp5/Base_executeJupyter

%notebook_set_reference NB-OTHER

%erp5_user zope

%erp5_password insecure

# Get ERP5 Object
context

# Get the data stream you uploaded the wav
# "1" is the ID at the ERP5
context.data_stream_module["1"]

# Import needed libraries to use later
import matplotlib.pyplot as plt
from scipy.fftpack import fft

# Import also scipy to read the audio file
import scipy
import scipy.io
from scipy.io.wavfile import read

# Not out of core, as you return all data as string (So you should avoid do like this)
datastream_as_string = context.data_stream_module["1"].getData()

print "Total is %s" % len(datastream_as_string)
print type(datastream_as_string)

# Now we try with out-of-core!!
stream = context.data_stream_module["1"].data 

l = 0
for chunk in stream.iterate(): # out of core
    l += len(chunk)
print "Total is %s" % l
print type(stream)

# Import StringIO as we need to manipulate file object at wavefile.read
from StringIO import StringIO

# Load the file from erp5 as string, which is not good here
# we will later try to improve this because this is not out-of-core
fs, data = read(StringIO(context.data_stream_module["1"].getData())) 

# Get only channel
array = data.T[0] # not out of core

# Save the array to erp5 to make it out-of-core
out_of_core_array = context.data_array_module.newContent(
    array=array, 
    portal_type="Data Array", 
    title="pydata-wav2")

cmplx = fft(out_of_core_array.getArray()) 
spectrum = abs(cmplx[:(len(cmplx)/2)-1]) # not out of core

# Save the spectrum array to make it now out of core
out_of_core_spectrum_array = context.data_array_module.newContent(
    array=spectrum, 
    portal_type="Data Array", 
    title="pydata-spectrum2")


figure = plt.figure() 

ax1 = figure.add_subplot(211)
ax2 = figure.add_subplot(212)

ax1.plot(out_of_core_array.getArray()) 
ax2.plot(out_of_core_spectrum_array.getArray())  

# figure.show() don't present inline rendering....
# So we use instead:
context.Base_renderAsHtml(plt)

# now quick save on image_module un a dummy way.
figure.savefig("/tmp/somenamec.png")

context.image_module.newContent(title="plot", portal_type="Image", data=open("/tmp/somenamec.png").read())

# Create an class for wrapper the file api
# So we can pass a out-of-core objects that behave like 
# average file.

class BigFileReader:

    def __init__(self, bigfile):
        self.bigfile = bigfile
        self.pos = 0


    def tell(self):
        return self.pos

    def seek(self, pos): # TODO whence
        # TODO check for out of range
        self.pos = pos

    def read(self, n):
        chunkv = []
        for chunk in self.bigfile.iterate(self.pos, n):
            chunkv.append(chunk)
        data = ''.join(chunkv)
        self.pos += len(data)
        return data



fs, data = read(BigFileReader(context.data_stream_module["1"].data)) 

# Get only channel
array = data.T[0] # not out of core

# Save the array to erp5 to make it out-of-core
out_of_core_array = context.data_array_module.newContent(
    array=array, 
    portal_type="Data Array", 
    title="pydata-wav2")

cmplx = fft(out_of_core_array.getArray()) 
spectrum = abs(cmplx[:(len(cmplx)/2)-1]) # not out of core

# Save the spectrum array to make it now out of core
out_of_core_spectrum_array = context.data_array_module.newContent(
    array=spectrum, 
    portal_type="Data Array", 
    title="pydata-spectrum2")

figure = plt.figure() 

ax1 = figure.add_subplot(211)
ax2 = figure.add_subplot(212)

ax1.plot(out_of_core_array.getArray()) 
ax2.plot(out_of_core_spectrum_array.getArray())

# figure.show() don't present inline rendering....
# So we use instead:
context.Base_renderAsHtml(plt)

class BigFileReader:

    def __init__(self, bigfile):
        self.bigfile = bigfile
        self.pos = 0


    def tell(self):
        return self.pos

    def seek(self, pos):    # TODO whence
        # TODO check for out of range
        self.pos = pos

    def read(self, n):
        chunkv = []
        for chunk in self.bigfile.iterate(self.pos, n):
            chunkv.append(chunk)
        data = ''.join(chunkv)
        self.pos += len(data)
        return data


# Load the file from erp5 as string, which is not good here
# we will later try to improve this because this is not out-of-core
fs, data = read(BigFileReader(context.data_stream_module["1"].data)) 

# Get only channel
array = data.T[0] # not out of core

# Save the array to erp5 to make it out-of-core
out_of_core_array = context.data_array_module.newContent(
    array=array, 
    portal_type="Data Array", 
    title="pydata-wav2")

cmplx = fft(out_of_core_array.getArray()) 
spectrum = abs(cmplx[:(len(cmplx)/2)-1]) # not out of core

# Save the spectrum array to make it now out of core
out_of_core_spectrum_array = context.data_array_module.newContent(
    array=spectrum, 
    portal_type="Data Array", 
    title="pydata-spectrum2-to-recover")

figure = plt.figure() 

ax1 = figure.add_subplot(211)
ax2 = figure.add_subplot(212)

ax1.plot(out_of_core_array.getArray()) 
ax2.plot(out_of_core_spectrum_array.getArray())  

# figure.show() don't present inline rendering....
# So we use instead:
context.Base_renderAsHtml(plt)

# We can recover an array saved into and re-replot the chart
# Query is assincronous, we must wait the object be catalogued
# So it can take a while to work in this example.
spectrum = context.portal_catalog.getResultValue(
      title="pydata-spectrum2-to-recover",
      portal_type="Data Array")

figure = plt.figure() 

ax1 = figure.add_subplot(211)

ax1.plot(spectrum.getArray()) 

# figure.show() don't present inline rendering....
# So we use instead:
context.Base_renderAsHtml(plt)