Pickle Data and store in Redis - Python

In [1]:
import pickle
In [2]:
data1 = {'firstname': [1, 2.0, 3, 4],
         'lastname': ('maiden', u'Smith'),
         'account': None}

pickle.dump(data1, open('data.pkl', 'wb'))

data2 = pickle.load(open('data.pkl', 'rb'))
In [3]:
data1 == data2
Out[3]:
True

Pickle a Person

In [4]:
class PicklePerson(object):
    def __init__(self, name, age, location):
        self.name = name
        self.age = age
        self.location = location
    
    def __repr__(self):
        return "name: " + self.name + "\n" + "age: " + self.age + \
            "\n" + "location: " + self.location
In [5]:
john = PicklePerson("John", "60", "Smith")
print john
name: John
age: 60
location: Smith
In [6]:
pickle.dump(john, open("pickle_john", "wb"))
zombie_john = pickle.load(open("pickle_john","r"))
In [7]:
zombie_john
Out[7]:
name: John
age: 60
location: Smith

cPickle

In [8]:
import cPickle, os
In [9]:
%timeit pickle.dump([data1 for x in xrange(1000)], open("pickle_john", "wb"))
100 loops, best of 3: 5.57 ms per loop
In [10]:
%timeit cPickle.dump([data1 for x in xrange(1000)], open("pickle_john", "wb"))
1000 loops, best of 3: 1.48 ms per loop
In [11]:
os.path.getsize('/Users/antigen/Downloads/Pickle-and-Redis/pickle_john')
Out[11]:
4100

Redis - Images and Pickled Python object under 512mb

A Python interface is redis is available at https://github.com/andymccurdy/redis-py

In [12]:
import redis
r = redis.StrictRedis(host='localhost', port=6379, db=0)
In [13]:
r.set('foo', 'bar')
Out[13]:
True
In [14]:
r.get('foo')
Out[14]:
'bar'
In [15]:
%timeit r.set('foo', 'bar')
10000 loops, best of 3: 141 us per loop
In [16]:
%timeit r.get('foo')
10000 loops, best of 3: 123 us per loop
In [17]:
import string

text = """Pipelines are a subclass of the base Redis class that provide support 
for buffering multiple commands to the server in a single request. They can be 
used to dramatically increase the performance of groups of commands by reducing 
the number of back-and-forth TCP packets between the client and server."""
In [18]:
for word in text.split():
    r.sadd("persistent", word)
    
print [r.srandmember('persistent') for i in xrange(10)]
print [r.srandmember('persistent') for i in xrange(10)]
['p', 'P', 'increase', 'and', 'commands', 'for', 'single', 'f', 'a', 'u']
['for', 'the', 'and', 'the', 'dramatically', 'be', 'f', 'R', 'used', 'packets']
In [19]:
for word in text.split():
    r.incr(word)
    
for word in set(text.split()):
    if int(r.get(word)) > 2:
        print word
        print "\t\t", r.get(word)
and
		4
Redis
		4
number
		4
TCP
		4
single
		4
are
		4
in
		4
reducing
		4
Pipelines
		4
for
		4
request.
		4
provide
		4
support
		4
subclass
		4
by
		4
increase
		4
to
		8
They
		4
between
		4
performance
		4
dramatically
		4
be
		4
used
		4
multiple
		4
that
		4
server.
		4
base
		4
groups
		4
buffering
		4
class
		4
a
		9
commands
		8
back-and-forth
		4
of
		16
packets
		4
server
		4
client
		4
can
		4
the
		20
In [20]:
bob = pickle.dumps(PicklePerson("bob","50","smith"))
print bob
ccopy_reg
_reconstructor
p0
(c__main__
PicklePerson
p1
c__builtin__
object
p2
Ntp3
Rp4
(dp5
S'age'
p6
S'50'
p7
sS'name'
p8
S'bob'
p9
sS'location'
p10
S'smith'
p11
sb.
In [21]:
r.set("bob", bob)
Out[21]:
True
In [22]:
pickle.loads(r.get("bob"))
Out[22]:
name: bob
age: 50
location: smith
In [23]:
import redisco
from redisco import connection_setup, models
redisco.connection_setup(host='localhost', port=6379, db=0)
In [25]:
class Person(models.Model):
    name = models.Attribute(required=True)
    age = models.Attribute(required=False)
    location = models.Attribute(required=False)
    
for x in Person.objects.filter(name="Tim"):
    x.delete()
In [26]:
tim_smith = Person(name="Tim",age="40",location="Vancouver")
tim_smiths = Person(name="Tim",age="70",location="Burnaby")
In [27]:
tim_smiths.save()
tim_smith.save()
Out[27]:
True
In [28]:
Person.objects.filter(name="Tim")
Out[28]:
[<Person:4 {'age': u'70', 'name': u'Tim', 'location': u'Burnaby'}>, <Person:5 {'age': u'40', 'name': u'Tim', 'location': u'Vancouver'}>]
In [29]:
Person.objects.filter(name="Tim", age="40")[0] == tim_smith
Out[29]:
True