https://www.youtube.com/watch?v=7lmCu8wz8ro
If you want to become an expert in Python, you should definitely watch this PyData talk from James Powell.
Definitions Python is a language orientated around protocols - Some behavior or syntax or bytecode or some top level function and there is a way to tell python how to implement that on an arbitrary object via underscore methods. The exact correspondance is usually guessable, but if you can't guess it you can it... google python data model
Metaclass Mechanism: Some hook into the class construction process. Questions: Do you have these methods implemented. Meaning: Library code & User code? How do you enforce a constraint?
Decorator Hooks into idea that everything creates a structure at run time. Wrap sets of functions with a before and after behavior.
Generators Take a single computation that would otherwise run eagerly from the injection of its parameters to the final computation and interleaving with other code by adding yield points where you can yield the intermediate result values or one small piece of the computation and also yield back to the caller. Think of a generator of a way to take one long piece of computation and break it up into small parts.
Context managers Two structures that allow you to tie two actions together. A setup action and a teardown action and make sure they always happen in concordance with each other.
# some behavior that I want to implement -> write some __ function __
# top-level function or top-level syntax -> corresponding __
# x + y -> __add__
# init x -> __init__
# repr(x) --> __repr__
# x() -> __call__
class Polynomial:
def __init__(self, *coeffs):
self.coeffs = coeffs
def __repr__(self):
return 'Polynomial(*{!r})'.format(self.coeffs)
def __add__(self, other):
return Polynomial(*(x + y for x, y in zip(self.coeffs, other.coeffs)))
def __len__(self):
return len(self.coeffs)
def __call__(self):
pass
p1 = Polynomial(1, 2, 3)
p2 = Polynomial(3, 4, 3)
p1 + p2
Polynomial(*(5, 7, 7))
len(p1)
3
# File 1 - library.py
class Base:
def food(self):
return 'foo'
# File2 - user.py
assert hasattr(Base, 'foo'), "you broke it, you fool!"
class Derived(Base):
def bar(self):
return self.foo
--------------------------------------------------------------------------- AssertionError Traceback (most recent call last) <ipython-input-27-08fa6af0fb76> in <module>() 1 # File2 - user.py 2 ----> 3 assert hasattr(Base, 'foo'), "you broke it, you fool!" 4 5 class Derived(Base): AssertionError: you broke it, you fool!
# File 1 - library.py
class Base:
def foo(self):
return self.bar()
# File2 - user.py
assert hasattr(Base, 'foo'), "you broke it, you fool!"
class Derived(Base):
def bar(self):
return 'bar'
Derived.bar
<function __main__.Derived.bar>
def _():
class Base:
pass
from dis import dis
dis(_) # LOAD_BUILD_CLASS
2 0 LOAD_BUILD_CLASS 2 LOAD_CONST 1 (<code object Base at 0x10f2daed0, file "<ipython-input-18-a194b247271c>", line 2>) 4 LOAD_CONST 2 ('Base') 6 MAKE_FUNCTION 0 8 LOAD_CONST 2 ('Base') 10 CALL_FUNCTION 2 12 STORE_FAST 0 (Base) 14 LOAD_CONST 0 (None) 16 RETURN_VALUE
# Catch Building of Classes
class Base:
def foo(self):
return self.bar()
old_bc = __build_class__
def my_bc(*a, **kw):
print('my buildclass ->', a, kw)
return old_bc(*a, **kw)
import builtins
builtins.__build_class__ = my_bc
# Catch Building of Classes
class Base:
def foo(self):
return self.bar()
old_bc = __build_class__
def my_bc(fun, name, base=None, **kw):
if base is Base:
print('Check if bar method defined')
if base is not None:
return old_bc(fun, name, base, **kw)
return old_bc(fun, name, **kw)
import builtins
builtins.__build_class__ = my_bc
import builtins
import importlib
importlib.reload(builtins)
<module 'builtins' (built-in)>
class BaseMeta(type):
def __new__(cls, name, bases, body):
print('BaseMeta.__new__', cls, name, bases, body)
return super().__new__(cls, name, bases, body)
class Base(metaclass=BaseMeta):
def foo(self):
return self.bar()
BaseMeta.__new__ <class '__main__.BaseMeta'> Base () {'__module__': '__main__', '__qualname__': 'Base', 'foo': <function Base.foo at 0x107378048>}
class BaseMeta(type):
def __new__(cls, name, bases, body):
if not 'bar' in body:
raise TypeError('bad user class')
return super().__new__(cls, name, bases, body)
class Base(metaclass=BaseMeta):
def foo(self):
return self.bar()
my buildclass -> (<function BaseMeta at 0x10749cf28>, 'BaseMeta', <class 'type'>) {}
--------------------------------------------------------------------------- RecursionError Traceback (most recent call last) <ipython-input-167-f2133308a9ce> in <module>() ----> 1 class BaseMeta(type): 2 def __new__(cls, name, bases, body): 3 if not 'bar' in body: 4 raise TypeError('bad user class') 5 return super().__new__(cls, name, bases, body) <ipython-input-165-f7e591caa0e8> in my_bc(*a, **kw) 8 def my_bc(*a, **kw): 9 print('my buildclass ->', a, kw) ---> 10 return old_bc(*a, **kw) 11 import builtins 12 builtins.__build_class__ = my_bc <ipython-input-1-8d1486f0ff44> in my_bc(fun, name, base, **kw) 10 print('Check if bar method defined') 11 if base is not None: ---> 12 return old_bc(fun, name, base, **kw) 13 return old_bc(fun, name, **kw) 14 ... last 1 frames repeated, from the frame below ... <ipython-input-1-8d1486f0ff44> in my_bc(fun, name, base, **kw) 10 print('Check if bar method defined') 11 if base is not None: ---> 12 return old_bc(fun, name, base, **kw) 13 return old_bc(fun, name, **kw) 14 RecursionError: maximum recursion depth exceeded
class BaseMeta(type):
def __new__(cls, name, bases, body):
if name != 'Base' and not 'bar' in body:
raise TypeError('bad user class')
return super().__new__(cls, name, bases, body)
class Base(metaclass=BaseMeta):
def foo(self):
return self.bar()
def __init_subclass__(*a, **kw):
print('init_subclass', a, kw)
return super().__init_subclass__(*a, **kw)
help(Base.__init_subclass__)
Help on method __init_subclass__ in module __main__: __init_subclass__(*a, **kw) method of __main__.BaseMeta instance This method is called when a class is subclassed. The default implementation does nothing. It may be overridden to extend subclasses.
# dec.py
def add(x, y=10):
return x + y
add(10, 20)
30
add
<function __main__.add>
# Name of function
add.__name__
'add'
# What module function is assigned to
add.__module__
'__main__'
# Default values
add.__defaults__
(10,)
# Byte code for function
add.__code__.co_code
b'|\x00|\x01\x17\x00S\x00'
# Variable names function interacts with
add.__code__.co_varnames
('x', 'y')
from inspect import getsource
getsource(add)
'def add(x, y=10):\n return x + y\n'
print(getsource(add))
def add(x, y=10): return x + y
# What file are you in?
from inspect import getfile
getfile(add)
'<ipython-input-19-3cec442ba064>'
from inspect import getmodule
getmodule(add)
<module '__main__'>
print('add(10)', add(10))
print('add(20, 30)', add(20, 30))
print('add("a", "b")', add("a", "b"))
add(10) 20 add(20, 30) 50 add("a", "b") ab
#Count how long it took to run
def add_timer(x, y=10):
before = time()
rv = x + y
after = time()
print('elapsed:', after - before)
return rv
print('add(10)', add_timer(10))
print('add(20, 30)', add_timer(20, 30))
print('add("a", "b")', add_timer("a", "b"))
elapsed: 0.0 add(10) 20 elapsed: 9.5367431640625e-07 add(20, 30) 50 elapsed: 9.5367431640625e-07 add("a", "b") ab
def sub(x, y=10):
return x - y
print('sub(10)', sub(10))
print('sub(20, 30)', sub(20, 30))
sub(10) 0 sub(20, 30) -10
def timer(func, x, y=10):
before = time()
rv = func(x, y)
after = time()
print('elapsed', after - before)
return rv
print('add(10)', timer(add, 10))
print('add(20, 30)', timer(add, 20, 30))
print('add("a", "b")', timer(add, "a", "b"))
elapsed 9.5367431640625e-07 add(10) 20 elapsed 9.5367431640625e-07 add(20, 30) 50 elapsed 9.5367431640625e-07 add("a", "b") ab
def timer(func):
def f(x, y=10):
before = time()
rv = func(x, y)
after = time()
print('elapsed', after - before)
return rv
return f
add = timer(add)
print('add(10)', add(10))
print('add(20, 30)', add(20, 30))
print('add("a", "b")', add("a", "b"))
elapsed 9.5367431640625e-07 add(10) 20 elapsed 9.5367431640625e-07 add(20, 30) 50 elapsed 9.5367431640625e-07 add("a", "b") ab
# Don't need to do add = timer(add) with decorators...
@timer
def add_dec(x, y=10):
return x + y
@timer
def sub_dec(x, y=10):
return x - y
print('add(10)', add_dec(10))
print('add(20, 30)', add_dec(20, 30))
print('add("a", "b")', add_dec("a", "b"))
print('sub(10)', sub_dec(10))
print('sub(20, 30)', sub_dec(20, 30))
elapsed 9.5367431640625e-07 add(10) 20 elapsed 1.1920928955078125e-06 add(20, 30) 50 elapsed 1.1920928955078125e-06 add("a", "b") ab elapsed 9.5367431640625e-07 sub(10) 0 elapsed 0.0 sub(20, 30) -10
# Don't hardcode parameters in decorator functions
def timer_k(func):
def f(*args, **kwargs):
before = time()
rv = func(*args, **kwargs)
after = time()
print('elapsed', after - before)
return rv
return f
@timer_k
def add_dec(x, y=10):
return x + y
@timer_k
def sub_dec(x, y=10):
return x - y
print('add(10)', add_dec(10))
print('add(20, 30)', add_dec(20, 30))
print('add("a", "b")', add_dec("a", "b"))
print('sub(10)', sub_dec(10))
print('sub(20, 30)', sub_dec(20, 30))
elapsed 9.5367431640625e-07 add(10) 20 elapsed 9.5367431640625e-07 add(20, 30) 50 elapsed 9.5367431640625e-07 add("a", "b") ab elapsed 0.0 sub(10) 0 elapsed 9.5367431640625e-07 sub(20, 30) -10
# What if I want to run a function n number of times
# Let's have add run 3 times in a row and sub run twice in a row
n = 2
def ntimes(f):
def wrapper(*args, **kwargs):
for _ in range(n):
print('running {.__name__}'.format(f))
rv = f(*args, **kwargs)
return rv
return wrapper
@ntimes
def add_dec(x, y=10):
return x + y
@ntimes
def sub_dec(x, y=10):
return x - y
print('add(10)', add_dec(10))
print('add(20, 30)', add_dec(20, 30))
print('add("a", "b")', add_dec("a", "b"))
print('sub(10)', sub_dec(10))
print('sub(20, 30)', sub_dec(20, 30))
running add_dec running add_dec add(10) 20 running add_dec running add_dec add(20, 30) 50 running add_dec running add_dec add("a", "b") ab running sub_dec running sub_dec sub(10) 0 running sub_dec running sub_dec sub(20, 30) -10
def ntimes(n):
def inner(f):
def wrapper(*args, **kwargs):
for _ in range(n):
print('running {.__name__}'.format(f))
rv = f(*args, **kwargs)
return rv
return wrapper
return inner
@ntimes(2)
def add_hdec(x, y=10):
return x + y
@ntimes(4)
def sub_hdec(x, y=10):
return x - y
print('add(10)', add_hdec(10))
print('add(20, 30)', add_hdec(20, 30))
print('add("a", "b")', add_hdec("a", "b"))
print('sub(10)', sub_hdec(10))
print('sub(20, 30)', sub_hdec(20, 30))
running add_hdec running add_hdec add(10) 20 running add_hdec running add_hdec add(20, 30) 50 running add_hdec running add_hdec add("a", "b") ab running sub_hdec running sub_hdec running sub_hdec running sub_hdec sub(10) 0 running sub_hdec running sub_hdec running sub_hdec running sub_hdec sub(20, 30) -10
# gen.py - use whenever sequencing is needd
# top-level syntax, function -> underscore method
# x() __call__
def add1(x, y):
return x + y
class Adder:
def __call__(self, x, y):
return x + y
add2 = Adder()
add1(10, 20)
30
add2(10, 20)
30
# top-level syntax, function -> underscore method
# x() __call__
def add1(x, y):
return x + y
class Adder:
def __init__(self):
self.z = 0
def __call__(self, x, y):
self.z += 1
return x + y + self.z
add2 = Adder()
from time import sleep
# This example has storage... and has eager return of the result sets
def compute():
rv = []
for i in range(10):
sleep(.5)
rv.append(i)
return rv
compute()
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
Wasteful because we have to wait for the entire action to complete and be read into memory, when we really just care about each number (one by one)
class Compute:
def __call__(self):
rv = []
for i in range(100000):
sleep(5)
rv.append(i)
return rv
def __iter__(self):
self.last = 0
return self
def __next__(self):
rv = self.last
self.last += 1
if self.last > 10:
raise StopIteration()
sleep(.5)
return self.last
compute = Compute()
# THIS IS UGLY... now let's make a generator
#This is a generator... don't eagerly compute. Return to user as they ask for it...
def compute():
for i in range(10):
sleep(.5)
yield i
# for x in xs:
# pass
# xi = iter(xs) -> __iter__
# while True:
# x = next(xi) -> __next__
for val in compute():
print(val)
0 1 2 3 4 5 6 7 8 9
class Api:
def run_this_first(self):
first()
def run_this_second(self):
second()
def run_this_last(self):
last()
def api():
first()
yield
second()
yield
last()
# cty.py
from sqlite3 import connect
# with ctx() as x:
# pass
# x = ctx().__enter__
# try:
# pass
# finally:
# x.__exit__
class temptable:
def __init__(self, cur):
self.cur = cur
def __enter__(self):
print('__enter__')
self.cur.execute('create table points(x int, y int)')
def __exit__(self, *args):
print('__exit__')
self.cur.execute('drop table points')
with connect('test.db') as conn:
cur = conn.cursor()
with temptable(cur):
cur.execute('insert into points (x, y) values(1, 1)')
cur.execute('insert into points (x, y) values(1, 2)')
cur.execute('insert into points (x, y) values(2, 1)')
cur.execute('insert into points (x, y) values(2, 2)')
for row in cur.execute("select x, y from points"):
print(row)
for row in cur.execute('select sum(x * y) from points'):
print(row)
__enter__ (1, 1) (1, 2) (2, 1) (2, 2) (9,) __exit__
rm test.db
def temptable(cur):
cur.execute('create table points(x int, y int)')
print('created table')
yield
cur.execute('drop table points')
print('dropped table')
class contextmanager:
def __init__(self, cur):
self.cur = cur
def __enter__(self):
self.gen = temptable(self.cur)
next(self.gen)
def __exit__(self, *args):
next(self.gen, None)
with connect('test.db') as conn:
cur = conn.cursor()
with contextmanager(cur):
cur.execute('insert into points (x, y) values(1, 1)')
cur.execute('insert into points (x, y) values(1, 2)')
cur.execute('insert into points (x, y) values(2, 1)')
cur.execute('insert into points (x, y) values(2, 2)')
for row in cur.execute("select x, y from points"):
print(row)
for row in cur.execute('select sum(x * y) from points'):
print(row)
created table (1, 1) (1, 2) (2, 1) (2, 2) (9,) dropped table
class contextmanager:
def __init__(self, gen):
self.gen = gen
def __call__(self, *args, **kwargs):
self.args, self.kwargs = args, kwargs
return self
def __enter__(self):
self.gen_inst = self.gen(*self.args, **self.kwargs)
next(self.gen_inst)
def __exit__(self, *args):
next(self.gen_inst, None)
def temptable(cur):
cur.execute('create table points(x int, y int)')
print('created table')
yield
cur.execute('drop table points')
print('dropped table')
temptable = contextmanager(temptable)
with connect('test.db') as conn:
cur = conn.cursor()
with temptable(cur):
cur.execute('insert into points (x, y) values(1, 1)')
cur.execute('insert into points (x, y) values(1, 2)')
cur.execute('insert into points (x, y) values(2, 1)')
cur.execute('insert into points (x, y) values(2, 2)')
for row in cur.execute("select x, y from points"):
print(row)
created table (1, 1) (1, 2) (2, 1) (2, 2) dropped table
class contextmanager:
def __init__(self, gen):
self.gen = gen
def __call__(self, *args, **kwargs):
self.args, self.kwargs = args, kwargs
return self
def __enter__(self):
self.gen_inst = self.gen(*self.args, **self.kwargs)
next(self.gen_inst)
def __exit__(self, *args):
next(self.gen_inst, None)
@contextmanager
def temptable(cur):
cur.execute('create table points(x int, y int)')
print('created table')
yield
cur.execute('drop table points')
print('dropped table')
with connect('test.db') as conn:
cur = conn.cursor()
with temptable(cur):
cur.execute('insert into points (x, y) values(1, 1)')
cur.execute('insert into points (x, y) values(1, 2)')
cur.execute('insert into points (x, y) values(2, 1)')
cur.execute('insert into points (x, y) values(2, 2)')
for row in cur.execute("select x, y from points"):
print(row)
created table (1, 1) (1, 2) (2, 1) (2, 2) dropped table
from sqlite3 import connect
from contextlib import contextmanager
@contextmanager
def temptable(cur):
cur.execute('create table points(x int, y int)')
print('created table')
try:
yield
finally:
cur.execute('drop table points')
print('dropped table')
with connect('test.db') as conn:
cur = conn.cursor()
with temptable(cur):
cur.execute('insert into points (x, y) values(1, 1)')
cur.execute('insert into points (x, y) values(1, 2)')
cur.execute('insert into points (x, y) values(2, 1)')
cur.execute('insert into points (x, y) values(2, 2)')
for row in cur.execute("select x, y from points"):
print(row)
created table (1, 1) (1, 2) (2, 1) (2, 2) dropped table