#default_exp foundation
#export
from fastcore.imports import *
from fastcore.basics import *
from functools import lru_cache
from contextlib import contextmanager
from copy import copy
from configparser import ConfigParser
import random,pickle
from fastcore.test import *
from nbdev.showdoc import *
from fastcore.nb_imports import *
import inspect
Basic functions used in the fastai library
#export
def copy_func(f):
"Copy a non-builtin function (NB `copy.copy` does not work for this)"
if not isinstance(f,FunctionType): return copy(f)
fn = FunctionType(f.__code__, f.__globals__, f.__name__, f.__defaults__, f.__closure__)
fn.__kwdefaults__ = f.__kwdefaults__
fn.__dict__.update(f.__dict__)
return fn
Sometimes it may be desirable to make a copy of a function that doesn't point to the original object. When you use Python's built in copy.copy
or copy.deepcopy
to copy a function, you get a reference to the original object:
import copy as cp
def foo(): pass
a = cp.copy(foo)
b = cp.deepcopy(foo)
a.someattr = 'hello' # since a and b point at the same object, updating a will update b
test_eq(b.someattr, 'hello')
assert a is foo and b is foo
However, with copy_func
, you can retrieve a copy of a function without a reference to the original object:
c = copy_func(foo) # c is an indpendent object
assert c is not foo
def g(x, *, y=3):
return x+y
test_eq(copy_func(g)(4), 7)
#export
def patch_to(cls, as_prop=False, cls_method=False):
"Decorator: add `f` to `cls`"
if not isinstance(cls, (tuple,list)): cls=(cls,)
def _inner(f):
for c_ in cls:
nf = copy_func(f)
# `functools.update_wrapper` when passing patched function to `Pipeline`, so we do it manually
for o in functools.WRAPPER_ASSIGNMENTS: setattr(nf, o, getattr(f,o))
nf.__qualname__ = f"{c_.__name__}.{f.__name__}"
if cls_method:
setattr(c_, f.__name__, MethodType(nf, c_))
else:
setattr(c_, f.__name__, property(nf) if as_prop else nf)
return f
return _inner
The @patch_to
decorator allows you to monkey patch a function into a class as a method:
class _T3(int): pass
@patch_to(_T3)
def func1(self, a): return self+a
t = _T3(1) # we initilized `t` to a type int = 1
test_eq(t.func1(2), 3) # we add 2 to `t`, so 2 + 1 = 3
You can access instance properties in the usual way via self
:
class _T4():
def __init__(self, g): self.g = g
@patch_to(_T4)
def greet(self, x): return self.g + x
t = _T4('hello ') # this sets self.g = 'helllo '
test_eq(t.greet('world'), 'hello world') #t.greet('world') will append 'world' to 'hello '
You can instead specify that the method should be a class method by setting cls_method=True
:
class _T5(int): attr = 3 # attr is a class attribute we will access in a later method
@patch_to(_T5, cls_method=True)
def func(cls, x): return cls.attr + x # you can access class attributes in the normal way
test_eq(_T5.func(4), 7)
Additionally you can specify that the function you want to patch should be a class attribute with as_prop
= False
@patch_to(_T5, as_prop=True)
def add_ten(self): return self + 10
t = _T5(4)
test_eq(t.add_ten, 14)
Instead of passing one class to the @patch_to
decorator, you can pass multiple classes in a tuple to simulteanously patch more than one class with the same method:
class _T6(int): pass
class _T7(int): pass
@patch_to((_T6,_T7))
def func_mult(self, a): return self*a
t = _T6(2)
test_eq(t.func_mult(4), 8)
t = _T7(2)
test_eq(t.func_mult(4), 8)
#export
def patch(f=None, *, as_prop=False, cls_method=False):
"Decorator: add `f` to the first parameter's class (based on f's type annotations)"
if f is None: return partial(patch, as_prop=as_prop, cls_method=cls_method)
cls = next(iter(f.__annotations__.values()))
return patch_to(cls, as_prop=as_prop, cls_method=cls_method)(f)
@patch
is an alternative to @patch_to
that allows you similarly monkey patch class(es) by using type annotations:
class _T8(int): pass
@patch
def func(self:_T8, a): return self+a
t = _T8(1) # we initilized `t` to a type int = 1
test_eq(t.func(3), 4) # we add 3 to `t`, so 3 + 1 = 4
test_eq(t.func.__qualname__, '_T8.func')
Similarly to patch_to
, you can supply a tuple of classes instead of a single class in your type annotations to patch multiple classes:
class _T9(int): pass
@patch
def func2(x:(_T8,_T9), a): return x*a # will patch both _T8 and _T9
t = _T8(2)
test_eq(t.func2(4), 8)
test_eq(t.func2.__qualname__, '_T8.func2')
t = _T9(2)
test_eq(t.func2(4), 8)
test_eq(t.func2.__qualname__, '_T9.func2')
Just like patch_to
decorator you can use as_propas_prop
and cls_method
parameters with patch
decorator:
@patch(as_prop=True)
def add_ten(self:_T5): return self + 10
t = _T5(4)
test_eq(t.add_ten, 14)
class _T5(int): attr = 3 # attr is a class attribute we will access in a later method
@patch(cls_method=True)
def func(cls:_T5, x): return cls.attr + x # you can access class attributes in the normal way
test_eq(_T5.func(4), 7)
#export
def patch_property(f):
"Deprecated; use `patch(as_prop=True)` instead"
warnings.warn("`patch_property` is deprecated and will be removed; use `patch(as_prop=True)` instead")
cls = next(iter(f.__annotations__.values()))
return patch_to(cls, as_prop=True)(f)
@contextmanager
def working_directory(path):
"Change working directory to `path` and return to previous on exit."
prev_cwd = Path.cwd()
os.chdir(path)
try: yield
finally: os.chdir(prev_cwd)
#export
def add_docs(cls, cls_doc=None, **docs):
"Copy values from `docs` to `cls` docstrings, and confirm all public methods are documented"
if cls_doc is not None: cls.__doc__ = cls_doc
for k,v in docs.items():
f = getattr(cls,k)
if hasattr(f,'__func__'): f = f.__func__ # required for class methods
f.__doc__ = v
# List of public callables without docstring
nodoc = [c for n,c in vars(cls).items() if callable(c)
and not n.startswith('_') and c.__doc__ is None]
assert not nodoc, f"Missing docs: {nodoc}"
assert cls.__doc__ is not None, f"Missing class docs: {cls}"
add_docs
allows you to add docstrings to a class and its associated methods. This function allows you to group docstrings together seperate from your code, which enables you to define one-line functions as well as organize your code more succintly. We believe this confers a number of benefits which we discuss in our style guide.
Suppose you have the following undocumented class:
class T:
def foo(self): pass
def bar(self): pass
You can add documentation to this class like so:
add_docs(T, cls_doc="A docstring for the class.",
foo="The foo method.",
bar="The bar method.")
Now, docstrings will appear as expected:
test_eq(T.__doc__, "A docstring for the class.")
test_eq(T.foo.__doc__, "The foo method.")
test_eq(T.bar.__doc__, "The bar method.")
add_docs
also validates that all of your public methods contain a docstring. If one of your methods is not documented, it will raise an error:
class T:
def foo(self): pass
def bar(self): pass
f=lambda: add_docs(T, "A docstring for the class.", foo="The foo method.")
test_fail(f, contains="Missing docs")
#hide
class _T:
def f(self): pass
@classmethod
def g(cls): pass
add_docs(_T, "a", f="f", g="g")
test_eq(_T.__doc__, "a")
test_eq(_T.f.__doc__, "f")
test_eq(_T.g.__doc__, "g")
#export
def docs(cls):
"Decorator version of `add_docs`, using `_docs` dict"
add_docs(cls, **cls._docs)
return cls
Instead of using add_docs
, you can use the decorator docs
as shown below. Note that the docstring for the class can be set with the argument cls_doc
:
@docs
class _T:
def f(self): pass
def g(cls): pass
_docs = dict(cls_doc="The class docstring",
f="The docstring for method f.",
g="A different docstring for method g.")
test_eq(_T.__doc__, "The class docstring")
test_eq(_T.f.__doc__, "The docstring for method f.")
test_eq(_T.g.__doc__, "A different docstring for method g.")
For either the docs
decorator or the add_docs
function, you can still define your docstrings in the normal way. Below we set the docstring for the class as usual, but define the method docstrings through the _docs
attribute:
@docs
class _T:
"The class docstring"
def f(self): pass
_docs = dict(f="The docstring for method f.")
test_eq(_T.__doc__, "The class docstring")
test_eq(_T.f.__doc__, "The docstring for method f.")
#export
def custom_dir(c, add:list):
"Implement custom `__dir__`, adding `add` to `cls`"
return dir(type(c)) + list(c.__dict__.keys()) + add
custom_dir
allows you extract the __dict__
property of a class and appends the list add
to it.
class _T:
def f(): pass
s = custom_dir(_T, add=['foo', 'bar']) # a list of everything in `__dict__` of `_T` with ['foo', 'bar'] appended.
assert {'foo', 'bar', 'f'}.issubset(s)
show_doc(is_iter)
assert is_iter([1])
assert not is_iter(array(1))
assert is_iter(array([1,2]))
assert (o for o in range(3))
#export
class _Arg:
def __init__(self,i): self.i = i
arg0 = _Arg(0)
arg1 = _Arg(1)
arg2 = _Arg(2)
arg3 = _Arg(3)
arg4 = _Arg(4)
# export
def coll_repr(c, max_n=10):
"String repr of up to `max_n` items of (possibly lazy) collection `c`"
return f'(#{len(c)}) [' + ','.join(itertools.islice(map(repr,c), max_n)) + (
'...' if len(c)>10 else '') + ']'
coll_repr
is used to provide a more informative __repr__
about list-like objects. coll_repr
and is used by L
to build a __repr__
that displays the length of a list in addition to a preview of a list.
Below is an example of the __repr__
string created for a list of 1000 elements:
test_eq(coll_repr(range(1000)), '(#1000) [0,1,2,3,4,5,6,7,8,9...]')
We can set the option max_n
to optionally preview a specified number of items instead of the default:
test_eq(coll_repr(range(1000), max_n=5), '(#1000) [0,1,2,3,4...]')
# export
def is_bool(x):
"Check whether `x` is a bool or None"
return isinstance(x,(bool,NoneType)) or isinstance_str(x, 'bool_')
# export
def mask2idxs(mask):
"Convert bool mask or index list to index `L`"
if isinstance(mask,slice): return mask
mask = list(mask)
if len(mask)==0: return []
it = mask[0]
if hasattr(it,'item'): it = it.item()
if is_bool(it): return [i for i,m in enumerate(mask) if m]
return [int(i) for i in mask]
test_eq(mask2idxs([False,True,False,True]), [1,3])
test_eq(mask2idxs(array([False,True,False,True])), [1,3])
test_eq(mask2idxs(array([1,2,3])), [1,2,3])
#export
def cycle(o):
"Like `itertools.cycle` except creates list of `None`s if `o` is empty"
o = listify(o)
return itertools.cycle(o) if o is not None and len(o) > 0 else itertools.cycle([None])
test_eq(itertools.islice(cycle([1,2,3]),5), [1,2,3,1,2])
test_eq(itertools.islice(cycle([]),3), [None]*3)
test_eq(itertools.islice(cycle(None),3), [None]*3)
test_eq(itertools.islice(cycle(1),3), [1,1,1])
#export
def zip_cycle(x, *args):
"Like `itertools.zip_longest` but `cycle`s through elements of all but first argument"
return zip(x, *map(cycle,args))
test_eq(zip_cycle([1,2,3,4],list('abc')), [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'a')])
#export
def is_indexer(idx):
"Test whether `idx` will index a single item in a list"
return isinstance(idx,int) or not getattr(idx,'ndim',1)
You can, for example index a single item in a list with an integer or a 0-dimensional numpy array:
assert is_indexer(1)
assert is_indexer(np.array(1))
However, you cannot index into single item in a list with another list or a numpy array with ndim > 0.
assert not is_indexer([1, 2])
assert not is_indexer(np.array([[1, 2], [3, 4]]))
#export
def negate_func(f):
"Create new function that negates result of `f`"
def _f(*args, **kwargs): return not f(*args, **kwargs)
return _f
def f(a): return a>0
test_eq(f(1),True)
test_eq(negate_func(f)(1),False)
test_eq(negate_func(f)(a=-1),True)
#export
class GetAttr:
"Inherit from this to have all attr accesses in `self._xtra` passed down to `self.default`"
_default='default'
def _component_attr_filter(self,k):
if k.startswith('__') or k in ('_xtra',self._default): return False
xtra = getattr(self,'_xtra',None)
return xtra is None or k in xtra
def _dir(self): return [k for k in dir(getattr(self,self._default)) if self._component_attr_filter(k)]
def __getattr__(self,k):
if self._component_attr_filter(k):
attr = getattr(self,self._default,None)
if attr is not None: return getattr(attr,k)
raise AttributeError(k)
def __dir__(self): return custom_dir(self,self._dir())
# def __getstate__(self): return self.__dict__
def __setstate__(self,data): self.__dict__.update(data)
show_doc(GetAttr, title_level=4)
class
GetAttr
[source]
GetAttr
()
Inherit from this to have all attr accesses in self._xtra
passed down to self.default
Inherit from GetAttr
to have attr access passed down to an instance attribute.
This makes it easy to create composites that don't require callers to know about their components. For a more detailed discussion of how this works as well as relevant context, we suggest reading the delegated composition section of this blog article.
You can customise the behaviour of GetAttr
in subclasses via;
_default
'default'
, so attr access is passed down to self.default
_default
can be set to the name of any instance attribute that does not start with dunder __
_xtra
None
, so all attr access is passed down_xtra
to a list of attribute namesTo illuminate the utility of GetAttr
, suppose we have the following two classes, _WebPage
which is a superclass of _ProductPage
, which we wish to compose like so:
class _WebPage:
def __init__(self, title, author="Jeremy"):
self.title,self.author = title,author
class _ProductPage:
def __init__(self, page, price): self.page,self.price = page,price
page = _WebPage('Soap', author="Sylvain")
p = _ProductPage(page, 15.0)
How do we make it so we can just write p.author
, instead of p.page.author
to access the author
attribute? We can use GetAttr
, of course! First, we subclass GetAttr
when defining _ProductPage
. Next, we set self.default
to the object whose attributes we want to be able to access directly, which in this case is the page
argument passed on initialization:
class _ProductPage(GetAttr):
def __init__(self, page, price): self.default,self.price = page,price #self.default allows you to access page directly.
p = _ProductPage(page, 15.0)
Now, we can access the author
attribute directly from the instance:
test_eq(p.author, 'Sylvain')
If you wish to store the object you are composing in an attribute other than self.default
, you can set the class attribute _data
as shown below. This is useful in the case where you might have a name collision with self.default
:
class _C(GetAttr):
_default = '_data' # use different component name; `self._data` rather than `self.default`
def __init__(self,a): self._data = a
def foo(self): noop
t = _C('Hi')
test_eq(t._data, 'Hi')
test_fail(lambda: t.default) # we no longer have self.default
test_eq(t.lower(), 'hi')
test_eq(t.upper(), 'HI')
assert 'lower' in dir(t)
assert 'upper' in dir(t)
By default, all attributes and methods of the object you are composing are retained. In the below example, we compose a str
object with the class _C
. This allows us to directly call string methods on instances of class _C
, such as str.lower()
or str.upper()
:
class _C(GetAttr):
# allow all attributes and methods to get passed to `self.default` (by leaving _xtra=None)
def __init__(self,a): self.default = a
def foo(self): noop
t = _C('Hi')
test_eq(t.lower(), 'hi')
test_eq(t.upper(), 'HI')
assert 'lower' in dir(t)
assert 'upper' in dir(t)
However, you can choose which attributes or methods to retain by defining a class attribute _xtra
, which is a list of allowed attribute and method names to delegate. In the below example, we only delegate the lower
method from the composed str
object when defining class _C
:
class _C(GetAttr):
_xtra = ['lower'] # specify which attributes get passed to `self.default`
def __init__(self,a): self.default = a
def foo(self): noop
t = _C('Hi')
test_eq(t.default, 'Hi')
test_eq(t.lower(), 'hi')
test_fail(lambda: t.upper()) # upper wasn't in _xtra, so it isn't available to be called
assert 'lower' in dir(t)
assert 'upper' not in dir(t)
You must be careful to properly set an instance attribute in __init__
that corresponds to the class attribute _default
. The below example sets the class attribute _default
to data
, but erroneously fails to define self.data
(and instead defines self.default
).
Failing to properly set instance attributes leads to errors when you try to access methods directly:
class _C(GetAttr):
_default = 'data' # use a bad component name; i.e. self.data does not exist
def __init__(self,a): self.default = a
def foo(self): noop
# TODO: should we raise an error when we create a new instance ...
t = _C('Hi')
test_eq(t.default, 'Hi')
# ... or is it enough for all GetAttr features to raise errors
test_fail(lambda: t.data)
test_fail(lambda: t.lower())
test_fail(lambda: t.upper())
test_fail(lambda: dir(t))
#hide
# I don't think this test is essential to the docs but it probably makes sense to
# check that everything works when we set both _xtra and _default to non-default values
class _C(GetAttr):
_xtra = ['lower', 'upper']
_default = 'data'
def __init__(self,a): self.data = a
def foo(self): noop
t = _C('Hi')
test_eq(t.data, 'Hi')
test_eq(t.lower(), 'hi')
test_eq(t.upper(), 'HI')
assert 'lower' in dir(t)
assert 'upper' in dir(t)
#hide
# when consolidating the filter logic, I choose the previous logic from
# __getattr__ k.startswith('__') rather than
# _dir k.startswith('_').
class _C(GetAttr):
def __init__(self): self.default = type('_D', (), {'_under': 1, '__dunder': 2})()
t = _C()
test_eq(t.default._under, 1)
test_eq(t._under, 1) # _ prefix attr access is allowed on component
assert '_under' in dir(t)
test_eq(t.default.__dunder, 2)
test_fail(lambda: t.__dunder) # __ prefix attr access is not allowed on component
assert '__dunder' not in dir(t)
assert t.__dir__ is not None # __ prefix attr access is allowed on composite
assert '__dir__' in dir(t)
#hide
#Failing test. TODO: make GetAttr pickle-safe
# class B:
# def __init__(self): self.a = A()
# @funcs_kwargs
# class A(GetAttr):
# wif=after_iter= noops
# _methods = 'wif after_iter'.split()
# _default = 'dataset'
# def __init__(self, **kwargs): pass
# a = A()
# b = A(wif=a.wif)
# a = A()
# b = A(wif=a.wif)
# tst = pickle.dumps(b)
# c = pickle.loads(tst)
#export
def delegate_attr(self, k, to):
"Use in `__getattr__` to delegate to attr `to` without inheriting from `GetAttr`"
if k.startswith('_') or k==to: raise AttributeError(k)
try: return getattr(getattr(self,to), k)
except AttributeError: raise AttributeError(k) from None
delegate_attr
is a functional way to delegate attributes, and is an alternative to GetAttr
. We recommend reading the documentation of GetAttr
for more details around delegation.
You can use achieve delegation when you define __getattr__
by using delegate_attr
:
#hide
import pandas as pd
class _C:
def __init__(self, o): self.o = o # self.o corresponds to the `to` argument in delegate_attr.
def __getattr__(self, k): return delegate_attr(self, k, to='o')
t = _C('HELLO') # delegates to a string
test_eq(t.lower(), 'hello')
t = _C(np.array([5,4,3])) # delegates to a numpy array
test_eq(t.sum(), 12)
t = _C(pd.DataFrame({'a': [1,2], 'b': [3,4]})) # delegates to a pandas.DataFrame
test_eq(t.b.max(), 4)
#export
class bind:
"Same as `partial`, except you can use `arg0` `arg1` etc param placeholders"
def __init__(self, fn, *pargs, **pkwargs):
self.fn,self.pargs,self.pkwargs = fn,pargs,pkwargs
self.maxi = max((x.i for x in pargs if isinstance(x, _Arg)), default=-1)
def __call__(self, *args, **kwargs):
args = list(args)
kwargs = {**self.pkwargs,**kwargs}
for k,v in kwargs.items():
if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
return self.fn(*fargs, **kwargs)
show_doc(bind, title_level=3)
bind
is the same as partial
, but also allows you to reorder positional arguments using variable name(s) arg{i}
where i refers to the zero-indexed positional argument. bind
as implemented currently only supports reordering of up to the first 5 positional arguments.
Consider the function myfunc
below, which has 3 positional arguments. These arguments can be referenced as arg0
, arg1
, and arg1
, respectively.
def myfn(a,b,c,d=1,e=2): return(a,b,c,d,e)
In the below example we bind the positional arguments of myfn
as follows:
14
, referenced by arg1
, is substituted for the first positional argument.17
for the second positional argument.19
, referenced by arg0
, is subsituted for the third positional argument.test_eq(bind(myfn, arg1, 17, arg0, e=3)(19,14), (14,17,19,1,3))
In this next example:
17
for the first positional argument.19
refrenced by arg0
, becomes the second positional argument.14
becomes the third positional argument.e
to 3
.test_eq(bind(myfn, 17, arg0, e=3)(19,14), (17,19,14,1,3))
This is an example of using bind
like partial
and do not reorder any arguments:
test_eq(bind(myfn)(17,19,14), (17,19,14,1,2))
bind
can also be used to change default values. In the below example, we use the first input 3
to override the default value of the named argument e
, and supply default values for the first three positional arguments:
test_eq(bind(myfn, 17,19,14,e=arg0)(3), (17,19,14,1,3))
L
helpers¶#export
def argwhere(iterable, f, negate=False, **kwargs):
"Like `filter_ex`, but return indices for matching items"
if kwargs: f = partial(f,**kwargs)
if negate: f = negate_func(f)
return [i for i,o in enumerate(iterable) if f(o)]
#export
def map_ex(iterable, f, *args, gen=False, **kwargs):
"Like `map`, but use `bind`, and supports `str` and indexing"
g = (bind(f,*args,**kwargs) if callable(f)
else f.format if isinstance(f,str)
else f.__getitem__)
res = map(g, iterable)
if gen: return res
return list(res)
#export
def filter_ex(iterable, f=noop, negate=False, gen=False, **kwargs):
"Like `filter`, but passing `kwargs` to `f`, defaulting `f` to `noop`, and adding `negate` and `gen`"
if kwargs: f = partial(f,**kwargs)
if negate: f = negate_func(f)
res = filter(f, iterable)
if gen: return res
return list(res)
#export
def range_of(a, b=None, step=None):
"All indices of collection `a`, if `a` is a collection, otherwise `range`"
if is_coll(a): a = len(a)
return list(range(a,b,step) if step is not None else range(a,b) if b is not None else range(a))
test_eq(range_of([1,1,1,1]), [0,1,2,3])
test_eq(range_of(4), [0,1,2,3])
#export
listable_types = typing.Collection,Generator,map,filter,zip
#export
def renumerate(iterable, start=0):
"Same as `enumerate`, but returns index as 2nd element instead of 1st"
return ((o,i) for i,o in enumerate(iterable, start=start))
test_eq(renumerate('abc'), (('a',0),('b',1),('c',2)))
#export
def first(x):
"First element of `x`, or None if missing"
try: return next(iter(x))
except StopIteration: return None
test_eq(first(['a', 'b', 'c', 'd', 'e']), 'a')
test_eq(first([]), None)
#export
def nested_attr(o, attr, default=None):
"Same as `getattr`, but if `attr` includes a `.`, then looks inside nested objects"
try:
for a in attr.split("."): o = getattr(o, a)
except AttributeError: return default
return o
a = SimpleNamespace(b=(SimpleNamespace(c=1)))
test_eq(nested_attr(a, 'b.c'), getattr(getattr(a, 'b'), 'c'))
test_eq(nested_attr(a, 'b.d'), None)
#export
class CollBase:
"Base class for composing a list of `items`"
def __init__(self, items): self.items = items
def __len__(self): return len(self.items)
def __getitem__(self, k): return self.items[list(k) if isinstance(k,CollBase) else k]
def __setitem__(self, k, v): self.items[list(k) if isinstance(k,CollBase) else k] = v
def __delitem__(self, i): del(self.items[i])
def __repr__(self): return self.items.__repr__()
def __iter__(self): return self.items.__iter__()
ColBase
is a base class that emulates the functionality of a python list
:
class _T(CollBase): pass
l = _T([1,2,3,4,5])
test_eq(len(l), 5) # __len__
test_eq(l[-1], 5); test_eq(l[0], 1) #__getitem__
l[2] = 100; test_eq(l[2], 100) # __set_item__
del l[0]; test_eq(len(l), 4) # __delitem__
test_eq(str(l), '[2, 100, 4, 5]') # __repr__
#export
class _L_Meta(type):
def __call__(cls, x=None, *args, **kwargs):
if not args and not kwargs and x is not None and isinstance(x,cls): return x
return super().__call__(x, *args, **kwargs)
#export
class L(GetAttr, CollBase, metaclass=_L_Meta):
"Behaves like a list of `items` but can also index with list of indices or masks"
_default='items'
def __init__(self, items=None, *rest, use_list=False, match=None):
if rest: items = (items,)+rest
if items is None: items = []
if (use_list is not None) or not is_array(items):
items = list(items) if use_list else listify(items)
if match is not None:
if is_coll(match): match = len(match)
if len(items)==1: items = items*match
else: assert len(items)==match, 'Match length mismatch'
super().__init__(items)
@property
def _xtra(self): return None
def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
def copy(self): return self._new(self.items.copy())
def _get(self, i):
if is_indexer(i) or isinstance(i,slice): return getattr(self.items,'iloc',self.items)[i]
i = mask2idxs(i)
return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')
else self.items.__array__()[(i,)] if hasattr(self.items,'__array__')
else [self.items[i_] for i_ in i])
def __setitem__(self, idx, o):
"Set `idx` (can be list of indices, or mask, or int) items to `o` (which is broadcast if not iterable)"
if isinstance(idx, int): self.items[idx] = o
else:
idx = idx if isinstance(idx,L) else listify(idx)
if not is_iter(o): o = [o]*len(idx)
for i,o_ in zip(idx,o): self.items[i] = o_
def __eq__(self,b):
if isinstance_str(b, 'ndarray'): return array_equal(b, self)
if isinstance(b, (str,dict)): return False
return all_equal(b,self)
def sorted(self, key=None, reverse=False): return self._new(sorted_ex(self, key=key, reverse=reverse))
def __iter__(self): return iter(self.items.itertuples() if hasattr(self.items,'iloc') else self.items)
def __contains__(self,b): return b in self.items
def __reversed__(self): return self._new(reversed(self.items))
def __invert__(self): return self._new(not i for i in self)
def __repr__(self): return repr(self.items)
def _repr_pretty_(self, p, cycle):
p.text('...' if cycle else repr(self.items) if is_array(self.items) else coll_repr(self))
def __mul__ (a,b): return a._new(a.items*b)
def __add__ (a,b): return a._new(a.items+listify(b))
def __radd__(a,b): return a._new(b)+a
def __addi__(a,b):
a.items += list(b)
return a
@classmethod
def split(cls, s, sep=None, maxsplit=-1): return cls(s.split(sep,maxsplit))
@classmethod
def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
def filter(self, f=noop, negate=False, gen=False, **kwargs):
return self._new(filter_ex(self, f=f, negate=negate, gen=gen, **kwargs))
def unique(self): return L(dict.fromkeys(self).keys())
def enumerate(self): return L(enumerate(self))
def renumerate(self): return L(renumerate(self))
def val2idx(self): return {v:k for k,v in self.enumerate()}
def itemgot(self, *idxs):
x = self
for idx in idxs: x = x.map(itemgetter(idx))
return x
def attrgot(self, k, default=None):
return self.map(lambda o: o.get(k,default) if isinstance(o, dict) else nested_attr(o,k,default))
def cycle(self): return cycle(self)
def map_dict(self, f=noop, *args, gen=False, **kwargs): return {k:f(k, *args,**kwargs) for k in self}
def map_filter(self, f=noop, g=noop, *args, gen=False, **kwargs):
res = filter(g, self.map(f, *args, gen=gen, **kwargs))
if gen: return res
return self._new(res)
def map_first(self, f=noop, g=noop, *args, **kwargs):
return first(self.map_filter(f, g, *args, gen=False, **kwargs))
def starmap(self, f, *args, **kwargs): return self._new(itertools.starmap(partial(f,*args,**kwargs), self))
def zip(self, cycled=False): return self._new((zip_cycle if cycled else zip)(*self))
def zipwith(self, *rest, cycled=False): return self._new([self, *rest]).zip(cycled=cycled)
def map_zip(self, f, *args, cycled=False, **kwargs): return self.zip(cycled=cycled).starmap(f, *args, **kwargs)
def map_zipwith(self, f, *rest, cycled=False, **kwargs):
return self.zipwith(*rest, cycled=cycled).starmap(f, **kwargs)
def shuffle(self):
it = copy(self.items)
random.shuffle(it)
return self._new(it)
def concat(self): return self._new(itertools.chain.from_iterable(self.map(L)))
def reduce(self, f, initial=None): return reduce(f, self) if initial is None else reduce(f, self, initial)
def sum(self): return self.reduce(operator.add)
def product(self): return self.reduce(operator.mul)
def setattrs(self, attr, val): [setattr(o,attr,val) for o in self]
#export
add_docs(L,
__getitem__="Retrieve `idx` (can be list of indices, or mask, or int) items",
range="Class Method: Same as `range`, but returns `L`. Can pass collection for `a`, to use `len(a)`",
split="Class Method: Same as `str.split`, but returns an `L`",
copy="Same as `list.copy`, but returns an `L`",
sorted="New `L` sorted by `key`. If key is str use `attrgetter`; if int use `itemgetter`",
unique="Unique items, in stable order",
val2idx="Dict from value to index",
filter="Create new `L` filtered by predicate `f`, passing `args` and `kwargs` to `f`",
argwhere="Like `filter`, but return indices for matching items",
map="Create new `L` with `f` applied to all `items`, passing `args` and `kwargs` to `f`",
map_filter="Same as `map` with `f` followed by `filter` with `g`",
map_first="First element of `map_filter`",
map_dict="Like `map`, but creates a dict from `items` to function results",
starmap="Like `map`, but use `itertools.starmap`",
itemgot="Create new `L` with item `idx` of all `items`",
attrgot="Create new `L` with attr `k` (or value `k` for dicts) of all `items`.",
cycle="Same as `itertools.cycle`",
enumerate="Same as `enumerate`",
renumerate="Same as `renumerate`",
zip="Create new `L` with `zip(*items)`",
zipwith="Create new `L` with `self` zip with each of `*rest`",
map_zip="Combine `zip` and `starmap`",
map_zipwith="Combine `zipwith` and `starmap`",
concat="Concatenate all elements of list",
shuffle="Same as `random.shuffle`, but not inplace",
reduce="Wrapper for `functools.reduce`",
sum="Sum of the items",
product="Product of the items",
setattrs="Call `setattr` on all items"
)
#hide
# Here we are fixing the signature of L. What happens is that
# the __call__ method on the MetaClass of L shadows the __init__
# giving the wrong signature [1].
# The solution adopted tries to not import the inspect module
# inside the lib code (notice that this cell is not exported)
# because it's really slow, but instead relies on pickling the
# correct signature and loading it.
# [1] https://stackoverflow.com/questions/49740290/call-from-metaclass-shadows-signature-of-init
def _f(items=None, *rest, use_list=False, match=None): ...
pickle.dumps(inspect.signature(_f))
b'\x80\x03cinspect\nSignature\nq\x00(cinspect\nParameter\nq\x01X\x05\x00\x00\x00itemsq\x02cinspect\n_ParameterKind\nq\x03K\x01\x85q\x04Rq\x05\x86q\x06Rq\x07}q\x08(X\x08\x00\x00\x00_defaultq\tNX\x0b\x00\x00\x00_annotationq\ncinspect\n_empty\nq\x0bubh\x01X\x04\x00\x00\x00restq\x0ch\x03K\x02\x85q\rRq\x0e\x86q\x0fRq\x10}q\x11(h\th\x0bh\nh\x0bubh\x01X\x08\x00\x00\x00use_listq\x12h\x03K\x03\x85q\x13Rq\x14\x86q\x15Rq\x16}q\x17(h\t\x89h\nh\x0bubh\x01X\x05\x00\x00\x00matchq\x18h\x14\x86q\x19Rq\x1a}q\x1b(h\tNh\nh\x0bubtq\x1c\x85q\x1dRq\x1e}q\x1fX\x12\x00\x00\x00_return_annotationq h\x0bsb.'
#export
#hide
L.__signature__ = pickle.loads(b'\x80\x03cinspect\nSignature\nq\x00(cinspect\nParameter\nq\x01X\x05\x00\x00\x00itemsq\x02cinspect\n_ParameterKind\nq\x03K\x01\x85q\x04Rq\x05\x86q\x06Rq\x07}q\x08(X\x08\x00\x00\x00_defaultq\tNX\x0b\x00\x00\x00_annotationq\ncinspect\n_empty\nq\x0bubh\x01X\x04\x00\x00\x00restq\x0ch\x03K\x02\x85q\rRq\x0e\x86q\x0fRq\x10}q\x11(h\th\x0bh\nh\x0bubh\x01X\x08\x00\x00\x00use_listq\x12h\x03K\x03\x85q\x13Rq\x14\x86q\x15Rq\x16}q\x17(h\t\x89h\nh\x0bubh\x01X\x05\x00\x00\x00matchq\x18h\x14\x86q\x19Rq\x1a}q\x1b(h\tNh\nh\x0bubtq\x1c\x85q\x1dRq\x1e}q\x1fX\x12\x00\x00\x00_return_annotationq h\x0bsb.')
#export
Sequence.register(L);
L
is a drop in replacement for a python list
. Inspired by NumPy, L
, supports advanced indexing and has additional methods (outlined below) that provide additional functionality and encourage simple expressive code. For example, the code below takes a list of pairs, selects the second item of each pair, takes its absolute value, filters items greater than 4, and adds them up:
#hide
from fastcore.utils import gt
d = dict(a=1,b=-5,d=6,e=9).items()
test_eq(L(d).itemgot(1).map(abs).filter(gt(4)).sum(), 20) # abs(-5) + abs(6) + abs(9) = 20; 1 was filtered out.
Read this overview section for a quick tutorial of L
, as well as background on the name.
You can create an L
from an existing iterable (e.g. a list, range, etc) and access or modify it with an int list/tuple index, mask, int, or slice. All list
methods can also be used with L
.
t = L(range(12))
test_eq(t, list(range(12)))
test_ne(t, list(range(11)))
t.reverse()
test_eq(t[0], 11)
t[3] = "h"
test_eq(t[3], "h")
t[3,5] = ("j","k")
test_eq(t[3,5], ["j","k"])
test_eq(t, L(t))
test_eq(L(L(1,2),[3,4]), ([1,2],[3,4]))
t
(#12) [11,10,9,'j',7,'k',5,4,3,2...]
Any L
is a Sequence
so you can use it with methods like random.sample
:
assert isinstance(t, Sequence)
import random
random.sample(t, 3)
[2, 'j', 1]
#hide
# test set items with L of collections
x = L([[1,2,3], [4,5], [6,7]])
x[0] = [1,2]
test_eq(x, L([[1,2], [4,5], [6,7]]))
There are optimized indexers for arrays, tensors, and DataFrames.
arr = np.arange(9).reshape(3,3)
t = L(arr, use_list=None)
test_eq(t[1,2], arr[[1,2]])
df = pd.DataFrame({'a':[1,2,3]})
t = L(df, use_list=None)
test_eq(t[1,2], L(pd.DataFrame({'a':[2,3]}, index=[1,2]), use_list=None))
You can also modify an L
with append
, +
, and *
.
t = L()
test_eq(t, [])
t.append(1)
test_eq(t, [1])
t += [3,2]
test_eq(t, [1,3,2])
t = t + [4]
test_eq(t, [1,3,2,4])
t = 5 + t
test_eq(t, [5,1,3,2,4])
test_eq(L(1,2,3), [1,2,3])
test_eq(L(1,2,3), L(1,2,3))
t = L(1)*5
t = t.map(operator.neg)
test_eq(t,[-1]*5)
test_eq(~L([True,False,False]), L([False,True,True]))
t = L(range(4))
test_eq(zip(t, L(1).cycle()), zip(range(4),(1,1,1,1)))
t = L.range(100)
test_shuffled(t,t.shuffle())
def _f(x,a=0): return x+a
t = L(1)*5
test_eq(t.map(_f), t)
test_eq(t.map(_f,1), [2]*5)
test_eq(t.map(_f,a=2), [3]*5)
An L
can be constructed from anything iterable, although tensors and arrays will not be iterated over on construction, unless you pass use_list
to the constructor.
test_eq(L([1,2,3]),[1,2,3])
test_eq(L(L([1,2,3])),[1,2,3])
test_ne(L([1,2,3]),[1,2,])
test_eq(L('abc'),['abc'])
test_eq(L(range(0,3)),[0,1,2])
test_eq(L(o for o in range(0,3)),[0,1,2])
test_eq(L(array(0)),[array(0)])
test_eq(L([array(0),array(1)]),[array(0),array(1)])
test_eq(L(array([0.,1.1]))[0],array([0.,1.1]))
test_eq(L(array([0.,1.1]), use_list=True), [array(0.),array(1.1)]) # `use_list=True` to unwrap arrays/arrays
If match
is not None
then the created list is same len as match
, either by:
len(items)==1
then items
is replicated,match
and items
are not already the same size.test_eq(L(1,match=[1,2,3]),[1,1,1])
test_eq(L([1,2],match=[2,3]),[1,2])
test_fail(lambda: L([1,2],match=[1,2,3]))
If you create an L
from an existing L
then you'll get back the original object (since L
uses the NewChkMeta
metaclass).
test_is(L(t), t)
An L
is considred equal to a list if they have the same elements. It's never considered equal to a str
a set
or a dict
even if they have the same elements/keys.
test_eq(L(['a', 'b']), ['a', 'b'])
test_ne(L(['a', 'b']), 'ab')
test_ne(L(['a', 'b']), {'a':1, 'b':2})
L
Methods¶show_doc(L.__getitem__)
t = L(range(12))
test_eq(t[1,2], [1,2]) # implicit tuple
test_eq(t[[1,2]], [1,2]) # list
test_eq(t[:3], [0,1,2]) # slice
test_eq(t[[False]*11 + [True]], [11]) # mask
test_eq(t[array(3)], 3)
show_doc(L.__setitem__)
L.__setitem__
[source]
L.setitem
(idx
,o
)
Set idx
(can be list of indices, or mask, or int) items to o
(which is broadcast if not iterable)
t[4,6] = 0
test_eq(t[4,6], [0,0])
t[4,6] = [1,2]
test_eq(t[4,6], [1,2])
show_doc(L.unique)
test_eq(L(1,2,3,4,4).unique(), [1,2,3,4])
show_doc(L.val2idx)
test_eq(L(1,2,3).val2idx(), {3:2,1:0,2:1})
show_doc(L.filter)
list(t)
[0, 1, 2, 3, 1, 5, 2, 7, 8, 9, 10, 11]
test_eq(t.filter(lambda o:o<5), [0,1,2,3,1,2])
test_eq(t.filter(lambda o:o<5, negate=True), [5,7,8,9,10,11])
show_doc(L.argwhere)
L.argwhere
[source]
L.argwhere
(f
,negate
=False
, ****kwargs
**)
Like filter
, but return indices for matching items
test_eq(t.argwhere(lambda o:o<5), [0,1,2,3,4,6])
show_doc(L.map)
test_eq(L.range(4).map(operator.neg), [0,-1,-2,-3])
If f
is a string then it is treated as a format string to create the mapping:
test_eq(L.range(4).map('#{}#'), ['#0#','#1#','#2#','#3#'])
If f
is a dictionary (or anything supporting __getitem__
) then it is indexed to create the mapping:
test_eq(L.range(4).map(list('abcd')), list('abcd'))
You can also pass the same arg
params that bind
accepts:
def f(a=None,b=None): return b
test_eq(L.range(4).map(f, b=arg0), range(4))
show_doc(L.map_dict)
L.map_dict
[source]
L.map_dict
(f
=noop
, ***args
,gen
=False
, **kwargs
**)
Like map
, but creates a dict from items
to function results
test_eq(L(range(1,5)).map_dict(), {1:1, 2:2, 3:3, 4:4})
test_eq(L(range(1,5)).map_dict(operator.neg), {1:-1, 2:-2, 3:-3, 4:-4})
t = L([[1,2,3],'abc'])
test_eq(t.zip(), [(1, 'a'),(2, 'b'),(3, 'c')])
t = L([[1,2,3,4],['a','b','c']])
test_eq(t.zip(cycled=True ), [(1, 'a'),(2, 'b'),(3, 'c'),(4, 'a')])
test_eq(t.zip(cycled=False), [(1, 'a'),(2, 'b'),(3, 'c')])
show_doc(L.map_zip)
t = L([1,2,3],[2,3,4])
test_eq(t.map_zip(operator.mul), [2,6,12])
show_doc(L.zipwith)
b = [[0],[1],[2,2]]
t = L([1,2,3]).zipwith(b)
test_eq(t, [(1,[0]), (2,[1]), (3,[2,2])])
show_doc(L.map_zipwith)
L.map_zipwith
[source]
L.map_zipwith
(f
, ***rest
,cycled
=False
, **kwargs
**)
Combine zipwith
and starmap
test_eq(L(1,2,3).map_zipwith(operator.mul, [2,3,4]), [2,6,12])
show_doc(L.itemgot)
test_eq(t.itemgot(1), b)
show_doc(L.attrgot)
# Example when items are not a dict
a = [SimpleNamespace(a=3,b=4),SimpleNamespace(a=1,b=2)]
test_eq(L(a).attrgot('b'), [4,2])
#Example of when items are a dict
b =[{'id': 15, 'name': 'nbdev'}, {'id': 17, 'name': 'fastcore'}]
test_eq(L(b).attrgot('id'), [15, 17])
show_doc(L.sorted)
test_eq(L(a).sorted('a').attrgot('b'), [2,4])
show_doc(L.split)
test_eq(L.split('a b c'), list('abc'))
show_doc(L.range)
test_eq_type(L.range([1,1,1]), L(range(3)))
test_eq_type(L.range(5,2,2), L(range(5,2,2)))
show_doc(L.concat)
test_eq(L([0,1,2,3],4,L(5,6)).concat(), range(7))
t = L([0,1,2,3],4,L(5,6)).copy()
test_eq(t.concat(), range(7))
show_doc(L.map_filter)
L.map_filter
[source]
L.map_filter
(f
=noop
,g
=noop
, ***args
,gen
=False
, **kwargs
**)
Same as map
with f
followed by filter
with g
t = L(0,1,2,3)
test_eq(t.map_filter(lambda o:o*2, lambda o:o<5), L(0,2,4))
show_doc(L.map_first)
t = L(0,1,2,3)
test_eq(t.map_first(lambda o:o*2 if o>2 else None), 6)
show_doc(L.setattrs)
t = L(SimpleNamespace(),SimpleNamespace())
t.setattrs('foo', 'bar')
test_eq(t.attrgot('foo'), ['bar','bar'])
#export
def save_config_file(file, d, **kwargs):
"Write settings dict to a new config file, or overwrite the existing one."
config = ConfigParser(**kwargs)
config['DEFAULT'] = d
config.write(open(file, 'w'))
#export
def read_config_file(file, **kwargs):
config = ConfigParser(**kwargs)
config.read(file)
return config
Config files are saved and read using Python's configparser.ConfigParser
, inside the DEFAULT
section.
_d = dict(user='fastai', lib_name='fastcore', some_path='test')
try:
save_config_file('tmp.ini', _d)
res = read_config_file('tmp.ini')
finally: os.unlink('tmp.ini')
test_eq(res['DEFAULT'], _d)
#export
def _add_new_defaults(cfg, file, **kwargs):
for k,v in kwargs.items():
if cfg.get(k, None) is None:
cfg[k] = v
save_config_file(file, cfg)
#export
@lru_cache(maxsize=None)
class Config:
"Reading and writing `settings.ini`"
def __init__(self, cfg_name='settings.ini'):
cfg_path = Path.cwd()
while cfg_path != cfg_path.parent and not (cfg_path/cfg_name).exists(): cfg_path = cfg_path.parent
self.config_path,self.config_file = cfg_path,cfg_path/cfg_name
assert self.config_file.exists(), f"Could not find {cfg_name}"
self.d = read_config_file(self.config_file)['DEFAULT']
_add_new_defaults(self.d, self.config_file,
host="github", doc_host="https://%(user)s.github.io", doc_baseurl="/%(lib_name)s/")
def __setitem__(self,k,v): self.d[k] = str(v)
def __contains__(self,k): return k in self.d
def save(self): save_config_file(self.config_file,self.d)
def __getattr__(self,k): return stop(AttributeError(k)) if k=='d' or k not in self.d else self.get(k)
def get(self,k,default=None): return self.d.get(k, default)
def path(self,k,default=None):
v = self.get(k, default)
return v if v is None else self.config_path/v
Config
searches parent directories for a config file, and provides direct access to the 'DEFAULT' section. Keys ending in _path
are converted to paths in the config file's directory.
try:
save_config_file('../tmp.ini', _d)
cfg = Config('tmp.ini')
finally: os.unlink('../tmp.ini')
test_eq(cfg.user,'fastai')
test_eq(cfg.doc_baseurl,'/fastcore/')
test_eq(cfg.get('some_path'), 'test')
test_eq(cfg.path('some_path'), Path('../test').resolve())
test_eq(cfg.get('foo','bar'),'bar')
#hide
from nbdev.export import notebook2script
notebook2script()
Converted 00_test.ipynb. Converted 01_basics.ipynb. Converted 02_foundation.ipynb. Converted 03_dispatch.ipynb. Converted 03_xtras.ipynb. Converted 04_transform.ipynb. Converted 05_logargs.ipynb. Converted 06_meta.ipynb. Converted 07_script.ipynb. Converted index.ipynb.