#default_exp core.foundation
#export
from local.test import *
from local.core.imports import *
from local.notebook.showdoc import *
Basic functions used in the fastai library
# export
defaults = SimpleNamespace()
See this blog post for more information about metaclasses.
PrePostInitMeta
ensures that the classes defined with it run __pre_init__
and __post_init__
(without having to write self.__pre_init__()
and self.__post_init__()
in the actual init
NewChkMeta
gives the PrePostInitMeta
functionality and ensures classes defined with it don't re-create an object of their type whenever it's passed to the constructorBypassNewMeta
ensures classes defined with it can easily be casted form objects they subclass.#export
class FixSigMeta(type):
"A metaclass that fixes the signature on classes that override __new__"
def __new__(cls, name, bases, dict):
res = super().__new__(cls, name, bases, dict)
if res.__init__ is not object.__init__: res.__signature__ = inspect.signature(res.__init__)
return res
#export
class PrePostInitMeta(FixSigMeta):
"A metaclass that calls optional `__pre_init__` and `__post_init__` methods"
def __call__(cls, *args, **kwargs):
res = cls.__new__(cls)
if type(res)==cls:
if hasattr(res,'__pre_init__'): res.__pre_init__(*args,**kwargs)
res.__init__(*args,**kwargs)
if hasattr(res,'__post_init__'): res.__post_init__(*args,**kwargs)
return res
show_doc(PrePostInitMeta, title_level=3)
class
PrePostInitMeta
[source]
PrePostInitMeta
(name
,bases
,dict
) ::FixSigMeta
A metaclass that calls optional __pre_init__
and __post_init__
methods
class _T(metaclass=PrePostInitMeta):
def __pre_init__(self): self.a = 0; assert self.a==0
def __init__(self,b=0): self.a += 1; assert self.a==1
def __post_init__(self): self.a += 1; assert self.a==2
t = _T()
test_eq(t.a, 2)
#export
class NewChkMeta(FixSigMeta):
"Metaclass to avoid recreating object passed to constructor"
def __call__(cls, x=None, *args, **kwargs):
if not args and not kwargs and x is not None and isinstance(x,cls):
x._newchk = 1
return x
res = super().__call__(*((x,) + args), **kwargs)
res._newchk = 0
return res
class _T(metaclass=NewChkMeta):
"Testing"
def __init__(self, o=None, b=1):
self.foo = getattr(o,'foo',0) + 1
self.b = b
class _T2():
def __init__(self, o): self.foo = getattr(o,'foo',0) + 1
t = _T(1)
test_eq(t.foo,1)
t2 = _T(t)
test_eq(t2.foo,1)
test_is(t,t2)
t3 = _T(t, b=2)
test_eq(t3.b, 2)
assert not t3 is t
t = _T2(1)
test_eq(t.foo,1)
t2 = _T2(t)
test_eq(t2.foo,2)
test_eq(_T.__doc__, "Testing")
# TODO: this shouldn't have "self, "
test_eq(str(inspect.signature(_T)), '(self, o=None, b=1)')
#export
class BypassNewMeta(FixSigMeta):
"Metaclass: casts `x` to this class if it's of type `cls._bypass_type`, initializing with `_new_meta` if available"
def __call__(cls, x=None, *args, **kwargs):
if hasattr(cls, '_new_meta'): x = cls._new_meta(x, *args, **kwargs)
elif not isinstance(x,getattr(cls,'_bypass_type',object)) or len(args) or len(kwargs):
x = super().__call__(*((x,)+args), **kwargs)
if cls!=x.__class__: x.__class__ = cls
return x
class T0: pass
class _T(T0, metaclass=BypassNewMeta):
_bypass_type=T0
def __init__(self,x): self.x=x
t = T0()
t.a = 1
t2 = _T(t)
test_eq(type(t2), _T)
test_eq(t2.a,1)
test_is(t2,t)
t = _T(2)
t.x = 2
#export
def copy_func(f):
"Copy a non-builtin function (NB `copy.copy` does not work for this)"
if not isinstance(f,types.FunctionType): return copy(f)
fn = types.FunctionType(f.__code__, f.__globals__, f.__name__, f.__defaults__, f.__closure__)
fn.__dict__.update(f.__dict__)
return fn
#export
def patch_to(cls, as_prop=False):
"Decorator: add `f` to `cls`"
if not isinstance(cls, (tuple,list)): cls=(cls,)
def _inner(f):
for c_ in cls:
nf = copy_func(f)
# `functools.update_wrapper` when passing patched function to `Pipeline`, so we do it manually
for o in functools.WRAPPER_ASSIGNMENTS: setattr(nf, o, getattr(f,o))
nf.__qualname__ = f"{c_.__name__}.{f.__name__}"
setattr(c_, f.__name__, property(nf) if as_prop else nf)
return f
return _inner
class _T3(int): pass
@patch_to(_T3)
def func1(x, a): return x+a
t = _T3(1)
test_eq(t.func1(2), 3)
If cls
is a tuple, f
is added to all types in the tuple.
class _T4(int): pass
@patch_to((_T3,_T4))
def func2(x, a): return x+2*a
t = _T3(1)
test_eq(t.func2(1), 3)
t = _T4(1)
test_eq(t.func2(1), 3)
#export
def patch(f):
"Decorator: add `f` to the first parameter's class (based on f's type annotations)"
cls = next(iter(f.__annotations__.values()))
return patch_to(cls)(f)
@patch
def func(x:_T3, a):
"test"
return x+2
t = _T3(1)
test_eq(t.func(2), 3)
test_eq(t.func.__qualname__, '_T3.func')
If annotation is a tuple, the function is added to all types in the tuple.
@patch
def func3(x:(_T3,_T4), a):
"test"
return x+2*a
t = _T3(1)
test_eq(t.func3(2), 5)
test_eq(t.func3.__qualname__, '_T3.func3')
t = _T4(1)
test_eq(t.func3(2), 5)
test_eq(t.func3.__qualname__, '_T4.func3')
#export
def patch_property(f):
"Decorator: add `f` as a property to the first parameter's class (based on f's type annotations)"
cls = next(iter(f.__annotations__.values()))
return patch_to(cls, as_prop=True)(f)
@patch_property
def prop(x:_T3): return x+1
t = _T3(1)
test_eq(t.prop, 2)
#export
def _mk_param(n,d=None): return inspect.Parameter(n, inspect.Parameter.KEYWORD_ONLY, default=d)
def test_sig(f, b): test_eq(str(inspect.signature(f)), b)
#export
def use_kwargs_dict(keep=False, **kwargs):
"Decorator: replace `**kwargs` in signature with `names` params"
def _f(f):
sig = inspect.signature(f)
sigd = dict(sig.parameters)
k = sigd.pop('kwargs')
s2 = {n:_mk_param(n,d) for n,d in kwargs.items() if n not in sigd}
sigd.update(s2)
if keep: sigd['kwargs'] = k
f.__signature__ = sig.replace(parameters=sigd.values())
return f
return _f
@use_kwargs_dict(y=1,z=None)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, *, y=1, z=None)')
#export
def use_kwargs(names, keep=False):
"Decorator: replace `**kwargs` in signature with `names` params"
def _f(f):
sig = inspect.signature(f)
sigd = dict(sig.parameters)
k = sigd.pop('kwargs')
s2 = {n:_mk_param(n) for n in names if n not in sigd}
sigd.update(s2)
if keep: sigd['kwargs'] = k
f.__signature__ = sig.replace(parameters=sigd.values())
return f
return _f
@use_kwargs(['y', 'z'])
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, *, y=None, z=None)')
@use_kwargs(['y', 'z'], keep=True)
def foo(a, *args, b=1, **kwargs): pass
test_sig(foo, '(a, *args, b=1, y=None, z=None, **kwargs)')
#export
def delegates(to=None, keep=False):
"Decorator: replace `**kwargs` in signature with params from `to`"
def _f(f):
if to is None: to_f,from_f = f.__base__.__init__,f.__init__
else: to_f,from_f = to,f
from_f = getattr(from_f,'__func__',from_f)
if hasattr(from_f,'__delwrap__'): return f
sig = inspect.signature(from_f)
sigd = dict(sig.parameters)
k = sigd.pop('kwargs')
s2 = {k:v for k,v in inspect.signature(to_f).parameters.items()
if v.default != inspect.Parameter.empty and k not in sigd}
sigd.update(s2)
if keep: sigd['kwargs'] = k
from_f.__signature__ = sig.replace(parameters=sigd.values())
from_f.__delwrap__ = to_f
return f
return _f
def basefoo(e, c=2): pass
@delegates(basefoo)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, c=2)')
@delegates(basefoo, keep=True)
def foo(a, b=1, **kwargs): pass
test_sig(foo, '(a, b=1, c=2, **kwargs)')
class BaseFoo:
def __init__(self, e, c=2): pass
@delegates()
class Foo(BaseFoo):
def __init__(self, a, b=1, **kwargs): super().__init__(**kwargs)
test_sig(Foo, '(a, b=1, c=2)')
#export
def funcs_kwargs(cls):
"Replace methods in `self._methods` with those from `kwargs`"
old_init = cls.__init__
def _init(self, *args, **kwargs):
for k in cls._methods:
arg = kwargs.pop(k,None)
if arg is not None:
if isinstance(arg,types.MethodType): arg = types.MethodType(arg.__func__, self)
setattr(self, k, arg)
old_init(self, *args, **kwargs)
functools.update_wrapper(_init, old_init)
cls.__init__ = use_kwargs(cls._methods)(_init)
return cls
#export
def method(f):
"Mark `f` as a method"
# `1` is a dummy instance since Py3 doesn't allow `None` any more
return types.MethodType(f, 1)
@funcs_kwargs
class T:
_methods=['b']
def __init__(self, f=1, **kwargs): assert not kwargs
def a(self): return 1
def b(self): return 2
t = T()
test_eq(t.a(), 1)
test_eq(t.b(), 2)
t = T(b = lambda:3)
test_eq(t.b(), 3)
test_sig(T, '(f=1, *, b=None)')
test_fail(lambda: T(a = lambda:3))
@method
def _f(self,a=1): return a+1
t = T(b = _f)
test_eq(t.b(2), 3)
class T2(T):
def __init__(self,a):
super().__init__(b = lambda:3)
self.a=a
t = T2(a=1)
test_eq(t.b(), 3)
test_sig(T2, '(a)')
def _g(a=1): return a+1
class T3(T): b = staticmethod(_g)
t = T3()
test_eq(t.b(2), 3)
Runtime type checking is handy, so let's make it easy!
@contextmanager
def working_directory(path):
"Change working directory to `path` and return to previous on exit."
prev_cwd = Path.cwd()
os.chdir(path)
try: yield
finally: os.chdir(prev_cwd)
#def is_listy(x): return isinstance(x,(list,tuple,Generator))
#export
def add_docs(cls, cls_doc=None, **docs):
"Copy values from `docs` to `cls` docstrings, and confirm all public methods are documented"
if cls_doc is not None: cls.__doc__ = cls_doc
for k,v in docs.items():
f = getattr(cls,k)
if hasattr(f,'__func__'): f = f.__func__ # required for class methods
f.__doc__ = v
# List of public callables without docstring
nodoc = [c for n,c in vars(cls).items() if callable(c)
and not n.startswith('_') and c.__doc__ is None]
assert not nodoc, f"Missing docs: {nodoc}"
assert cls.__doc__ is not None, f"Missing class docs: {cls}"
#export
def docs(cls):
"Decorator version of `add_docs`, using `_docs` dict"
add_docs(cls, **cls._docs)
return cls
class _T:
def f(self): pass
@classmethod
def g(cls): pass
add_docs(_T, "a", f="f", g="g")
test_eq(_T.__doc__, "a")
test_eq(_T.f.__doc__, "f")
test_eq(_T.g.__doc__, "g")
#export
def custom_dir(c, add:list):
"Implement custom `__dir__`, adding `add` to `cls`"
return dir(type(c)) + list(c.__dict__.keys()) + add
show_doc(is_iter)
assert is_iter([1])
assert not is_iter(array(1))
assert is_iter(array([1,2]))
assert (o for o in range(3))
#export
class _Arg:
def __init__(self,i): self.i = i
arg0 = _Arg(0)
arg1 = _Arg(1)
arg2 = _Arg(2)
arg3 = _Arg(3)
arg4 = _Arg(4)
#export
class bind:
"Same as `partial`, except you can use `arg0` `arg1` etc param placeholders"
def __init__(self, fn, *pargs, **pkwargs):
self.fn,self.pargs,self.pkwargs = fn,pargs,pkwargs
self.maxi = max((x.i for x in pargs if isinstance(x, _Arg)), default=-1)
def __call__(self, *args, **kwargs):
args = list(args)
kwargs = {**self.pkwargs,**kwargs}
for k,v in kwargs.items():
if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
return self.fn(*fargs, **kwargs)
def myfn(a,b,c,d=1,e=2): return(a,b,c,d,e)
test_eq(bind(myfn, arg1, 17, arg0, e=3)(19,14), (14,17,19,1,3))
test_eq(bind(myfn, 17, arg0, e=3)(19,14), (17,19,14,1,3))
test_eq(bind(myfn, 17, e=3)(19,14), (17,19,14,1,3))
test_eq(bind(myfn)(17,19,14), (17,19,14,1,2))
test_eq(bind(myfn, 17,19,14,e=arg0)(3), (17,19,14,1,3))
#export
class GetAttr:
"Inherit from this to have all attr accesses in `self._xtra` passed down to `self.default`"
_default='default'
@property
def _xtra(self): return self._dir()
def _dir(self): return [o for o in dir(getattr(self,self._default)) if not o.startswith('_')]
def __getattr__(self,k):
if k.startswith('__') or k in ('_xtra',self._default): raise AttributeError(k)
xtra = getattr(self, '_xtra', None)
if xtra is None or k in xtra:
attr = getattr(self,self._default,None)
if attr is not None: return getattr(attr, k)
raise AttributeError(k)
def __dir__(self): return custom_dir(self, self._dir() if self._xtra is None else self._dir())
# def __getstate__(self): return self.__dict__
def __setstate__(self,data): self.__dict__.update(data)
class B:
def __init__(self): self.a = A()
@funcs_kwargs
class A(GetAttr):
wif=after_iter= noops
_methods = 'wif after_iter'.split()
_default = 'dataset'
def __init__(self, **kwargs): pass
a = A()
a.wif
<bound method noops of <__main__.A object at 0x7f353a300910>>
b = A(wif=a.wif)
tst = pickle.dumps(b)
c = pickle.loads(tst)
a.a.a
<bound method noops of <__main__.A object at 0x7f353a3b3f90>>
class _C(GetAttr):
_xtra = ['lower']
def __init__(self,a): self.default = a
def foo(self): noop
t = _C('Hi')
test_eq(t.lower(), 'hi')
test_fail(lambda: t.upper())
assert 'lower' in dir(t)
#export
def delegate_attr(self, k, to):
"Use in `__getattr__` to delegate to attr `to` without inheriting from `GetAttr`"
if k.startswith('_') or k==to: raise AttributeError(k)
try: return getattr(getattr(self,to), k)
except AttributeError: raise AttributeError(k) from None
class _C:
f = 'Hi'
def __getattr__(self, k): return delegate_attr(self, k, 'f')
t = _C()
test_eq(t.lower(), 'hi')
#export
def _is_array(x): return hasattr(x,'__array__') or hasattr(x,'iloc')
def _listify(o):
if o is None: return []
if isinstance(o, list): return o
if isinstance(o, str) or _is_array(o): return [o]
if is_iter(o): return list(o)
return [o]
# export
def coll_repr(c, max_n=10):
"String repr of up to `max_n` items of (possibly lazy) collection `c`"
return f'(#{len(c)}) [' + ','.join(itertools.islice(map(str,c), max_n)) + (
'...' if len(c)>10 else '') + ']'
test_eq(coll_repr(range(1000), 5), '(#1000) [0,1,2,3,4...]')
# export
def mask2idxs(mask):
"Convert bool mask or index list to index `L`"
if isinstance(mask,slice): return mask
mask = list(mask)
if len(mask)==0: return []
it = mask[0]
if hasattr(it,'item'): it = it.item()
if isinstance(it,(bool,NoneType,np.bool_)): return [i for i,m in enumerate(mask) if m]
return [int(i) for i in mask]
# just for tests
import torch
test_eq(mask2idxs([False,True,False,True]), [1,3])
test_eq(mask2idxs(array([False,True,False,True])), [1,3])
test_eq(mask2idxs(torch.tensor([False,True,False,True])), [1,3])
test_eq(mask2idxs(array([1,2,3])), [1,2,3])
#export
listable_types = typing.Collection,Generator,map,filter,zip
#export
class CollBase:
"Base class for composing a list of `items`"
def __init__(self, items): self.items = items
def __len__(self): return len(self.items)
def __getitem__(self, k): return self.items[k]
def __setitem__(self, k, v): self.items[list(k) if isinstance(k,CollBase) else k] = v
def __delitem__(self, i): del(self.items[i])
def __repr__(self): return self.items.__repr__()
def __iter__(self): return self.items.__iter__()
#export
def cycle(o):
"Like `itertools.cycle` except creates list of `None`s if `o` is empty"
o = _listify(o)
return itertools.cycle(o) if o is not None and len(o) > 0 else itertools.cycle([None])
test_eq(itertools.islice(cycle([1,2,3]),5), [1,2,3,1,2])
test_eq(itertools.islice(cycle([]),3), [None]*3)
test_eq(itertools.islice(cycle(None),3), [None]*3)
test_eq(itertools.islice(cycle(1),3), [1,1,1])
#export
def zip_cycle(x, *args):
"Like `itertools.zip_longest` but `cycle`s through elements of all but first argument"
return zip(x, *map(cycle,args))
test_eq(zip_cycle([1,2,3,4],list('abc')), [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'a')])
#export
def is_indexer(idx):
"Test whether `idx` will index a single item in a list"
return isinstance(idx,int) or not getattr(idx,'ndim',1)
#export
def negate_func(f):
"Create new function that negates result of `f`"
def _f(*args, **kwargs): return not f(*args, **kwargs)
return _f
def f(a): return a>0
test_eq(f(1),True)
test_eq(negate_func(f)(1),False)
test_eq(negate_func(f)(a=-1),True)
#export
class L(CollBase, metaclass=NewChkMeta):
"Behaves like a list of `items` but can also index with list of indices or masks"
_default='items'
def __init__(self, items=None, *rest, use_list=False, match=None):
if rest: items = (items,)+rest
if items is None: items = []
if (use_list is not None) or not _is_array(items):
items = list(items) if use_list else _listify(items)
if match is not None:
if is_coll(match): match = len(match)
if len(items)==1: items = items*match
else: assert len(items)==match, 'Match length mismatch'
super().__init__(items)
@property
def _xtra(self): return None
def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
def copy(self): return self._new(self.items.copy())
def _get(self, i):
if is_indexer(i) or isinstance(i,slice): return getattr(self.items,'iloc',self.items)[i]
i = mask2idxs(i)
return (self.items.iloc[list(i)] if hasattr(self.items,'iloc')
else self.items.__array__()[(i,)] if hasattr(self.items,'__array__')
else [self.items[i_] for i_ in i])
def __setitem__(self, idx, o):
"Set `idx` (can be list of indices, or mask, or int) items to `o` (which is broadcast if not iterable)"
idx = idx if isinstance(idx,L) else _listify(idx)
if not is_iter(o): o = [o]*len(idx)
for i,o_ in zip(idx,o): self.items[i] = o_
def __iter__(self): return iter(self.items.itertuples() if hasattr(self.items,'iloc') else self.items)
def __contains__(self,b): return b in self.items
def __invert__(self): return self._new(not i for i in self)
def __eq__(self,b): return False if isinstance(b, (str,dict,set)) else all_equal(b,self)
def __repr__(self): return repr(self.items) if _is_array(self.items) else coll_repr(self)
def __mul__ (a,b): return a._new(a.items*b)
def __add__ (a,b): return a._new(a.items+_listify(b))
def __radd__(a,b): return a._new(b)+a
def __addi__(a,b):
a.items += list(b)
return a
def sorted(self, key=None, reverse=False):
if isinstance(key,str): k=lambda o:getattr(o,key,0)
elif isinstance(key,int): k=itemgetter(key)
else: k=key
return self._new(sorted(self.items, key=k, reverse=reverse))
@classmethod
def split(cls, s, sep=None, maxsplit=-1): return cls(s.split(sep,maxsplit))
@classmethod
def range(cls, a, b=None, step=None):
if is_coll(a): a = len(a)
return cls(range(a,b,step) if step is not None else range(a,b) if b is not None else range(a))
def map(self, f, *args, **kwargs):
g = (bind(f,*args,**kwargs) if callable(f)
else f.format if isinstance(f,str)
else f.__getitem__)
return self._new(map(g, self))
def filter(self, f, negate=False, **kwargs):
if kwargs: f = partial(f,**kwargs)
if negate: f = negate_func(f)
return self._new(filter(f, self))
def unique(self): return L(dict.fromkeys(self).keys())
def enumerate(self): return L(enumerate(self))
def val2idx(self): return {v:k for k,v in self.enumerate()}
def itemgot(self, *idxs):
x = self
for idx in idxs: x = x.map(itemgetter(idx))
return x
def attrgot(self, k, default=None): return self.map(lambda o:getattr(o,k,default))
def cycle(self): return cycle(self)
def map_dict(self, f=noop, *args, **kwargs): return {k:f(k, *args,**kwargs) for k in self}
def starmap(self, f, *args, **kwargs): return self._new(itertools.starmap(partial(f,*args,**kwargs), self))
def zip(self, cycled=False): return self._new((zip_cycle if cycled else zip)(*self))
def zipwith(self, *rest, cycled=False): return self._new([self, *rest]).zip(cycled=cycled)
def map_zip(self, f, *args, cycled=False, **kwargs): return self.zip(cycled=cycled).starmap(f, *args, **kwargs)
def map_zipwith(self, f, *rest, cycled=False, **kwargs): return self.zipwith(*rest, cycled=cycled).starmap(f, **kwargs)
def concat(self): return self._new(itertools.chain.from_iterable(self.map(L)))
def shuffle(self):
it = copy(self.items)
random.shuffle(it)
return self._new(it)
def append(self,o): return self.items.append(o)
def remove(self,o): return self.items.remove(o)
def count (self,o): return self.items.count(o)
def reverse(self ): return self.items.reverse()
def pop(self,o=-1): return self.items.pop(o)
def clear(self ): return self.items.clear()
def index(self, value, start=0, stop=sys.maxsize): return self.items.index(value, start=start, stop=stop)
def sort(self, key=None, reverse=False): return self.items.sort(key=key, reverse=reverse)
#export
_docs = {o:"Passthru to `list` method" for o in
'append count remove reverse sort pop clear index'.split()}
add_docs(L,
__getitem__="Retrieve `idx` (can be list of indices, or mask, or int) items",
range="Same as `range`, but returns an `L`. Can pass a collection for `a`, to use `len(a)`",
split="Same as `str.split`, but returns an `L`",
copy="Same as `list.copy`, but returns an `L`",
sorted="New `L` sorted by `key`. If key is str then use `attrgetter`. If key is int then use `itemgetter`",
unique="Unique items, in stable order",
val2idx="Dict from value to index",
filter="Create new `L` filtered by predicate `f`, passing `args` and `kwargs` to `f`",
map="Create new `L` with `f` applied to all `items`, passing `args` and `kwargs` to `f`",
map_dict="Like `map`, but creates a dict from `items` to function results",
starmap="Like `map`, but use `itertools.starmap`",
itemgot="Create new `L` with item `idx` of all `items`",
attrgot="Create new `L` with attr `k` of all `items`",
cycle="Same as `itertools.cycle`",
enumerate="Same as `enumerate`",
zip="Create new `L` with `zip(*items)`",
zipwith="Create new `L` with `self` zip with each of `*rest`",
map_zip="Combine `zip` and `starmap`",
map_zipwith="Combine `zipwith` and `starmap`",
concat="Concatenate all elements of list",
shuffle="Same as `random.shuffle`, but not inplace",
**_docs
)
You can create an L
from an existing iterable (e.g. a list, range, etc) and access or modify it with an int list/tuple index, mask, int, or slice. All list
methods can also be used with L
.
t = L(range(12))
test_eq(t, list(range(12)))
test_ne(t, list(range(11)))
t.reverse()
test_eq(t[0], 11)
t[3] = "h"
test_eq(t[3], "h")
t[3,5] = ("j","k")
test_eq(t[3,5], ["j","k"])
test_eq(t, L(t))
test_eq(L(L(1,2),[3,4]), ([1,2],[3,4]))
t
(#12) [11,10,9,j,7,k,5,4,3,2...]
There are optimized indexers for arrays, tensors, and DataFrames.
arr = np.arange(9).reshape(3,3)
t = L(arr, use_list=None)
test_eq(t[1,2], arr[[1,2]])
arr = np.arange(9).reshape(3,3)
t = L(arr, use_list=None)
test_eq(t[1,2], arr[[1,2]])
df = pd.DataFrame({'a':[1,2,3]})
t = L(df, use_list=None)
test_eq(t[1,2], L(pd.DataFrame({'a':[2,3]}, index=[1,2]), use_list=None))
You can also modify an L
with append
, +
, and *
.
t = L()
test_eq(t, [])
t.append(1)
test_eq(t, [1])
t += [3,2]
test_eq(t, [1,3,2])
t = t + [4]
test_eq(t, [1,3,2,4])
t = 5 + t
test_eq(t, [5,1,3,2,4])
test_eq(L(1,2,3), [1,2,3])
test_eq(L(1,2,3), L(1,2,3))
t = L(1)*5
t = t.map(operator.neg)
test_eq(t,[-1]*5)
test_eq(~L([True,False,False]), L([False,True,True]))
t = L(range(4))
test_eq(zip(t, L(1).cycle()), zip(range(4),(1,1,1,1)))
t = L.range(100)
test_shuffled(t,t.shuffle())
def _f(x,a=0): return x+a
t = L(1)*5
test_eq(t.map(_f), t)
test_eq(t.map(_f,1), [2]*5)
test_eq(t.map(_f,a=2), [3]*5)
An L
can be constructed from anything iterable, although tensors and arrays will not be iterated over on construction, unless you pass use_list
to the constructor.
test_eq(L([1,2,3]),[1,2,3])
test_eq(L(L([1,2,3])),[1,2,3])
test_ne(L([1,2,3]),[1,2,])
test_eq(L('abc'),['abc'])
test_eq(L(range(0,3)),[0,1,2])
test_eq(L(o for o in range(0,3)),[0,1,2])
test_eq(L(array(0)),[array(0)])
test_eq(L([array(0),array(1)]),[array(0),array(1)])
test_eq(L(array([0.,1.1]))[0],array([0.,1.1]))
test_eq(L(array([0.,1.1]), use_list=True), [array(0.),array(1.1)]) # `use_list=True` to unwrap arrays/arrays
If match
is not None
then the created list is same len as match
, either by:
len(items)==1
then items
is replicated,match
and items
are not already the same size.test_eq(L(1,match=[1,2,3]),[1,1,1])
test_eq(L([1,2],match=[2,3]),[1,2])
test_fail(lambda: L([1,2],match=[1,2,3]))
If you create an L
from an existing L
then you'll get back the original object (since L
uses the NewChkMeta
metaclass).
test_is(L(t), t)
An L
is considred equal to a list if they have the same elements. It's never considered equal to a str
a set
or a dict
even if they have the same elements/keys.
test_eq(L(['a', 'b']), ['a', 'b'])
test_ne(L(['a', 'b']), 'ab')
test_ne(L(['a', 'b']), {'a', 'b'})
test_ne(L(['a', 'b']), {'a':1, 'b':2})
show_doc(L.__getitem__)
t = L(range(12))
test_eq(t[1,2], [1,2]) # implicit tuple
test_eq(t[[1,2]], [1,2]) # list
test_eq(t[:3], [0,1,2]) # slice
test_eq(t[[False]*11 + [True]], [11]) # mask
test_eq(t[array(3)], 3)
show_doc(L.__setitem__)
L.__setitem__
[source]
L.setitem
(idx
,o
)
Set idx
(can be list of indices, or mask, or int) items to o
(which is broadcast if not iterable)
t[4,6] = 0
test_eq(t[4,6], [0,0])
t[4,6] = [1,2]
test_eq(t[4,6], [1,2])
show_doc(L.unique)
test_eq(L(1,2,3,4,4).unique(), [1,2,3,4])
show_doc(L.val2idx)
test_eq(L(1,2,3).val2idx(), {3:2,1:0,2:1})
show_doc(L.filter)
list(t)
[0, 1, 2, 3, 1, 5, 2, 7, 8, 9, 10, 11]
test_eq(t.filter(lambda o:o<5), [0,1,2,3,1,2])
test_eq(t.filter(lambda o:o<5, negate=True), [5,7,8,9,10,11])
show_doc(L.map)
test_eq(L.range(4).map(operator.neg), [0,-1,-2,-3])
If f
is a string then it is treated as a format string to create the mapping:
test_eq(L.range(4).map('#{}#'), ['#0#','#1#','#2#','#3#'])
If f
is a dictionary (or anything supporting __getitem__
) then it is indexed to create the mapping:
test_eq(L.range(4).map(list('abcd')), list('abcd'))
If the special argument _arg
is passed, then that is the kwarg used in the map.
#What is this? TODO Jeremy: fix
#L.range(4).map(f, b=arg0)
def f(a=None,b=None): return b
test_eq(L.range(4).map(f, b=arg0), range(4))
show_doc(L.map_dict)
L.map_dict
[source]
L.map_dict
(f
='noop'
, ***args
, **kwargs
**)
Like map
, but creates a dict from items
to function results
test_eq(L(range(1,5)).map_dict(), {1:1, 2:2, 3:3, 4:4})
test_eq(L(range(1,5)).map_dict(operator.neg), {1:-1, 2:-2, 3:-3, 4:-4})
t = L([[1,2,3],'abc'])
test_eq(t.zip(), [(1, 'a'),(2, 'b'),(3, 'c')])
t = L([[1,2,3,4],['a','b','c']])
test_eq(t.zip(cycled=True ), [(1, 'a'),(2, 'b'),(3, 'c'),(4, 'a')])
test_eq(t.zip(cycled=False), [(1, 'a'),(2, 'b'),(3, 'c')])
show_doc(L.map_zip)
t = L([1,2,3],[2,3,4])
test_eq(t.map_zip(operator.mul), [2,6,12])
show_doc(L.zipwith)
b = [[0],[1],[2,2]]
t = L([1,2,3]).zipwith(b)
test_eq(t, [(1,[0]), (2,[1]), (3,[2,2])])
show_doc(L.map_zipwith)
L.map_zipwith
[source]
L.map_zipwith
(f
, ***rest
,cycled
=False
, **kwargs
**)
Combine zipwith
and starmap
test_eq(L(1,2,3).map_zipwith(operator.mul, [2,3,4]), [2,6,12])
show_doc(L.itemgot)
test_eq(t.itemgot(1), b)
show_doc(L.attrgot)
a = [SimpleNamespace(a=3,b=4),SimpleNamespace(a=1,b=2)]
test_eq(L(a).attrgot('b'), [4,2])
show_doc(L.sorted)
test_eq(L(a).sorted('a').attrgot('b'), [2,4])
show_doc(L.split)
test_eq(L.split('a b c'), list('abc'))
show_doc(L.range)
test_eq_type(L.range([1,1,1]), L(range(3)))
test_eq_type(L.range(5,2,2), L(range(5,2,2)))
show_doc(L.concat)
test_eq(L([0,1,2,3],4,L(5,6)).concat(), range(7))
t = L([0,1,2,3],4,L(5,6)).copy()
test_eq(t.concat(), range(7))
#hide
from local.notebook.export import notebook2script
notebook2script(all_fs=True)
Converted 00_test.ipynb. Converted 01_core_foundation.ipynb. Converted 01a_core_utils.ipynb. Converted 01b_core_dispatch.ipynb. Converted 01c_core_transform.ipynb. Converted 02_core_script.ipynb. Converted 03_torchcore.ipynb. Converted 03a_layers.ipynb. Converted 04_data_load.ipynb. Converted 05_data_core.ipynb. Converted 06_data_transforms.ipynb. Converted 07_data_block.ipynb. Converted 08_vision_core.ipynb. Converted 09_vision_augment.ipynb. Converted 09a_vision_data.ipynb. Converted 09b_vision_utils.ipynb. Converted 10_pets_tutorial.ipynb. Converted 11_vision_models_xresnet.ipynb. Converted 12_optimizer.ipynb. Converted 13_learner.ipynb. Converted 13a_metrics.ipynb. Converted 14_callback_schedule.ipynb. Converted 14a_callback_data.ipynb. Converted 15_callback_hook.ipynb. Converted 15a_vision_models_unet.ipynb. Converted 16_callback_progress.ipynb. Converted 17_callback_tracker.ipynb. Converted 18_callback_fp16.ipynb. Converted 19_callback_mixup.ipynb. Converted 20_interpret.ipynb. Converted 20a_distributed.ipynb. Converted 21_vision_learner.ipynb. Converted 22_tutorial_imagenette.ipynb. Converted 23_tutorial_transfer_learning.ipynb. Converted 30_text_core.ipynb. Converted 31_text_data.ipynb. Converted 32_text_models_awdlstm.ipynb. Converted 33_text_models_core.ipynb. Converted 34_callback_rnn.ipynb. Converted 35_tutorial_wikitext.ipynb. Converted 36_text_models_qrnn.ipynb. Converted 37_text_learner.ipynb. Converted 38_tutorial_ulmfit.ipynb. Converted 40_tabular_core.ipynb. Converted 41_tabular_model.ipynb. Converted 42_tabular_rapids.ipynb. Converted 50_data_block_examples.ipynb. Converted 60_medical_imaging.ipynb. Converted 65_medical_text.ipynb. Converted 70_callback_wandb.ipynb. Converted 71_callback_tensorboard.ipynb. Converted 90_notebook_core.ipynb. Converted 91_notebook_export.ipynb. Converted 92_notebook_showdoc.ipynb. Converted 93_notebook_export2html.ipynb. Converted 94_notebook_test.ipynb. Converted 95_index.ipynb. Converted 96_data_external.ipynb. Converted 97_utils_test.ipynb. Converted notebook2jekyll.ipynb. Converted xse_resnext.ipynb.