Parameterize
notebooks.¶Notebooks should be reusable. At least the very least, a notebook that will restart and run all could be reused.
This post explores a default behavior to making modules reusable. We establish the opinion that any lowercase variables name * become a parameter of the module if its assignment evaluates literally.
Create an argumentparser from a give module. This class applies the semantics for does and does not become a variable. We rely on the argparser to capture the metadata about the application.
import argparse, ast, inspect
class CreateParser(ast.NodeTransformer):
def __init__(self, parser): self.parser = parser
def visit_Assign(self, node):
if len(node.targets) == 1:
target, parameter = node.targets[0].id, node.value
try:
parameter = ast.literal_eval(parameter)
if target[0].lower(): self.parser.add_argument(
'--%s'%target, type=ast.literal_eval, default=parameter,
help="{} : {} = {}".format(target, type(parameter).__name__, parameter))
except: ...
return node
visit_Module = ast.NodeTransformer.generic_visit
def generic_visit(self, node): return node
Update the ast.Module
based on a set of assignments passed from either a function call or command line tool.
class FindAndReplace(ast.NodeTransformer):
def __init__(self, **values): self.assignments = values
def visit_Assign(self, node):
if len(node.targets) == 1:
target, parameter = node.targets[0].id, node.value
if target in self.assignments:
value = self.assignments[target]
if isinstance(value, str): node.value = ast.Str(value)
else: node.value = ast.parse(str(value)).body[0].value
return node
Parameterization uses features from importlib
to load the module appropriately.
from importlib.util import spec_from_file_location, spec_from_loader, module_from_spec, find_spec
from inspect import Signature, Parameter
from copy import deepcopy
Our parameterizer can discover a spec from a module or string.
def get_spec(object): return (
spec_from_loader(object.__name__, object.__loader__)
if isinstance(object, __import__('types').ModuleType) else find_spec(object))
parameterize
is called for a string or function to return a called version of the module with the assignment replacement.
FindAndReplace
transformer. def parameterize(object):
spec = get_spec(object)
source = spec.loader.get_source(spec.loader.path)
nodes = ast.parse(source)
parser = argparse.ArgumentParser(prog=spec.name, description=ast.get_docstring(nodes))
CreateParser(parser).visit(nodes)
def call(argv=None, **kwargs):
module = module_from_spec(spec)
if argv is not None: kwargs = {**vars(parser.parse_args(argv)), **kwargs}
return exec(compile(
FindAndReplace(**kwargs).visit(deepcopy(nodes)), '<Parameterized>', 'exec'
), module.__dict__, module.__dict__) or module
call.__signature__ = Signature([
Parameter(key, Parameter.KEYWORD_ONLY, default=value)
for key, value in vars(parser.parse_args([])).items()
])
call.__doc__ = ast.get_docstring(nodes)
return call
from IPython import get_ipython
def _test_the_parameterized_module():
%reload_ext deathbeds.__Importing_notebooks_with_proper_source
f = parameterize('Untitled')
f(), f(a=99)
try: f(['-h'])
except SystemExit: ...