#default_exp notebook.showdoc
# export
from local.core.imports import *
from local.notebook.core import *
from local.notebook.export import *
import inspect,enum,nbconvert
from IPython.display import Markdown,display
from IPython.core import page
from nbconvert import HTMLExporter
Functions to show the doc cells in notebooks
from local.core.foundation import add_docs, patch
from local.core.utils import compose
from local.core.transform import Pipeline
from local.data.external import untar_data
test_cases = [
Pipeline, #Basic class
compose, #Func with star args and type annotation
untar_data, #Func with defaults
add_docs, #Func with kwargs
Path.ls #Monkey-patched
]
The inspect module lets us know quickly if an object is a function or a class but it doesn't distinguish classes and enums.
# export
def is_enum(cls):
"Check if `cls` is an enum or another type of class"
return type(cls) in (enum.Enum, enum.EnumMeta)
e = enum.Enum('e', 'a b')
assert is_enum(e)
assert not is_enum(Pipeline)
#hide
#Tricking jupyter notebook to have a __file__ attribute. All _file_ will be replaced by __file__
_file_ = Path('local').absolute()/'notebook'/'show_doc.py'
We don't link to all PyTorch functions, just the ones in an index we keep. We can easily add a reference with the following convenience function when writing the docs.
# export
def _get_pytorch_index():
if not (Path(_file_).parent/'index_pytorch.txt').exists(): return {}
return json.load(open(Path(_file_).parent/'index_pytorch.txt', 'r'))
def add_pytorch_index(func_name, url):
"Add `func_name` in the PyTorch index for automatic links."
index = _get_pytorch_index()
if not url.startswith("https://pytorch.org/docs/stable/"):
url = "https://pytorch.org/docs/stable/" + url
index[func_name] = url
json.dump(index, open(Path(_file_).parent/'index_pytorch.txt', 'w'), indent=2)
url
can be the full url or just the part after https://pytorch.org/docs/stable/
, see the example below.
#hide
ind,ind_bak = Path(_file_).parent/'index_pytorch.txt',Path(_file_).parent/'index_pytorch.bak'
if ind.exists(): shutil.move(ind, ind_bak)
assert _get_pytorch_index() == {}
add_pytorch_index('Tensor', 'tensors.html#torch-tensor')
assert _get_pytorch_index() == {'Tensor':'https://pytorch.org/docs/stable/tensors.html#torch-tensor'}
if ind_bak.exists(): shutil.move(ind_bak, ind)
add_pytorch_index('Tensor', 'tensors.html#torch-tensor')
add_pytorch_index('device', 'tensor_attributes.html#torch-device')
add_pytorch_index('DataLoader', 'data.html#torch.utils.data.DataLoader')
# export
def is_fastai_module(name):
"Test if `name` is a fastai module."
dir_name = os.path.sep.join(name.split('.'))
return (Path(_file_).parent.parent/f"{dir_name}.py").exists()
assert is_fastai_module('data.external')
assert is_fastai_module('learner')
assert not is_fastai_module('export')
# export
#Might change once the library is renamed fastai.
def _is_fastai_class(ft): return belongs_to_module(ft, 'fastai_source')
def _strip_fastai(s): return re.sub(r'^local\.', '', s)
FASTAI_DOCS = ''
# export
def doc_link(name, include_bt:bool=True):
"Create link to documentation for `name`."
cname = f'`{name}`' if include_bt else name
#Link to modules
if is_fastai_module(name): return f'[{cname}]({FASTAI_DOCS}/{name}.html)'
#Link to fastai functions
try_fastai = source_nb(name, is_name=True)
if try_fastai:
page = '.'.join(try_fastai.split('_')[1:]).replace('.ipynb', '.html')
return f'[{cname}]({FASTAI_DOCS}/{page}#{name})'
#Link to PyTorch
try_pytorch = _get_pytorch_index().get(name, None)
if try_pytorch: return f'[{cname}]({try_pytorch})'
#Leave as is
return cname
assert doc_link('core.transform') == f'[`core.transform`]({FASTAI_DOCS}/core.transform.html)'
assert doc_link('Pipeline') == f'[`Pipeline`]({FASTAI_DOCS}/core.transform.html#Pipeline)'
assert doc_link('Transform.create') == f'[`Transform.create`]({FASTAI_DOCS}/core.transform.html#Transform.create)'
assert doc_link('Tensor') == '[`Tensor`](https://pytorch.org/docs/stable/tensors.html#torch-tensor)'
assert doc_link('Tenso') == '`Tenso`'
#export
_re_backticks = re.compile(r"""
# Catches any link of the form \[`obj`\](old_link) or just `obj` to either update old links or add the link to the docs of obj
\[` # Opening [ and `
([^`]*) # Catching group with anything but a `
`\] # ` then closing ]
(?: # Beginning of non-catching group
\( # Opening (
[^)]* # Anything but a closing )
\) # Closing )
) # End of non-catching group
| # OR
` # Opening `
([^`]*) # Antyhing but a `
` # Closing `
""", re.VERBOSE)
# export
def add_doc_links(text):
"Search for doc links for any item between backticks in `text`."
def _replace_link(m): return doc_link(m.group(1) or m.group(2))
return _re_backticks.sub(_replace_link, text)
This function not only add links to backstick keywords, it also update the links that are already in the text.
tst = add_doc_links('This is an example of `Pipeline`')
assert tst == "This is an example of [`Pipeline`](/core.transform.html#Pipeline)"
tst = add_doc_links('Here we alread add a link in [`Tensor`](fake)')
assert tst == "Here we alread add a link in [`Tensor`](https://pytorch.org/docs/stable/tensors.html#torch-tensor)"
#export
def _is_type_dispatch(x): return type(x).__name__ == "TypeDispatch"
def _unwrapped_type_dispatch_func(x): return x.first() if _is_type_dispatch(x) else x
def _is_property(x): return type(x)==property
def _has_property_getter(x): return _is_property(x) and hasattr(x, 'fget') and hasattr(x.fget, 'func')
def _property_getter(x): return x.fget.func if _has_property_getter(x) else x
def _unwrapped_func(x):
x = _unwrapped_type_dispatch_func(x)
x = _property_getter(x)
return x
#export
SOURCE_URL = "https://github.com/fastai/fastai_dev/tree/master/dev/"
def get_source_link(func):
"Return link to `func` in source code"
func = _unwrapped_func(func)
try: line = inspect.getsourcelines(func)[1]
except Exception: return ''
module = inspect.getmodule(func).__name__.replace('.', '/') + '.py'
return f"{SOURCE_URL}{module}#L{line}"
#hide
from local.data.transforms import Categorize, DataBunch
assert get_source_link(Categorize.encodes).startswith(SOURCE_URL + 'local/data/transforms.py')
assert get_source_link(DataBunch.train_dl).startswith(SOURCE_URL + 'local/data/core.py')
#hide
assert get_source_link(Pipeline).startswith(SOURCE_URL + 'local/core/transform.py')
As important as the source code, we want to quickly jump to where the function is defined in a dev notebook.
#export
_re_header = re.compile(r"""
# Catches any header in markdown with the title in group 1
^\s* # Beginning of text followed by any number of whitespace
\#+ # One # or more
\s* # Any number of whitespace
(.*) # Catching group with anything
$ # End of text
""", re.VERBOSE)
#export
FASTAI_NB_DEV = 'https://nbviewer.jupyter.org/github/fastai/fastai_docs/blob/master/dev/'
def get_nb_source_link(func, local=False, is_name=None):
"Return a link to the notebook where `func` is defined."
func = _unwrapped_type_dispatch_func(func)
pref = '' if local else FASTAI_NB_DEV
is_name = is_name or isinstance(func, str)
src = source_nb(func, is_name=is_name, return_all=True)
if src is None: return '' if is_name else get_source_link(func)
find_name,nb_name = src
nb = read_nb(nb_name)
pat = re.compile(f'^{find_name}\s+=|^(def|class)\s+{find_name}\s*\(', re.MULTILINE)
if len(find_name.split('.')) == 2:
clas,func = find_name.split('.')
pat2 = re.compile(f'@patch\s*\ndef\s+{func}\s*\([^:]*:\s*{clas}\s*(?:,|\))')
else: pat2 = None
for i,cell in enumerate(nb['cells']):
if cell['cell_type'] == 'code':
if re.search(pat, cell['source']): break
if pat2 is not None and re.search(pat2, cell['source']): break
if re.search(pat, cell['source']) is None and (pat2 is not None and re.search(pat2, cell['source']) is None):
return '' if is_name else get_function_source(func)
header_pat = re.compile(r'^\s*#+\s*(.*)$')
while i >= 0:
cell = nb['cells'][i]
if cell['cell_type'] == 'markdown' and _re_header.search(cell['source']):
title = _re_header.search(cell['source']).groups()[0]
anchor = '-'.join([s for s in title.split(' ') if len(s) > 0])
return f'{pref}{nb_name}#{anchor}'
i -= 1
return f'{pref}{nb_name}'
assert get_nb_source_link(Pipeline.decode) == get_nb_source_link(Pipeline)
assert get_nb_source_link('Pipeline') == get_nb_source_link(Pipeline)
assert get_nb_source_link(patch) == f'{FASTAI_NB_DEV}01_core_foundation.ipynb#Foundational-functions'
assert get_nb_source_link(patch, local=True) == f'01_core_foundation.ipynb#Foundational-functions'
assert get_nb_source_link('Path.ls') == f'{FASTAI_NB_DEV}01a_core_utils.ipynb#File-and-network-functions'
You can either pass an object or its name (by default is_name
will look if func
is a string or not, but you can override if there is some inconsistent behavior). local
will return a local link.
# export
def nb_source_link(func, is_name=None, disp=True):
"Show a relative link to the notebook where `func` is defined"
is_name = is_name or isinstance(func, str)
func_name = func if is_name else qual_name(func)
link = get_nb_source_link(func, local=True, is_name=is_name)
if disp: display(Markdown(f'[{func_name}]({link})'))
else: return link
This function assumes you are in one notebook in the dev folder, otherwise you use disp=False
to get the relative link. You can either pass an object or its name (by default is_name
will look if func
is a string or not, but you can override if there is some inconsistent behavior).
nb_source_link(Pipeline)
assert nb_source_link(patch, disp=False) == f'01_core_foundation.ipynb#Foundational-functions'
assert nb_source_link('patch', disp=False) == f'01_core_foundation.ipynb#Foundational-functions'
# export
def type_repr(t):
"Representation of type `t` (in a type annotation)"
if getattr(t, '__args__', None):
args = t.__args__
if len(args)==2 and args[1] == type(None):
return f'`Optional`\[{type_repr(args[0])}\]'
reprs = ', '.join([type_repr(o) for o in args])
return f'{doc_link(get_name(t))}\[{reprs}\]'
else: return doc_link(get_name(t))
The representation tries to find doc links if possible.
from torch import Tensor
tst = type_repr(Optional[Tensor])
assert tst == '`Optional`\\[[`Tensor`](https://pytorch.org/docs/stable/tensors.html#torch-tensor)\\]'
tst = type_repr(Union[Tensor, float])
assert tst == '`Union`\\[[`Tensor`](https://pytorch.org/docs/stable/tensors.html#torch-tensor), `float`\\]'
# export
_arg_prefixes = {inspect._VAR_POSITIONAL: '\*', inspect._VAR_KEYWORD:'\*\*'}
def format_param(p):
"Formats function param to `param1:Type=val`. Font weights: param1=bold, val=italic"
arg_prefix = _arg_prefixes.get(p.kind, '') # asterisk prefix for *args and **kwargs
res = f"**{arg_prefix}`{p.name}`**"
if hasattr(p, 'annotation') and p.annotation != p.empty: res += f':{type_repr(p.annotation)}'
if p.default != p.empty:
default = getattr(p.default, 'func', p.default) #For partials
default = getattr(default, '__name__', default) #Tries to find a name
if is_enum(default.__class__): #Enum have a crappy repr
res += f'=*`{default.__class__.__name__}.{default.name}`*'
else: res += f'=*`{repr(default)}`*'
return res
sig = inspect.signature(untar_data)
params = [format_param(p) for _,p in sig.parameters.items()]
assert params == [
'**`url`**',
'**`fname`**=*`None`*',
'**`dest`**=*`None`*',
"**`c_key`**=*`'data'`*",
'**`force_download`**=*`False`*',
"**`extract_func`**=*`'tar_extract'`*"]
sig = inspect.signature(compose)
params = [format_param(p) for _,p in sig.parameters.items()]
assert params[0] == '**\\*`funcs`**'
params
['**\\*`funcs`**', '**`order`**=*`None`*']
# export
def _format_enum_doc(enum, full_name):
"Formatted `enum` definition to show in documentation"
vals = ', '.join(enum.__members__.keys())
return f'<code>{full_name}</code>',f'<code>Enum</code> = [{vals}]'
tst = _format_enum_doc(e, 'e')
assert tst == ('<code>e</code>', '<code>Enum</code> = [a, b]'),tst
# export
def _escape_chars(s):
return s.replace('_', '\_')
def _format_func_doc(func, full_name=None):
"Formatted `func` definition to show in documentation"
try:
sig = inspect.signature(func)
fmt_params = [format_param(param) for name,param
in sig.parameters.items() if name not in ('self','cls')]
except: fmt_params = []
name = f'<code>{full_name or func.__name__}</code>'
arg_str = f"({', '.join(fmt_params)})"
f_name = f"<code>class</code> {name}" if inspect.isclass(func) else name
return f'{f_name}',f'{name}{arg_str}'
assert _format_func_doc(compose) == ('<code>compose</code>',
'<code>compose</code>(**\\*`funcs`**, **`order`**=*`None`*)')
# export
def _format_cls_doc(cls, full_name):
"Formatted `cls` definition to show in documentation"
parent_class = inspect.getclasstree([cls])[-1][0][1][0]
name,args = _format_func_doc(cls, full_name)
if parent_class != object: args += f' :: {doc_link(get_name(parent_class))}'
return name,args
assert _format_cls_doc(Pipeline, 'Pipeline') == ('<code>class</code> <code>Pipeline</code>',
'<code>Pipeline</code>(**`funcs`**=*`None`*, **`as_item`**=*`False`*, **`split_idx`**=*`None`*)')
# export
def show_doc(elt, doc_string=True, name=None, title_level=None, disp=True, default_cls_level=2):
"Show documentation for element `elt`. Supported types: class, function, and enum."
elt = getattr(elt, '__func__', elt)
qname = name or qual_name(elt)
if inspect.isclass(elt):
if is_enum(elt.__class__): name,args = _format_enum_doc(elt, qname)
else: name,args = _format_cls_doc (elt, qname)
elif callable(elt): name,args = _format_func_doc(elt, qname)
else: name,args = f"<code>{qname}</code>", ''
link = get_source_link(elt) #TODO: use get_source_link when it works
source_link = f'<a href="{link}" class="source_link" style="float:right">[source]</a>'
title_level = title_level or (default_cls_level if inspect.isclass(elt) else 4)
doc = f'<h{title_level} id="{qname}" class="doc_header">{name}{source_link}</h{title_level}>'
doc += f'\n\n> {args}\n\n' if len(args) > 0 else '\n\n'
if doc_string and inspect.getdoc(elt): doc += add_doc_links(inspect.getdoc(elt))
if disp: display(Markdown(doc))
else: return doc
doc_string
determines if we show the docstring of the function or not. name
can be used to provide an alternative to the name automatically found. title_level
determines the level of the anchor (default 3 for classes and 4 for functions). If disp
is False
, the function returns the markdown code instead of displaying it.
For instance
show_doc(untar_data)
will display
untar_data
[source]
untar_data
(url
,fname
=None
,dest
=None
,c_key
=ConfigKey.Data
,force_download
=False
,extract_func
=\'tar_extract\'
)
Download url
to fname
if dest
doesn't exist, and un-tgz to folder dest
.
#hide
show_doc(Pipeline)
class
Pipeline
[source]
Pipeline
(funcs
=None
,as_item
=False
,split_idx
=None
)
A pipeline of composed (for encode/decode) transforms, setup with types
#hide
show_doc(Pipeline.decode)
Pipeline.decode
[source]
Pipeline.decode
(o
,full
=True
)
#hide
show_doc(compose)
compose
[source]
compose
(***funcs
,order
*=None
*)
Create a function that composes all functions in funcs
, passing along remaining *args
and **kwargs
to all
#hide
show_doc(untar_data)
untar_data
[source]
untar_data
(url
,fname
=None
,dest
=None
,c_key
='data'
,force_download
=False
,extract_func
='tar_extract'
)
Download url
to fname
if dest
doesn't exist, and un-tgz to folder dest
.
#hide
show_doc(add_docs)
#hide
show_doc(Pipeline.__call__)
#hide
from local.data.core import DataBunch
show_doc(DataBunch.train_dl, name='DataBunch.train_dl')
DataBunch.train_dl
[source]Training DataLoader
# hide
show_doc(Path.ls)
#export
def md2html(md):
"Convert markdown `md` to HTML code"
if nbconvert.__version__ < '5.5.0': return HTMLExporter().markdown2html(md)
else: return HTMLExporter().markdown2html(defaultdict(lambda: defaultdict(dict)), md)
#export
def doc(elt):
"Show `show_doc` info in preview window"
md = show_doc(elt, disp=False)
output = md2html(md)
if IN_COLAB: get_ipython().run_cell_magic(u'html', u'', output)
else:
try: page.page({'text/html': output})
except: display(Markdown(md))
#hide
notebook2script(all_fs=True)
Converted 00_test.ipynb. Converted 01_core_foundation.ipynb. Converted 01a_core_utils.ipynb. Converted 01b_core_dispatch.ipynb. Converted 01c_core_transform.ipynb. Converted 02_core_script.ipynb. Converted 03_torchcore.ipynb. Converted 03a_layers.ipynb. Converted 04_data_load.ipynb. Converted 05_data_core.ipynb. Converted 06_data_transforms.ipynb. Converted 07_data_block.ipynb. Converted 08_vision_core.ipynb. Converted 09_vision_augment.ipynb. Converted 09a_vision_data.ipynb. Converted 10_pets_tutorial.ipynb. Converted 11_vision_models_xresnet.ipynb. Converted 12_optimizer.ipynb. Converted 13_learner.ipynb. Converted 13a_metrics.ipynb. Converted 14_callback_schedule.ipynb. Converted 14a_callback_data.ipynb. Converted 15_callback_hook.ipynb. Converted 15a_vision_models_unet.ipynb. Converted 16_callback_progress.ipynb. Converted 17_callback_tracker.ipynb. Converted 18_callback_fp16.ipynb. Converted 19_callback_mixup.ipynb. Converted 20_interpret.ipynb. Converted 20a_distributed.ipynb. Converted 21_vision_learner.ipynb. Converted 22_tutorial_imagenette.ipynb. Converted 23_tutorial_transfer_learning.ipynb. Converted 30_text_core.ipynb. Converted 31_text_data.ipynb. Converted 32_text_models_awdlstm.ipynb. Converted 33_text_models_core.ipynb. Converted 34_callback_rnn.ipynb. Converted 35_tutorial_wikitext.ipynb. Converted 36_text_models_qrnn.ipynb. Converted 37_text_learner.ipynb. Converted 38_tutorial_ulmfit.ipynb. Converted 40_tabular_core.ipynb. Converted 41_tabular_model.ipynb. Converted 42_tabular_rapids.ipynb. Converted 50_data_block_examples.ipynb. Converted 60_medical_imaging.ipynb. Converted 65_medical_text.ipynb. Converted 70_callback_wandb.ipynb. Converted 90_notebook_core.ipynb. Converted 91_notebook_export.ipynb. Converted 92_notebook_showdoc.ipynb. Converted 93_notebook_export2html.ipynb. Converted 94_notebook_test.ipynb. Converted 95_index.ipynb. Converted 96_data_external.ipynb. Converted 97_utils_test.ipynb. Converted notebook2jekyll.ipynb.