Discovering Computable Math on the Web

In [173]:
import re
from operator import is_not
from functools import partial, lru_cache

from sympy import init_printing
from sympy.plotting import plot
from latex2sympy import process_sympy
from pyquery import PyQuery
from notebook.utils import url_path_join as ujoin

We're going to do some scraping... at least do some caching.

In [177]:
@lru_cache(maxsize=128)
def pq(url):
    return PyQuery(url)

There's a lot of crazy Latex out there, and a lot of math that latex2sympy can't parse yet, so here's a helper function to winnow that down some.

This PR handles the various style expressions!

In [178]:
STYLE_RE = r'{\\(display|text|script)style (.*)\}'
In [179]:
def parses(raw):
    match = re.match(STYLE_RE, raw, re.M)
    
    if match is not None:
        raw = match[2]
    try:
        return process_sympy(raw)
    except:
        pass

Wikipedia (or any other MediaWiki page with the Math extension.

TODO: How might we search for MediaWiki pages that use a given extension?

Here are some URLs and HTML selectors that have lots of neat stuff.

In [180]:
WIKIPEDIA = "https://en.wikipedia.org/wiki"
MEDIAWIKI_SELECTOR = ".mwe-math-fallback-image-inline[alt]"
MATH_WIKI = "http://math.wikia.com/wiki"
WIKIA_SELECTOR = ".tex[alt]"
In [181]:
def wiki_maths(topic, wiki=WIKIPEDIA, selector=MEDIAWIKI_SELECTOR, attrib="alt"):
    url = ujoin(wiki, topic.replace(" ", "_"))
    return list(filter(partial(is_not, None), [
        parses(img.attrib[attrib])
        for img in pq(url)(selector)
    ]))

🚀 Rockets

In [183]:
wiki_maths("Tsiolkovsky rocket equation")
Out[183]:
$$\left [ \Delta v = v_{e*text} \log{\left (\frac{m_{0}}{m_{f}},e \right )}, \quad m_{0}, \quad m_{f}, \quad v_{e*text}, \quad v_{e*text} = I_{text*(p*s)} g_{0}, \quad I_{text*(p*s)}, \quad g_{0}, \quad t = 0, \quad P_{1} = V \operatorname{left}{\left (right \left(\Delta m + m\right) \right )}, \quad P_{2} = \Delta V_{e} m + m \operatorname{left}{\left (\Delta V right + V \right )}, \quad t = 0, \quad t = 0, \quad V_{e}, \quad v_{e}, \quad V_{e} = V - v_{e}, \quad dm = - \Delta m, \quad \Delta m, \quad m \frac{d}{d t} V = - v_{e} \frac{d}{d t} m, \quad m_{0}, \quad m_{1}, \quad v_{e}, \quad m_{0} - m_{1}, \quad M_{f}, \quad \Delta v, \quad m_{1}, \quad m_{0}, \quad c, \quad \frac{m_{0}}{m_{1}} = left \left(right \frac{1 + \frac{\Delta v}{c}}{- \frac{\Delta v}{c} + 1}\right)^{\frac{c}{2 v_{e}}}, \quad \frac{m_{0}}{m_{1}}, \quad R, \quad \frac{\Delta v}{c} = \frac{R^{\frac{2 v_{e}}{c}} - 1}{R^{\frac{2 v_{e}}{c}} + 1}, \quad R^{\frac{2 v_{e}}{c}} = exp left \frac{2 v_{e}}{c} \log{\left (R right,e \right )}, \quad \tanh{\left (x \right )} = \frac{e^{2 x} - 1}{e^{2 x} + 1}, \quad \Delta v = c \tanh{\left (\operatorname{left}{\left (\frac{v_{e}}{c} \log{\left (right \frac{m_{0}}{m_{1}},e \right )} \right )} \right )}, \quad m_{0}, \quad \Delta m, \quad \frac{J}{\Delta m} = \frac{F}{p} = V_{rm*(e*(h*x))}, \quad v_{e*text} = I_{text*(p*s)} g_{0}, \quad I_{text*(p*s)}, \quad v_{e*text}, \quad g_{0}, \quad \Delta v, \quad \Delta v, \quad - \frac{1}{e^{2.15555555555556}} + 1, \quad \Delta v, \quad - \frac{1}{e^{1.11111111111111}} + 1, \quad \Delta v, \quad - \frac{1}{e^{1.04444444444444}} + 1, \quad v_{e}, \quad m{\left (t \right )}, \quad v_{e}\right ]$$

Some advanced applied concepts

In [184]:
wiki_maths("Radar")
Out[184]:
$$\left [ P_{r} = \frac{P_{t} G_{t} A_{r} F^{4} \sigma}{\left(4 \pi\right)^{2} R_{r}^{2} R_{t}^{2}}, \quad G_{r} \lambda^{2} over 4 \pi, \quad \lambda, \quad F_{D}, \quad F_{T}, \quad V_{R}, \quad C, \quad F_{D} = 2 F_{T} \operatorname{left}{\left (right \frac{V_{R}}{C} \right )}, \quad F_{D} = F_{T} \operatorname{left}{\left (right \frac{V_{R}}{C} \right )}, \quad F_{T}, \quad F_{R}, \quad F_{T}, \quad F_{R}, \quad F_{R}, \quad \left|{F_{D}}\right| < \frac{F_{R}}{2}, \quad F_{D}, \quad \left|{V_{R}}\right| < \frac{F_{R} \frac{C}{F_{T}}}{4}, \quad \sqrt{2}\right ]$$
In [185]:
wiki_maths("Kalman filter")
Out[185]:
$$\left [ hat x, \quad P_{k*(k*mid) - 1}, \quad mathbf x, \quad N \mathcal{math}, \quad mathbf w, \quad mathbf z, \quad mathbf v, \quad hat mathbf x, \quad mathbf x, \quad hat mathbf x, \quad P mathbf, \quad hat mathbf x, \quad P mathbf, \quad tilde mathbf y, \quad S mathbf, \quad K mathbf, \quad hat mathbf x, \quad P mathbf, \quad tilde mathbf y, \quad hat mathbf x, \quad P mathbf, \quad operatorname E \xi, \quad \xi, \quad F mathbf, \quad mathbf x, \quad dot x, \quad mathbf x, \quad mathbf B u, \quad G mathbf, \quad mathbf x, \quad Q mathbf, \quad \Delta t 0 neq, \quad N{\left (0,Q mathbf \right )}, \quad mathbf w, \quad mathbf z, \quad R mathbf = mathrm E left, \quad hat mathbf x, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad hat mathbf x, \quad P mathbf, \quad tilde mathbf y, \quad P mathbf, \quad mathbf z, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad mathbf x, \quad P mathbf, \quad K mathbf, \quad P mathbf, \quad P mathbf, \quad hat mathbf x, \quad P mathbf, \quad Q mathbf, \quad R mathbf, \quad P mathbf, \quad P mathbf, \quad F mathbf, \quad H mathbf, \quad Q mathbf, \quad R mathbf, \quad Q mathbf, \quad R mathbf, \quad P mathbf, \quad P mathbf, \quad Q mathbf, \quad R mathbf, \quad P mathbf, \quad P mathbf, \quad \hat{wide} mathbf x, \quad E left, \quad E left, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad P mathbf, \quad E left, \quad E left, \quad P mathbf, \quad P mathbf, \quad Q mathbf, \quad R mathbf, \quad Q mathbf, \quad R mathbf, \quad p, \quad p, \quad p, \quad x_{k - 1}, \quad p, \quad Z mathbf, \quad p, \quad p, \quad mathbf x, \quad Z mathbf, \quad mathbf x, \quad p, \quad mathbf z, \quad p, \quad k = 1, \quad mathbf x, \quad p, \quad mathbf z, \quad p, \quad p{\left (mathbf z \right )} = \prod_{k=0}^{T} p, \quad hat mathbf x, \quad H mathbf, \quad \ell = \log{\left (p{\left (mathbf z \right )},10 \right )}, \quad \frac{1}{\ell} = 0, \quad \ell^{k} = \ell^{k - 1} - \frac{left}{2}, \quad d_{y}, \quad hat mathbf x, \quad N, \quad mathbf z, \quad mathbf z, \quad hat mathbf x, \quad mathbf y, \quad hat mathbf x, \quad i = 1, \quad K^{i} mathbf = mathbf P^{i} mathbf H^{T mathrm} \frac{left}{mathbf R right + mathbf H mathbf P H^{T mathrm} mathbf}, \quad P^{i} mathbf = mathbf P left \left(right \operatorname{left}^{T mathrm}{\left (F mathbf - H K mathbf^{2} right \right )}\right)^{i}, \quad P mathbf, \quad K mathbf, \quad P mathbf, \quad P mathbf, \quad mathbf x, \quad - P mathbf + P mathbf, \quad hat mathbf x, \quad k < n, \quad mathbf z, \quad mathbf z, \quad hat mathbf x, \quad hat mathbf x, \quad P mathbf, \quad P mathbf, \quad hat mathbf x, \quad P mathbf, \quad hat mathbf x, \quad P mathbf, \quad C mathbf, \quad mathbf x, \quad k, \quad mathbf x, \quad k + 1, \quad S mathbf, \quad hat C mathbf, \quad hat mathbf x, \quad \alpha, \quad beta_{k}, \quad alpha_{k}, \quad hat mathbf y, \quad hat mathbf y, \quad - hat mathbf y + mathbf y, \quad W mathbf, \quad mathbf W \operatorname{left}{\left (- hat mathbf right y + mathbf y \right )}, \quad mathbf \frac{hat}{W} mathbf y, \quad W mathbf, \quad W mathbf, \quad mathbf x, \quad mathbf z, \quad f, \quad h, \quad left, \quad P mathbf \left(L + \lambda\right), \quad A mathbf, \quad B mathbf, \quad chi_{k*(k*mid) - 1}^{i} = f i quad \operatorname{left}{\left (chi_{-k*mid + k - 1}^{i} right \right )} = 0, \quad \frac{f}{R^{L}}, \quad \alpha, \quad \kappa, \quad \beta, \quad x, \quad \alpha = 10^{- 3}, \quad \kappa = 0, \quad \beta = 2, \quad x, \quad \beta = 2, \quad chi_{k*(k*mid) - 1}, \quad R mathbf, \quad gamma_{k}^{i} = i quad h{\left (chi_{k*(k*mid) - 1}^{i} \right )} = 0, \quad P mathbf, \quad K_{k} = P mathbf, \quad hat mathbf x, \quad P mathbf, \quad mathbf Q t, \quad mathbf R t, \quad mathbf t w, \quad mathbf t v, \quad mathbf K t = mathbf P t mathbf H^{T mathrm} t mathbf \frac{t}{R}, \quad mathbf K t, \quad mathbf R t, \quad t tilde mathbf y = - H hat mathbf^{2} t^{2} x + mathbf t z, \quad mathbf x, \quad hat mathbf x, \quad mathbf K t = 0\right ]$$

More Math

In [186]:
wiki_maths("Differential equation", MATH_WIKI, WIKIA_SELECTOR)
Out[186]:
$$\left [ \frac{d}{d x} y = x^{2} - x - 2, \quad x, \quad y{\left (x \right )}, \quad x, \quad x, \quad y{\left (x \right )}, \quad y{\left (x \right )}, \quad y{\left (x \right )} = e^{3 x} + 2 e^{2 x}\right ]$$

Math in Jupyter Markdown

The Jupyter Notebook uses Jupyter Markdown, which is a subset of Github-flavored Markdown plus inline and block math, wrapped in $ or $$. You can drop a bit of the URL for nbviewer in this function and see all the defined equations from a notebook.

In [191]:
def notebook_math(nbviewer_fragment):
    url = ujoin("https://nbviewer.jupyter.org", nbviewer_fragment)
    r = requests.get(url)
    matches = sum([
        re.findall(r'\$\$(.*?)\$\$', p.text, flags=re.M) 
        for p in pq(url)("p") if p.text
    ], [])
    return list(filter(partial(is_not, None), [
        parses(match)
        for match in matches
    ]))
In [190]:
notebook_math("github/waltherg/notebooks/blob/master/2013-12-03-Crank_Nicolson.ipynb")
Out[190]:
$$\left [ \frac{d}{d t} u = D \frac{partial^{2} u}{partial x^{2}} + f{\left (u \right )}, \quad Bigg \frac{d}{d x} u, \quad t_{n} = n \Delta t, \quad x_{j} = j \Delta x, \quad \Delta t = \frac{T}{N}, \quad \Delta x = \frac{L}{J}, \quad approx u{\left (j \Delta x,n \Delta t \right )} U{\left (j \Delta x,n \Delta t \right )}, \quad Bigg \frac{d}{d t} u, \quad Bigg \frac{partial^{2} u}{partial x^{2}}, \quad \frac{- U_{0}^{n} + U_{1}^{n}}{\Delta x} = 0, \quad \frac{d}{d t} u = D_{u} \frac{partial^{2} u}{partial x^{2}} + f{\left (u,v \right )}, \quad \frac{d}{d t} v = D_{v} \frac{partial^{2} v}{partial x^{2}} - f{\left (u,v \right )}, \quad Bigg \frac{d}{d x} u, \quad Bigg \frac{d}{d x} v, \quad A_{u} U^{n + 1} mathbf = B_{u} U^{n} mathbf + f^{n} mathbf, \quad A_{v} V^{n + 1} mathbf = B_{v} V^{n} mathbf - f^{n} mathbf\right ]$$