#default_exp page

#export
from fastcore.utils import *
from fastcore.foundation import *
from ghapi.core import *

import re
from urllib.parse import parse_qs,urlsplit

api = GhApi()

gists = api.gists.list_public()
len(gists)

api.gists.list_public

len(api.gists.list_public(per_page=100))

#export
def paged(oper, *args, per_page=30, max_pages=9999, **kwargs):
    "Convert operation `oper(*args,**kwargs)` into an iterator"
    yield from itertools.takewhile(noop, (oper(*args, per_page=per_page, page=i, **kwargs) for i in range(1,max_pages+1)))

api.repos.list_for_org

repos = api.repos.list_for_org('fastai')
len(repos),repos[0].name

repos = paged(api.repos.list_for_org, 'fastai')

for page in repos: print(len(page), page[0].name)

#export
class _Scanner:
    def __init__(self, buf): self.buf,self.match = buf,None
    def __getitem__(self, key): return self.match.group(key)
    def scan(self, pattern):
        self.match = re.compile(pattern).match(self.buf)
        if self.match: self.buf = self.buf[self.match.end():]
        return self.match

_QUOTED        = r'"((?:[^"\\]|\\.)*)"'
_TOKEN         = r'([^()<>@,;:\"\[\]?={}\s]+)'
_RE_COMMA_HREF = r' *,? *< *([^>]*) *> *'
_RE_ATTR       = rf'{_TOKEN} *(?:= *({_TOKEN}|{_QUOTED}))? *'

#export
def _parse_link_hdr(header):
    "Parse an RFC 5988 link header, returning a `list` of `tuple`s of URL and attr `dict`"
    scanner,links = _Scanner(header),[]
    while scanner.scan(_RE_COMMA_HREF):
        href,attrs = scanner[1],[]
        while scanner.scan('; *'):
            if scanner.scan(_RE_ATTR):
                attr_name, token, quoted = scanner[1], scanner[3], scanner[4]
                if quoted is not None: attrs.append([attr_name, quoted.replace(r'\"', '"')])
                elif token is not None: attrs.append([attr_name, token])
                else: attrs.append([attr_name, None])
        links.append((href,dict(attrs)))
    if scanner.buf: raise Exception(f"parse() failed at {scanner.buf!r}")
    return links

#export
def parse_link_hdr(header):
    "Parse an RFC 5988 link header, returning a `dict` from rels to a `tuple` of URL and attrs `dict`"
    return {a.pop('rel'):(u,a) for u,a in _parse_link_hdr(header)}

parse_link_hdr('<http://example.com>; rel="foo bar"; type=text/html')

links = parse_link_hdr('<http://example.com>; rel="foo bar"; type=text/html')
link = links['foo bar']
test_eq(link[0], 'http://example.com')
test_eq(link[1]['type'], 'text/html')

api.recv_hdrs['Link']

parse_link_hdr(api.recv_hdrs['Link'])

#export
@patch
def last_page(self:GhApi):
    "Parse RFC 5988 link header from most recent operation, and extract the last page"
    header = self.recv_hdrs.get('Link', '')
    last = nested_idx(parse_link_hdr(header), 'last', 0) or ''
    qs = parse_qs(urlsplit(last).query)
    return int(nested_idx(qs,'page',0) or 0)

api.repos.list_for_org('github', per_page=100)
api.last_page()

#export
def _call_page(i, oper, args, kwargs, per_page):
    return oper(*args, per_page=per_page, page=i, **kwargs)

#export
def pages(oper, n_pages, *args, n_workers=None, per_page=100, **kwargs):
    "Get `n_pages` pages from `oper(*args,**kwargs)`"
    return parallel(_call_page, range(1,n_pages+1), oper=oper, per_page=per_page, args=args, kwargs=kwargs,
                    progress=False, n_workers=ifnone(n_workers,n_pages), threadpool=True)

gh_repos = pages(api.repos.list_for_org, api.last_page(), 'github').concat()
len(gh_repos)

gists = pages(api.gists.list_public, 30).concat()
len(gists)

api.activity.list_public_events()
api.last_page()

evts = pages(api.activity.list_public_events, api.last_page(), per_page=30).concat()
len(evts)

#hide
from nbdev.export import notebook2script
notebook2script()