407 lines
14 KiB
Python
407 lines
14 KiB
Python
"""English Wikipedia API client and small wikitext helpers (standard library only)."""
|
||
import html
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
API = "https://en.wikipedia.org/w/api.php"
|
||
USER_AGENT = "us-officeholders-map/1.0"
|
||
DELAY = float(os.environ.get("WIKI_DELAY", "1.5")) # seconds between requests
|
||
CACHE_DIR = os.environ.get("WIKI_CACHE_DIR") # development only: reuse fetched pages
|
||
|
||
|
||
class WikiError(Exception):
|
||
pass
|
||
|
||
|
||
class Client:
|
||
def __init__(self, progress=None):
|
||
self.progress = progress or (lambda msg: None)
|
||
self._last = 0.0
|
||
self.requests = 0
|
||
self.revisions = {} # title -> revision timestamp or id
|
||
|
||
def _get(self, params):
|
||
params = dict(params, format="json", formatversion="2", maxlag="5")
|
||
url = API + "?" + urllib.parse.urlencode(params)
|
||
for attempt in range(6):
|
||
wait = self._last + DELAY - time.time()
|
||
if wait > 0:
|
||
time.sleep(wait)
|
||
self._last = time.time()
|
||
self.requests += 1
|
||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
|
||
retry_after = None
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||
body = json.loads(resp.read().decode("utf-8"))
|
||
err = body.get("error")
|
||
if err and err.get("code") == "maxlag":
|
||
retry_after = 5
|
||
elif err:
|
||
raise WikiError(f"Wikipedia API error: {err.get('info') or err.get('code')}")
|
||
else:
|
||
return body
|
||
except urllib.error.HTTPError as e:
|
||
if e.code == 429 or e.code >= 500:
|
||
ra = e.headers.get("Retry-After")
|
||
retry_after = int(ra) if ra and ra.isdigit() else None
|
||
else:
|
||
raise WikiError(f"Wikipedia answered HTTP {e.code}") from e
|
||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
||
if attempt == 5:
|
||
raise WikiError(f"Could not reach Wikipedia ({e})") from e
|
||
delay = retry_after if retry_after is not None else min(60, 2 * 2 ** attempt)
|
||
self.progress(f"Wikipedia is busy, retrying in {delay} s…")
|
||
time.sleep(delay)
|
||
raise WikiError("Wikipedia did not answer after several attempts")
|
||
|
||
def _cache(self, title, text=None):
|
||
if not CACHE_DIR:
|
||
return None
|
||
p = os.path.join(CACHE_DIR, re.sub(r"[^\w.-]+", "_", title) + ".wiki")
|
||
if text is None:
|
||
if os.path.exists(p):
|
||
with open(p, encoding="utf-8") as f:
|
||
return f.read()
|
||
return None
|
||
os.makedirs(CACHE_DIR, exist_ok=True)
|
||
with open(p, "w", encoding="utf-8") as f:
|
||
f.write(text)
|
||
return text
|
||
|
||
def page(self, title):
|
||
"""Wikitext of one page (action=parse), following redirects."""
|
||
cached = self._cache(title)
|
||
if cached is not None:
|
||
return cached
|
||
self.progress(f"Reading “{title}”")
|
||
body = self._get({"action": "parse", "page": title, "prop": "wikitext|revid", "redirects": "1"})
|
||
p = body.get("parse") or {}
|
||
text = p.get("wikitext")
|
||
if text is None:
|
||
raise WikiError(f"No text returned for “{title}”")
|
||
self.revisions[title] = p.get("revid")
|
||
self._cache(title, text)
|
||
return text
|
||
|
||
def pages(self, titles, label="pages"):
|
||
"""Wikitext of many pages, 50 per request (action=query). Returns {requested title: text or None}."""
|
||
out = {}
|
||
todo = []
|
||
for t in titles:
|
||
c = self._cache(t)
|
||
if c is not None:
|
||
out[t] = c
|
||
else:
|
||
todo.append(t)
|
||
for i in range(0, len(todo), 50):
|
||
chunk = todo[i:i + 50]
|
||
self.progress(f"Reading {label} {i + 1}–{i + len(chunk)} of {len(todo)}")
|
||
body = self._get({"action": "query", "prop": "revisions", "rvprop": "content|ids", "rvslots": "main",
|
||
"titles": "|".join(chunk), "redirects": "1"})
|
||
q = body.get("query", {})
|
||
norm = {n["from"]: n["to"] for n in q.get("normalized", [])}
|
||
redir = {r["from"]: r["to"] for r in q.get("redirects", [])}
|
||
by_title = {p["title"]: p for p in q.get("pages", [])}
|
||
for t in chunk:
|
||
t2 = norm.get(t, t)
|
||
t2 = redir.get(t2, t2)
|
||
p = by_title.get(t2)
|
||
if not p or p.get("missing") or not p.get("revisions"):
|
||
out[t] = None
|
||
continue
|
||
rev = p["revisions"][0]
|
||
out[t] = rev["slots"]["main"]["content"]
|
||
self.revisions[t] = rev.get("revid")
|
||
self._cache(t, out[t])
|
||
return out
|
||
|
||
|
||
# ---------------------------------------------------------------- wikitext helpers
|
||
|
||
def strip_noise(text):
|
||
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
|
||
text = re.sub(r"<ref[^>/]*/>", "", text)
|
||
text = re.sub(r"<ref[^>]*>.*?</ref>", "", text, flags=re.S)
|
||
return text
|
||
|
||
|
||
def find_template(text, name, start=0):
|
||
"""Balanced {{name ...}} starting at or after `start`; returns (start, end) or None."""
|
||
m = re.compile(r"\{\{\s*" + name, re.I).search(text, start)
|
||
if not m:
|
||
return None
|
||
i = m.start()
|
||
depth = 0
|
||
j = i
|
||
while j < len(text):
|
||
if text.startswith("{{", j):
|
||
depth += 1
|
||
j += 2
|
||
elif text.startswith("}}", j):
|
||
depth -= 1
|
||
j += 2
|
||
if depth == 0:
|
||
return i, j
|
||
else:
|
||
j += 1
|
||
return i, len(text)
|
||
|
||
|
||
def split_top(s, sep="|"):
|
||
"""Split on `sep` outside [[ ]] and {{ }}."""
|
||
parts, cur, depth, i = [], [], 0, 0
|
||
while i < len(s):
|
||
two = s[i:i + 2]
|
||
if two in ("[[", "{{"):
|
||
depth += 1
|
||
cur.append(two)
|
||
i += 2
|
||
continue
|
||
if two in ("]]", "}}") and depth > 0:
|
||
depth -= 1
|
||
cur.append(two)
|
||
i += 2
|
||
continue
|
||
if depth == 0 and s.startswith(sep, i):
|
||
parts.append("".join(cur))
|
||
cur = []
|
||
i += len(sep)
|
||
continue
|
||
cur.append(s[i])
|
||
i += 1
|
||
parts.append("".join(cur))
|
||
return parts
|
||
|
||
|
||
def template_params(tpl):
|
||
body = tpl[2:-2]
|
||
parts = split_top(body)
|
||
named, positional = {}, []
|
||
for p in parts[1:]:
|
||
if "=" in p and re.match(r"^\s*[\w ]+\s*=", p):
|
||
k, v = p.split("=", 1)
|
||
named[k.strip()] = v.strip()
|
||
else:
|
||
positional.append(p.strip())
|
||
return named, positional
|
||
|
||
|
||
def infobox(text, name="Infobox"):
|
||
loc = find_template(strip_noise(text), name)
|
||
if not loc:
|
||
return {}
|
||
t = strip_noise(text)
|
||
named, _ = template_params(t[loc[0]:loc[1]])
|
||
return named
|
||
|
||
|
||
_KEEP_FIRST = {"nowrap", "no wrap", "small", "smaller", "resize", "center", "nobold", "big", "nobr", "abbr", "lang", "notatypo", "sic", "tooltip", "h", "strong"}
|
||
|
||
|
||
def _template_plain(m):
|
||
inner = m.group(1)
|
||
parts = inner.split("|")
|
||
name = parts[0].strip().lower()
|
||
args = [p for p in parts[1:] if not re.match(r"^\s*\w+\s*=", p)]
|
||
if name == "sortname" and len(args) >= 2:
|
||
return f"{args[0].strip()} {args[1].strip()}"
|
||
if name in _KEEP_FIRST and args:
|
||
return args[0]
|
||
if name == "!":
|
||
return "|"
|
||
if name in ("dts", "start date", "birth date and age", "birth date"):
|
||
return " ".join(a.strip() for a in args[:3])
|
||
if name in ("legend", "legend0") and len(args) >= 2:
|
||
return args[1]
|
||
return ""
|
||
|
||
|
||
def plain(s):
|
||
"""Rough wikitext -> text."""
|
||
if not s:
|
||
return ""
|
||
s = strip_noise(s)
|
||
s = re.sub(r"\[\[(?:File|Image):[^\[\]]*(?:\[\[[^\]]*\]\][^\[\]]*)*\]\]", "", s, flags=re.I)
|
||
for _ in range(4):
|
||
s2 = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]|]*)\]\]", r"\1", s)
|
||
s2 = re.sub(r"\{\{([^{}]*)\}\}", _template_plain, s2)
|
||
if s2 == s:
|
||
break
|
||
s = s2
|
||
s = re.sub(r"\[https?://[^\s\]]+\s*([^\]]*)\]", r"\1", s)
|
||
s = re.sub(r"'{2,}", "", s)
|
||
s = re.sub(r"<br\s*/?>", " ", s, flags=re.I)
|
||
s = re.sub(r"<[^>]+>", "", s)
|
||
s = html.unescape(s).replace(" ", " ")
|
||
return re.sub(r"\s+", " ", s).strip()
|
||
|
||
|
||
def sections(text, level=2):
|
||
"""[(heading, body)] for headings of exactly `level` (body runs to the next heading of that level or higher)."""
|
||
eq = "=" * level
|
||
pat = re.compile(r"^" + eq + r"(?!=)\s*(.+?)\s*" + eq + r"\s*$", re.M)
|
||
heads = list(pat.finditer(text))
|
||
out = []
|
||
for i, m in enumerate(heads):
|
||
end = heads[i + 1].start() if i + 1 < len(heads) else len(text)
|
||
body = text[m.end():end]
|
||
# stop at a higher-level heading
|
||
for lv in range(1, level):
|
||
hm = re.search(r"^" + "=" * lv + r"(?!=)[^=\n].*?" + "=" * lv + r"\s*$", body, re.M)
|
||
if hm:
|
||
body = body[:hm.start()]
|
||
out.append((plain(m.group(1)), body))
|
||
return out
|
||
|
||
|
||
def section(text, title, level=2):
|
||
for h, body in sections(text, level):
|
||
if h.lower() == title.lower():
|
||
return body
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------- tables
|
||
|
||
class Cell:
|
||
__slots__ = ("raw", "text", "header", "attrs", "rowspan", "colspan")
|
||
|
||
def __init__(self, raw, header, attrs):
|
||
self.raw = raw.strip()
|
||
self.text = plain(raw)
|
||
self.header = header
|
||
self.attrs = attrs
|
||
rs = re.search(r"rowspan\s*=\s*\"?(\d+)", attrs)
|
||
cs = re.search(r"colspan\s*=\s*\"?(\d+)", attrs)
|
||
self.rowspan = int(rs.group(1)) if rs else 1
|
||
self.colspan = int(cs.group(1)) if cs else 1
|
||
|
||
def __repr__(self):
|
||
return f"Cell({self.text!r})"
|
||
|
||
|
||
_ATTR_TEMPLATES = re.compile(r"\{\{\s*(party shading|party color|shading pvi|usraterating)[^{}]*\}\}", re.I)
|
||
|
||
|
||
def _split_attrs(cell):
|
||
parts = split_top(cell, "|")
|
||
if len(parts) >= 2:
|
||
left = _ATTR_TEMPLATES.sub(" ", parts[0])
|
||
left = re.sub(r"[\w-]+\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s]+)", " ", left)
|
||
if not left.strip():
|
||
return parts[0], "|".join(parts[1:])
|
||
return "", cell
|
||
|
||
|
||
class Table:
|
||
def __init__(self, raw, start):
|
||
self.raw = raw
|
||
self.start = start
|
||
self.caption = ""
|
||
self.rows = [] # list of lists of Cell (spans expanded)
|
||
self.row_raw = []
|
||
self._parse()
|
||
|
||
def _parse(self):
|
||
lines = self.raw.split("\n")
|
||
rows, raws = [], []
|
||
cur, cur_raw = None, None
|
||
depth = 0
|
||
for ln in lines[1:]:
|
||
s = ln.strip()
|
||
if depth > 0:
|
||
if cur:
|
||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||
if s.startswith("{|"):
|
||
depth += 1
|
||
if s.startswith("|}"):
|
||
depth -= 1
|
||
continue
|
||
if s.startswith("{|"):
|
||
depth += 1
|
||
if cur:
|
||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||
continue
|
||
if s.startswith("|}"):
|
||
break
|
||
if s.startswith("|+"):
|
||
cap = s[2:]
|
||
a, content = _split_attrs(cap)
|
||
self.caption = plain(content)
|
||
continue
|
||
if s.startswith("|-"):
|
||
cur, cur_raw = [], []
|
||
rows.append(cur)
|
||
raws.append(cur_raw)
|
||
continue
|
||
if cur is None:
|
||
cur, cur_raw = [], []
|
||
rows.append(cur)
|
||
raws.append(cur_raw)
|
||
cur_raw.append(ln)
|
||
if s.startswith("!"):
|
||
for c in split_top(s[1:], "!!"):
|
||
for c2 in split_top(c, "||"):
|
||
a, content = _split_attrs(c2)
|
||
cur.append((content, True, a))
|
||
elif s.startswith("|"):
|
||
for c in split_top(s[1:], "||"):
|
||
a, content = _split_attrs(c)
|
||
cur.append((content, False, a))
|
||
elif cur:
|
||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||
grid, grid_raw, pending = [], [], {}
|
||
for r, rr in zip(rows, raws):
|
||
if not r and not pending:
|
||
continue
|
||
cells = [Cell(c, h, a) for c, h, a in r]
|
||
line, col, ci = [], 0, 0
|
||
while ci < len(cells) or col in pending:
|
||
if col in pending:
|
||
rem, c = pending[col]
|
||
line.append(c)
|
||
if rem <= 1:
|
||
del pending[col]
|
||
else:
|
||
pending[col] = (rem - 1, c)
|
||
col += 1
|
||
continue
|
||
c = cells[ci]
|
||
ci += 1
|
||
for _ in range(min(c.colspan, 50)):
|
||
line.append(c)
|
||
if c.rowspan > 1:
|
||
pending[col] = (c.rowspan - 1, c)
|
||
col += 1
|
||
if line:
|
||
grid.append(line)
|
||
grid_raw.append("\n".join(rr))
|
||
self.rows = grid
|
||
self.row_raw = grid_raw
|
||
|
||
def texts(self, i):
|
||
return [c.text for c in self.rows[i]]
|
||
|
||
|
||
def tables(text):
|
||
"""All tables (outer and nested) with their start offsets, in document order."""
|
||
out, stack = [], []
|
||
for m in re.finditer(r"^[ \t]*(\{\||\|\})", text, re.M):
|
||
if m.group(1) == "{|":
|
||
stack.append(m.start())
|
||
elif stack:
|
||
s = stack.pop()
|
||
out.append(Table(text[s:m.end()], s))
|
||
return sorted(out, key=lambda t: t.start)
|
||
|
||
|
||
def split_rows(table_text):
|
||
"""Raw row chunks of a wikitext table (split on |- lines)."""
|
||
return re.split(r"\n[ \t]*\|-[^\n]*", table_text)
|