Files
2026-11-13 17:35:10 +01:00

407 lines
14 KiB
Python
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""English Wikipedia API client and small wikitext helpers (standard library only)."""
import html
import json
import os
import re
import time
import urllib.error
import urllib.parse
import urllib.request
API = "https://en.wikipedia.org/w/api.php"
USER_AGENT = "us-officeholders-map/1.0"
DELAY = float(os.environ.get("WIKI_DELAY", "1.5")) # seconds between requests
CACHE_DIR = os.environ.get("WIKI_CACHE_DIR") # development only: reuse fetched pages
class WikiError(Exception):
pass
class Client:
def __init__(self, progress=None):
self.progress = progress or (lambda msg: None)
self._last = 0.0
self.requests = 0
self.revisions = {} # title -> revision timestamp or id
def _get(self, params):
params = dict(params, format="json", formatversion="2", maxlag="5")
url = API + "?" + urllib.parse.urlencode(params)
for attempt in range(6):
wait = self._last + DELAY - time.time()
if wait > 0:
time.sleep(wait)
self._last = time.time()
self.requests += 1
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
retry_after = None
try:
with urllib.request.urlopen(req, timeout=60) as resp:
body = json.loads(resp.read().decode("utf-8"))
err = body.get("error")
if err and err.get("code") == "maxlag":
retry_after = 5
elif err:
raise WikiError(f"Wikipedia API error: {err.get('info') or err.get('code')}")
else:
return body
except urllib.error.HTTPError as e:
if e.code == 429 or e.code >= 500:
ra = e.headers.get("Retry-After")
retry_after = int(ra) if ra and ra.isdigit() else None
else:
raise WikiError(f"Wikipedia answered HTTP {e.code}") from e
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
if attempt == 5:
raise WikiError(f"Could not reach Wikipedia ({e})") from e
delay = retry_after if retry_after is not None else min(60, 2 * 2 ** attempt)
self.progress(f"Wikipedia is busy, retrying in {delay} s…")
time.sleep(delay)
raise WikiError("Wikipedia did not answer after several attempts")
def _cache(self, title, text=None):
if not CACHE_DIR:
return None
p = os.path.join(CACHE_DIR, re.sub(r"[^\w.-]+", "_", title) + ".wiki")
if text is None:
if os.path.exists(p):
with open(p, encoding="utf-8") as f:
return f.read()
return None
os.makedirs(CACHE_DIR, exist_ok=True)
with open(p, "w", encoding="utf-8") as f:
f.write(text)
return text
def page(self, title):
"""Wikitext of one page (action=parse), following redirects."""
cached = self._cache(title)
if cached is not None:
return cached
self.progress(f"Reading “{title}”")
body = self._get({"action": "parse", "page": title, "prop": "wikitext|revid", "redirects": "1"})
p = body.get("parse") or {}
text = p.get("wikitext")
if text is None:
raise WikiError(f"No text returned for “{title}”")
self.revisions[title] = p.get("revid")
self._cache(title, text)
return text
def pages(self, titles, label="pages"):
"""Wikitext of many pages, 50 per request (action=query). Returns {requested title: text or None}."""
out = {}
todo = []
for t in titles:
c = self._cache(t)
if c is not None:
out[t] = c
else:
todo.append(t)
for i in range(0, len(todo), 50):
chunk = todo[i:i + 50]
self.progress(f"Reading {label} {i + 1}–{i + len(chunk)} of {len(todo)}")
body = self._get({"action": "query", "prop": "revisions", "rvprop": "content|ids", "rvslots": "main",
"titles": "|".join(chunk), "redirects": "1"})
q = body.get("query", {})
norm = {n["from"]: n["to"] for n in q.get("normalized", [])}
redir = {r["from"]: r["to"] for r in q.get("redirects", [])}
by_title = {p["title"]: p for p in q.get("pages", [])}
for t in chunk:
t2 = norm.get(t, t)
t2 = redir.get(t2, t2)
p = by_title.get(t2)
if not p or p.get("missing") or not p.get("revisions"):
out[t] = None
continue
rev = p["revisions"][0]
out[t] = rev["slots"]["main"]["content"]
self.revisions[t] = rev.get("revid")
self._cache(t, out[t])
return out
# ---------------------------------------------------------------- wikitext helpers
def strip_noise(text):
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
text = re.sub(r"<ref[^>/]*/>", "", text)
text = re.sub(r"<ref[^>]*>.*?</ref>", "", text, flags=re.S)
return text
def find_template(text, name, start=0):
"""Balanced {{name ...}} starting at or after `start`; returns (start, end) or None."""
m = re.compile(r"\{\{\s*" + name, re.I).search(text, start)
if not m:
return None
i = m.start()
depth = 0
j = i
while j < len(text):
if text.startswith("{{", j):
depth += 1
j += 2
elif text.startswith("}}", j):
depth -= 1
j += 2
if depth == 0:
return i, j
else:
j += 1
return i, len(text)
def split_top(s, sep="|"):
"""Split on `sep` outside [[ ]] and {{ }}."""
parts, cur, depth, i = [], [], 0, 0
while i < len(s):
two = s[i:i + 2]
if two in ("[[", "{{"):
depth += 1
cur.append(two)
i += 2
continue
if two in ("]]", "}}") and depth > 0:
depth -= 1
cur.append(two)
i += 2
continue
if depth == 0 and s.startswith(sep, i):
parts.append("".join(cur))
cur = []
i += len(sep)
continue
cur.append(s[i])
i += 1
parts.append("".join(cur))
return parts
def template_params(tpl):
body = tpl[2:-2]
parts = split_top(body)
named, positional = {}, []
for p in parts[1:]:
if "=" in p and re.match(r"^\s*[\w ]+\s*=", p):
k, v = p.split("=", 1)
named[k.strip()] = v.strip()
else:
positional.append(p.strip())
return named, positional
def infobox(text, name="Infobox"):
loc = find_template(strip_noise(text), name)
if not loc:
return {}
t = strip_noise(text)
named, _ = template_params(t[loc[0]:loc[1]])
return named
_KEEP_FIRST = {"nowrap", "no wrap", "small", "smaller", "resize", "center", "nobold", "big", "nobr", "abbr", "lang", "notatypo", "sic", "tooltip", "h", "strong"}
def _template_plain(m):
inner = m.group(1)
parts = inner.split("|")
name = parts[0].strip().lower()
args = [p for p in parts[1:] if not re.match(r"^\s*\w+\s*=", p)]
if name == "sortname" and len(args) >= 2:
return f"{args[0].strip()} {args[1].strip()}"
if name in _KEEP_FIRST and args:
return args[0]
if name == "!":
return "|"
if name in ("dts", "start date", "birth date and age", "birth date"):
return " ".join(a.strip() for a in args[:3])
if name in ("legend", "legend0") and len(args) >= 2:
return args[1]
return ""
def plain(s):
"""Rough wikitext -> text."""
if not s:
return ""
s = strip_noise(s)
s = re.sub(r"\[\[(?:File|Image):[^\[\]]*(?:\[\[[^\]]*\]\][^\[\]]*)*\]\]", "", s, flags=re.I)
for _ in range(4):
s2 = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]|]*)\]\]", r"\1", s)
s2 = re.sub(r"\{\{([^{}]*)\}\}", _template_plain, s2)
if s2 == s:
break
s = s2
s = re.sub(r"\[https?://[^\s\]]+\s*([^\]]*)\]", r"\1", s)
s = re.sub(r"'{2,}", "", s)
s = re.sub(r"<br\s*/?>", " ", s, flags=re.I)
s = re.sub(r"<[^>]+>", "", s)
s = html.unescape(s).replace(" ", " ")
return re.sub(r"\s+", " ", s).strip()
def sections(text, level=2):
"""[(heading, body)] for headings of exactly `level` (body runs to the next heading of that level or higher)."""
eq = "=" * level
pat = re.compile(r"^" + eq + r"(?!=)\s*(.+?)\s*" + eq + r"\s*$", re.M)
heads = list(pat.finditer(text))
out = []
for i, m in enumerate(heads):
end = heads[i + 1].start() if i + 1 < len(heads) else len(text)
body = text[m.end():end]
# stop at a higher-level heading
for lv in range(1, level):
hm = re.search(r"^" + "=" * lv + r"(?!=)[^=\n].*?" + "=" * lv + r"\s*$", body, re.M)
if hm:
body = body[:hm.start()]
out.append((plain(m.group(1)), body))
return out
def section(text, title, level=2):
for h, body in sections(text, level):
if h.lower() == title.lower():
return body
return None
# ---------------------------------------------------------------- tables
class Cell:
__slots__ = ("raw", "text", "header", "attrs", "rowspan", "colspan")
def __init__(self, raw, header, attrs):
self.raw = raw.strip()
self.text = plain(raw)
self.header = header
self.attrs = attrs
rs = re.search(r"rowspan\s*=\s*\"?(\d+)", attrs)
cs = re.search(r"colspan\s*=\s*\"?(\d+)", attrs)
self.rowspan = int(rs.group(1)) if rs else 1
self.colspan = int(cs.group(1)) if cs else 1
def __repr__(self):
return f"Cell({self.text!r})"
_ATTR_TEMPLATES = re.compile(r"\{\{\s*(party shading|party color|shading pvi|usraterating)[^{}]*\}\}", re.I)
def _split_attrs(cell):
parts = split_top(cell, "|")
if len(parts) >= 2:
left = _ATTR_TEMPLATES.sub(" ", parts[0])
left = re.sub(r"[\w-]+\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s]+)", " ", left)
if not left.strip():
return parts[0], "|".join(parts[1:])
return "", cell
class Table:
def __init__(self, raw, start):
self.raw = raw
self.start = start
self.caption = ""
self.rows = [] # list of lists of Cell (spans expanded)
self.row_raw = []
self._parse()
def _parse(self):
lines = self.raw.split("\n")
rows, raws = [], []
cur, cur_raw = None, None
depth = 0
for ln in lines[1:]:
s = ln.strip()
if depth > 0:
if cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
if s.startswith("{|"):
depth += 1
if s.startswith("|}"):
depth -= 1
continue
if s.startswith("{|"):
depth += 1
if cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
continue
if s.startswith("|}"):
break
if s.startswith("|+"):
cap = s[2:]
a, content = _split_attrs(cap)
self.caption = plain(content)
continue
if s.startswith("|-"):
cur, cur_raw = [], []
rows.append(cur)
raws.append(cur_raw)
continue
if cur is None:
cur, cur_raw = [], []
rows.append(cur)
raws.append(cur_raw)
cur_raw.append(ln)
if s.startswith("!"):
for c in split_top(s[1:], "!!"):
for c2 in split_top(c, "||"):
a, content = _split_attrs(c2)
cur.append((content, True, a))
elif s.startswith("|"):
for c in split_top(s[1:], "||"):
a, content = _split_attrs(c)
cur.append((content, False, a))
elif cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
grid, grid_raw, pending = [], [], {}
for r, rr in zip(rows, raws):
if not r and not pending:
continue
cells = [Cell(c, h, a) for c, h, a in r]
line, col, ci = [], 0, 0
while ci < len(cells) or col in pending:
if col in pending:
rem, c = pending[col]
line.append(c)
if rem <= 1:
del pending[col]
else:
pending[col] = (rem - 1, c)
col += 1
continue
c = cells[ci]
ci += 1
for _ in range(min(c.colspan, 50)):
line.append(c)
if c.rowspan > 1:
pending[col] = (c.rowspan - 1, c)
col += 1
if line:
grid.append(line)
grid_raw.append("\n".join(rr))
self.rows = grid
self.row_raw = grid_raw
def texts(self, i):
return [c.text for c in self.rows[i]]
def tables(text):
"""All tables (outer and nested) with their start offsets, in document order."""
out, stack = [], []
for m in re.finditer(r"^[ \t]*(\{\||\|\})", text, re.M):
if m.group(1) == "{|":
stack.append(m.start())
elif stack:
s = stack.pop()
out.append(Table(text[s:m.end()], s))
return sorted(out, key=lambda t: t.start)
def split_rows(table_text):
"""Raw row chunks of a wikitext table (split on |- lines)."""
return re.split(r"\n[ \t]*\|-[^\n]*", table_text)