Initial commit

This commit is contained in:
ebagot committed 2026-11-13 17:35:10 +01:00
commit a206bfb5db
21 files changed
+16747

No files matched your search

+406
View File
@@ -0,0 +1,406 @@
"""English Wikipedia API client and small wikitext helpers (standard library only)."""
import html
import json
import os
import re
import time
import urllib.error
import urllib.parse
import urllib.request
API = "https://en.wikipedia.org/w/api.php"
USER_AGENT = "us-officeholders-map/1.0"
DELAY = float(os.environ.get("WIKI_DELAY", "1.5")) # seconds between requests
CACHE_DIR = os.environ.get("WIKI_CACHE_DIR") # development only: reuse fetched pages
class WikiError(Exception):
pass
class Client:
def __init__(self, progress=None):
self.progress = progress or (lambda msg: None)
self._last = 0.0
self.requests = 0
self.revisions = {} # title -> revision timestamp or id
def _get(self, params):
params = dict(params, format="json", formatversion="2", maxlag="5")
url = API + "?" + urllib.parse.urlencode(params)
for attempt in range(6):
wait = self._last + DELAY - time.time()
if wait > 0:
time.sleep(wait)
self._last = time.time()
self.requests += 1
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
retry_after = None
try:
with urllib.request.urlopen(req, timeout=60) as resp:
body = json.loads(resp.read().decode("utf-8"))
err = body.get("error")
if err and err.get("code") == "maxlag":
retry_after = 5
elif err:
raise WikiError(f"Wikipedia API error: {err.get('info') or err.get('code')}")
else:
return body
except urllib.error.HTTPError as e:
if e.code == 429 or e.code >= 500:
ra = e.headers.get("Retry-After")
retry_after = int(ra) if ra and ra.isdigit() else None
else:
raise WikiError(f"Wikipedia answered HTTP {e.code}") from e
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
if attempt == 5:
raise WikiError(f"Could not reach Wikipedia ({e})") from e
delay = retry_after if retry_after is not None else min(60, 2 * 2 ** attempt)
self.progress(f"Wikipedia is busy, retrying in {delay} s…")
time.sleep(delay)
raise WikiError("Wikipedia did not answer after several attempts")
def _cache(self, title, text=None):
if not CACHE_DIR:
return None
p = os.path.join(CACHE_DIR, re.sub(r"[^\w.-]+", "_", title) + ".wiki")
if text is None:
if os.path.exists(p):
with open(p, encoding="utf-8") as f:
return f.read()
return None
os.makedirs(CACHE_DIR, exist_ok=True)
with open(p, "w", encoding="utf-8") as f:
f.write(text)
return text
def page(self, title):
"""Wikitext of one page (action=parse), following redirects."""
cached = self._cache(title)
if cached is not None:
return cached
self.progress(f"Reading “{title}”")
body = self._get({"action": "parse", "page": title, "prop": "wikitext|revid", "redirects": "1"})
p = body.get("parse") or {}
text = p.get("wikitext")
if text is None:
raise WikiError(f"No text returned for “{title}”")
self.revisions[title] = p.get("revid")
self._cache(title, text)
return text
def pages(self, titles, label="pages"):
"""Wikitext of many pages, 50 per request (action=query). Returns {requested title: text or None}."""
out = {}
todo = []
for t in titles:
c = self._cache(t)
if c is not None:
out[t] = c
else:
todo.append(t)
for i in range(0, len(todo), 50):
chunk = todo[i:i + 50]
self.progress(f"Reading {label} {i + 1}–{i + len(chunk)} of {len(todo)}")
body = self._get({"action": "query", "prop": "revisions", "rvprop": "content|ids", "rvslots": "main",
"titles": "|".join(chunk), "redirects": "1"})
q = body.get("query", {})
norm = {n["from"]: n["to"] for n in q.get("normalized", [])}
redir = {r["from"]: r["to"] for r in q.get("redirects", [])}
by_title = {p["title"]: p for p in q.get("pages", [])}
for t in chunk:
t2 = norm.get(t, t)
t2 = redir.get(t2, t2)
p = by_title.get(t2)
if not p or p.get("missing") or not p.get("revisions"):
out[t] = None
continue
rev = p["revisions"][0]
out[t] = rev["slots"]["main"]["content"]
self.revisions[t] = rev.get("revid")
self._cache(t, out[t])
return out
# ---------------------------------------------------------------- wikitext helpers
def strip_noise(text):
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
text = re.sub(r"<ref[^>/]*/>", "", text)
text = re.sub(r"<ref[^>]*>.*?</ref>", "", text, flags=re.S)
return text
def find_template(text, name, start=0):
"""Balanced {{name ...}} starting at or after `start`; returns (start, end) or None."""
m = re.compile(r"\{\{\s*" + name, re.I).search(text, start)
if not m:
return None
i = m.start()
depth = 0
j = i
while j < len(text):
if text.startswith("{{", j):
depth += 1
j += 2
elif text.startswith("}}", j):
depth -= 1
j += 2
if depth == 0:
return i, j
else:
j += 1
return i, len(text)
def split_top(s, sep="|"):
"""Split on `sep` outside [[ ]] and {{ }}."""
parts, cur, depth, i = [], [], 0, 0
while i < len(s):
two = s[i:i + 2]
if two in ("[[", "{{"):
depth += 1
cur.append(two)
i += 2
continue
if two in ("]]", "}}") and depth > 0:
depth -= 1
cur.append(two)
i += 2
continue
if depth == 0 and s.startswith(sep, i):
parts.append("".join(cur))
cur = []
i += len(sep)
continue
cur.append(s[i])
i += 1
parts.append("".join(cur))
return parts
def template_params(tpl):
body = tpl[2:-2]
parts = split_top(body)
named, positional = {}, []
for p in parts[1:]:
if "=" in p and re.match(r"^\s*[\w ]+\s*=", p):
k, v = p.split("=", 1)
named[k.strip()] = v.strip()
else:
positional.append(p.strip())
return named, positional
def infobox(text, name="Infobox"):
loc = find_template(strip_noise(text), name)
if not loc:
return {}
t = strip_noise(text)
named, _ = template_params(t[loc[0]:loc[1]])
return named
_KEEP_FIRST = {"nowrap", "no wrap", "small", "smaller", "resize", "center", "nobold", "big", "nobr", "abbr", "lang", "notatypo", "sic", "tooltip", "h", "strong"}
def _template_plain(m):
inner = m.group(1)
parts = inner.split("|")
name = parts[0].strip().lower()
args = [p for p in parts[1:] if not re.match(r"^\s*\w+\s*=", p)]
if name == "sortname" and len(args) >= 2:
return f"{args[0].strip()} {args[1].strip()}"
if name in _KEEP_FIRST and args:
return args[0]
if name == "!":
return "|"
if name in ("dts", "start date", "birth date and age", "birth date"):
return " ".join(a.strip() for a in args[:3])
if name in ("legend", "legend0") and len(args) >= 2:
return args[1]
return ""
def plain(s):
"""Rough wikitext -> text."""
if not s:
return ""
s = strip_noise(s)
s = re.sub(r"\[\[(?:File|Image):[^\[\]]*(?:\[\[[^\]]*\]\][^\[\]]*)*\]\]", "", s, flags=re.I)
for _ in range(4):
s2 = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]|]*)\]\]", r"\1", s)
s2 = re.sub(r"\{\{([^{}]*)\}\}", _template_plain, s2)
if s2 == s:
break
s = s2
s = re.sub(r"\[https?://[^\s\]]+\s*([^\]]*)\]", r"\1", s)
s = re.sub(r"'{2,}", "", s)
s = re.sub(r"<br\s*/?>", " ", s, flags=re.I)
s = re.sub(r"<[^>]+>", "", s)
s = html.unescape(s).replace(" ", " ")
return re.sub(r"\s+", " ", s).strip()
def sections(text, level=2):
"""[(heading, body)] for headings of exactly `level` (body runs to the next heading of that level or higher)."""
eq = "=" * level
pat = re.compile(r"^" + eq + r"(?!=)\s*(.+?)\s*" + eq + r"\s*$", re.M)
heads = list(pat.finditer(text))
out = []
for i, m in enumerate(heads):
end = heads[i + 1].start() if i + 1 < len(heads) else len(text)
body = text[m.end():end]
# stop at a higher-level heading
for lv in range(1, level):
hm = re.search(r"^" + "=" * lv + r"(?!=)[^=\n].*?" + "=" * lv + r"\s*$", body, re.M)
if hm:
body = body[:hm.start()]
out.append((plain(m.group(1)), body))
return out
def section(text, title, level=2):
for h, body in sections(text, level):
if h.lower() == title.lower():
return body
return None
# ---------------------------------------------------------------- tables
class Cell:
__slots__ = ("raw", "text", "header", "attrs", "rowspan", "colspan")
def __init__(self, raw, header, attrs):
self.raw = raw.strip()
self.text = plain(raw)
self.header = header
self.attrs = attrs
rs = re.search(r"rowspan\s*=\s*\"?(\d+)", attrs)
cs = re.search(r"colspan\s*=\s*\"?(\d+)", attrs)
self.rowspan = int(rs.group(1)) if rs else 1
self.colspan = int(cs.group(1)) if cs else 1
def __repr__(self):
return f"Cell({self.text!r})"
_ATTR_TEMPLATES = re.compile(r"\{\{\s*(party shading|party color|shading pvi|usraterating)[^{}]*\}\}", re.I)
def _split_attrs(cell):
parts = split_top(cell, "|")
if len(parts) >= 2:
left = _ATTR_TEMPLATES.sub(" ", parts[0])
left = re.sub(r"[\w-]+\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s]+)", " ", left)
if not left.strip():
return parts[0], "|".join(parts[1:])
return "", cell
class Table:
def __init__(self, raw, start):
self.raw = raw
self.start = start
self.caption = ""
self.rows = [] # list of lists of Cell (spans expanded)
self.row_raw = []
self._parse()
def _parse(self):
lines = self.raw.split("\n")
rows, raws = [], []
cur, cur_raw = None, None
depth = 0
for ln in lines[1:]:
s = ln.strip()
if depth > 0:
if cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
if s.startswith("{|"):
depth += 1
if s.startswith("|}"):
depth -= 1
continue
if s.startswith("{|"):
depth += 1
if cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
continue
if s.startswith("|}"):
break
if s.startswith("|+"):
cap = s[2:]
a, content = _split_attrs(cap)
self.caption = plain(content)
continue
if s.startswith("|-"):
cur, cur_raw = [], []
rows.append(cur)
raws.append(cur_raw)
continue
if cur is None:
cur, cur_raw = [], []
rows.append(cur)
raws.append(cur_raw)
cur_raw.append(ln)
if s.startswith("!"):
for c in split_top(s[1:], "!!"):
for c2 in split_top(c, "||"):
a, content = _split_attrs(c2)
cur.append((content, True, a))
elif s.startswith("|"):
for c in split_top(s[1:], "||"):
a, content = _split_attrs(c)
cur.append((content, False, a))
elif cur:
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
grid, grid_raw, pending = [], [], {}
for r, rr in zip(rows, raws):
if not r and not pending:
continue
cells = [Cell(c, h, a) for c, h, a in r]
line, col, ci = [], 0, 0
while ci < len(cells) or col in pending:
if col in pending:
rem, c = pending[col]
line.append(c)
if rem <= 1:
del pending[col]
else:
pending[col] = (rem - 1, c)
col += 1
continue
c = cells[ci]
ci += 1
for _ in range(min(c.colspan, 50)):
line.append(c)
if c.rowspan > 1:
pending[col] = (c.rowspan - 1, c)
col += 1
if line:
grid.append(line)
grid_raw.append("\n".join(rr))
self.rows = grid
self.row_raw = grid_raw
def texts(self, i):
return [c.text for c in self.rows[i]]
def tables(text):
"""All tables (outer and nested) with their start offsets, in document order."""
out, stack = [], []
for m in re.finditer(r"^[ \t]*(\{\||\|\})", text, re.M):
if m.group(1) == "{|":
stack.append(m.start())
elif stack:
s = stack.pop()
out.append(Table(text[s:m.end()], s))
return sorted(out, key=lambda t: t.start)
def split_rows(table_text):
"""Raw row chunks of a wikitext table (split on |- lines)."""
return re.split(r"\n[ \t]*\|-[^\n]*", table_text)