"""English Wikipedia API client and small wikitext helpers (standard library only).""" import html import json import os import re import time import urllib.error import urllib.parse import urllib.request API = "https://en.wikipedia.org/w/api.php" USER_AGENT = "us-officeholders-map/1.0" DELAY = float(os.environ.get("WIKI_DELAY", "1.5")) # seconds between requests CACHE_DIR = os.environ.get("WIKI_CACHE_DIR") # development only: reuse fetched pages class WikiError(Exception): pass class Client: def __init__(self, progress=None): self.progress = progress or (lambda msg: None) self._last = 0.0 self.requests = 0 self.revisions = {} # title -> revision timestamp or id def _get(self, params): params = dict(params, format="json", formatversion="2", maxlag="5") url = API + "?" + urllib.parse.urlencode(params) for attempt in range(6): wait = self._last + DELAY - time.time() if wait > 0: time.sleep(wait) self._last = time.time() self.requests += 1 req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"}) retry_after = None try: with urllib.request.urlopen(req, timeout=60) as resp: body = json.loads(resp.read().decode("utf-8")) err = body.get("error") if err and err.get("code") == "maxlag": retry_after = 5 elif err: raise WikiError(f"Wikipedia API error: {err.get('info') or err.get('code')}") else: return body except urllib.error.HTTPError as e: if e.code == 429 or e.code >= 500: ra = e.headers.get("Retry-After") retry_after = int(ra) if ra and ra.isdigit() else None else: raise WikiError(f"Wikipedia answered HTTP {e.code}") from e except (urllib.error.URLError, TimeoutError, ConnectionError) as e: if attempt == 5: raise WikiError(f"Could not reach Wikipedia ({e})") from e delay = retry_after if retry_after is not None else min(60, 2 * 2 ** attempt) self.progress(f"Wikipedia is busy, retrying in {delay} s…") time.sleep(delay) raise WikiError("Wikipedia did not answer after several attempts") def _cache(self, title, text=None): if not CACHE_DIR: return None p = os.path.join(CACHE_DIR, re.sub(r"[^\w.-]+", "_", title) + ".wiki") if text is None: if os.path.exists(p): with open(p, encoding="utf-8") as f: return f.read() return None os.makedirs(CACHE_DIR, exist_ok=True) with open(p, "w", encoding="utf-8") as f: f.write(text) return text def page(self, title): """Wikitext of one page (action=parse), following redirects.""" cached = self._cache(title) if cached is not None: return cached self.progress(f"Reading “{title}”") body = self._get({"action": "parse", "page": title, "prop": "wikitext|revid", "redirects": "1"}) p = body.get("parse") or {} text = p.get("wikitext") if text is None: raise WikiError(f"No text returned for “{title}”") self.revisions[title] = p.get("revid") self._cache(title, text) return text def pages(self, titles, label="pages"): """Wikitext of many pages, 50 per request (action=query). Returns {requested title: text or None}.""" out = {} todo = [] for t in titles: c = self._cache(t) if c is not None: out[t] = c else: todo.append(t) for i in range(0, len(todo), 50): chunk = todo[i:i + 50] self.progress(f"Reading {label} {i + 1}–{i + len(chunk)} of {len(todo)}") body = self._get({"action": "query", "prop": "revisions", "rvprop": "content|ids", "rvslots": "main", "titles": "|".join(chunk), "redirects": "1"}) q = body.get("query", {}) norm = {n["from"]: n["to"] for n in q.get("normalized", [])} redir = {r["from"]: r["to"] for r in q.get("redirects", [])} by_title = {p["title"]: p for p in q.get("pages", [])} for t in chunk: t2 = norm.get(t, t) t2 = redir.get(t2, t2) p = by_title.get(t2) if not p or p.get("missing") or not p.get("revisions"): out[t] = None continue rev = p["revisions"][0] out[t] = rev["slots"]["main"]["content"] self.revisions[t] = rev.get("revid") self._cache(t, out[t]) return out # ---------------------------------------------------------------- wikitext helpers def strip_noise(text): text = re.sub(r"", "", text, flags=re.S) text = re.sub(r"/]*/>", "", text) text = re.sub(r"]*>.*?", "", text, flags=re.S) return text def find_template(text, name, start=0): """Balanced {{name ...}} starting at or after `start`; returns (start, end) or None.""" m = re.compile(r"\{\{\s*" + name, re.I).search(text, start) if not m: return None i = m.start() depth = 0 j = i while j < len(text): if text.startswith("{{", j): depth += 1 j += 2 elif text.startswith("}}", j): depth -= 1 j += 2 if depth == 0: return i, j else: j += 1 return i, len(text) def split_top(s, sep="|"): """Split on `sep` outside [[ ]] and {{ }}.""" parts, cur, depth, i = [], [], 0, 0 while i < len(s): two = s[i:i + 2] if two in ("[[", "{{"): depth += 1 cur.append(two) i += 2 continue if two in ("]]", "}}") and depth > 0: depth -= 1 cur.append(two) i += 2 continue if depth == 0 and s.startswith(sep, i): parts.append("".join(cur)) cur = [] i += len(sep) continue cur.append(s[i]) i += 1 parts.append("".join(cur)) return parts def template_params(tpl): body = tpl[2:-2] parts = split_top(body) named, positional = {}, [] for p in parts[1:]: if "=" in p and re.match(r"^\s*[\w ]+\s*=", p): k, v = p.split("=", 1) named[k.strip()] = v.strip() else: positional.append(p.strip()) return named, positional def infobox(text, name="Infobox"): loc = find_template(strip_noise(text), name) if not loc: return {} t = strip_noise(text) named, _ = template_params(t[loc[0]:loc[1]]) return named _KEEP_FIRST = {"nowrap", "no wrap", "small", "smaller", "resize", "center", "nobold", "big", "nobr", "abbr", "lang", "notatypo", "sic", "tooltip", "h", "strong"} def _template_plain(m): inner = m.group(1) parts = inner.split("|") name = parts[0].strip().lower() args = [p for p in parts[1:] if not re.match(r"^\s*\w+\s*=", p)] if name == "sortname" and len(args) >= 2: return f"{args[0].strip()} {args[1].strip()}" if name in _KEEP_FIRST and args: return args[0] if name == "!": return "|" if name in ("dts", "start date", "birth date and age", "birth date"): return " ".join(a.strip() for a in args[:3]) if name in ("legend", "legend0") and len(args) >= 2: return args[1] return "" def plain(s): """Rough wikitext -> text.""" if not s: return "" s = strip_noise(s) s = re.sub(r"\[\[(?:File|Image):[^\[\]]*(?:\[\[[^\]]*\]\][^\[\]]*)*\]\]", "", s, flags=re.I) for _ in range(4): s2 = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]|]*)\]\]", r"\1", s) s2 = re.sub(r"\{\{([^{}]*)\}\}", _template_plain, s2) if s2 == s: break s = s2 s = re.sub(r"\[https?://[^\s\]]+\s*([^\]]*)\]", r"\1", s) s = re.sub(r"'{2,}", "", s) s = re.sub(r"", " ", s, flags=re.I) s = re.sub(r"<[^>]+>", "", s) s = html.unescape(s).replace(" ", " ") return re.sub(r"\s+", " ", s).strip() def sections(text, level=2): """[(heading, body)] for headings of exactly `level` (body runs to the next heading of that level or higher).""" eq = "=" * level pat = re.compile(r"^" + eq + r"(?!=)\s*(.+?)\s*" + eq + r"\s*$", re.M) heads = list(pat.finditer(text)) out = [] for i, m in enumerate(heads): end = heads[i + 1].start() if i + 1 < len(heads) else len(text) body = text[m.end():end] # stop at a higher-level heading for lv in range(1, level): hm = re.search(r"^" + "=" * lv + r"(?!=)[^=\n].*?" + "=" * lv + r"\s*$", body, re.M) if hm: body = body[:hm.start()] out.append((plain(m.group(1)), body)) return out def section(text, title, level=2): for h, body in sections(text, level): if h.lower() == title.lower(): return body return None # ---------------------------------------------------------------- tables class Cell: __slots__ = ("raw", "text", "header", "attrs", "rowspan", "colspan") def __init__(self, raw, header, attrs): self.raw = raw.strip() self.text = plain(raw) self.header = header self.attrs = attrs rs = re.search(r"rowspan\s*=\s*\"?(\d+)", attrs) cs = re.search(r"colspan\s*=\s*\"?(\d+)", attrs) self.rowspan = int(rs.group(1)) if rs else 1 self.colspan = int(cs.group(1)) if cs else 1 def __repr__(self): return f"Cell({self.text!r})" _ATTR_TEMPLATES = re.compile(r"\{\{\s*(party shading|party color|shading pvi|usraterating)[^{}]*\}\}", re.I) def _split_attrs(cell): parts = split_top(cell, "|") if len(parts) >= 2: left = _ATTR_TEMPLATES.sub(" ", parts[0]) left = re.sub(r"[\w-]+\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s]+)", " ", left) if not left.strip(): return parts[0], "|".join(parts[1:]) return "", cell class Table: def __init__(self, raw, start): self.raw = raw self.start = start self.caption = "" self.rows = [] # list of lists of Cell (spans expanded) self.row_raw = [] self._parse() def _parse(self): lines = self.raw.split("\n") rows, raws = [], [] cur, cur_raw = None, None depth = 0 for ln in lines[1:]: s = ln.strip() if depth > 0: if cur: cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2]) if s.startswith("{|"): depth += 1 if s.startswith("|}"): depth -= 1 continue if s.startswith("{|"): depth += 1 if cur: cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2]) continue if s.startswith("|}"): break if s.startswith("|+"): cap = s[2:] a, content = _split_attrs(cap) self.caption = plain(content) continue if s.startswith("|-"): cur, cur_raw = [], [] rows.append(cur) raws.append(cur_raw) continue if cur is None: cur, cur_raw = [], [] rows.append(cur) raws.append(cur_raw) cur_raw.append(ln) if s.startswith("!"): for c in split_top(s[1:], "!!"): for c2 in split_top(c, "||"): a, content = _split_attrs(c2) cur.append((content, True, a)) elif s.startswith("|"): for c in split_top(s[1:], "||"): a, content = _split_attrs(c) cur.append((content, False, a)) elif cur: cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2]) grid, grid_raw, pending = [], [], {} for r, rr in zip(rows, raws): if not r and not pending: continue cells = [Cell(c, h, a) for c, h, a in r] line, col, ci = [], 0, 0 while ci < len(cells) or col in pending: if col in pending: rem, c = pending[col] line.append(c) if rem <= 1: del pending[col] else: pending[col] = (rem - 1, c) col += 1 continue c = cells[ci] ci += 1 for _ in range(min(c.colspan, 50)): line.append(c) if c.rowspan > 1: pending[col] = (c.rowspan - 1, c) col += 1 if line: grid.append(line) grid_raw.append("\n".join(rr)) self.rows = grid self.row_raw = grid_raw def texts(self, i): return [c.text for c in self.rows[i]] def tables(text): """All tables (outer and nested) with their start offsets, in document order.""" out, stack = [], [] for m in re.finditer(r"^[ \t]*(\{\||\|\})", text, re.M): if m.group(1) == "{|": stack.append(m.start()) elif stack: s = stack.pop() out.append(Table(text[s:m.end()], s)) return sorted(out, key=lambda t: t.start) def split_rows(table_text): """Raw row chunks of a wikitext table (split on |- lines).""" return re.split(r"\n[ \t]*\|-[^\n]*", table_text)