Initial commit
This commit is contained in:
commit
a206bfb5db
21 files changed
+16747
No files matched your search
+406
@@ -0,0 +1,406 @@
|
||||
"""English Wikipedia API client and small wikitext helpers (standard library only)."""
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
API = "https://en.wikipedia.org/w/api.php"
|
||||
USER_AGENT = "us-officeholders-map/1.0"
|
||||
DELAY = float(os.environ.get("WIKI_DELAY", "1.5")) # seconds between requests
|
||||
CACHE_DIR = os.environ.get("WIKI_CACHE_DIR") # development only: reuse fetched pages
|
||||
|
||||
|
||||
class WikiError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, progress=None):
|
||||
self.progress = progress or (lambda msg: None)
|
||||
self._last = 0.0
|
||||
self.requests = 0
|
||||
self.revisions = {} # title -> revision timestamp or id
|
||||
|
||||
def _get(self, params):
|
||||
params = dict(params, format="json", formatversion="2", maxlag="5")
|
||||
url = API + "?" + urllib.parse.urlencode(params)
|
||||
for attempt in range(6):
|
||||
wait = self._last + DELAY - time.time()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self._last = time.time()
|
||||
self.requests += 1
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"})
|
||||
retry_after = None
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
body = json.loads(resp.read().decode("utf-8"))
|
||||
err = body.get("error")
|
||||
if err and err.get("code") == "maxlag":
|
||||
retry_after = 5
|
||||
elif err:
|
||||
raise WikiError(f"Wikipedia API error: {err.get('info') or err.get('code')}")
|
||||
else:
|
||||
return body
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429 or e.code >= 500:
|
||||
ra = e.headers.get("Retry-After")
|
||||
retry_after = int(ra) if ra and ra.isdigit() else None
|
||||
else:
|
||||
raise WikiError(f"Wikipedia answered HTTP {e.code}") from e
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
||||
if attempt == 5:
|
||||
raise WikiError(f"Could not reach Wikipedia ({e})") from e
|
||||
delay = retry_after if retry_after is not None else min(60, 2 * 2 ** attempt)
|
||||
self.progress(f"Wikipedia is busy, retrying in {delay} s…")
|
||||
time.sleep(delay)
|
||||
raise WikiError("Wikipedia did not answer after several attempts")
|
||||
|
||||
def _cache(self, title, text=None):
|
||||
if not CACHE_DIR:
|
||||
return None
|
||||
p = os.path.join(CACHE_DIR, re.sub(r"[^\w.-]+", "_", title) + ".wiki")
|
||||
if text is None:
|
||||
if os.path.exists(p):
|
||||
with open(p, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
return None
|
||||
os.makedirs(CACHE_DIR, exist_ok=True)
|
||||
with open(p, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
return text
|
||||
|
||||
def page(self, title):
|
||||
"""Wikitext of one page (action=parse), following redirects."""
|
||||
cached = self._cache(title)
|
||||
if cached is not None:
|
||||
return cached
|
||||
self.progress(f"Reading “{title}”")
|
||||
body = self._get({"action": "parse", "page": title, "prop": "wikitext|revid", "redirects": "1"})
|
||||
p = body.get("parse") or {}
|
||||
text = p.get("wikitext")
|
||||
if text is None:
|
||||
raise WikiError(f"No text returned for “{title}”")
|
||||
self.revisions[title] = p.get("revid")
|
||||
self._cache(title, text)
|
||||
return text
|
||||
|
||||
def pages(self, titles, label="pages"):
|
||||
"""Wikitext of many pages, 50 per request (action=query). Returns {requested title: text or None}."""
|
||||
out = {}
|
||||
todo = []
|
||||
for t in titles:
|
||||
c = self._cache(t)
|
||||
if c is not None:
|
||||
out[t] = c
|
||||
else:
|
||||
todo.append(t)
|
||||
for i in range(0, len(todo), 50):
|
||||
chunk = todo[i:i + 50]
|
||||
self.progress(f"Reading {label} {i + 1}–{i + len(chunk)} of {len(todo)}")
|
||||
body = self._get({"action": "query", "prop": "revisions", "rvprop": "content|ids", "rvslots": "main",
|
||||
"titles": "|".join(chunk), "redirects": "1"})
|
||||
q = body.get("query", {})
|
||||
norm = {n["from"]: n["to"] for n in q.get("normalized", [])}
|
||||
redir = {r["from"]: r["to"] for r in q.get("redirects", [])}
|
||||
by_title = {p["title"]: p for p in q.get("pages", [])}
|
||||
for t in chunk:
|
||||
t2 = norm.get(t, t)
|
||||
t2 = redir.get(t2, t2)
|
||||
p = by_title.get(t2)
|
||||
if not p or p.get("missing") or not p.get("revisions"):
|
||||
out[t] = None
|
||||
continue
|
||||
rev = p["revisions"][0]
|
||||
out[t] = rev["slots"]["main"]["content"]
|
||||
self.revisions[t] = rev.get("revid")
|
||||
self._cache(t, out[t])
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- wikitext helpers
|
||||
|
||||
def strip_noise(text):
|
||||
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
|
||||
text = re.sub(r"<ref[^>/]*/>", "", text)
|
||||
text = re.sub(r"<ref[^>]*>.*?</ref>", "", text, flags=re.S)
|
||||
return text
|
||||
|
||||
|
||||
def find_template(text, name, start=0):
|
||||
"""Balanced {{name ...}} starting at or after `start`; returns (start, end) or None."""
|
||||
m = re.compile(r"\{\{\s*" + name, re.I).search(text, start)
|
||||
if not m:
|
||||
return None
|
||||
i = m.start()
|
||||
depth = 0
|
||||
j = i
|
||||
while j < len(text):
|
||||
if text.startswith("{{", j):
|
||||
depth += 1
|
||||
j += 2
|
||||
elif text.startswith("}}", j):
|
||||
depth -= 1
|
||||
j += 2
|
||||
if depth == 0:
|
||||
return i, j
|
||||
else:
|
||||
j += 1
|
||||
return i, len(text)
|
||||
|
||||
|
||||
def split_top(s, sep="|"):
|
||||
"""Split on `sep` outside [[ ]] and {{ }}."""
|
||||
parts, cur, depth, i = [], [], 0, 0
|
||||
while i < len(s):
|
||||
two = s[i:i + 2]
|
||||
if two in ("[[", "{{"):
|
||||
depth += 1
|
||||
cur.append(two)
|
||||
i += 2
|
||||
continue
|
||||
if two in ("]]", "}}") and depth > 0:
|
||||
depth -= 1
|
||||
cur.append(two)
|
||||
i += 2
|
||||
continue
|
||||
if depth == 0 and s.startswith(sep, i):
|
||||
parts.append("".join(cur))
|
||||
cur = []
|
||||
i += len(sep)
|
||||
continue
|
||||
cur.append(s[i])
|
||||
i += 1
|
||||
parts.append("".join(cur))
|
||||
return parts
|
||||
|
||||
|
||||
def template_params(tpl):
|
||||
body = tpl[2:-2]
|
||||
parts = split_top(body)
|
||||
named, positional = {}, []
|
||||
for p in parts[1:]:
|
||||
if "=" in p and re.match(r"^\s*[\w ]+\s*=", p):
|
||||
k, v = p.split("=", 1)
|
||||
named[k.strip()] = v.strip()
|
||||
else:
|
||||
positional.append(p.strip())
|
||||
return named, positional
|
||||
|
||||
|
||||
def infobox(text, name="Infobox"):
|
||||
loc = find_template(strip_noise(text), name)
|
||||
if not loc:
|
||||
return {}
|
||||
t = strip_noise(text)
|
||||
named, _ = template_params(t[loc[0]:loc[1]])
|
||||
return named
|
||||
|
||||
|
||||
_KEEP_FIRST = {"nowrap", "no wrap", "small", "smaller", "resize", "center", "nobold", "big", "nobr", "abbr", "lang", "notatypo", "sic", "tooltip", "h", "strong"}
|
||||
|
||||
|
||||
def _template_plain(m):
|
||||
inner = m.group(1)
|
||||
parts = inner.split("|")
|
||||
name = parts[0].strip().lower()
|
||||
args = [p for p in parts[1:] if not re.match(r"^\s*\w+\s*=", p)]
|
||||
if name == "sortname" and len(args) >= 2:
|
||||
return f"{args[0].strip()} {args[1].strip()}"
|
||||
if name in _KEEP_FIRST and args:
|
||||
return args[0]
|
||||
if name == "!":
|
||||
return "|"
|
||||
if name in ("dts", "start date", "birth date and age", "birth date"):
|
||||
return " ".join(a.strip() for a in args[:3])
|
||||
if name in ("legend", "legend0") and len(args) >= 2:
|
||||
return args[1]
|
||||
return ""
|
||||
|
||||
|
||||
def plain(s):
|
||||
"""Rough wikitext -> text."""
|
||||
if not s:
|
||||
return ""
|
||||
s = strip_noise(s)
|
||||
s = re.sub(r"\[\[(?:File|Image):[^\[\]]*(?:\[\[[^\]]*\]\][^\[\]]*)*\]\]", "", s, flags=re.I)
|
||||
for _ in range(4):
|
||||
s2 = re.sub(r"\[\[(?:[^|\]]*\|)?([^\]|]*)\]\]", r"\1", s)
|
||||
s2 = re.sub(r"\{\{([^{}]*)\}\}", _template_plain, s2)
|
||||
if s2 == s:
|
||||
break
|
||||
s = s2
|
||||
s = re.sub(r"\[https?://[^\s\]]+\s*([^\]]*)\]", r"\1", s)
|
||||
s = re.sub(r"'{2,}", "", s)
|
||||
s = re.sub(r"<br\s*/?>", " ", s, flags=re.I)
|
||||
s = re.sub(r"<[^>]+>", "", s)
|
||||
s = html.unescape(s).replace(" ", " ")
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def sections(text, level=2):
|
||||
"""[(heading, body)] for headings of exactly `level` (body runs to the next heading of that level or higher)."""
|
||||
eq = "=" * level
|
||||
pat = re.compile(r"^" + eq + r"(?!=)\s*(.+?)\s*" + eq + r"\s*$", re.M)
|
||||
heads = list(pat.finditer(text))
|
||||
out = []
|
||||
for i, m in enumerate(heads):
|
||||
end = heads[i + 1].start() if i + 1 < len(heads) else len(text)
|
||||
body = text[m.end():end]
|
||||
# stop at a higher-level heading
|
||||
for lv in range(1, level):
|
||||
hm = re.search(r"^" + "=" * lv + r"(?!=)[^=\n].*?" + "=" * lv + r"\s*$", body, re.M)
|
||||
if hm:
|
||||
body = body[:hm.start()]
|
||||
out.append((plain(m.group(1)), body))
|
||||
return out
|
||||
|
||||
|
||||
def section(text, title, level=2):
|
||||
for h, body in sections(text, level):
|
||||
if h.lower() == title.lower():
|
||||
return body
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- tables
|
||||
|
||||
class Cell:
|
||||
__slots__ = ("raw", "text", "header", "attrs", "rowspan", "colspan")
|
||||
|
||||
def __init__(self, raw, header, attrs):
|
||||
self.raw = raw.strip()
|
||||
self.text = plain(raw)
|
||||
self.header = header
|
||||
self.attrs = attrs
|
||||
rs = re.search(r"rowspan\s*=\s*\"?(\d+)", attrs)
|
||||
cs = re.search(r"colspan\s*=\s*\"?(\d+)", attrs)
|
||||
self.rowspan = int(rs.group(1)) if rs else 1
|
||||
self.colspan = int(cs.group(1)) if cs else 1
|
||||
|
||||
def __repr__(self):
|
||||
return f"Cell({self.text!r})"
|
||||
|
||||
|
||||
_ATTR_TEMPLATES = re.compile(r"\{\{\s*(party shading|party color|shading pvi|usraterating)[^{}]*\}\}", re.I)
|
||||
|
||||
|
||||
def _split_attrs(cell):
|
||||
parts = split_top(cell, "|")
|
||||
if len(parts) >= 2:
|
||||
left = _ATTR_TEMPLATES.sub(" ", parts[0])
|
||||
left = re.sub(r"[\w-]+\s*=\s*(\"[^\"]*\"|'[^']*'|[^\s]+)", " ", left)
|
||||
if not left.strip():
|
||||
return parts[0], "|".join(parts[1:])
|
||||
return "", cell
|
||||
|
||||
|
||||
class Table:
|
||||
def __init__(self, raw, start):
|
||||
self.raw = raw
|
||||
self.start = start
|
||||
self.caption = ""
|
||||
self.rows = [] # list of lists of Cell (spans expanded)
|
||||
self.row_raw = []
|
||||
self._parse()
|
||||
|
||||
def _parse(self):
|
||||
lines = self.raw.split("\n")
|
||||
rows, raws = [], []
|
||||
cur, cur_raw = None, None
|
||||
depth = 0
|
||||
for ln in lines[1:]:
|
||||
s = ln.strip()
|
||||
if depth > 0:
|
||||
if cur:
|
||||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||||
if s.startswith("{|"):
|
||||
depth += 1
|
||||
if s.startswith("|}"):
|
||||
depth -= 1
|
||||
continue
|
||||
if s.startswith("{|"):
|
||||
depth += 1
|
||||
if cur:
|
||||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||||
continue
|
||||
if s.startswith("|}"):
|
||||
break
|
||||
if s.startswith("|+"):
|
||||
cap = s[2:]
|
||||
a, content = _split_attrs(cap)
|
||||
self.caption = plain(content)
|
||||
continue
|
||||
if s.startswith("|-"):
|
||||
cur, cur_raw = [], []
|
||||
rows.append(cur)
|
||||
raws.append(cur_raw)
|
||||
continue
|
||||
if cur is None:
|
||||
cur, cur_raw = [], []
|
||||
rows.append(cur)
|
||||
raws.append(cur_raw)
|
||||
cur_raw.append(ln)
|
||||
if s.startswith("!"):
|
||||
for c in split_top(s[1:], "!!"):
|
||||
for c2 in split_top(c, "||"):
|
||||
a, content = _split_attrs(c2)
|
||||
cur.append((content, True, a))
|
||||
elif s.startswith("|"):
|
||||
for c in split_top(s[1:], "||"):
|
||||
a, content = _split_attrs(c)
|
||||
cur.append((content, False, a))
|
||||
elif cur:
|
||||
cur[-1] = (cur[-1][0] + "\n" + ln, cur[-1][1], cur[-1][2])
|
||||
grid, grid_raw, pending = [], [], {}
|
||||
for r, rr in zip(rows, raws):
|
||||
if not r and not pending:
|
||||
continue
|
||||
cells = [Cell(c, h, a) for c, h, a in r]
|
||||
line, col, ci = [], 0, 0
|
||||
while ci < len(cells) or col in pending:
|
||||
if col in pending:
|
||||
rem, c = pending[col]
|
||||
line.append(c)
|
||||
if rem <= 1:
|
||||
del pending[col]
|
||||
else:
|
||||
pending[col] = (rem - 1, c)
|
||||
col += 1
|
||||
continue
|
||||
c = cells[ci]
|
||||
ci += 1
|
||||
for _ in range(min(c.colspan, 50)):
|
||||
line.append(c)
|
||||
if c.rowspan > 1:
|
||||
pending[col] = (c.rowspan - 1, c)
|
||||
col += 1
|
||||
if line:
|
||||
grid.append(line)
|
||||
grid_raw.append("\n".join(rr))
|
||||
self.rows = grid
|
||||
self.row_raw = grid_raw
|
||||
|
||||
def texts(self, i):
|
||||
return [c.text for c in self.rows[i]]
|
||||
|
||||
|
||||
def tables(text):
|
||||
"""All tables (outer and nested) with their start offsets, in document order."""
|
||||
out, stack = [], []
|
||||
for m in re.finditer(r"^[ \t]*(\{\||\|\})", text, re.M):
|
||||
if m.group(1) == "{|":
|
||||
stack.append(m.start())
|
||||
elif stack:
|
||||
s = stack.pop()
|
||||
out.append(Table(text[s:m.end()], s))
|
||||
return sorted(out, key=lambda t: t.start)
|
||||
|
||||
|
||||
def split_rows(table_text):
|
||||
"""Raw row chunks of a wikitext table (split on |- lines)."""
|
||||
return re.split(r"\n[ \t]*\|-[^\n]*", table_text)
|
||||
Reference in new issue
Block a user