Initial commit

This commit is contained in:
ebagot committed 2026-11-13 17:35:10 +01:00
commit a206bfb5db
21 files changed
+16747

No files matched your search

+379
View File
@@ -0,0 +1,379 @@
"""Refresh the map data from English Wikipedia.
Usage (inside the container): python update.py
Mode: UPDATE_MODE=auto|election|current (auto = election results between Election Day and January 3, else current officeholders).
Each category (senate, house, governors, legislatures) is checked before it replaces the stored file;
if reading or checking fails, the previous data for that category is kept and a warning is recorded.
"""
import datetime as dt
import json
import os
import sys
import time
import traceback
import common as C
import parsers as P
from wiki import Client, WikiError
LOCK = ".update.lock"
LOCK_STALE_SECONDS = 300 # the running update touches the lock at every step, so an older lock is left over from a crash
class Busy(Exception):
pass
def acquire_lock():
C.ensure_dir()
p = C.path(LOCK)
try:
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
try:
age = time.time() - os.path.getmtime(p)
except OSError:
age = 0
if age < LOCK_STALE_SECONDS:
raise Busy("An update is already running.")
os.unlink(p)
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
with os.fdopen(fd, "w") as f:
f.write(f"{os.getpid()} {dt.datetime.now(dt.timezone.utc).isoformat()}\n")
def release_lock():
try:
os.unlink(C.path(LOCK))
except OSError:
pass
def lock_held():
p = C.path(LOCK)
return os.path.exists(p) and time.time() - os.path.getmtime(p) < LOCK_STALE_SECONDS
# ---------------------------------------------------------------- validation
def validate_senate(data):
errs = []
if len(data) != 50 or set(data) != set(C.NAMES):
errs.append(f"{len(data)} states instead of 50")
total = sum(len(v) for v in data.values())
if total != 100:
errs.append(f"{total} senators instead of 100")
for code, v in sorted(data.items()):
if len(v) != 2:
errs.append(f"{C.NAMES.get(code, code)} has {len(v)} senators")
elif v[0]["class"] == v[1]["class"]:
errs.append(f"{C.NAMES[code]} has two class {v[0]['class']} senators")
for s in v:
if not s["name"] and s["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"a {C.NAMES.get(code, code)} senator has no name")
if s["name"] and s["party"] not in ("D", "R", "I"):
errs.append(f"no party for {s['name']}")
return errs
def validate_house(data):
errs = []
if set(data) != set(C.NAMES):
missing = sorted(set(C.NAMES) - set(data))
errs.append(f"{len(data)} states instead of 50" + (f" (missing {', '.join(missing[:5])})" if missing else ""))
total = sum(len(v) for v in data.values())
if total != 435:
errs.append(f"{total} representatives instead of 435")
for code, n in C.APPORTIONMENT.items():
v = data.get(code, [])
if len(v) != n:
errs.append(f"{C.NAMES[code]}: {len(v)} seats instead of {n}")
continue
expected = [0] if n == 1 else list(range(1, n + 1))
if sorted(r["district"] for r in v) != expected and not (n == 1 and [r["district"] for r in v] == [1]):
errs.append(f"{C.NAMES[code]}: unexpected district numbers")
for r in v:
if r["name"] and r["party"] not in ("D", "R", "I"):
errs.append(f"no party for {r['name']}")
if not r["name"] and r["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"{C.NAMES[code]} district {r['district']} has no name")
return errs
def validate_governors(data):
errs = []
if set(data) != set(C.NAMES):
errs.append(f"{len(data)} governors instead of 50")
for code, g in data.items():
if not g["name"] and g["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"no governor name for {C.NAMES.get(code, code)}")
if g["name"] and g["party"] not in ("D", "R", "I"):
errs.append(f"no party for governor {g['name']}")
return errs
def chamber_errors(code, which, ch):
size = C.CHAMBERS[code][0 if which == "upper" else 1][1]
total = ch["D"] + ch["R"] + ch["I"] + ch["vacant"] + ch["uncalled"]
if ch["seats"] != size or total != size:
return f"seats add up to {total} instead of {size}"
return None
def validate_legislatures(data):
errs = []
if set(data) != set(C.NAMES):
errs.append(f"{len(data)} states instead of 50")
n = 0
for code, legs in data.items():
for which in ("upper", "lower"):
ch = legs.get(which)
if ch is None:
if not (code == "NE" and which == "lower"):
errs.append(f"{C.NAMES[code]} {which} chamber missing")
continue
n += 1
e = chamber_errors(code, which, ch)
if e:
errs.append(f"{ch['name']}: {e}")
if n != 99:
errs.append(f"{n} chambers instead of 99")
return errs
VALIDATORS = {"senate": validate_senate, "house": validate_house, "governors": validate_governors,
"legislatures": validate_legislatures}
LABEL = {"senate": "U.S. Senate", "house": "U.S. House", "governors": "Governors", "legislatures": "State legislatures"}
# ---------------------------------------------------------------- totals (for messages and the CLI)
def party_totals(entries):
t = {"D": 0, "R": 0, "I": 0, "unknown": 0}
for e in entries:
if not e or e.get("status") in ("uncalled", "runoff") or e.get("party") not in ("D", "R", "I"):
t["unknown"] += 1
else:
t[e["party"]] += 1
return t
def totals(cat, data):
if cat == "senate":
return party_totals([s for v in data.values() for s in v])
if cat == "house":
return party_totals([r for v in data.values() for r in v])
if cat == "governors":
return party_totals(list(data.values()))
seats = {"D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "chambers": 0}
for legs in data.values():
for ch in legs.values():
if ch:
seats["chambers"] += 1
for k in ("D", "R", "I", "vacant", "uncalled"):
seats[k] += ch[k]
return seats
def fmt_totals(t):
return ", ".join(f"{v} {k}" for k, v in t.items() if v)
# ---------------------------------------------------------------- the run
def now_iso():
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
def run_update(progress=print, requested_mode=None):
"""Run a full update. Returns a summary dict. Raises Busy if another update holds the lock."""
C.ensure_seed()
acquire_lock()
def progress_and_touch(msg):
try:
os.utime(C.path(LOCK))
except OSError:
pass
progress(msg)
try:
return _run(progress_and_touch, requested_mode)
finally:
release_lock()
def _run(progress, requested_mode):
day = C.today()
mode, requested = C.choose_mode(day, requested_mode)
year = C.election_window(day) or C.last_election_year(day)
started = now_iso()
warnings, results = [], {}
client = Client(progress)
progress(f"Mode: {'election results ' + str(year) if mode == 'election' else 'current officeholders'}"
f"{' (forced by UPDATE_MODE)' if requested != 'auto' else ' (chosen from today’s date)'}")
def attempt(cat, fn):
progress(f"{LABEL[cat]}: reading Wikipedia…")
try:
data, sources = fn()
except (P.ParseError, WikiError, KeyError, ValueError, IndexError, StopIteration) as e:
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
progress(f"{LABEL[cat]}: failed, previous data kept")
traceback.print_exc(file=sys.stderr)
return None
errs = VALIDATORS[cat](data)
if errs:
more = f" and {len(errs) - 4} more problem(s)" if len(errs) > 4 else ""
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: "
f"{'; '.join(errs[:4])}{more}.")
progress(f"{LABEL[cat]}: checks failed, previous data kept")
return None
return data, sources
meta_base = {"mode": mode, "electionYear": year if mode == "election" else None}
# Governors first: the Senate step uses them to flag senators who were elected governor.
if mode == "election":
g = attempt("governors", lambda: P.governors_election(client, year))
else:
g = attempt("governors", lambda: P.governors_current(client, day))
if g:
save("governors", g[0], g[1], meta_base, progress, results)
if mode == "election":
s = attempt("senate", lambda: P.senate_election(client, year))
else:
s = attempt("senate", lambda: P.senate_current(client, day))
if s:
if mode == "election" and g:
flag_senators_elected_governor(s[0], g[0], year)
save("senate", s[0], s[1], meta_base, progress, results)
if mode == "election":
h = attempt("house", lambda: P.house_election(client, year))
else:
h = attempt("house", lambda: P.house_current(client, day))
if h:
save("house", h[0], h[1], meta_base, progress, results)
update_legislatures(client, mode, day, year, meta_base, progress, warnings, results)
finished = now_iso()
run = {"startedAt": started, "finishedAt": finished, "mode": mode, "requestedMode": requested,
"electionYear": meta_base["electionYear"], "updated": sorted(results), "warnings": warnings,
"requests": client.requests}
meta = C.read_json(C.path("meta.json"), {}) or {}
meta["lastRun"] = run
C.write_json_atomic(C.path("meta.json"), meta)
run["totals"] = {cat: fmt_totals(t) for cat, t in results.items()}
progress(f"Done: {len(results)} of 4 categories updated, {len(warnings)} warning(s).")
return run
def save(cat, data, sources, meta_base, progress, results):
meta = dict(meta_base, asOf=now_iso(), sources=sources)
C.save_category(cat, data, meta)
results[cat] = totals(cat, data)
progress(f"{LABEL[cat]}: updated ({fmt_totals(results[cat])})")
def flag_senators_elected_governor(senate, governors, year):
for code, sens in senate.items():
g = governors.get(code)
if not g or g.get("status") != f"elected-{year}" or not g.get("name"):
continue
for s in sens:
if s["status"] == "continuing" and s["name"] and s["name"].lower() == g["name"].lower():
s["note"] = (f"Elected Governor of {C.NAMES[code]} in {year}; the Senate seat will be filled "
f"after the new governor takes office. " + s["note"]).strip()
def update_legislatures(client, mode, day, year, meta_base, progress, warnings, results):
cat = "legislatures"
progress(f"{LABEL[cat]}: reading Wikipedia…")
try:
data, sources, problems = P.legislatures(client, mode, day, year)
except (P.ParseError, WikiError, KeyError, ValueError, IndexError) as e:
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
traceback.print_exc(file=sys.stderr)
return
prev_file = C.load_category(cat) or {}
prev = prev_file.get("data", {})
def same_cycle(meta):
meta = meta or {}
return meta.get("mode") == mode and (mode != "election" or meta.get("electionYear") == year)
# Earlier figures for the same election (stored data first, then the seed shipped with the app):
# used while Wikipedia has no post-election figures yet for a chamber.
earlier = [f.get("data", {}) for f in (prev_file, C.read_json(os.path.join(C.SEED_DIR, cat + ".json"), {}) or {})
if f and same_cycle(f.get("meta"))]
kept, unknown_kept = [], []
for code, legs in data.items():
for which in ("upper", "lower"):
ch = legs.get(which)
if ch is None:
continue
reason = problems.get((code, which)) or chamber_errors(code, which, ch)
old = (prev.get(code) or {}).get(which)
if reason:
if old:
legs[which] = old
kept.append(f"{ch['name']} ({reason})")
else:
kept.append(f"{ch['name']} ({reason}; no previous data either)")
continue
same = [e for e in ((d.get(code) or {}).get(which) for d in earlier) if e]
if ch.pop("_unknown", False):
better = [e for e in same if e.get("uncalled", 0) < ch["uncalled"]]
if better:
# Wikipedia has no post-election figures yet: keep the earlier, more complete figures.
legs[which] = better[0]
unknown_kept.append(ch["name"])
continue
named = [e["leader"] for e in same if (e.get("leader") or {}).get("name") and e["leader"].get("status") != "current"]
if mode == "election" and named and (ch.get("leader") or {}).get("status") == "current":
# an earlier read recorded the leader chosen for the new session: better than the outgoing one
ch["leader"] = named[0]
for legs in data.values():
for ch in legs.values():
if ch:
ch.pop("_unknown", None)
if len(kept) > 20:
warnings.append(f"{LABEL[cat]}: kept all previous data because {len(kept)} chambers could not be read "
f"(for example {'; '.join(kept[:3])}).")
return
errs = validate_legislatures(data)
if errs:
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: {'; '.join(errs[:4])}.")
return
if kept:
warnings.append(f"{LABEL[cat]}: previous figures kept for {len(kept)} chamber(s) whose Wikipedia data did not add up: "
+ "; ".join(kept) + ".")
if unknown_kept:
warnings.append(f"{LABEL[cat]}: Wikipedia has no {year} results yet for {len(unknown_kept)} chamber(s); "
f"the earlier figures were kept: {', '.join(unknown_kept)}.")
save(cat, data, sources, meta_base, progress, results)
def main():
try:
run = run_update(progress=lambda m: print(m, flush=True))
except Busy as e:
print(str(e))
return 2
print()
print("Totals:")
for cat, t in run["totals"].items():
print(f" {LABEL[cat]}: {t}")
if run["warnings"]:
print("Warnings:")
for w in run["warnings"]:
print(" - " + w)
return 0
if __name__ == "__main__":
sys.exit(main())