Files
2026-11-13 17:35:10 +01:00

380 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Refresh the map data from English Wikipedia.
Usage (inside the container): python update.py
Mode: UPDATE_MODE=auto|election|current (auto = election results between Election Day and January 3, else current officeholders).
Each category (senate, house, governors, legislatures) is checked before it replaces the stored file;
if reading or checking fails, the previous data for that category is kept and a warning is recorded.
"""
import datetime as dt
import json
import os
import sys
import time
import traceback
import common as C
import parsers as P
from wiki import Client, WikiError
LOCK = ".update.lock"
LOCK_STALE_SECONDS = 300 # the running update touches the lock at every step, so an older lock is left over from a crash
class Busy(Exception):
pass
def acquire_lock():
C.ensure_dir()
p = C.path(LOCK)
try:
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
try:
age = time.time() - os.path.getmtime(p)
except OSError:
age = 0
if age < LOCK_STALE_SECONDS:
raise Busy("An update is already running.")
os.unlink(p)
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
with os.fdopen(fd, "w") as f:
f.write(f"{os.getpid()} {dt.datetime.now(dt.timezone.utc).isoformat()}\n")
def release_lock():
try:
os.unlink(C.path(LOCK))
except OSError:
pass
def lock_held():
p = C.path(LOCK)
return os.path.exists(p) and time.time() - os.path.getmtime(p) < LOCK_STALE_SECONDS
# ---------------------------------------------------------------- validation
def validate_senate(data):
errs = []
if len(data) != 50 or set(data) != set(C.NAMES):
errs.append(f"{len(data)} states instead of 50")
total = sum(len(v) for v in data.values())
if total != 100:
errs.append(f"{total} senators instead of 100")
for code, v in sorted(data.items()):
if len(v) != 2:
errs.append(f"{C.NAMES.get(code, code)} has {len(v)} senators")
elif v[0]["class"] == v[1]["class"]:
errs.append(f"{C.NAMES[code]} has two class {v[0]['class']} senators")
for s in v:
if not s["name"] and s["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"a {C.NAMES.get(code, code)} senator has no name")
if s["name"] and s["party"] not in ("D", "R", "I"):
errs.append(f"no party for {s['name']}")
return errs
def validate_house(data):
errs = []
if set(data) != set(C.NAMES):
missing = sorted(set(C.NAMES) - set(data))
errs.append(f"{len(data)} states instead of 50" + (f" (missing {', '.join(missing[:5])})" if missing else ""))
total = sum(len(v) for v in data.values())
if total != 435:
errs.append(f"{total} representatives instead of 435")
for code, n in C.APPORTIONMENT.items():
v = data.get(code, [])
if len(v) != n:
errs.append(f"{C.NAMES[code]}: {len(v)} seats instead of {n}")
continue
expected = [0] if n == 1 else list(range(1, n + 1))
if sorted(r["district"] for r in v) != expected and not (n == 1 and [r["district"] for r in v] == [1]):
errs.append(f"{C.NAMES[code]}: unexpected district numbers")
for r in v:
if r["name"] and r["party"] not in ("D", "R", "I"):
errs.append(f"no party for {r['name']}")
if not r["name"] and r["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"{C.NAMES[code]} district {r['district']} has no name")
return errs
def validate_governors(data):
errs = []
if set(data) != set(C.NAMES):
errs.append(f"{len(data)} governors instead of 50")
for code, g in data.items():
if not g["name"] and g["status"] not in ("uncalled", "runoff", "vacant"):
errs.append(f"no governor name for {C.NAMES.get(code, code)}")
if g["name"] and g["party"] not in ("D", "R", "I"):
errs.append(f"no party for governor {g['name']}")
return errs
def chamber_errors(code, which, ch):
size = C.CHAMBERS[code][0 if which == "upper" else 1][1]
total = ch["D"] + ch["R"] + ch["I"] + ch["vacant"] + ch["uncalled"]
if ch["seats"] != size or total != size:
return f"seats add up to {total} instead of {size}"
return None
def validate_legislatures(data):
errs = []
if set(data) != set(C.NAMES):
errs.append(f"{len(data)} states instead of 50")
n = 0
for code, legs in data.items():
for which in ("upper", "lower"):
ch = legs.get(which)
if ch is None:
if not (code == "NE" and which == "lower"):
errs.append(f"{C.NAMES[code]} {which} chamber missing")
continue
n += 1
e = chamber_errors(code, which, ch)
if e:
errs.append(f"{ch['name']}: {e}")
if n != 99:
errs.append(f"{n} chambers instead of 99")
return errs
VALIDATORS = {"senate": validate_senate, "house": validate_house, "governors": validate_governors,
"legislatures": validate_legislatures}
LABEL = {"senate": "U.S. Senate", "house": "U.S. House", "governors": "Governors", "legislatures": "State legislatures"}
# ---------------------------------------------------------------- totals (for messages and the CLI)
def party_totals(entries):
t = {"D": 0, "R": 0, "I": 0, "unknown": 0}
for e in entries:
if not e or e.get("status") in ("uncalled", "runoff") or e.get("party") not in ("D", "R", "I"):
t["unknown"] += 1
else:
t[e["party"]] += 1
return t
def totals(cat, data):
if cat == "senate":
return party_totals([s for v in data.values() for s in v])
if cat == "house":
return party_totals([r for v in data.values() for r in v])
if cat == "governors":
return party_totals(list(data.values()))
seats = {"D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "chambers": 0}
for legs in data.values():
for ch in legs.values():
if ch:
seats["chambers"] += 1
for k in ("D", "R", "I", "vacant", "uncalled"):
seats[k] += ch[k]
return seats
def fmt_totals(t):
return ", ".join(f"{v} {k}" for k, v in t.items() if v)
# ---------------------------------------------------------------- the run
def now_iso():
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
def run_update(progress=print, requested_mode=None):
"""Run a full update. Returns a summary dict. Raises Busy if another update holds the lock."""
C.ensure_seed()
acquire_lock()
def progress_and_touch(msg):
try:
os.utime(C.path(LOCK))
except OSError:
pass
progress(msg)
try:
return _run(progress_and_touch, requested_mode)
finally:
release_lock()
def _run(progress, requested_mode):
day = C.today()
mode, requested = C.choose_mode(day, requested_mode)
year = C.election_window(day) or C.last_election_year(day)
started = now_iso()
warnings, results = [], {}
client = Client(progress)
progress(f"Mode: {'election results ' + str(year) if mode == 'election' else 'current officeholders'}"
f"{' (forced by UPDATE_MODE)' if requested != 'auto' else ' (chosen from today’s date)'}")
def attempt(cat, fn):
progress(f"{LABEL[cat]}: reading Wikipedia…")
try:
data, sources = fn()
except (P.ParseError, WikiError, KeyError, ValueError, IndexError, StopIteration) as e:
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
progress(f"{LABEL[cat]}: failed, previous data kept")
traceback.print_exc(file=sys.stderr)
return None
errs = VALIDATORS[cat](data)
if errs:
more = f" and {len(errs) - 4} more problem(s)" if len(errs) > 4 else ""
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: "
f"{'; '.join(errs[:4])}{more}.")
progress(f"{LABEL[cat]}: checks failed, previous data kept")
return None
return data, sources
meta_base = {"mode": mode, "electionYear": year if mode == "election" else None}
# Governors first: the Senate step uses them to flag senators who were elected governor.
if mode == "election":
g = attempt("governors", lambda: P.governors_election(client, year))
else:
g = attempt("governors", lambda: P.governors_current(client, day))
if g:
save("governors", g[0], g[1], meta_base, progress, results)
if mode == "election":
s = attempt("senate", lambda: P.senate_election(client, year))
else:
s = attempt("senate", lambda: P.senate_current(client, day))
if s:
if mode == "election" and g:
flag_senators_elected_governor(s[0], g[0], year)
save("senate", s[0], s[1], meta_base, progress, results)
if mode == "election":
h = attempt("house", lambda: P.house_election(client, year))
else:
h = attempt("house", lambda: P.house_current(client, day))
if h:
save("house", h[0], h[1], meta_base, progress, results)
update_legislatures(client, mode, day, year, meta_base, progress, warnings, results)
finished = now_iso()
run = {"startedAt": started, "finishedAt": finished, "mode": mode, "requestedMode": requested,
"electionYear": meta_base["electionYear"], "updated": sorted(results), "warnings": warnings,
"requests": client.requests}
meta = C.read_json(C.path("meta.json"), {}) or {}
meta["lastRun"] = run
C.write_json_atomic(C.path("meta.json"), meta)
run["totals"] = {cat: fmt_totals(t) for cat, t in results.items()}
progress(f"Done: {len(results)} of 4 categories updated, {len(warnings)} warning(s).")
return run
def save(cat, data, sources, meta_base, progress, results):
meta = dict(meta_base, asOf=now_iso(), sources=sources)
C.save_category(cat, data, meta)
results[cat] = totals(cat, data)
progress(f"{LABEL[cat]}: updated ({fmt_totals(results[cat])})")
def flag_senators_elected_governor(senate, governors, year):
for code, sens in senate.items():
g = governors.get(code)
if not g or g.get("status") != f"elected-{year}" or not g.get("name"):
continue
for s in sens:
if s["status"] == "continuing" and s["name"] and s["name"].lower() == g["name"].lower():
s["note"] = (f"Elected Governor of {C.NAMES[code]} in {year}; the Senate seat will be filled "
f"after the new governor takes office. " + s["note"]).strip()
def update_legislatures(client, mode, day, year, meta_base, progress, warnings, results):
cat = "legislatures"
progress(f"{LABEL[cat]}: reading Wikipedia…")
try:
data, sources, problems = P.legislatures(client, mode, day, year)
except (P.ParseError, WikiError, KeyError, ValueError, IndexError) as e:
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
traceback.print_exc(file=sys.stderr)
return
prev_file = C.load_category(cat) or {}
prev = prev_file.get("data", {})
def same_cycle(meta):
meta = meta or {}
return meta.get("mode") == mode and (mode != "election" or meta.get("electionYear") == year)
# Earlier figures for the same election (stored data first, then the seed shipped with the app):
# used while Wikipedia has no post-election figures yet for a chamber.
earlier = [f.get("data", {}) for f in (prev_file, C.read_json(os.path.join(C.SEED_DIR, cat + ".json"), {}) or {})
if f and same_cycle(f.get("meta"))]
kept, unknown_kept = [], []
for code, legs in data.items():
for which in ("upper", "lower"):
ch = legs.get(which)
if ch is None:
continue
reason = problems.get((code, which)) or chamber_errors(code, which, ch)
old = (prev.get(code) or {}).get(which)
if reason:
if old:
legs[which] = old
kept.append(f"{ch['name']} ({reason})")
else:
kept.append(f"{ch['name']} ({reason}; no previous data either)")
continue
same = [e for e in ((d.get(code) or {}).get(which) for d in earlier) if e]
if ch.pop("_unknown", False):
better = [e for e in same if e.get("uncalled", 0) < ch["uncalled"]]
if better:
# Wikipedia has no post-election figures yet: keep the earlier, more complete figures.
legs[which] = better[0]
unknown_kept.append(ch["name"])
continue
named = [e["leader"] for e in same if (e.get("leader") or {}).get("name") and e["leader"].get("status") != "current"]
if mode == "election" and named and (ch.get("leader") or {}).get("status") == "current":
# an earlier read recorded the leader chosen for the new session: better than the outgoing one
ch["leader"] = named[0]
for legs in data.values():
for ch in legs.values():
if ch:
ch.pop("_unknown", None)
if len(kept) > 20:
warnings.append(f"{LABEL[cat]}: kept all previous data because {len(kept)} chambers could not be read "
f"(for example {'; '.join(kept[:3])}).")
return
errs = validate_legislatures(data)
if errs:
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: {'; '.join(errs[:4])}.")
return
if kept:
warnings.append(f"{LABEL[cat]}: previous figures kept for {len(kept)} chamber(s) whose Wikipedia data did not add up: "
+ "; ".join(kept) + ".")
if unknown_kept:
warnings.append(f"{LABEL[cat]}: Wikipedia has no {year} results yet for {len(unknown_kept)} chamber(s); "
f"the earlier figures were kept: {', '.join(unknown_kept)}.")
save(cat, data, sources, meta_base, progress, results)
def main():
try:
run = run_update(progress=lambda m: print(m, flush=True))
except Busy as e:
print(str(e))
return 2
print()
print("Totals:")
for cat, t in run["totals"].items():
print(f" {LABEL[cat]}: {t}")
if run["warnings"]:
print("Warnings:")
for w in run["warnings"]:
print(" - " + w)
return 0
if __name__ == "__main__":
sys.exit(main())