"""Refresh the map data from English Wikipedia. Usage (inside the container): python update.py Mode: UPDATE_MODE=auto|election|current (auto = election results between Election Day and January 3, else current officeholders). Each category (senate, house, governors, legislatures) is checked before it replaces the stored file; if reading or checking fails, the previous data for that category is kept and a warning is recorded. """ import datetime as dt import json import os import sys import time import traceback import common as C import parsers as P from wiki import Client, WikiError LOCK = ".update.lock" LOCK_STALE_SECONDS = 300 # the running update touches the lock at every step, so an older lock is left over from a crash class Busy(Exception): pass def acquire_lock(): C.ensure_dir() p = C.path(LOCK) try: fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY) except FileExistsError: try: age = time.time() - os.path.getmtime(p) except OSError: age = 0 if age < LOCK_STALE_SECONDS: raise Busy("An update is already running.") os.unlink(p) fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY) with os.fdopen(fd, "w") as f: f.write(f"{os.getpid()} {dt.datetime.now(dt.timezone.utc).isoformat()}\n") def release_lock(): try: os.unlink(C.path(LOCK)) except OSError: pass def lock_held(): p = C.path(LOCK) return os.path.exists(p) and time.time() - os.path.getmtime(p) < LOCK_STALE_SECONDS # ---------------------------------------------------------------- validation def validate_senate(data): errs = [] if len(data) != 50 or set(data) != set(C.NAMES): errs.append(f"{len(data)} states instead of 50") total = sum(len(v) for v in data.values()) if total != 100: errs.append(f"{total} senators instead of 100") for code, v in sorted(data.items()): if len(v) != 2: errs.append(f"{C.NAMES.get(code, code)} has {len(v)} senators") elif v[0]["class"] == v[1]["class"]: errs.append(f"{C.NAMES[code]} has two class {v[0]['class']} senators") for s in v: if not s["name"] and s["status"] not in ("uncalled", "runoff", "vacant"): errs.append(f"a {C.NAMES.get(code, code)} senator has no name") if s["name"] and s["party"] not in ("D", "R", "I"): errs.append(f"no party for {s['name']}") return errs def validate_house(data): errs = [] if set(data) != set(C.NAMES): missing = sorted(set(C.NAMES) - set(data)) errs.append(f"{len(data)} states instead of 50" + (f" (missing {', '.join(missing[:5])})" if missing else "")) total = sum(len(v) for v in data.values()) if total != 435: errs.append(f"{total} representatives instead of 435") for code, n in C.APPORTIONMENT.items(): v = data.get(code, []) if len(v) != n: errs.append(f"{C.NAMES[code]}: {len(v)} seats instead of {n}") continue expected = [0] if n == 1 else list(range(1, n + 1)) if sorted(r["district"] for r in v) != expected and not (n == 1 and [r["district"] for r in v] == [1]): errs.append(f"{C.NAMES[code]}: unexpected district numbers") for r in v: if r["name"] and r["party"] not in ("D", "R", "I"): errs.append(f"no party for {r['name']}") if not r["name"] and r["status"] not in ("uncalled", "runoff", "vacant"): errs.append(f"{C.NAMES[code]} district {r['district']} has no name") return errs def validate_governors(data): errs = [] if set(data) != set(C.NAMES): errs.append(f"{len(data)} governors instead of 50") for code, g in data.items(): if not g["name"] and g["status"] not in ("uncalled", "runoff", "vacant"): errs.append(f"no governor name for {C.NAMES.get(code, code)}") if g["name"] and g["party"] not in ("D", "R", "I"): errs.append(f"no party for governor {g['name']}") return errs def chamber_errors(code, which, ch): size = C.CHAMBERS[code][0 if which == "upper" else 1][1] total = ch["D"] + ch["R"] + ch["I"] + ch["vacant"] + ch["uncalled"] if ch["seats"] != size or total != size: return f"seats add up to {total} instead of {size}" return None def validate_legislatures(data): errs = [] if set(data) != set(C.NAMES): errs.append(f"{len(data)} states instead of 50") n = 0 for code, legs in data.items(): for which in ("upper", "lower"): ch = legs.get(which) if ch is None: if not (code == "NE" and which == "lower"): errs.append(f"{C.NAMES[code]} {which} chamber missing") continue n += 1 e = chamber_errors(code, which, ch) if e: errs.append(f"{ch['name']}: {e}") if n != 99: errs.append(f"{n} chambers instead of 99") return errs VALIDATORS = {"senate": validate_senate, "house": validate_house, "governors": validate_governors, "legislatures": validate_legislatures} LABEL = {"senate": "U.S. Senate", "house": "U.S. House", "governors": "Governors", "legislatures": "State legislatures"} # ---------------------------------------------------------------- totals (for messages and the CLI) def party_totals(entries): t = {"D": 0, "R": 0, "I": 0, "unknown": 0} for e in entries: if not e or e.get("status") in ("uncalled", "runoff") or e.get("party") not in ("D", "R", "I"): t["unknown"] += 1 else: t[e["party"]] += 1 return t def totals(cat, data): if cat == "senate": return party_totals([s for v in data.values() for s in v]) if cat == "house": return party_totals([r for v in data.values() for r in v]) if cat == "governors": return party_totals(list(data.values())) seats = {"D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "chambers": 0} for legs in data.values(): for ch in legs.values(): if ch: seats["chambers"] += 1 for k in ("D", "R", "I", "vacant", "uncalled"): seats[k] += ch[k] return seats def fmt_totals(t): return ", ".join(f"{v} {k}" for k, v in t.items() if v) # ---------------------------------------------------------------- the run def now_iso(): return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() def run_update(progress=print, requested_mode=None): """Run a full update. Returns a summary dict. Raises Busy if another update holds the lock.""" C.ensure_seed() acquire_lock() def progress_and_touch(msg): try: os.utime(C.path(LOCK)) except OSError: pass progress(msg) try: return _run(progress_and_touch, requested_mode) finally: release_lock() def _run(progress, requested_mode): day = C.today() mode, requested = C.choose_mode(day, requested_mode) year = C.election_window(day) or C.last_election_year(day) started = now_iso() warnings, results = [], {} client = Client(progress) progress(f"Mode: {'election results ' + str(year) if mode == 'election' else 'current officeholders'}" f"{' (forced by UPDATE_MODE)' if requested != 'auto' else ' (chosen from today’s date)'}") def attempt(cat, fn): progress(f"{LABEL[cat]}: reading Wikipedia…") try: data, sources = fn() except (P.ParseError, WikiError, KeyError, ValueError, IndexError, StopIteration) as e: warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).") progress(f"{LABEL[cat]}: failed, previous data kept") traceback.print_exc(file=sys.stderr) return None errs = VALIDATORS[cat](data) if errs: more = f" and {len(errs) - 4} more problem(s)" if len(errs) > 4 else "" warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: " f"{'; '.join(errs[:4])}{more}.") progress(f"{LABEL[cat]}: checks failed, previous data kept") return None return data, sources meta_base = {"mode": mode, "electionYear": year if mode == "election" else None} # Governors first: the Senate step uses them to flag senators who were elected governor. if mode == "election": g = attempt("governors", lambda: P.governors_election(client, year)) else: g = attempt("governors", lambda: P.governors_current(client, day)) if g: save("governors", g[0], g[1], meta_base, progress, results) if mode == "election": s = attempt("senate", lambda: P.senate_election(client, year)) else: s = attempt("senate", lambda: P.senate_current(client, day)) if s: if mode == "election" and g: flag_senators_elected_governor(s[0], g[0], year) save("senate", s[0], s[1], meta_base, progress, results) if mode == "election": h = attempt("house", lambda: P.house_election(client, year)) else: h = attempt("house", lambda: P.house_current(client, day)) if h: save("house", h[0], h[1], meta_base, progress, results) update_legislatures(client, mode, day, year, meta_base, progress, warnings, results) finished = now_iso() run = {"startedAt": started, "finishedAt": finished, "mode": mode, "requestedMode": requested, "electionYear": meta_base["electionYear"], "updated": sorted(results), "warnings": warnings, "requests": client.requests} meta = C.read_json(C.path("meta.json"), {}) or {} meta["lastRun"] = run C.write_json_atomic(C.path("meta.json"), meta) run["totals"] = {cat: fmt_totals(t) for cat, t in results.items()} progress(f"Done: {len(results)} of 4 categories updated, {len(warnings)} warning(s).") return run def save(cat, data, sources, meta_base, progress, results): meta = dict(meta_base, asOf=now_iso(), sources=sources) C.save_category(cat, data, meta) results[cat] = totals(cat, data) progress(f"{LABEL[cat]}: updated ({fmt_totals(results[cat])})") def flag_senators_elected_governor(senate, governors, year): for code, sens in senate.items(): g = governors.get(code) if not g or g.get("status") != f"elected-{year}" or not g.get("name"): continue for s in sens: if s["status"] == "continuing" and s["name"] and s["name"].lower() == g["name"].lower(): s["note"] = (f"Elected Governor of {C.NAMES[code]} in {year}; the Senate seat will be filled " f"after the new governor takes office. " + s["note"]).strip() def update_legislatures(client, mode, day, year, meta_base, progress, warnings, results): cat = "legislatures" progress(f"{LABEL[cat]}: reading Wikipedia…") try: data, sources, problems = P.legislatures(client, mode, day, year) except (P.ParseError, WikiError, KeyError, ValueError, IndexError) as e: warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).") traceback.print_exc(file=sys.stderr) return prev_file = C.load_category(cat) or {} prev = prev_file.get("data", {}) def same_cycle(meta): meta = meta or {} return meta.get("mode") == mode and (mode != "election" or meta.get("electionYear") == year) # Earlier figures for the same election (stored data first, then the seed shipped with the app): # used while Wikipedia has no post-election figures yet for a chamber. earlier = [f.get("data", {}) for f in (prev_file, C.read_json(os.path.join(C.SEED_DIR, cat + ".json"), {}) or {}) if f and same_cycle(f.get("meta"))] kept, unknown_kept = [], [] for code, legs in data.items(): for which in ("upper", "lower"): ch = legs.get(which) if ch is None: continue reason = problems.get((code, which)) or chamber_errors(code, which, ch) old = (prev.get(code) or {}).get(which) if reason: if old: legs[which] = old kept.append(f"{ch['name']} ({reason})") else: kept.append(f"{ch['name']} ({reason}; no previous data either)") continue same = [e for e in ((d.get(code) or {}).get(which) for d in earlier) if e] if ch.pop("_unknown", False): better = [e for e in same if e.get("uncalled", 0) < ch["uncalled"]] if better: # Wikipedia has no post-election figures yet: keep the earlier, more complete figures. legs[which] = better[0] unknown_kept.append(ch["name"]) continue named = [e["leader"] for e in same if (e.get("leader") or {}).get("name") and e["leader"].get("status") != "current"] if mode == "election" and named and (ch.get("leader") or {}).get("status") == "current": # an earlier read recorded the leader chosen for the new session: better than the outgoing one ch["leader"] = named[0] for legs in data.values(): for ch in legs.values(): if ch: ch.pop("_unknown", None) if len(kept) > 20: warnings.append(f"{LABEL[cat]}: kept all previous data because {len(kept)} chambers could not be read " f"(for example {'; '.join(kept[:3])}).") return errs = validate_legislatures(data) if errs: warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: {'; '.join(errs[:4])}.") return if kept: warnings.append(f"{LABEL[cat]}: previous figures kept for {len(kept)} chamber(s) whose Wikipedia data did not add up: " + "; ".join(kept) + ".") if unknown_kept: warnings.append(f"{LABEL[cat]}: Wikipedia has no {year} results yet for {len(unknown_kept)} chamber(s); " f"the earlier figures were kept: {', '.join(unknown_kept)}.") save(cat, data, sources, meta_base, progress, results) def main(): try: run = run_update(progress=lambda m: print(m, flush=True)) except Busy as e: print(str(e)) return 2 print() print("Totals:") for cat, t in run["totals"].items(): print(f" {LABEL[cat]}: {t}") if run["warnings"]: print("Warnings:") for w in run["warnings"]: print(" - " + w) return 0 if __name__ == "__main__": sys.exit(main())