380 lines
15 KiB
Python
380 lines
15 KiB
Python
"""Refresh the map data from English Wikipedia.
|
||
|
||
Usage (inside the container): python update.py
|
||
Mode: UPDATE_MODE=auto|election|current (auto = election results between Election Day and January 3, else current officeholders).
|
||
Each category (senate, house, governors, legislatures) is checked before it replaces the stored file;
|
||
if reading or checking fails, the previous data for that category is kept and a warning is recorded.
|
||
"""
|
||
import datetime as dt
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
import traceback
|
||
|
||
import common as C
|
||
import parsers as P
|
||
from wiki import Client, WikiError
|
||
|
||
LOCK = ".update.lock"
|
||
LOCK_STALE_SECONDS = 300 # the running update touches the lock at every step, so an older lock is left over from a crash
|
||
|
||
|
||
class Busy(Exception):
|
||
pass
|
||
|
||
|
||
def acquire_lock():
|
||
C.ensure_dir()
|
||
p = C.path(LOCK)
|
||
try:
|
||
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||
except FileExistsError:
|
||
try:
|
||
age = time.time() - os.path.getmtime(p)
|
||
except OSError:
|
||
age = 0
|
||
if age < LOCK_STALE_SECONDS:
|
||
raise Busy("An update is already running.")
|
||
os.unlink(p)
|
||
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||
with os.fdopen(fd, "w") as f:
|
||
f.write(f"{os.getpid()} {dt.datetime.now(dt.timezone.utc).isoformat()}\n")
|
||
|
||
|
||
def release_lock():
|
||
try:
|
||
os.unlink(C.path(LOCK))
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def lock_held():
|
||
p = C.path(LOCK)
|
||
return os.path.exists(p) and time.time() - os.path.getmtime(p) < LOCK_STALE_SECONDS
|
||
|
||
|
||
# ---------------------------------------------------------------- validation
|
||
|
||
def validate_senate(data):
|
||
errs = []
|
||
if len(data) != 50 or set(data) != set(C.NAMES):
|
||
errs.append(f"{len(data)} states instead of 50")
|
||
total = sum(len(v) for v in data.values())
|
||
if total != 100:
|
||
errs.append(f"{total} senators instead of 100")
|
||
for code, v in sorted(data.items()):
|
||
if len(v) != 2:
|
||
errs.append(f"{C.NAMES.get(code, code)} has {len(v)} senators")
|
||
elif v[0]["class"] == v[1]["class"]:
|
||
errs.append(f"{C.NAMES[code]} has two class {v[0]['class']} senators")
|
||
for s in v:
|
||
if not s["name"] and s["status"] not in ("uncalled", "runoff", "vacant"):
|
||
errs.append(f"a {C.NAMES.get(code, code)} senator has no name")
|
||
if s["name"] and s["party"] not in ("D", "R", "I"):
|
||
errs.append(f"no party for {s['name']}")
|
||
return errs
|
||
|
||
|
||
def validate_house(data):
|
||
errs = []
|
||
if set(data) != set(C.NAMES):
|
||
missing = sorted(set(C.NAMES) - set(data))
|
||
errs.append(f"{len(data)} states instead of 50" + (f" (missing {', '.join(missing[:5])})" if missing else ""))
|
||
total = sum(len(v) for v in data.values())
|
||
if total != 435:
|
||
errs.append(f"{total} representatives instead of 435")
|
||
for code, n in C.APPORTIONMENT.items():
|
||
v = data.get(code, [])
|
||
if len(v) != n:
|
||
errs.append(f"{C.NAMES[code]}: {len(v)} seats instead of {n}")
|
||
continue
|
||
expected = [0] if n == 1 else list(range(1, n + 1))
|
||
if sorted(r["district"] for r in v) != expected and not (n == 1 and [r["district"] for r in v] == [1]):
|
||
errs.append(f"{C.NAMES[code]}: unexpected district numbers")
|
||
for r in v:
|
||
if r["name"] and r["party"] not in ("D", "R", "I"):
|
||
errs.append(f"no party for {r['name']}")
|
||
if not r["name"] and r["status"] not in ("uncalled", "runoff", "vacant"):
|
||
errs.append(f"{C.NAMES[code]} district {r['district']} has no name")
|
||
return errs
|
||
|
||
|
||
def validate_governors(data):
|
||
errs = []
|
||
if set(data) != set(C.NAMES):
|
||
errs.append(f"{len(data)} governors instead of 50")
|
||
for code, g in data.items():
|
||
if not g["name"] and g["status"] not in ("uncalled", "runoff", "vacant"):
|
||
errs.append(f"no governor name for {C.NAMES.get(code, code)}")
|
||
if g["name"] and g["party"] not in ("D", "R", "I"):
|
||
errs.append(f"no party for governor {g['name']}")
|
||
return errs
|
||
|
||
|
||
def chamber_errors(code, which, ch):
|
||
size = C.CHAMBERS[code][0 if which == "upper" else 1][1]
|
||
total = ch["D"] + ch["R"] + ch["I"] + ch["vacant"] + ch["uncalled"]
|
||
if ch["seats"] != size or total != size:
|
||
return f"seats add up to {total} instead of {size}"
|
||
return None
|
||
|
||
|
||
def validate_legislatures(data):
|
||
errs = []
|
||
if set(data) != set(C.NAMES):
|
||
errs.append(f"{len(data)} states instead of 50")
|
||
n = 0
|
||
for code, legs in data.items():
|
||
for which in ("upper", "lower"):
|
||
ch = legs.get(which)
|
||
if ch is None:
|
||
if not (code == "NE" and which == "lower"):
|
||
errs.append(f"{C.NAMES[code]} {which} chamber missing")
|
||
continue
|
||
n += 1
|
||
e = chamber_errors(code, which, ch)
|
||
if e:
|
||
errs.append(f"{ch['name']}: {e}")
|
||
if n != 99:
|
||
errs.append(f"{n} chambers instead of 99")
|
||
return errs
|
||
|
||
|
||
VALIDATORS = {"senate": validate_senate, "house": validate_house, "governors": validate_governors,
|
||
"legislatures": validate_legislatures}
|
||
|
||
LABEL = {"senate": "U.S. Senate", "house": "U.S. House", "governors": "Governors", "legislatures": "State legislatures"}
|
||
|
||
|
||
# ---------------------------------------------------------------- totals (for messages and the CLI)
|
||
|
||
def party_totals(entries):
|
||
t = {"D": 0, "R": 0, "I": 0, "unknown": 0}
|
||
for e in entries:
|
||
if not e or e.get("status") in ("uncalled", "runoff") or e.get("party") not in ("D", "R", "I"):
|
||
t["unknown"] += 1
|
||
else:
|
||
t[e["party"]] += 1
|
||
return t
|
||
|
||
|
||
def totals(cat, data):
|
||
if cat == "senate":
|
||
return party_totals([s for v in data.values() for s in v])
|
||
if cat == "house":
|
||
return party_totals([r for v in data.values() for r in v])
|
||
if cat == "governors":
|
||
return party_totals(list(data.values()))
|
||
seats = {"D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "chambers": 0}
|
||
for legs in data.values():
|
||
for ch in legs.values():
|
||
if ch:
|
||
seats["chambers"] += 1
|
||
for k in ("D", "R", "I", "vacant", "uncalled"):
|
||
seats[k] += ch[k]
|
||
return seats
|
||
|
||
|
||
def fmt_totals(t):
|
||
return ", ".join(f"{v} {k}" for k, v in t.items() if v)
|
||
|
||
|
||
# ---------------------------------------------------------------- the run
|
||
|
||
def now_iso():
|
||
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
|
||
|
||
|
||
def run_update(progress=print, requested_mode=None):
|
||
"""Run a full update. Returns a summary dict. Raises Busy if another update holds the lock."""
|
||
C.ensure_seed()
|
||
acquire_lock()
|
||
|
||
def progress_and_touch(msg):
|
||
try:
|
||
os.utime(C.path(LOCK))
|
||
except OSError:
|
||
pass
|
||
progress(msg)
|
||
|
||
try:
|
||
return _run(progress_and_touch, requested_mode)
|
||
finally:
|
||
release_lock()
|
||
|
||
|
||
def _run(progress, requested_mode):
|
||
day = C.today()
|
||
mode, requested = C.choose_mode(day, requested_mode)
|
||
year = C.election_window(day) or C.last_election_year(day)
|
||
started = now_iso()
|
||
warnings, results = [], {}
|
||
client = Client(progress)
|
||
progress(f"Mode: {'election results ' + str(year) if mode == 'election' else 'current officeholders'}"
|
||
f"{' (forced by UPDATE_MODE)' if requested != 'auto' else ' (chosen from today’s date)'}")
|
||
|
||
def attempt(cat, fn):
|
||
progress(f"{LABEL[cat]}: reading Wikipedia…")
|
||
try:
|
||
data, sources = fn()
|
||
except (P.ParseError, WikiError, KeyError, ValueError, IndexError, StopIteration) as e:
|
||
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
|
||
progress(f"{LABEL[cat]}: failed, previous data kept")
|
||
traceback.print_exc(file=sys.stderr)
|
||
return None
|
||
errs = VALIDATORS[cat](data)
|
||
if errs:
|
||
more = f" and {len(errs) - 4} more problem(s)" if len(errs) > 4 else ""
|
||
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: "
|
||
f"{'; '.join(errs[:4])}{more}.")
|
||
progress(f"{LABEL[cat]}: checks failed, previous data kept")
|
||
return None
|
||
return data, sources
|
||
|
||
meta_base = {"mode": mode, "electionYear": year if mode == "election" else None}
|
||
|
||
# Governors first: the Senate step uses them to flag senators who were elected governor.
|
||
if mode == "election":
|
||
g = attempt("governors", lambda: P.governors_election(client, year))
|
||
else:
|
||
g = attempt("governors", lambda: P.governors_current(client, day))
|
||
if g:
|
||
save("governors", g[0], g[1], meta_base, progress, results)
|
||
|
||
if mode == "election":
|
||
s = attempt("senate", lambda: P.senate_election(client, year))
|
||
else:
|
||
s = attempt("senate", lambda: P.senate_current(client, day))
|
||
if s:
|
||
if mode == "election" and g:
|
||
flag_senators_elected_governor(s[0], g[0], year)
|
||
save("senate", s[0], s[1], meta_base, progress, results)
|
||
|
||
if mode == "election":
|
||
h = attempt("house", lambda: P.house_election(client, year))
|
||
else:
|
||
h = attempt("house", lambda: P.house_current(client, day))
|
||
if h:
|
||
save("house", h[0], h[1], meta_base, progress, results)
|
||
|
||
update_legislatures(client, mode, day, year, meta_base, progress, warnings, results)
|
||
|
||
finished = now_iso()
|
||
run = {"startedAt": started, "finishedAt": finished, "mode": mode, "requestedMode": requested,
|
||
"electionYear": meta_base["electionYear"], "updated": sorted(results), "warnings": warnings,
|
||
"requests": client.requests}
|
||
meta = C.read_json(C.path("meta.json"), {}) or {}
|
||
meta["lastRun"] = run
|
||
C.write_json_atomic(C.path("meta.json"), meta)
|
||
run["totals"] = {cat: fmt_totals(t) for cat, t in results.items()}
|
||
progress(f"Done: {len(results)} of 4 categories updated, {len(warnings)} warning(s).")
|
||
return run
|
||
|
||
|
||
def save(cat, data, sources, meta_base, progress, results):
|
||
meta = dict(meta_base, asOf=now_iso(), sources=sources)
|
||
C.save_category(cat, data, meta)
|
||
results[cat] = totals(cat, data)
|
||
progress(f"{LABEL[cat]}: updated ({fmt_totals(results[cat])})")
|
||
|
||
|
||
def flag_senators_elected_governor(senate, governors, year):
|
||
for code, sens in senate.items():
|
||
g = governors.get(code)
|
||
if not g or g.get("status") != f"elected-{year}" or not g.get("name"):
|
||
continue
|
||
for s in sens:
|
||
if s["status"] == "continuing" and s["name"] and s["name"].lower() == g["name"].lower():
|
||
s["note"] = (f"Elected Governor of {C.NAMES[code]} in {year}; the Senate seat will be filled "
|
||
f"after the new governor takes office. " + s["note"]).strip()
|
||
|
||
|
||
def update_legislatures(client, mode, day, year, meta_base, progress, warnings, results):
|
||
cat = "legislatures"
|
||
progress(f"{LABEL[cat]}: reading Wikipedia…")
|
||
try:
|
||
data, sources, problems = P.legislatures(client, mode, day, year)
|
||
except (P.ParseError, WikiError, KeyError, ValueError, IndexError) as e:
|
||
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
|
||
traceback.print_exc(file=sys.stderr)
|
||
return
|
||
prev_file = C.load_category(cat) or {}
|
||
prev = prev_file.get("data", {})
|
||
|
||
def same_cycle(meta):
|
||
meta = meta or {}
|
||
return meta.get("mode") == mode and (mode != "election" or meta.get("electionYear") == year)
|
||
|
||
# Earlier figures for the same election (stored data first, then the seed shipped with the app):
|
||
# used while Wikipedia has no post-election figures yet for a chamber.
|
||
earlier = [f.get("data", {}) for f in (prev_file, C.read_json(os.path.join(C.SEED_DIR, cat + ".json"), {}) or {})
|
||
if f and same_cycle(f.get("meta"))]
|
||
kept, unknown_kept = [], []
|
||
for code, legs in data.items():
|
||
for which in ("upper", "lower"):
|
||
ch = legs.get(which)
|
||
if ch is None:
|
||
continue
|
||
reason = problems.get((code, which)) or chamber_errors(code, which, ch)
|
||
old = (prev.get(code) or {}).get(which)
|
||
if reason:
|
||
if old:
|
||
legs[which] = old
|
||
kept.append(f"{ch['name']} ({reason})")
|
||
else:
|
||
kept.append(f"{ch['name']} ({reason}; no previous data either)")
|
||
continue
|
||
same = [e for e in ((d.get(code) or {}).get(which) for d in earlier) if e]
|
||
if ch.pop("_unknown", False):
|
||
better = [e for e in same if e.get("uncalled", 0) < ch["uncalled"]]
|
||
if better:
|
||
# Wikipedia has no post-election figures yet: keep the earlier, more complete figures.
|
||
legs[which] = better[0]
|
||
unknown_kept.append(ch["name"])
|
||
continue
|
||
named = [e["leader"] for e in same if (e.get("leader") or {}).get("name") and e["leader"].get("status") != "current"]
|
||
if mode == "election" and named and (ch.get("leader") or {}).get("status") == "current":
|
||
# an earlier read recorded the leader chosen for the new session: better than the outgoing one
|
||
ch["leader"] = named[0]
|
||
for legs in data.values():
|
||
for ch in legs.values():
|
||
if ch:
|
||
ch.pop("_unknown", None)
|
||
if len(kept) > 20:
|
||
warnings.append(f"{LABEL[cat]}: kept all previous data because {len(kept)} chambers could not be read "
|
||
f"(for example {'; '.join(kept[:3])}).")
|
||
return
|
||
errs = validate_legislatures(data)
|
||
if errs:
|
||
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: {'; '.join(errs[:4])}.")
|
||
return
|
||
if kept:
|
||
warnings.append(f"{LABEL[cat]}: previous figures kept for {len(kept)} chamber(s) whose Wikipedia data did not add up: "
|
||
+ "; ".join(kept) + ".")
|
||
if unknown_kept:
|
||
warnings.append(f"{LABEL[cat]}: Wikipedia has no {year} results yet for {len(unknown_kept)} chamber(s); "
|
||
f"the earlier figures were kept: {', '.join(unknown_kept)}.")
|
||
save(cat, data, sources, meta_base, progress, results)
|
||
|
||
|
||
def main():
|
||
try:
|
||
run = run_update(progress=lambda m: print(m, flush=True))
|
||
except Busy as e:
|
||
print(str(e))
|
||
return 2
|
||
print()
|
||
print("Totals:")
|
||
for cat, t in run["totals"].items():
|
||
print(f" {LABEL[cat]}: {t}")
|
||
if run["warnings"]:
|
||
print("Warnings:")
|
||
for w in run["warnings"]:
|
||
print(" - " + w)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|