Initial commit
This commit is contained in:
commit
a206bfb5db
21 files changed
+16747
No files matched your search
+379
@@ -0,0 +1,379 @@
|
||||
"""Refresh the map data from English Wikipedia.
|
||||
|
||||
Usage (inside the container): python update.py
|
||||
Mode: UPDATE_MODE=auto|election|current (auto = election results between Election Day and January 3, else current officeholders).
|
||||
Each category (senate, house, governors, legislatures) is checked before it replaces the stored file;
|
||||
if reading or checking fails, the previous data for that category is kept and a warning is recorded.
|
||||
"""
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import common as C
|
||||
import parsers as P
|
||||
from wiki import Client, WikiError
|
||||
|
||||
LOCK = ".update.lock"
|
||||
LOCK_STALE_SECONDS = 300 # the running update touches the lock at every step, so an older lock is left over from a crash
|
||||
|
||||
|
||||
class Busy(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def acquire_lock():
|
||||
C.ensure_dir()
|
||||
p = C.path(LOCK)
|
||||
try:
|
||||
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
except FileExistsError:
|
||||
try:
|
||||
age = time.time() - os.path.getmtime(p)
|
||||
except OSError:
|
||||
age = 0
|
||||
if age < LOCK_STALE_SECONDS:
|
||||
raise Busy("An update is already running.")
|
||||
os.unlink(p)
|
||||
fd = os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(f"{os.getpid()} {dt.datetime.now(dt.timezone.utc).isoformat()}\n")
|
||||
|
||||
|
||||
def release_lock():
|
||||
try:
|
||||
os.unlink(C.path(LOCK))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def lock_held():
|
||||
p = C.path(LOCK)
|
||||
return os.path.exists(p) and time.time() - os.path.getmtime(p) < LOCK_STALE_SECONDS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- validation
|
||||
|
||||
def validate_senate(data):
|
||||
errs = []
|
||||
if len(data) != 50 or set(data) != set(C.NAMES):
|
||||
errs.append(f"{len(data)} states instead of 50")
|
||||
total = sum(len(v) for v in data.values())
|
||||
if total != 100:
|
||||
errs.append(f"{total} senators instead of 100")
|
||||
for code, v in sorted(data.items()):
|
||||
if len(v) != 2:
|
||||
errs.append(f"{C.NAMES.get(code, code)} has {len(v)} senators")
|
||||
elif v[0]["class"] == v[1]["class"]:
|
||||
errs.append(f"{C.NAMES[code]} has two class {v[0]['class']} senators")
|
||||
for s in v:
|
||||
if not s["name"] and s["status"] not in ("uncalled", "runoff", "vacant"):
|
||||
errs.append(f"a {C.NAMES.get(code, code)} senator has no name")
|
||||
if s["name"] and s["party"] not in ("D", "R", "I"):
|
||||
errs.append(f"no party for {s['name']}")
|
||||
return errs
|
||||
|
||||
|
||||
def validate_house(data):
|
||||
errs = []
|
||||
if set(data) != set(C.NAMES):
|
||||
missing = sorted(set(C.NAMES) - set(data))
|
||||
errs.append(f"{len(data)} states instead of 50" + (f" (missing {', '.join(missing[:5])})" if missing else ""))
|
||||
total = sum(len(v) for v in data.values())
|
||||
if total != 435:
|
||||
errs.append(f"{total} representatives instead of 435")
|
||||
for code, n in C.APPORTIONMENT.items():
|
||||
v = data.get(code, [])
|
||||
if len(v) != n:
|
||||
errs.append(f"{C.NAMES[code]}: {len(v)} seats instead of {n}")
|
||||
continue
|
||||
expected = [0] if n == 1 else list(range(1, n + 1))
|
||||
if sorted(r["district"] for r in v) != expected and not (n == 1 and [r["district"] for r in v] == [1]):
|
||||
errs.append(f"{C.NAMES[code]}: unexpected district numbers")
|
||||
for r in v:
|
||||
if r["name"] and r["party"] not in ("D", "R", "I"):
|
||||
errs.append(f"no party for {r['name']}")
|
||||
if not r["name"] and r["status"] not in ("uncalled", "runoff", "vacant"):
|
||||
errs.append(f"{C.NAMES[code]} district {r['district']} has no name")
|
||||
return errs
|
||||
|
||||
|
||||
def validate_governors(data):
|
||||
errs = []
|
||||
if set(data) != set(C.NAMES):
|
||||
errs.append(f"{len(data)} governors instead of 50")
|
||||
for code, g in data.items():
|
||||
if not g["name"] and g["status"] not in ("uncalled", "runoff", "vacant"):
|
||||
errs.append(f"no governor name for {C.NAMES.get(code, code)}")
|
||||
if g["name"] and g["party"] not in ("D", "R", "I"):
|
||||
errs.append(f"no party for governor {g['name']}")
|
||||
return errs
|
||||
|
||||
|
||||
def chamber_errors(code, which, ch):
|
||||
size = C.CHAMBERS[code][0 if which == "upper" else 1][1]
|
||||
total = ch["D"] + ch["R"] + ch["I"] + ch["vacant"] + ch["uncalled"]
|
||||
if ch["seats"] != size or total != size:
|
||||
return f"seats add up to {total} instead of {size}"
|
||||
return None
|
||||
|
||||
|
||||
def validate_legislatures(data):
|
||||
errs = []
|
||||
if set(data) != set(C.NAMES):
|
||||
errs.append(f"{len(data)} states instead of 50")
|
||||
n = 0
|
||||
for code, legs in data.items():
|
||||
for which in ("upper", "lower"):
|
||||
ch = legs.get(which)
|
||||
if ch is None:
|
||||
if not (code == "NE" and which == "lower"):
|
||||
errs.append(f"{C.NAMES[code]} {which} chamber missing")
|
||||
continue
|
||||
n += 1
|
||||
e = chamber_errors(code, which, ch)
|
||||
if e:
|
||||
errs.append(f"{ch['name']}: {e}")
|
||||
if n != 99:
|
||||
errs.append(f"{n} chambers instead of 99")
|
||||
return errs
|
||||
|
||||
|
||||
VALIDATORS = {"senate": validate_senate, "house": validate_house, "governors": validate_governors,
|
||||
"legislatures": validate_legislatures}
|
||||
|
||||
LABEL = {"senate": "U.S. Senate", "house": "U.S. House", "governors": "Governors", "legislatures": "State legislatures"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- totals (for messages and the CLI)
|
||||
|
||||
def party_totals(entries):
|
||||
t = {"D": 0, "R": 0, "I": 0, "unknown": 0}
|
||||
for e in entries:
|
||||
if not e or e.get("status") in ("uncalled", "runoff") or e.get("party") not in ("D", "R", "I"):
|
||||
t["unknown"] += 1
|
||||
else:
|
||||
t[e["party"]] += 1
|
||||
return t
|
||||
|
||||
|
||||
def totals(cat, data):
|
||||
if cat == "senate":
|
||||
return party_totals([s for v in data.values() for s in v])
|
||||
if cat == "house":
|
||||
return party_totals([r for v in data.values() for r in v])
|
||||
if cat == "governors":
|
||||
return party_totals(list(data.values()))
|
||||
seats = {"D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "chambers": 0}
|
||||
for legs in data.values():
|
||||
for ch in legs.values():
|
||||
if ch:
|
||||
seats["chambers"] += 1
|
||||
for k in ("D", "R", "I", "vacant", "uncalled"):
|
||||
seats[k] += ch[k]
|
||||
return seats
|
||||
|
||||
|
||||
def fmt_totals(t):
|
||||
return ", ".join(f"{v} {k}" for k, v in t.items() if v)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the run
|
||||
|
||||
def now_iso():
|
||||
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def run_update(progress=print, requested_mode=None):
|
||||
"""Run a full update. Returns a summary dict. Raises Busy if another update holds the lock."""
|
||||
C.ensure_seed()
|
||||
acquire_lock()
|
||||
|
||||
def progress_and_touch(msg):
|
||||
try:
|
||||
os.utime(C.path(LOCK))
|
||||
except OSError:
|
||||
pass
|
||||
progress(msg)
|
||||
|
||||
try:
|
||||
return _run(progress_and_touch, requested_mode)
|
||||
finally:
|
||||
release_lock()
|
||||
|
||||
|
||||
def _run(progress, requested_mode):
|
||||
day = C.today()
|
||||
mode, requested = C.choose_mode(day, requested_mode)
|
||||
year = C.election_window(day) or C.last_election_year(day)
|
||||
started = now_iso()
|
||||
warnings, results = [], {}
|
||||
client = Client(progress)
|
||||
progress(f"Mode: {'election results ' + str(year) if mode == 'election' else 'current officeholders'}"
|
||||
f"{' (forced by UPDATE_MODE)' if requested != 'auto' else ' (chosen from today’s date)'}")
|
||||
|
||||
def attempt(cat, fn):
|
||||
progress(f"{LABEL[cat]}: reading Wikipedia…")
|
||||
try:
|
||||
data, sources = fn()
|
||||
except (P.ParseError, WikiError, KeyError, ValueError, IndexError, StopIteration) as e:
|
||||
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
|
||||
progress(f"{LABEL[cat]}: failed, previous data kept")
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
return None
|
||||
errs = VALIDATORS[cat](data)
|
||||
if errs:
|
||||
more = f" and {len(errs) - 4} more problem(s)" if len(errs) > 4 else ""
|
||||
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: "
|
||||
f"{'; '.join(errs[:4])}{more}.")
|
||||
progress(f"{LABEL[cat]}: checks failed, previous data kept")
|
||||
return None
|
||||
return data, sources
|
||||
|
||||
meta_base = {"mode": mode, "electionYear": year if mode == "election" else None}
|
||||
|
||||
# Governors first: the Senate step uses them to flag senators who were elected governor.
|
||||
if mode == "election":
|
||||
g = attempt("governors", lambda: P.governors_election(client, year))
|
||||
else:
|
||||
g = attempt("governors", lambda: P.governors_current(client, day))
|
||||
if g:
|
||||
save("governors", g[0], g[1], meta_base, progress, results)
|
||||
|
||||
if mode == "election":
|
||||
s = attempt("senate", lambda: P.senate_election(client, year))
|
||||
else:
|
||||
s = attempt("senate", lambda: P.senate_current(client, day))
|
||||
if s:
|
||||
if mode == "election" and g:
|
||||
flag_senators_elected_governor(s[0], g[0], year)
|
||||
save("senate", s[0], s[1], meta_base, progress, results)
|
||||
|
||||
if mode == "election":
|
||||
h = attempt("house", lambda: P.house_election(client, year))
|
||||
else:
|
||||
h = attempt("house", lambda: P.house_current(client, day))
|
||||
if h:
|
||||
save("house", h[0], h[1], meta_base, progress, results)
|
||||
|
||||
update_legislatures(client, mode, day, year, meta_base, progress, warnings, results)
|
||||
|
||||
finished = now_iso()
|
||||
run = {"startedAt": started, "finishedAt": finished, "mode": mode, "requestedMode": requested,
|
||||
"electionYear": meta_base["electionYear"], "updated": sorted(results), "warnings": warnings,
|
||||
"requests": client.requests}
|
||||
meta = C.read_json(C.path("meta.json"), {}) or {}
|
||||
meta["lastRun"] = run
|
||||
C.write_json_atomic(C.path("meta.json"), meta)
|
||||
run["totals"] = {cat: fmt_totals(t) for cat, t in results.items()}
|
||||
progress(f"Done: {len(results)} of 4 categories updated, {len(warnings)} warning(s).")
|
||||
return run
|
||||
|
||||
|
||||
def save(cat, data, sources, meta_base, progress, results):
|
||||
meta = dict(meta_base, asOf=now_iso(), sources=sources)
|
||||
C.save_category(cat, data, meta)
|
||||
results[cat] = totals(cat, data)
|
||||
progress(f"{LABEL[cat]}: updated ({fmt_totals(results[cat])})")
|
||||
|
||||
|
||||
def flag_senators_elected_governor(senate, governors, year):
|
||||
for code, sens in senate.items():
|
||||
g = governors.get(code)
|
||||
if not g or g.get("status") != f"elected-{year}" or not g.get("name"):
|
||||
continue
|
||||
for s in sens:
|
||||
if s["status"] == "continuing" and s["name"] and s["name"].lower() == g["name"].lower():
|
||||
s["note"] = (f"Elected Governor of {C.NAMES[code]} in {year}; the Senate seat will be filled "
|
||||
f"after the new governor takes office. " + s["note"]).strip()
|
||||
|
||||
|
||||
def update_legislatures(client, mode, day, year, meta_base, progress, warnings, results):
|
||||
cat = "legislatures"
|
||||
progress(f"{LABEL[cat]}: reading Wikipedia…")
|
||||
try:
|
||||
data, sources, problems = P.legislatures(client, mode, day, year)
|
||||
except (P.ParseError, WikiError, KeyError, ValueError, IndexError) as e:
|
||||
warnings.append(f"{LABEL[cat]}: kept the previous data because Wikipedia could not be read as expected ({e}).")
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
return
|
||||
prev_file = C.load_category(cat) or {}
|
||||
prev = prev_file.get("data", {})
|
||||
|
||||
def same_cycle(meta):
|
||||
meta = meta or {}
|
||||
return meta.get("mode") == mode and (mode != "election" or meta.get("electionYear") == year)
|
||||
|
||||
# Earlier figures for the same election (stored data first, then the seed shipped with the app):
|
||||
# used while Wikipedia has no post-election figures yet for a chamber.
|
||||
earlier = [f.get("data", {}) for f in (prev_file, C.read_json(os.path.join(C.SEED_DIR, cat + ".json"), {}) or {})
|
||||
if f and same_cycle(f.get("meta"))]
|
||||
kept, unknown_kept = [], []
|
||||
for code, legs in data.items():
|
||||
for which in ("upper", "lower"):
|
||||
ch = legs.get(which)
|
||||
if ch is None:
|
||||
continue
|
||||
reason = problems.get((code, which)) or chamber_errors(code, which, ch)
|
||||
old = (prev.get(code) or {}).get(which)
|
||||
if reason:
|
||||
if old:
|
||||
legs[which] = old
|
||||
kept.append(f"{ch['name']} ({reason})")
|
||||
else:
|
||||
kept.append(f"{ch['name']} ({reason}; no previous data either)")
|
||||
continue
|
||||
same = [e for e in ((d.get(code) or {}).get(which) for d in earlier) if e]
|
||||
if ch.pop("_unknown", False):
|
||||
better = [e for e in same if e.get("uncalled", 0) < ch["uncalled"]]
|
||||
if better:
|
||||
# Wikipedia has no post-election figures yet: keep the earlier, more complete figures.
|
||||
legs[which] = better[0]
|
||||
unknown_kept.append(ch["name"])
|
||||
continue
|
||||
named = [e["leader"] for e in same if (e.get("leader") or {}).get("name") and e["leader"].get("status") != "current"]
|
||||
if mode == "election" and named and (ch.get("leader") or {}).get("status") == "current":
|
||||
# an earlier read recorded the leader chosen for the new session: better than the outgoing one
|
||||
ch["leader"] = named[0]
|
||||
for legs in data.values():
|
||||
for ch in legs.values():
|
||||
if ch:
|
||||
ch.pop("_unknown", None)
|
||||
if len(kept) > 20:
|
||||
warnings.append(f"{LABEL[cat]}: kept all previous data because {len(kept)} chambers could not be read "
|
||||
f"(for example {'; '.join(kept[:3])}).")
|
||||
return
|
||||
errs = validate_legislatures(data)
|
||||
if errs:
|
||||
warnings.append(f"{LABEL[cat]}: kept the previous data because the new data did not pass the checks: {'; '.join(errs[:4])}.")
|
||||
return
|
||||
if kept:
|
||||
warnings.append(f"{LABEL[cat]}: previous figures kept for {len(kept)} chamber(s) whose Wikipedia data did not add up: "
|
||||
+ "; ".join(kept) + ".")
|
||||
if unknown_kept:
|
||||
warnings.append(f"{LABEL[cat]}: Wikipedia has no {year} results yet for {len(unknown_kept)} chamber(s); "
|
||||
f"the earlier figures were kept: {', '.join(unknown_kept)}.")
|
||||
save(cat, data, sources, meta_base, progress, results)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
run = run_update(progress=lambda m: print(m, flush=True))
|
||||
except Busy as e:
|
||||
print(str(e))
|
||||
return 2
|
||||
print()
|
||||
print("Totals:")
|
||||
for cat, t in run["totals"].items():
|
||||
print(f" {LABEL[cat]}: {t}")
|
||||
if run["warnings"]:
|
||||
print("Warnings:")
|
||||
for w in run["warnings"]:
|
||||
print(" - " + w)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user