221 lines
8.8 KiB
Python
221 lines
8.8 KiB
Python
"""Small web server: the map page, its data, and a button-triggered update from Wikipedia."""
|
|
import datetime as dt
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
import common as C
|
|
import update as U
|
|
|
|
STATIC = os.path.join(C.APP_DIR, "static")
|
|
PORT = int(os.environ.get("PORT", "8080"))
|
|
|
|
MODE_LABEL = {"election": "{year} election results", "current": "current officeholders"}
|
|
|
|
|
|
class UpdateJob:
|
|
"""One update at a time, run in a background thread; keeps its progress for the status endpoint."""
|
|
|
|
def __init__(self):
|
|
self.lock = threading.Lock()
|
|
self.state = "idle"
|
|
self.messages = []
|
|
self.warnings = []
|
|
self.error = None
|
|
self.started = None
|
|
self.finished = None
|
|
self.thread = None
|
|
|
|
def start(self):
|
|
with self.lock:
|
|
if self.state == "running" or U.lock_held():
|
|
return False
|
|
self.state, self.messages, self.warnings, self.error = "running", [], [], None
|
|
self.started, self.finished = U.now_iso(), None
|
|
self.thread = threading.Thread(target=self._run, daemon=True)
|
|
self.thread.start()
|
|
return True
|
|
|
|
def _progress(self, msg):
|
|
print(msg, flush=True)
|
|
with self.lock:
|
|
self.messages.append(msg)
|
|
del self.messages[:-200]
|
|
|
|
def _run(self):
|
|
try:
|
|
run = U.run_update(progress=self._progress)
|
|
with self.lock:
|
|
self.warnings = run["warnings"]
|
|
self.state = "done"
|
|
except U.Busy as e:
|
|
with self.lock:
|
|
self.state, self.error = "failed", str(e)
|
|
except Exception as e: # report anything unexpected in plain words, details go to the log
|
|
traceback.print_exc()
|
|
with self.lock:
|
|
self.state = "failed"
|
|
self.error = f"The update stopped because of an unexpected error ({type(e).__name__}: {e}). Previous data was kept."
|
|
finally:
|
|
with self.lock:
|
|
self.finished = U.now_iso()
|
|
|
|
def status(self):
|
|
with self.lock:
|
|
running_elsewhere = self.state != "running" and U.lock_held()
|
|
return {"state": "running" if running_elsewhere else self.state,
|
|
"runningElsewhere": running_elsewhere,
|
|
"startedAt": self.started, "finishedAt": self.finished,
|
|
"messages": list(self.messages), "warnings": list(self.warnings), "error": self.error}
|
|
|
|
|
|
JOB = UpdateJob()
|
|
|
|
|
|
def long_date(iso):
|
|
d = dt.date.fromisoformat(iso[:10])
|
|
return f"{d:%B} {d.day}, {d.year}"
|
|
|
|
|
|
def merged_data():
|
|
cats = {cat: (C.load_category(cat) or {"meta": {}, "data": {}}) for cat in C.CATEGORIES}
|
|
meta_file = C.read_json(C.path("meta.json"), {}) or {}
|
|
powers = C.seed_governor_powers()
|
|
data = {}
|
|
for code in C.NAMES:
|
|
gov = cats["governors"]["data"].get(code)
|
|
if gov is not None and not gov.get("powers"):
|
|
gov = dict(gov, powers=powers.get(code, []))
|
|
leg = cats["legislatures"]["data"].get(code)
|
|
data[code] = {"senators": cats["senate"]["data"].get(code, []),
|
|
"house": cats["house"]["data"].get(code, []),
|
|
"gov": gov, "leg": leg}
|
|
metas = {cat: cats[cat]["meta"] for cat in C.CATEGORIES}
|
|
modes = {m.get("mode") for m in metas.values()}
|
|
years = {m.get("electionYear") for m in metas.values() if m.get("mode") == "election"}
|
|
dates = sorted(m.get("asOf", "")[:10] for m in metas.values() if m.get("asOf"))
|
|
mode = modes.pop() if len(modes) == 1 else "mixed"
|
|
year = max(years) if years else None
|
|
last = meta_file.get("lastRun") or {}
|
|
sources = []
|
|
for m in metas.values():
|
|
for s in m.get("sources", []):
|
|
if s not in sources:
|
|
sources.append(s)
|
|
if all(m.get("seed") for m in metas.values()):
|
|
footer = meta_file.get("seedFooter", "")
|
|
else:
|
|
if dates and dates[0] != dates[-1]:
|
|
read = f", read between {long_date(dates[0])} and {long_date(dates[-1])}."
|
|
else:
|
|
read = f", read on {long_date(dates[0])}." if dates else "."
|
|
footer = "Data: English Wikipedia (" + "; ".join(sources) + ")" + read
|
|
if mode in ("election", "mixed"):
|
|
footer += " Results that were not final on that date are marked “Not yet called”. Where a chamber's leader for the new session was not chosen yet, the current leader is shown and noted."
|
|
footer += " Map shapes: us-atlas (U.S. Census Bureau boundaries)."
|
|
meta = {
|
|
"mode": mode,
|
|
"electionYear": year,
|
|
"electionDay": C.election_day(year).isoformat() if year else None,
|
|
"modeLabel": "mixed sources (see warnings)" if mode == "mixed" else MODE_LABEL.get(mode, mode).format(year=year),
|
|
"asOf": dates[0] if dates else None,
|
|
"newest": dates[-1] if dates else None,
|
|
"categories": {cat: {"mode": m.get("mode"), "asOf": m.get("asOf"), "electionYear": m.get("electionYear")}
|
|
for cat, m in metas.items()},
|
|
"lastRun": last or None,
|
|
"warnings": (last.get("warnings") if last else []) or [],
|
|
"footer": footer,
|
|
}
|
|
return {"data": data, "meta": meta}
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = "us-officeholders-map/1.0"
|
|
|
|
def log_message(self, fmt, *args):
|
|
sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
|
|
|
|
def _send(self, code, body, ctype="application/json; charset=utf-8", extra=None):
|
|
if isinstance(body, (dict, list)):
|
|
body = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
|
elif isinstance(body, str):
|
|
body = body.encode("utf-8")
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Cache-Control", "no-store" if ctype.startswith("application/json") else "public, max-age=300")
|
|
self.send_header("X-Content-Type-Options", "nosniff")
|
|
for k, v in (extra or {}).items():
|
|
self.send_header(k, v)
|
|
self.end_headers()
|
|
if self.command != "HEAD":
|
|
self.wfile.write(body)
|
|
|
|
def _static(self, rel):
|
|
full = os.path.realpath(os.path.join(STATIC, rel))
|
|
if not full.startswith(os.path.realpath(STATIC) + os.sep) or not os.path.isfile(full):
|
|
return self._send(404, {"error": "Not found"})
|
|
ctype = mimetypes.guess_type(full)[0] or "application/octet-stream"
|
|
if ctype.startswith("text/") or ctype in ("application/javascript", "application/json"):
|
|
ctype += "; charset=utf-8"
|
|
with open(full, "rb") as f:
|
|
body = f.read()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Cache-Control", "no-cache" if rel == "index.html" else "public, max-age=3600")
|
|
self.send_header("X-Content-Type-Options", "nosniff")
|
|
self.end_headers()
|
|
if self.command != "HEAD":
|
|
self.wfile.write(body)
|
|
|
|
def do_HEAD(self):
|
|
self.do_GET()
|
|
|
|
def do_GET(self):
|
|
p = self.path.split("?", 1)[0]
|
|
if p in ("/", "/index.html"):
|
|
return self._static("index.html")
|
|
if p.startswith("/static/"):
|
|
return self._static(p[len("/static/"):])
|
|
if p == "/api/data":
|
|
try:
|
|
return self._send(200, merged_data())
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
return self._send(500, {"error": f"Could not read the stored data ({e})."})
|
|
if p == "/api/update/status":
|
|
return self._send(200, JOB.status())
|
|
return self._send(404, {"error": "Not found"})
|
|
|
|
def do_POST(self):
|
|
p = self.path.split("?", 1)[0]
|
|
if p != "/api/update":
|
|
return self._send(404, {"error": "Not found"})
|
|
# Plain cross-site form posts cannot set this header, so another web page cannot start updates.
|
|
if self.headers.get("X-Requested-With") != "us-officeholders-map":
|
|
return self._send(403, {"error": "Missing request header."})
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if length:
|
|
self.rfile.read(min(length, 4096))
|
|
if JOB.start():
|
|
return self._send(202, {"started": True})
|
|
return self._send(409, {"started": False, "error": "An update is already running."})
|
|
|
|
|
|
def main():
|
|
copied = C.ensure_seed()
|
|
if copied:
|
|
print(f"Seeded {C.DATA_DIR} with: {', '.join(copied)}", flush=True)
|
|
httpd = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
|
|
print(f"Serving on port {PORT}, data in {C.DATA_DIR}", flush=True)
|
|
httpd.serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|