Initial commit
This commit is contained in:
commit
a206bfb5db
21 files changed
+16747
No files matched your search
+220
@@ -0,0 +1,220 @@
|
||||
"""Small web server: the map page, its data, and a button-triggered update from Wikipedia."""
|
||||
import datetime as dt
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import common as C
|
||||
import update as U
|
||||
|
||||
STATIC = os.path.join(C.APP_DIR, "static")
|
||||
PORT = int(os.environ.get("PORT", "8080"))
|
||||
|
||||
MODE_LABEL = {"election": "{year} election results", "current": "current officeholders"}
|
||||
|
||||
|
||||
class UpdateJob:
|
||||
"""One update at a time, run in a background thread; keeps its progress for the status endpoint."""
|
||||
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.state = "idle"
|
||||
self.messages = []
|
||||
self.warnings = []
|
||||
self.error = None
|
||||
self.started = None
|
||||
self.finished = None
|
||||
self.thread = None
|
||||
|
||||
def start(self):
|
||||
with self.lock:
|
||||
if self.state == "running" or U.lock_held():
|
||||
return False
|
||||
self.state, self.messages, self.warnings, self.error = "running", [], [], None
|
||||
self.started, self.finished = U.now_iso(), None
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
self.thread.start()
|
||||
return True
|
||||
|
||||
def _progress(self, msg):
|
||||
print(msg, flush=True)
|
||||
with self.lock:
|
||||
self.messages.append(msg)
|
||||
del self.messages[:-200]
|
||||
|
||||
def _run(self):
|
||||
try:
|
||||
run = U.run_update(progress=self._progress)
|
||||
with self.lock:
|
||||
self.warnings = run["warnings"]
|
||||
self.state = "done"
|
||||
except U.Busy as e:
|
||||
with self.lock:
|
||||
self.state, self.error = "failed", str(e)
|
||||
except Exception as e: # report anything unexpected in plain words, details go to the log
|
||||
traceback.print_exc()
|
||||
with self.lock:
|
||||
self.state = "failed"
|
||||
self.error = f"The update stopped because of an unexpected error ({type(e).__name__}: {e}). Previous data was kept."
|
||||
finally:
|
||||
with self.lock:
|
||||
self.finished = U.now_iso()
|
||||
|
||||
def status(self):
|
||||
with self.lock:
|
||||
running_elsewhere = self.state != "running" and U.lock_held()
|
||||
return {"state": "running" if running_elsewhere else self.state,
|
||||
"runningElsewhere": running_elsewhere,
|
||||
"startedAt": self.started, "finishedAt": self.finished,
|
||||
"messages": list(self.messages), "warnings": list(self.warnings), "error": self.error}
|
||||
|
||||
|
||||
JOB = UpdateJob()
|
||||
|
||||
|
||||
def long_date(iso):
|
||||
d = dt.date.fromisoformat(iso[:10])
|
||||
return f"{d:%B} {d.day}, {d.year}"
|
||||
|
||||
|
||||
def merged_data():
|
||||
cats = {cat: (C.load_category(cat) or {"meta": {}, "data": {}}) for cat in C.CATEGORIES}
|
||||
meta_file = C.read_json(C.path("meta.json"), {}) or {}
|
||||
powers = C.seed_governor_powers()
|
||||
data = {}
|
||||
for code in C.NAMES:
|
||||
gov = cats["governors"]["data"].get(code)
|
||||
if gov is not None and not gov.get("powers"):
|
||||
gov = dict(gov, powers=powers.get(code, []))
|
||||
leg = cats["legislatures"]["data"].get(code)
|
||||
data[code] = {"senators": cats["senate"]["data"].get(code, []),
|
||||
"house": cats["house"]["data"].get(code, []),
|
||||
"gov": gov, "leg": leg}
|
||||
metas = {cat: cats[cat]["meta"] for cat in C.CATEGORIES}
|
||||
modes = {m.get("mode") for m in metas.values()}
|
||||
years = {m.get("electionYear") for m in metas.values() if m.get("mode") == "election"}
|
||||
dates = sorted(m.get("asOf", "")[:10] for m in metas.values() if m.get("asOf"))
|
||||
mode = modes.pop() if len(modes) == 1 else "mixed"
|
||||
year = max(years) if years else None
|
||||
last = meta_file.get("lastRun") or {}
|
||||
sources = []
|
||||
for m in metas.values():
|
||||
for s in m.get("sources", []):
|
||||
if s not in sources:
|
||||
sources.append(s)
|
||||
if all(m.get("seed") for m in metas.values()):
|
||||
footer = meta_file.get("seedFooter", "")
|
||||
else:
|
||||
if dates and dates[0] != dates[-1]:
|
||||
read = f", read between {long_date(dates[0])} and {long_date(dates[-1])}."
|
||||
else:
|
||||
read = f", read on {long_date(dates[0])}." if dates else "."
|
||||
footer = "Data: English Wikipedia (" + "; ".join(sources) + ")" + read
|
||||
if mode in ("election", "mixed"):
|
||||
footer += " Results that were not final on that date are marked “Not yet called”. Where a chamber's leader for the new session was not chosen yet, the current leader is shown and noted."
|
||||
footer += " Map shapes: us-atlas (U.S. Census Bureau boundaries)."
|
||||
meta = {
|
||||
"mode": mode,
|
||||
"electionYear": year,
|
||||
"electionDay": C.election_day(year).isoformat() if year else None,
|
||||
"modeLabel": "mixed sources (see warnings)" if mode == "mixed" else MODE_LABEL.get(mode, mode).format(year=year),
|
||||
"asOf": dates[0] if dates else None,
|
||||
"newest": dates[-1] if dates else None,
|
||||
"categories": {cat: {"mode": m.get("mode"), "asOf": m.get("asOf"), "electionYear": m.get("electionYear")}
|
||||
for cat, m in metas.items()},
|
||||
"lastRun": last or None,
|
||||
"warnings": (last.get("warnings") if last else []) or [],
|
||||
"footer": footer,
|
||||
}
|
||||
return {"data": data, "meta": meta}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "us-officeholders-map/1.0"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args))
|
||||
|
||||
def _send(self, code, body, ctype="application/json; charset=utf-8", extra=None):
|
||||
if isinstance(body, (dict, list)):
|
||||
body = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
elif isinstance(body, str):
|
||||
body = body.encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store" if ctype.startswith("application/json") else "public, max-age=300")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
for k, v in (extra or {}).items():
|
||||
self.send_header(k, v)
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def _static(self, rel):
|
||||
full = os.path.realpath(os.path.join(STATIC, rel))
|
||||
if not full.startswith(os.path.realpath(STATIC) + os.sep) or not os.path.isfile(full):
|
||||
return self._send(404, {"error": "Not found"})
|
||||
ctype = mimetypes.guess_type(full)[0] or "application/octet-stream"
|
||||
if ctype.startswith("text/") or ctype in ("application/javascript", "application/json"):
|
||||
ctype += "; charset=utf-8"
|
||||
with open(full, "rb") as f:
|
||||
body = f.read()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-cache" if rel == "index.html" else "public, max-age=3600")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_HEAD(self):
|
||||
self.do_GET()
|
||||
|
||||
def do_GET(self):
|
||||
p = self.path.split("?", 1)[0]
|
||||
if p in ("/", "/index.html"):
|
||||
return self._static("index.html")
|
||||
if p.startswith("/static/"):
|
||||
return self._static(p[len("/static/"):])
|
||||
if p == "/api/data":
|
||||
try:
|
||||
return self._send(200, merged_data())
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return self._send(500, {"error": f"Could not read the stored data ({e})."})
|
||||
if p == "/api/update/status":
|
||||
return self._send(200, JOB.status())
|
||||
return self._send(404, {"error": "Not found"})
|
||||
|
||||
def do_POST(self):
|
||||
p = self.path.split("?", 1)[0]
|
||||
if p != "/api/update":
|
||||
return self._send(404, {"error": "Not found"})
|
||||
# Plain cross-site form posts cannot set this header, so another web page cannot start updates.
|
||||
if self.headers.get("X-Requested-With") != "us-officeholders-map":
|
||||
return self._send(403, {"error": "Missing request header."})
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
if length:
|
||||
self.rfile.read(min(length, 4096))
|
||||
if JOB.start():
|
||||
return self._send(202, {"started": True})
|
||||
return self._send(409, {"started": False, "error": "An update is already running."})
|
||||
|
||||
|
||||
def main():
|
||||
copied = C.ensure_seed()
|
||||
if copied:
|
||||
print(f"Seeded {C.DATA_DIR} with: {', '.join(copied)}", flush=True)
|
||||
httpd = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
|
||||
print(f"Serving on port {PORT}, data in {C.DATA_DIR}", flush=True)
|
||||
httpd.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user