"""Turn Wikipedia pages into the data the map uses. One function per category and mode. Every parser returns (data, sources, notes) and raises ParseError when the page does not look as expected. Validation of totals happens in update.py, not here. """ import datetime as dt import re import common as C from wiki import plain, section, sections, split_rows, strip_noise, tables, infobox STATE_BY_NAME = {n.lower(): c for n, c in C.STATES.items()} class ParseError(Exception): pass def party_code(text): t = plain(text or "").strip().lower() if not t: return None t = re.sub(r"\s*party.*$", "", t) if "party" in t and not t.startswith("independent") else t if t in ("d", "dfl") or re.match(r"^(d|dfl)\b", t): return "D" if t == "r" or re.match(r"^r\b", t): return "R" if t == "i" or re.match(r"^i\b", t): return "I" if "independent" in t: return "I" if "democrat" in t or "farmer" in t or "dfl" in t: return "D" if "republican" in t: return "R" return "I" def state_code(name): return STATE_BY_NAME.get(plain(name).strip().lower()) # ---------------------------------------------------------------- race rows (Senate, House, governors) AYE = re.compile(r"\{\{\s*aye\s*\}\}", re.I) MBY = re.compile(r"\{\{\s*mby\s*\}\}", re.I) # "advanced to runoff" mark STRIPE = re.compile(r"\{\{\s*party stripe\s*\|([^}|]*)", re.I) def candidates(row): """[(name, party code, won, has a Wikipedia article, advanced to a runoff)] from a results row.""" out = [] for line in row.split("\n"): st = STRIPE.search(line) if not st: continue won = bool(AYE.search(line)) rest = AYE.sub("", line[st.end():].split("}}", 1)[-1]) bold = re.search(r"'''(.+?)'''", rest) if bold: name_src = bold.group(1) name = plain(name_src) else: name_src = rest name = re.sub(r"\s*\([^()]*\)[\s\d.,%]*$", "", plain(rest)).strip() pm = re.search(r"(?:'''|\]\])\s*\(([^()]*)\)", rest) or re.search(r"\(([^()]*)\)[\s\d.,%]*$", plain(rest)) party = party_code(pm.group(1)) if pm else party_code(st.group(1)) name = re.sub(r"\s*\((incumbent|write-in)\)", "", name, flags=re.I).strip() if name: out.append((name, party, won, "[[" in name_src, bool(MBY.search(line)))) return out def race_result(row, term_end, elected_status, extra_note=""): """Officeholder entry for one race row: winner, or not-yet-called / runoff.""" cands = candidates(row) winners = [c for c in cands if c[2]] if len(winners) > 1: raise ParseError("several winners marked in one race") if winners: name, party = winners[0][0], winners[0][1] return {"name": name, "party": party, "termEnd": term_end, "status": elected_status, "note": extra_note} text_no_cands = "\n".join(l for l in row.split("\n") if not STRIPE.search(l)) advanced = [c for c in cands if c[4]] runoff = advanced or re.search(r"run-?off", plain(text_no_cands), re.I) # the two main candidates: those in the runoff, else notable (linked) Democrats and Republicans first shown = advanced[:2] or sorted(cands, key=lambda c: (c[1] not in ("D", "R"), not c[3]))[:2] who = ", ".join(f"{c[0]} ({c[1]})" for c in shown) parties = {c[1] for c in shown} if runoff: m = re.search(r"run-?off[^.]*?((?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}(?:, \d{4})?)", plain(text_no_cands), re.I) note = "Runoff" + (f" on {m.group(1)}" if m else "") + (f": {who.replace(', ', ' vs ')}" if who else "") status = "runoff" else: note = "Not yet called" + (f"; {who.replace(', ', ' vs ')}" if who else "") status = "uncalled" return {"name": None, "party": parties.pop() if len(parties) == 1 and len(shown) > 1 else None, "termEnd": term_end, "status": status, "note": (note + ". " + extra_note).strip()} def header_lines(row): return [l for l in row.split("\n") if l.lstrip().startswith("!")] # ---------------------------------------------------------------- Senate def current_senators(client): title = "List of current United States senators" text = strip_noise(client.page(title)) start = text.find('id="senators"') if start < 0: raise ParseError(f"senator table not found in “{title}”") start = text.rfind("{|", 0, start) end = text.find("\n|}", start) table = text[start:end] # footnotes saying an independent sits with the Democrats dem_caucus_efn = set() for m in re.finditer(r"\{\{efn\|name=([^|}]+)\|(.*?)\}\}\s*$", text, re.M | re.S): if re.search(r"Democratic Caucus|caucus(es)? with (the )?Democrat", m.group(2), re.I): dem_caucus_efn.add(m.group(1).strip()) out, state = [], None for row in split_rows(table)[1:]: sm = re.search(r"\[\[List of United States senators from ([^|\]]+)\|", row) if sm: state = state_code(sm.group(1)) name_line = next((l for l in row.split("\n") if l.startswith("!")), None) if not name_line or state is None: continue name_cell = name_line[1:].split("|", 1)[-1] if re.match(r"!\s*scope", name_line) else name_line[1:] name = plain(name_cell) pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", row, re.I) cm = re.search(r"Class (\d)", row) if not cm: continue caucus = any(f"name={n}" in row for n in dem_caucus_efn) or re.search(r"Democratic Caucus", row) out.append({"state": state, "name": name or None, "party": party_code(pc.group(1)) if pc else None, "class": int(cm.group(1)), "vacant": not name or name.lower().startswith("vacant"), "demCaucus": bool(caucus)}) if len(out) < 90: raise ParseError(f"only {len(out)} senators found in “{title}”") return out, title def senate_current(client, day): sens, title = current_senators(client) data = {} for s in sens: entry = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"], "termEnd": C.senate_term_end(s["class"], day), "class": s["class"], "status": "vacant" if s["vacant"] else "continuing", "note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""} data.setdefault(s["state"], []).append(entry) for v in data.values(): v.sort(key=lambda e: e["class"]) return data, [title] def senate_election(client, year): title = f"{year} United States Senate elections" text = strip_noise(client.page(title)) body = section(text, "Race summary") if body is None: raise ParseError(f"“Race summary” section not found in “{title}”") new_congress = dt.date(year + 1, 1, 3) regular = C.senate_class_up(year) races = {} for row in split_rows(body): head = "\n".join(header_lines(row)) m = re.search(r"\[\[\s*%d United States Senate (special )?elections? in ([^|\]]+)\|" % year, head) if not m: continue code = state_code(m.group(2)) if not code: continue special = bool(m.group(1)) cm = re.search(r"Class (\d)", head) cls = int(cm.group(1)) if cm else (None if special else regular) if cls is None: raise ParseError(f"class of the special election in {C.NAMES[code]} not found") note = f"Won {year} special election for the remainder of the Class {cls} term." if special else "" e = race_result(row, C.senate_term_end(cls, new_congress), f"elected-{year}", note) e["class"] = cls races[(code, cls)] = e if len(races) < 30: raise ParseError(f"only {len(races)} Senate races found in “{title}”") sens, list_title = current_senators(client) data = {} for s in sens: key = (s["state"], s["class"]) if key in races: e = dict(races.pop(key)) else: e = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"], "termEnd": C.senate_term_end(s["class"], new_congress), "class": s["class"], "status": "vacant" if s["vacant"] else "continuing", "note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""} data.setdefault(s["state"], []).append({k: e[k] for k in ("name", "party", "termEnd", "class", "status", "note")}) if races: raise ParseError("races without a matching seat in the senator list: " + ", ".join(f"{c} class {k}" for c, k in races)) for v in data.values(): v.sort(key=lambda e: e["class"]) return data, [title, list_title] # ---------------------------------------------------------------- House def ushr(row): for l in header_lines(row): m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", l, re.I) if m: d = m.group(2) return m.group(1).upper(), 0 if not d.isdigit() else int(d) return None def house_election(client, year): title = f"{year} United States House of Representatives elections" text = strip_noise(client.page(title)) term_end = C.house_term_end(dt.date(year + 1, 1, 3)) data = {} for head, body in sections(text, 2): code = state_code(head) if not code: continue reps = [] for row in split_rows(body): u = ushr(row) if not u: continue e = race_result(row, term_end, f"elected-{year}") reps.append({"district": u[1], **{k: e[k] for k in ("name", "party", "termEnd", "status", "note")}}) data[code] = reps if len(data) < 45: raise ParseError(f"only {len(data)} state sections found in “{title}”") return data, [title] def house_current(client, day): title = "List of current members of the United States House of Representatives" text = strip_noise(client.page(title)) term_end = C.house_term_end(day) best = None for t in tables(text): if not t.rows: continue head = " ".join(c.text.lower() for c in t.rows[0]) if "district" in head and "member" in head and "party" in head and len(t.rows) > 400: best = t break if best is None: raise ParseError(f"member table not found in “{title}”") head = [c.text.lower() for c in best.rows[0]] i_dist = next(i for i, h in enumerate(head) if h.startswith("district")) i_mem = next(i for i, h in enumerate(head) if h.startswith("member")) i_party = max(i for i, h in enumerate(head) if h.startswith("party")) data = {} for row, raw in zip(best.rows[1:], best.row_raw[1:]): if len(row) <= max(i_dist, i_mem, i_party) or row[i_dist].header and row[i_dist] is best.rows[0][i_dist]: continue dist_raw = row[i_dist].raw m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", dist_raw, re.I) if m: code, d = m.group(1).upper(), m.group(2) else: dt_txt = row[i_dist].text m2 = re.match(r"(.+?)\s+(at-large|\d+)", dt_txt, re.I) if not m2 or not state_code(m2.group(1)): continue code, d = state_code(m2.group(1)), m2.group(2) if code not in C.APPORTIONMENT: continue district = int(d) if d.isdigit() else 0 member = row[i_mem].text vacant = not member or member.lower().startswith("vacant") pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.raw + " " + c.attrs for c in row), re.I) party = None if vacant else (party_code(row[i_party].text) or (party_code(pc.group(1)) if pc else None)) note = "" if vacant: note = plain(re.sub(r"(?i)^vacant", "", member)).strip(" .–-") or "" entry = {"district": district, "name": None if vacant else member, "party": party, "termEnd": term_end, "status": "vacant" if vacant else "continuing", "note": note} lst = data.setdefault(code, []) if not any(r["district"] == district for r in lst): lst.append(entry) return data, [title] # ---------------------------------------------------------------- Governors def current_governors(client): title = "List of current United States governors" text = strip_noise(client.page(title)) out = {} for t in tables(text): if not t.rows: continue head = [c.text.lower() for c in t.rows[0]] if not any(h.startswith("governor") for h in head) or not any("term end" in h for h in head): continue i_gov = next(i for i, h in enumerate(head) if h.startswith("governor")) i_end = next(i for i, h in enumerate(head) if "term end" in h) i_party = max((i for i, h in enumerate(head) if h.startswith("party")), default=None) for row in t.rows[1:]: st = re.search(r"\[\[\s*Governor of ([^|\]]+)\|", row[0].raw) if row else None code = state_code(st.group(1)) if st else state_code(row[0].text.split("(")[0]) if row else None if not code or len(row) <= max(i_gov, i_end): continue name = row[i_gov].text party = None if i_party is not None and len(row) > i_party: party = party_code(row[i_party].text) if not party: pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.attrs for c in row), re.I) party = party_code(pc.group(1)) if pc else None end_txt = row[i_end].text ym = re.search(r"(\d{4})", end_txt) out[code] = {"name": name or None, "party": party, "endYear": int(ym.group(1)) if ym else None, "termLimited": "term limit" in end_txt.lower(), "endText": end_txt} if len(out) >= 50: break if len(out) < 45: raise ParseError(f"only {len(out)} governors found in “{title}”") return out, title def _gov_entry(code, g, status, powers): note = "Term-limited." if g.get("termLimited") else "" term_end = C.governor_term_end(code, g["endYear"]) if g.get("endYear") else None return {"name": g["name"], "party": g["party"], "termEnd": term_end, "status": status, "note": note, "powers": powers.get(code, [])} def governors_current(client, day): govs, title = current_governors(client) powers = C.seed_governor_powers() data = {code: _gov_entry(code, g, "continuing", powers) for code, g in govs.items()} return data, [title] def governors_election(client, year): title = f"{year} United States gubernatorial elections" text = strip_noise(client.page(title)) body = section(text, "Race summary") if body is None: raise ParseError(f"“Race summary” section not found in “{title}”") powers = C.seed_governor_powers() races = {} for row in split_rows(body): head = "\n".join(header_lines(row)) m = re.search(r"\[\[\s*%d ([A-Z][A-Za-z .]+?) gubernatorial (?:special )?election\|" % year, head) if not m: continue code = state_code(m.group(1)) if not code: continue term = C.GOVERNOR_TERM_YEARS.get(code, 4) starts_december = C.GOVERNOR_INAUGURATION.get(code, "01").startswith("12") end_year = year + term if starts_december else year + 1 + term e = race_result(row, C.governor_term_end(code, end_year), f"elected-{year}", "End of term deduced from the term length.") e["powers"] = powers.get(code, []) races[code] = e if len(races) < 20: raise ParseError(f"only {len(races)} governor races found in “{title}”") govs, list_title = current_governors(client) data = {} for code in C.NAMES: if code in races: data[code] = races[code] elif code in govs: data[code] = _gov_entry(code, govs[code], "continuing", powers) return data, [title, list_title] # ---------------------------------------------------------------- State legislatures LOWER_WORDS = re.compile(r"house|assembly|delegates", re.I) def chamber_titles(): out = [] for code, (up, low) in C.CHAMBERS.items(): out.append((code, "upper", up[0], up[1])) if low: out.append((code, "lower", low[0], low[1])) return out GROUP_WORDS = re.compile(r"\b(majority|minority|caucus|coalition|total|officially|nonpartisan)\b", re.I) PARTY_WORDS = re.compile(r"democrat|republican|independent|vacan|libertarian|green|progressive|dfl", re.I) def infobox_composition(box): """{D, R, I, vacant} from political_groups1 of an Infobox legislature. Lines are read as groups: a heading (bold or ';' line) with an optional count, then bullet lines. A bullet without its own count takes the heading's count when it is the heading's only bullet. """ pg = box.get("political_groups1") or box.get("political_groups") or "" groups = [] # [heading label, heading count, [[label, count]]] for line in pg.split("\n"): s = line.strip() if not s: continue is_bullet = s.startswith("*") or (s.startswith("|") and not s.startswith("|}")) txt = plain(s.lstrip("*|; ")) m = re.search(r"^(.*?)\(\s*(\d+)\s*\)\s*$", txt) label, count = (m.group(1).strip(), int(m.group(2))) if m else (txt.strip(), None) if is_bullet and (count is not None or PARTY_WORDS.search(label)): if not groups: groups.append(["", None, []]) groups[-1][2].append([label, count]) elif s.startswith(("''", ";")) or GROUP_WORDS.search(label) or count is not None: groups.append([label, count, []]) counts = {"D": 0, "R": 0, "I": 0, "vacant": 0} found = 0 for head, head_count, bullets in groups: if len(bullets) == 1 and bullets[0][1] is None and head_count is not None: bullets[0][1] = head_count if not bullets and head_count is not None and PARTY_WORDS.search(head) and not GROUP_WORDS.search(head): bullets = [[head, head_count]] for label, n in bullets: if n is None: continue if re.search(r"vacan", label, re.I): counts["vacant"] += n else: counts[party_code(label) or "I"] += n found += 1 return counts if found else None def chamber_leader(box, which, code): leaders = [] for i in range(1, 12): if f"leader{i}" in box: leaders.append((box.get(f"leader{i}_type", ""), box.get(f"leader{i}", ""))) pick, title, note = None, None, "" if which == "lower" or code == "NE": for t, n in leaders: tp = plain(t) if re.match(r"speaker\b", tp, re.I) and "pro tem" not in tp.lower(): pick, title = (t, n), ("Speaker of the Legislature" if code == "NE" else "Speaker") break if not pick and which == "upper": lg = False for t, n in leaders: tp = plain(t).lower() is_lg = "lieutenant governor" in t.lower() if re.match(r"speaker\b", tp) and "pro tem" not in tp: # Tennessee: the Senate elects its Speaker, who is also lieutenant governor pick, title = (t, n), "Speaker of the Senate" note = "Also lieutenant governor." if is_lg else "" break if re.match(r"president\b", tp) and "pro tem" not in tp: if is_lg: lg = True continue pick, title = (t, n), "President of the Senate" break if not pick: for t, n in leaders: tp = plain(t) if re.search(r"pro tem|temporary president", tp, re.I): pick = (t, n) title = "President pro tempore" if "pro tem" in tp.lower() else "Temporary President" break if pick and lg: note = "Lieutenant governor presides." if not pick: others = [l for l in leaders if "lieutenant governor" not in l[0].lower()] or leaders if not others: return None pick = others[0] title = plain(pick[0]) or "Leader" txt = plain(pick[1]) m = re.match(r"^(.*?)\s*\(([^()]*)\)\s*$", txt) name, party = (m.group(1), party_code(m.group(2))) if m else (txt, None) if not name or name.lower().startswith("vacant"): return {"title": title, "name": None, "party": None, "termEnd": None, "note": "Vacant.", "status": "vacant"} if not party: bg = re.search(r"party color\s*\|([^}]*)\}\}", box.get("background_color", ""), re.I) if bg and party_code(bg.group(1)) in ("D", "R"): party = party_code(bg.group(1)) note = (note + " Party deduced from the chamber's majority (not given in the infobox).").strip() return {"title": title, "name": name.strip(), "party": party, "termEnd": None, "note": note} def summary_tables(client, year): """Post-election compositions from the main state legislative elections page: {(code, which): info}.""" title = f"{year} United States state legislative elections" text = strip_noise(client.page(title)) out = {} heads = [(m.start(), plain(m.group(1))) for m in re.finditer(r"^===(?!=)\s*(.+?)\s*===\s*$", text, re.M)] for t in tables(text): if not t.caption or not t.rows: continue hi = next((i for i, r in enumerate(t.rows[:3]) if any(c.text == "Before" for c in r) and any(c.text == "After" for c in r)), None) if hi is None: continue state = None for pos, h in heads: if pos < t.start: state = h code = state_code(state or "") if not code: continue which = "upper" if code == "NE" or not LOWER_WORDS.search(t.caption) else "lower" hdr = [c.text for c in t.rows[hi]] i_b, i_a = hdr.index("Before"), hdr.index("After") i_p = max(i for i, h in enumerate(hdr) if h == "Party") before = {"D": 0, "R": 0, "I": 0, "vacant": 0} after = {"D": 0, "R": 0, "I": 0, "vacant": 0} after_known = False after_missing = False # a party that held seats before has no "After" figure yet seats = None for r in t.rows[hi + 1:]: if len(r) <= max(i_b, i_a, i_p): continue label = r[i_p].text if any(c.text.lower() == "total" for c in r): nb = re.search(r"\d+", r[i_b].text) seats = int(nb.group(0)) if nb else None continue if not label: continue key = "vacant" if re.search(r"vacan", label, re.I) else party_code(label) nb = re.search(r"\d+", r[i_b].text) na = re.search(r"\d+", r[i_a].text) if nb: before[key] += int(nb.group(0)) if na: after[key] += int(na.group(0)) after_known = True elif nb and int(nb.group(0)) > 0 and key != "vacant": after_missing = True complete = seats is not None and sum(after.values()) == seats out[(code, which)] = {"caption": t.caption, "seats": seats, "before": before, "after": after if after_known and (complete or not after_missing) else None} if len(out) < 60: raise ParseError(f"only {len(out)} chamber tables found in “{title}”") return out, title def majority_party(c): top = sorted(("D", "R", "I"), key=lambda k: -c.get(k, 0)) if c.get(top[0], 0) == c.get(top[1], 0): return None return top[0] def fmt_counts(c): return ", ".join(f"{c[k]} {k}" for k in ("D", "R", "I") if c.get(k)) PARTY_WORD = {"D": "Democrats", "R": "Republicans", "I": "Independents"} def legislatures(client, mode, day, year=None): """Returns ({code: {upper, lower}}, sources, per-chamber problems {(code, which): reason}).""" titles = chamber_titles() articles = client.pages([t for _, _, t, _ in titles], label="state chamber articles") summary, summary_title = (summary_tables(client, year) if mode == "election" else ({}, None)) data, problems = {}, {} for code, which, title, size in titles: text = articles.get(title) box = infobox(text, "Infobox legislature") if text else {} name = plain(box.get("name", "")) or title leader = chamber_leader(box, which, code) if box else None next_year = C.next_legislative_election(code, day) if leader: leader["termEnd"] = str(next_year) parts = [leader["note"]] if mode == "election": parts.append("Current leader; the leader for the new session may not be chosen yet.") leader.setdefault("status", "current") parts.append(f"Leaders serve one legislative session; end deduced from the next regular election ({next_year}).") leader["note"] = " ".join(p for p in parts if p) ch = {"name": name, "seats": size, "D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0, "compositionNote": "", "leader": leader} s = summary.get((code, which)) if mode == "election" and s: if s["seats"] and s["seats"] != size: problems[(code, which)] = f"the summary table gives {s['seats']} seats instead of {size}" elif s["after"]: a = s["after"] known = a["D"] + a["R"] + a["I"] + a["vacant"] if known > size: problems[(code, which)] = f"post-election seats add up to {known}, more than {size}" ch.update({k: a[k] for k in ("D", "R", "I", "vacant")}) ch["uncalled"] = max(0, size - known) notes = [] if ch["uncalled"]: notes.append(f"{ch['uncalled']} seat(s) not yet called on Wikipedia.") mb, ma = majority_party(s["before"]), majority_party(a) if not ch["uncalled"] else None if mb and ma and mb != ma: notes.append(f"{PARTY_WORD[ma]} flipped the chamber.") ch["compositionNote"] = " ".join(notes) else: ch["uncalled"] = size ch["compositionNote"] = (f"Wikipedia has not published {year} results for this chamber yet" + (f" (before the election: {fmt_counts(s['before'])})." if fmt_counts(s["before"]) else ".")) ch["_unknown"] = True else: comp = infobox_composition(box) if box else None if not text: problems[(code, which)] = f"article “{title}” not found" elif comp is None: problems[(code, which)] = f"no party composition in the infobox of “{title}”" else: ch.update(comp) total = sum(comp.values()) if total != size: problems[(code, which)] = f"infobox seats add up to {total} instead of {size}" if mode == "election": ch["compositionNote"] = f"No regular election for this chamber in {year}; current composition from the chamber article." if not leader and (code, which) not in problems: problems[(code, which)] = f"no presiding officer found in the infobox of “{title}”" if code == "NE": ch["compositionNote"] = (ch["compositionNote"] + " Unicameral and officially nonpartisan; counts use members' party affiliation as Wikipedia reports it.").strip() data.setdefault(code, {"upper": None, "lower": None})[which] = ch sources = ([summary_title] if summary_title else []) + ["each state chamber's article"] return data, sources, problems