Initial commit
This commit is contained in:
commit
a206bfb5db
21 files changed
+16747
No files matched your search
+624
@@ -0,0 +1,624 @@
|
||||
"""Turn Wikipedia pages into the data the map uses. One function per category and mode.
|
||||
|
||||
Every parser returns (data, sources, notes) and raises ParseError when the page does not look as expected.
|
||||
Validation of totals happens in update.py, not here.
|
||||
"""
|
||||
import datetime as dt
|
||||
import re
|
||||
|
||||
import common as C
|
||||
from wiki import plain, section, sections, split_rows, strip_noise, tables, infobox
|
||||
|
||||
STATE_BY_NAME = {n.lower(): c for n, c in C.STATES.items()}
|
||||
|
||||
|
||||
class ParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def party_code(text):
|
||||
t = plain(text or "").strip().lower()
|
||||
if not t:
|
||||
return None
|
||||
t = re.sub(r"\s*party.*$", "", t) if "party" in t and not t.startswith("independent") else t
|
||||
if t in ("d", "dfl") or re.match(r"^(d|dfl)\b", t):
|
||||
return "D"
|
||||
if t == "r" or re.match(r"^r\b", t):
|
||||
return "R"
|
||||
if t == "i" or re.match(r"^i\b", t):
|
||||
return "I"
|
||||
if "independent" in t:
|
||||
return "I"
|
||||
if "democrat" in t or "farmer" in t or "dfl" in t:
|
||||
return "D"
|
||||
if "republican" in t:
|
||||
return "R"
|
||||
return "I"
|
||||
|
||||
|
||||
def state_code(name):
|
||||
return STATE_BY_NAME.get(plain(name).strip().lower())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- race rows (Senate, House, governors)
|
||||
|
||||
AYE = re.compile(r"\{\{\s*aye\s*\}\}", re.I)
|
||||
MBY = re.compile(r"\{\{\s*mby\s*\}\}", re.I) # "advanced to runoff" mark
|
||||
STRIPE = re.compile(r"\{\{\s*party stripe\s*\|([^}|]*)", re.I)
|
||||
|
||||
|
||||
def candidates(row):
|
||||
"""[(name, party code, won, has a Wikipedia article, advanced to a runoff)] from a results row."""
|
||||
out = []
|
||||
for line in row.split("\n"):
|
||||
st = STRIPE.search(line)
|
||||
if not st:
|
||||
continue
|
||||
won = bool(AYE.search(line))
|
||||
rest = AYE.sub("", line[st.end():].split("}}", 1)[-1])
|
||||
bold = re.search(r"'''(.+?)'''", rest)
|
||||
if bold:
|
||||
name_src = bold.group(1)
|
||||
name = plain(name_src)
|
||||
else:
|
||||
name_src = rest
|
||||
name = re.sub(r"\s*\([^()]*\)[\s\d.,%]*$", "", plain(rest)).strip()
|
||||
pm = re.search(r"(?:'''|\]\])\s*\(([^()]*)\)", rest) or re.search(r"\(([^()]*)\)[\s\d.,%]*$", plain(rest))
|
||||
party = party_code(pm.group(1)) if pm else party_code(st.group(1))
|
||||
name = re.sub(r"\s*\((incumbent|write-in)\)", "", name, flags=re.I).strip()
|
||||
if name:
|
||||
out.append((name, party, won, "[[" in name_src, bool(MBY.search(line))))
|
||||
return out
|
||||
|
||||
|
||||
def race_result(row, term_end, elected_status, extra_note=""):
|
||||
"""Officeholder entry for one race row: winner, or not-yet-called / runoff."""
|
||||
cands = candidates(row)
|
||||
winners = [c for c in cands if c[2]]
|
||||
if len(winners) > 1:
|
||||
raise ParseError("several winners marked in one race")
|
||||
if winners:
|
||||
name, party = winners[0][0], winners[0][1]
|
||||
return {"name": name, "party": party, "termEnd": term_end, "status": elected_status, "note": extra_note}
|
||||
text_no_cands = "\n".join(l for l in row.split("\n") if not STRIPE.search(l))
|
||||
advanced = [c for c in cands if c[4]]
|
||||
runoff = advanced or re.search(r"run-?off", plain(text_no_cands), re.I)
|
||||
# the two main candidates: those in the runoff, else notable (linked) Democrats and Republicans first
|
||||
shown = advanced[:2] or sorted(cands, key=lambda c: (c[1] not in ("D", "R"), not c[3]))[:2]
|
||||
who = ", ".join(f"{c[0]} ({c[1]})" for c in shown)
|
||||
parties = {c[1] for c in shown}
|
||||
if runoff:
|
||||
m = re.search(r"run-?off[^.]*?((?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}(?:, \d{4})?)",
|
||||
plain(text_no_cands), re.I)
|
||||
note = "Runoff" + (f" on {m.group(1)}" if m else "") + (f": {who.replace(', ', ' vs ')}" if who else "")
|
||||
status = "runoff"
|
||||
else:
|
||||
note = "Not yet called" + (f"; {who.replace(', ', ' vs ')}" if who else "")
|
||||
status = "uncalled"
|
||||
return {"name": None, "party": parties.pop() if len(parties) == 1 and len(shown) > 1 else None,
|
||||
"termEnd": term_end, "status": status, "note": (note + ". " + extra_note).strip()}
|
||||
|
||||
|
||||
def header_lines(row):
|
||||
return [l for l in row.split("\n") if l.lstrip().startswith("!")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Senate
|
||||
|
||||
def current_senators(client):
|
||||
title = "List of current United States senators"
|
||||
text = strip_noise(client.page(title))
|
||||
start = text.find('id="senators"')
|
||||
if start < 0:
|
||||
raise ParseError(f"senator table not found in “{title}”")
|
||||
start = text.rfind("{|", 0, start)
|
||||
end = text.find("\n|}", start)
|
||||
table = text[start:end]
|
||||
# footnotes saying an independent sits with the Democrats
|
||||
dem_caucus_efn = set()
|
||||
for m in re.finditer(r"\{\{efn\|name=([^|}]+)\|(.*?)\}\}\s*$", text, re.M | re.S):
|
||||
if re.search(r"Democratic Caucus|caucus(es)? with (the )?Democrat", m.group(2), re.I):
|
||||
dem_caucus_efn.add(m.group(1).strip())
|
||||
out, state = [], None
|
||||
for row in split_rows(table)[1:]:
|
||||
sm = re.search(r"\[\[List of United States senators from ([^|\]]+)\|", row)
|
||||
if sm:
|
||||
state = state_code(sm.group(1))
|
||||
name_line = next((l for l in row.split("\n") if l.startswith("!")), None)
|
||||
if not name_line or state is None:
|
||||
continue
|
||||
name_cell = name_line[1:].split("|", 1)[-1] if re.match(r"!\s*scope", name_line) else name_line[1:]
|
||||
name = plain(name_cell)
|
||||
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", row, re.I)
|
||||
cm = re.search(r"Class (\d)", row)
|
||||
if not cm:
|
||||
continue
|
||||
caucus = any(f"name={n}" in row for n in dem_caucus_efn) or re.search(r"Democratic Caucus", row)
|
||||
out.append({"state": state, "name": name or None, "party": party_code(pc.group(1)) if pc else None,
|
||||
"class": int(cm.group(1)), "vacant": not name or name.lower().startswith("vacant"),
|
||||
"demCaucus": bool(caucus)})
|
||||
if len(out) < 90:
|
||||
raise ParseError(f"only {len(out)} senators found in “{title}”")
|
||||
return out, title
|
||||
|
||||
|
||||
def senate_current(client, day):
|
||||
sens, title = current_senators(client)
|
||||
data = {}
|
||||
for s in sens:
|
||||
entry = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"],
|
||||
"termEnd": C.senate_term_end(s["class"], day), "class": s["class"],
|
||||
"status": "vacant" if s["vacant"] else "continuing",
|
||||
"note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""}
|
||||
data.setdefault(s["state"], []).append(entry)
|
||||
for v in data.values():
|
||||
v.sort(key=lambda e: e["class"])
|
||||
return data, [title]
|
||||
|
||||
|
||||
def senate_election(client, year):
|
||||
title = f"{year} United States Senate elections"
|
||||
text = strip_noise(client.page(title))
|
||||
body = section(text, "Race summary")
|
||||
if body is None:
|
||||
raise ParseError(f"“Race summary” section not found in “{title}”")
|
||||
new_congress = dt.date(year + 1, 1, 3)
|
||||
regular = C.senate_class_up(year)
|
||||
races = {}
|
||||
for row in split_rows(body):
|
||||
head = "\n".join(header_lines(row))
|
||||
m = re.search(r"\[\[\s*%d United States Senate (special )?elections? in ([^|\]]+)\|" % year, head)
|
||||
if not m:
|
||||
continue
|
||||
code = state_code(m.group(2))
|
||||
if not code:
|
||||
continue
|
||||
special = bool(m.group(1))
|
||||
cm = re.search(r"Class (\d)", head)
|
||||
cls = int(cm.group(1)) if cm else (None if special else regular)
|
||||
if cls is None:
|
||||
raise ParseError(f"class of the special election in {C.NAMES[code]} not found")
|
||||
note = f"Won {year} special election for the remainder of the Class {cls} term." if special else ""
|
||||
e = race_result(row, C.senate_term_end(cls, new_congress), f"elected-{year}", note)
|
||||
e["class"] = cls
|
||||
races[(code, cls)] = e
|
||||
if len(races) < 30:
|
||||
raise ParseError(f"only {len(races)} Senate races found in “{title}”")
|
||||
sens, list_title = current_senators(client)
|
||||
data = {}
|
||||
for s in sens:
|
||||
key = (s["state"], s["class"])
|
||||
if key in races:
|
||||
e = dict(races.pop(key))
|
||||
else:
|
||||
e = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"],
|
||||
"termEnd": C.senate_term_end(s["class"], new_congress), "class": s["class"],
|
||||
"status": "vacant" if s["vacant"] else "continuing",
|
||||
"note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""}
|
||||
data.setdefault(s["state"], []).append({k: e[k] for k in ("name", "party", "termEnd", "class", "status", "note")})
|
||||
if races:
|
||||
raise ParseError("races without a matching seat in the senator list: " + ", ".join(f"{c} class {k}" for c, k in races))
|
||||
for v in data.values():
|
||||
v.sort(key=lambda e: e["class"])
|
||||
return data, [title, list_title]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- House
|
||||
|
||||
def ushr(row):
|
||||
for l in header_lines(row):
|
||||
m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", l, re.I)
|
||||
if m:
|
||||
d = m.group(2)
|
||||
return m.group(1).upper(), 0 if not d.isdigit() else int(d)
|
||||
return None
|
||||
|
||||
|
||||
def house_election(client, year):
|
||||
title = f"{year} United States House of Representatives elections"
|
||||
text = strip_noise(client.page(title))
|
||||
term_end = C.house_term_end(dt.date(year + 1, 1, 3))
|
||||
data = {}
|
||||
for head, body in sections(text, 2):
|
||||
code = state_code(head)
|
||||
if not code:
|
||||
continue
|
||||
reps = []
|
||||
for row in split_rows(body):
|
||||
u = ushr(row)
|
||||
if not u:
|
||||
continue
|
||||
e = race_result(row, term_end, f"elected-{year}")
|
||||
reps.append({"district": u[1], **{k: e[k] for k in ("name", "party", "termEnd", "status", "note")}})
|
||||
data[code] = reps
|
||||
if len(data) < 45:
|
||||
raise ParseError(f"only {len(data)} state sections found in “{title}”")
|
||||
return data, [title]
|
||||
|
||||
|
||||
def house_current(client, day):
|
||||
title = "List of current members of the United States House of Representatives"
|
||||
text = strip_noise(client.page(title))
|
||||
term_end = C.house_term_end(day)
|
||||
best = None
|
||||
for t in tables(text):
|
||||
if not t.rows:
|
||||
continue
|
||||
head = " ".join(c.text.lower() for c in t.rows[0])
|
||||
if "district" in head and "member" in head and "party" in head and len(t.rows) > 400:
|
||||
best = t
|
||||
break
|
||||
if best is None:
|
||||
raise ParseError(f"member table not found in “{title}”")
|
||||
head = [c.text.lower() for c in best.rows[0]]
|
||||
i_dist = next(i for i, h in enumerate(head) if h.startswith("district"))
|
||||
i_mem = next(i for i, h in enumerate(head) if h.startswith("member"))
|
||||
i_party = max(i for i, h in enumerate(head) if h.startswith("party"))
|
||||
data = {}
|
||||
for row, raw in zip(best.rows[1:], best.row_raw[1:]):
|
||||
if len(row) <= max(i_dist, i_mem, i_party) or row[i_dist].header and row[i_dist] is best.rows[0][i_dist]:
|
||||
continue
|
||||
dist_raw = row[i_dist].raw
|
||||
m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", dist_raw, re.I)
|
||||
if m:
|
||||
code, d = m.group(1).upper(), m.group(2)
|
||||
else:
|
||||
dt_txt = row[i_dist].text
|
||||
m2 = re.match(r"(.+?)\s+(at-large|\d+)", dt_txt, re.I)
|
||||
if not m2 or not state_code(m2.group(1)):
|
||||
continue
|
||||
code, d = state_code(m2.group(1)), m2.group(2)
|
||||
if code not in C.APPORTIONMENT:
|
||||
continue
|
||||
district = int(d) if d.isdigit() else 0
|
||||
member = row[i_mem].text
|
||||
vacant = not member or member.lower().startswith("vacant")
|
||||
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.raw + " " + c.attrs for c in row), re.I)
|
||||
party = None if vacant else (party_code(row[i_party].text) or (party_code(pc.group(1)) if pc else None))
|
||||
note = ""
|
||||
if vacant:
|
||||
note = plain(re.sub(r"(?i)^vacant", "", member)).strip(" .–-") or ""
|
||||
entry = {"district": district, "name": None if vacant else member, "party": party,
|
||||
"termEnd": term_end, "status": "vacant" if vacant else "continuing", "note": note}
|
||||
lst = data.setdefault(code, [])
|
||||
if not any(r["district"] == district for r in lst):
|
||||
lst.append(entry)
|
||||
return data, [title]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Governors
|
||||
|
||||
def current_governors(client):
|
||||
title = "List of current United States governors"
|
||||
text = strip_noise(client.page(title))
|
||||
out = {}
|
||||
for t in tables(text):
|
||||
if not t.rows:
|
||||
continue
|
||||
head = [c.text.lower() for c in t.rows[0]]
|
||||
if not any(h.startswith("governor") for h in head) or not any("term end" in h for h in head):
|
||||
continue
|
||||
i_gov = next(i for i, h in enumerate(head) if h.startswith("governor"))
|
||||
i_end = next(i for i, h in enumerate(head) if "term end" in h)
|
||||
i_party = max((i for i, h in enumerate(head) if h.startswith("party")), default=None)
|
||||
for row in t.rows[1:]:
|
||||
st = re.search(r"\[\[\s*Governor of ([^|\]]+)\|", row[0].raw) if row else None
|
||||
code = state_code(st.group(1)) if st else state_code(row[0].text.split("(")[0]) if row else None
|
||||
if not code or len(row) <= max(i_gov, i_end):
|
||||
continue
|
||||
name = row[i_gov].text
|
||||
party = None
|
||||
if i_party is not None and len(row) > i_party:
|
||||
party = party_code(row[i_party].text)
|
||||
if not party:
|
||||
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.attrs for c in row), re.I)
|
||||
party = party_code(pc.group(1)) if pc else None
|
||||
end_txt = row[i_end].text
|
||||
ym = re.search(r"(\d{4})", end_txt)
|
||||
out[code] = {"name": name or None, "party": party, "endYear": int(ym.group(1)) if ym else None,
|
||||
"termLimited": "term limit" in end_txt.lower(), "endText": end_txt}
|
||||
if len(out) >= 50:
|
||||
break
|
||||
if len(out) < 45:
|
||||
raise ParseError(f"only {len(out)} governors found in “{title}”")
|
||||
return out, title
|
||||
|
||||
|
||||
def _gov_entry(code, g, status, powers):
|
||||
note = "Term-limited." if g.get("termLimited") else ""
|
||||
term_end = C.governor_term_end(code, g["endYear"]) if g.get("endYear") else None
|
||||
return {"name": g["name"], "party": g["party"], "termEnd": term_end, "status": status, "note": note,
|
||||
"powers": powers.get(code, [])}
|
||||
|
||||
|
||||
def governors_current(client, day):
|
||||
govs, title = current_governors(client)
|
||||
powers = C.seed_governor_powers()
|
||||
data = {code: _gov_entry(code, g, "continuing", powers) for code, g in govs.items()}
|
||||
return data, [title]
|
||||
|
||||
|
||||
def governors_election(client, year):
|
||||
title = f"{year} United States gubernatorial elections"
|
||||
text = strip_noise(client.page(title))
|
||||
body = section(text, "Race summary")
|
||||
if body is None:
|
||||
raise ParseError(f"“Race summary” section not found in “{title}”")
|
||||
powers = C.seed_governor_powers()
|
||||
races = {}
|
||||
for row in split_rows(body):
|
||||
head = "\n".join(header_lines(row))
|
||||
m = re.search(r"\[\[\s*%d ([A-Z][A-Za-z .]+?) gubernatorial (?:special )?election\|" % year, head)
|
||||
if not m:
|
||||
continue
|
||||
code = state_code(m.group(1))
|
||||
if not code:
|
||||
continue
|
||||
term = C.GOVERNOR_TERM_YEARS.get(code, 4)
|
||||
starts_december = C.GOVERNOR_INAUGURATION.get(code, "01").startswith("12")
|
||||
end_year = year + term if starts_december else year + 1 + term
|
||||
e = race_result(row, C.governor_term_end(code, end_year), f"elected-{year}",
|
||||
"End of term deduced from the term length.")
|
||||
e["powers"] = powers.get(code, [])
|
||||
races[code] = e
|
||||
if len(races) < 20:
|
||||
raise ParseError(f"only {len(races)} governor races found in “{title}”")
|
||||
govs, list_title = current_governors(client)
|
||||
data = {}
|
||||
for code in C.NAMES:
|
||||
if code in races:
|
||||
data[code] = races[code]
|
||||
elif code in govs:
|
||||
data[code] = _gov_entry(code, govs[code], "continuing", powers)
|
||||
return data, [title, list_title]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- State legislatures
|
||||
|
||||
LOWER_WORDS = re.compile(r"house|assembly|delegates", re.I)
|
||||
|
||||
|
||||
def chamber_titles():
|
||||
out = []
|
||||
for code, (up, low) in C.CHAMBERS.items():
|
||||
out.append((code, "upper", up[0], up[1]))
|
||||
if low:
|
||||
out.append((code, "lower", low[0], low[1]))
|
||||
return out
|
||||
|
||||
|
||||
GROUP_WORDS = re.compile(r"\b(majority|minority|caucus|coalition|total|officially|nonpartisan)\b", re.I)
|
||||
PARTY_WORDS = re.compile(r"democrat|republican|independent|vacan|libertarian|green|progressive|dfl", re.I)
|
||||
|
||||
|
||||
def infobox_composition(box):
|
||||
"""{D, R, I, vacant} from political_groups1 of an Infobox legislature.
|
||||
|
||||
Lines are read as groups: a heading (bold or ';' line) with an optional count, then bullet lines.
|
||||
A bullet without its own count takes the heading's count when it is the heading's only bullet.
|
||||
"""
|
||||
pg = box.get("political_groups1") or box.get("political_groups") or ""
|
||||
groups = [] # [heading label, heading count, [[label, count]]]
|
||||
for line in pg.split("\n"):
|
||||
s = line.strip()
|
||||
if not s:
|
||||
continue
|
||||
is_bullet = s.startswith("*") or (s.startswith("|") and not s.startswith("|}"))
|
||||
txt = plain(s.lstrip("*|; "))
|
||||
m = re.search(r"^(.*?)\(\s*(\d+)\s*\)\s*$", txt)
|
||||
label, count = (m.group(1).strip(), int(m.group(2))) if m else (txt.strip(), None)
|
||||
if is_bullet and (count is not None or PARTY_WORDS.search(label)):
|
||||
if not groups:
|
||||
groups.append(["", None, []])
|
||||
groups[-1][2].append([label, count])
|
||||
elif s.startswith(("''", ";")) or GROUP_WORDS.search(label) or count is not None:
|
||||
groups.append([label, count, []])
|
||||
counts = {"D": 0, "R": 0, "I": 0, "vacant": 0}
|
||||
found = 0
|
||||
for head, head_count, bullets in groups:
|
||||
if len(bullets) == 1 and bullets[0][1] is None and head_count is not None:
|
||||
bullets[0][1] = head_count
|
||||
if not bullets and head_count is not None and PARTY_WORDS.search(head) and not GROUP_WORDS.search(head):
|
||||
bullets = [[head, head_count]]
|
||||
for label, n in bullets:
|
||||
if n is None:
|
||||
continue
|
||||
if re.search(r"vacan", label, re.I):
|
||||
counts["vacant"] += n
|
||||
else:
|
||||
counts[party_code(label) or "I"] += n
|
||||
found += 1
|
||||
return counts if found else None
|
||||
|
||||
|
||||
def chamber_leader(box, which, code):
|
||||
leaders = []
|
||||
for i in range(1, 12):
|
||||
if f"leader{i}" in box:
|
||||
leaders.append((box.get(f"leader{i}_type", ""), box.get(f"leader{i}", "")))
|
||||
pick, title, note = None, None, ""
|
||||
if which == "lower" or code == "NE":
|
||||
for t, n in leaders:
|
||||
tp = plain(t)
|
||||
if re.match(r"speaker\b", tp, re.I) and "pro tem" not in tp.lower():
|
||||
pick, title = (t, n), ("Speaker of the Legislature" if code == "NE" else "Speaker")
|
||||
break
|
||||
if not pick and which == "upper":
|
||||
lg = False
|
||||
for t, n in leaders:
|
||||
tp = plain(t).lower()
|
||||
is_lg = "lieutenant governor" in t.lower()
|
||||
if re.match(r"speaker\b", tp) and "pro tem" not in tp:
|
||||
# Tennessee: the Senate elects its Speaker, who is also lieutenant governor
|
||||
pick, title = (t, n), "Speaker of the Senate"
|
||||
note = "Also lieutenant governor." if is_lg else ""
|
||||
break
|
||||
if re.match(r"president\b", tp) and "pro tem" not in tp:
|
||||
if is_lg:
|
||||
lg = True
|
||||
continue
|
||||
pick, title = (t, n), "President of the Senate"
|
||||
break
|
||||
if not pick:
|
||||
for t, n in leaders:
|
||||
tp = plain(t)
|
||||
if re.search(r"pro tem|temporary president", tp, re.I):
|
||||
pick = (t, n)
|
||||
title = "President pro tempore" if "pro tem" in tp.lower() else "Temporary President"
|
||||
break
|
||||
if pick and lg:
|
||||
note = "Lieutenant governor presides."
|
||||
if not pick:
|
||||
others = [l for l in leaders if "lieutenant governor" not in l[0].lower()] or leaders
|
||||
if not others:
|
||||
return None
|
||||
pick = others[0]
|
||||
title = plain(pick[0]) or "Leader"
|
||||
txt = plain(pick[1])
|
||||
m = re.match(r"^(.*?)\s*\(([^()]*)\)\s*$", txt)
|
||||
name, party = (m.group(1), party_code(m.group(2))) if m else (txt, None)
|
||||
if not name or name.lower().startswith("vacant"):
|
||||
return {"title": title, "name": None, "party": None, "termEnd": None, "note": "Vacant.", "status": "vacant"}
|
||||
if not party:
|
||||
bg = re.search(r"party color\s*\|([^}]*)\}\}", box.get("background_color", ""), re.I)
|
||||
if bg and party_code(bg.group(1)) in ("D", "R"):
|
||||
party = party_code(bg.group(1))
|
||||
note = (note + " Party deduced from the chamber's majority (not given in the infobox).").strip()
|
||||
return {"title": title, "name": name.strip(), "party": party, "termEnd": None, "note": note}
|
||||
|
||||
|
||||
def summary_tables(client, year):
|
||||
"""Post-election compositions from the main state legislative elections page: {(code, which): info}."""
|
||||
title = f"{year} United States state legislative elections"
|
||||
text = strip_noise(client.page(title))
|
||||
out = {}
|
||||
heads = [(m.start(), plain(m.group(1))) for m in re.finditer(r"^===(?!=)\s*(.+?)\s*===\s*$", text, re.M)]
|
||||
for t in tables(text):
|
||||
if not t.caption or not t.rows:
|
||||
continue
|
||||
hi = next((i for i, r in enumerate(t.rows[:3]) if any(c.text == "Before" for c in r) and any(c.text == "After" for c in r)), None)
|
||||
if hi is None:
|
||||
continue
|
||||
state = None
|
||||
for pos, h in heads:
|
||||
if pos < t.start:
|
||||
state = h
|
||||
code = state_code(state or "")
|
||||
if not code:
|
||||
continue
|
||||
which = "upper" if code == "NE" or not LOWER_WORDS.search(t.caption) else "lower"
|
||||
hdr = [c.text for c in t.rows[hi]]
|
||||
i_b, i_a = hdr.index("Before"), hdr.index("After")
|
||||
i_p = max(i for i, h in enumerate(hdr) if h == "Party")
|
||||
before = {"D": 0, "R": 0, "I": 0, "vacant": 0}
|
||||
after = {"D": 0, "R": 0, "I": 0, "vacant": 0}
|
||||
after_known = False
|
||||
after_missing = False # a party that held seats before has no "After" figure yet
|
||||
seats = None
|
||||
for r in t.rows[hi + 1:]:
|
||||
if len(r) <= max(i_b, i_a, i_p):
|
||||
continue
|
||||
label = r[i_p].text
|
||||
if any(c.text.lower() == "total" for c in r):
|
||||
nb = re.search(r"\d+", r[i_b].text)
|
||||
seats = int(nb.group(0)) if nb else None
|
||||
continue
|
||||
if not label:
|
||||
continue
|
||||
key = "vacant" if re.search(r"vacan", label, re.I) else party_code(label)
|
||||
nb = re.search(r"\d+", r[i_b].text)
|
||||
na = re.search(r"\d+", r[i_a].text)
|
||||
if nb:
|
||||
before[key] += int(nb.group(0))
|
||||
if na:
|
||||
after[key] += int(na.group(0))
|
||||
after_known = True
|
||||
elif nb and int(nb.group(0)) > 0 and key != "vacant":
|
||||
after_missing = True
|
||||
complete = seats is not None and sum(after.values()) == seats
|
||||
out[(code, which)] = {"caption": t.caption, "seats": seats, "before": before,
|
||||
"after": after if after_known and (complete or not after_missing) else None}
|
||||
if len(out) < 60:
|
||||
raise ParseError(f"only {len(out)} chamber tables found in “{title}”")
|
||||
return out, title
|
||||
|
||||
|
||||
def majority_party(c):
|
||||
top = sorted(("D", "R", "I"), key=lambda k: -c.get(k, 0))
|
||||
if c.get(top[0], 0) == c.get(top[1], 0):
|
||||
return None
|
||||
return top[0]
|
||||
|
||||
|
||||
def fmt_counts(c):
|
||||
return ", ".join(f"{c[k]} {k}" for k in ("D", "R", "I") if c.get(k))
|
||||
|
||||
|
||||
PARTY_WORD = {"D": "Democrats", "R": "Republicans", "I": "Independents"}
|
||||
|
||||
|
||||
def legislatures(client, mode, day, year=None):
|
||||
"""Returns ({code: {upper, lower}}, sources, per-chamber problems {(code, which): reason})."""
|
||||
titles = chamber_titles()
|
||||
articles = client.pages([t for _, _, t, _ in titles], label="state chamber articles")
|
||||
summary, summary_title = (summary_tables(client, year) if mode == "election" else ({}, None))
|
||||
data, problems = {}, {}
|
||||
for code, which, title, size in titles:
|
||||
text = articles.get(title)
|
||||
box = infobox(text, "Infobox legislature") if text else {}
|
||||
name = plain(box.get("name", "")) or title
|
||||
leader = chamber_leader(box, which, code) if box else None
|
||||
next_year = C.next_legislative_election(code, day)
|
||||
if leader:
|
||||
leader["termEnd"] = str(next_year)
|
||||
parts = [leader["note"]]
|
||||
if mode == "election":
|
||||
parts.append("Current leader; the leader for the new session may not be chosen yet.")
|
||||
leader.setdefault("status", "current")
|
||||
parts.append(f"Leaders serve one legislative session; end deduced from the next regular election ({next_year}).")
|
||||
leader["note"] = " ".join(p for p in parts if p)
|
||||
ch = {"name": name, "seats": size, "D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0,
|
||||
"compositionNote": "", "leader": leader}
|
||||
s = summary.get((code, which))
|
||||
if mode == "election" and s:
|
||||
if s["seats"] and s["seats"] != size:
|
||||
problems[(code, which)] = f"the summary table gives {s['seats']} seats instead of {size}"
|
||||
elif s["after"]:
|
||||
a = s["after"]
|
||||
known = a["D"] + a["R"] + a["I"] + a["vacant"]
|
||||
if known > size:
|
||||
problems[(code, which)] = f"post-election seats add up to {known}, more than {size}"
|
||||
ch.update({k: a[k] for k in ("D", "R", "I", "vacant")})
|
||||
ch["uncalled"] = max(0, size - known)
|
||||
notes = []
|
||||
if ch["uncalled"]:
|
||||
notes.append(f"{ch['uncalled']} seat(s) not yet called on Wikipedia.")
|
||||
mb, ma = majority_party(s["before"]), majority_party(a) if not ch["uncalled"] else None
|
||||
if mb and ma and mb != ma:
|
||||
notes.append(f"{PARTY_WORD[ma]} flipped the chamber.")
|
||||
ch["compositionNote"] = " ".join(notes)
|
||||
else:
|
||||
ch["uncalled"] = size
|
||||
ch["compositionNote"] = (f"Wikipedia has not published {year} results for this chamber yet"
|
||||
+ (f" (before the election: {fmt_counts(s['before'])})." if fmt_counts(s["before"]) else "."))
|
||||
ch["_unknown"] = True
|
||||
else:
|
||||
comp = infobox_composition(box) if box else None
|
||||
if not text:
|
||||
problems[(code, which)] = f"article “{title}” not found"
|
||||
elif comp is None:
|
||||
problems[(code, which)] = f"no party composition in the infobox of “{title}”"
|
||||
else:
|
||||
ch.update(comp)
|
||||
total = sum(comp.values())
|
||||
if total != size:
|
||||
problems[(code, which)] = f"infobox seats add up to {total} instead of {size}"
|
||||
if mode == "election":
|
||||
ch["compositionNote"] = f"No regular election for this chamber in {year}; current composition from the chamber article."
|
||||
if not leader and (code, which) not in problems:
|
||||
problems[(code, which)] = f"no presiding officer found in the infobox of “{title}”"
|
||||
if code == "NE":
|
||||
ch["compositionNote"] = (ch["compositionNote"] + " Unicameral and officially nonpartisan; counts use members' party affiliation as Wikipedia reports it.").strip()
|
||||
data.setdefault(code, {"upper": None, "lower": None})[which] = ch
|
||||
sources = ([summary_title] if summary_title else []) + ["each state chamber's article"]
|
||||
return data, sources, problems
|
||||
Reference in new issue
Block a user