Files
us-officeholders-map/app/parsers.py
T
2026-11-13 17:35:10 +01:00

625 lines
28 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Turn Wikipedia pages into the data the map uses. One function per category and mode.
Every parser returns (data, sources, notes) and raises ParseError when the page does not look as expected.
Validation of totals happens in update.py, not here.
"""
import datetime as dt
import re
import common as C
from wiki import plain, section, sections, split_rows, strip_noise, tables, infobox
STATE_BY_NAME = {n.lower(): c for n, c in C.STATES.items()}
class ParseError(Exception):
pass
def party_code(text):
t = plain(text or "").strip().lower()
if not t:
return None
t = re.sub(r"\s*party.*$", "", t) if "party" in t and not t.startswith("independent") else t
if t in ("d", "dfl") or re.match(r"^(d|dfl)\b", t):
return "D"
if t == "r" or re.match(r"^r\b", t):
return "R"
if t == "i" or re.match(r"^i\b", t):
return "I"
if "independent" in t:
return "I"
if "democrat" in t or "farmer" in t or "dfl" in t:
return "D"
if "republican" in t:
return "R"
return "I"
def state_code(name):
return STATE_BY_NAME.get(plain(name).strip().lower())
# ---------------------------------------------------------------- race rows (Senate, House, governors)
AYE = re.compile(r"\{\{\s*aye\s*\}\}", re.I)
MBY = re.compile(r"\{\{\s*mby\s*\}\}", re.I) # "advanced to runoff" mark
STRIPE = re.compile(r"\{\{\s*party stripe\s*\|([^}|]*)", re.I)
def candidates(row):
"""[(name, party code, won, has a Wikipedia article, advanced to a runoff)] from a results row."""
out = []
for line in row.split("\n"):
st = STRIPE.search(line)
if not st:
continue
won = bool(AYE.search(line))
rest = AYE.sub("", line[st.end():].split("}}", 1)[-1])
bold = re.search(r"'''(.+?)'''", rest)
if bold:
name_src = bold.group(1)
name = plain(name_src)
else:
name_src = rest
name = re.sub(r"\s*\([^()]*\)[\s\d.,%]*$", "", plain(rest)).strip()
pm = re.search(r"(?:'''|\]\])\s*\(([^()]*)\)", rest) or re.search(r"\(([^()]*)\)[\s\d.,%]*$", plain(rest))
party = party_code(pm.group(1)) if pm else party_code(st.group(1))
name = re.sub(r"\s*\((incumbent|write-in)\)", "", name, flags=re.I).strip()
if name:
out.append((name, party, won, "[[" in name_src, bool(MBY.search(line))))
return out
def race_result(row, term_end, elected_status, extra_note=""):
"""Officeholder entry for one race row: winner, or not-yet-called / runoff."""
cands = candidates(row)
winners = [c for c in cands if c[2]]
if len(winners) > 1:
raise ParseError("several winners marked in one race")
if winners:
name, party = winners[0][0], winners[0][1]
return {"name": name, "party": party, "termEnd": term_end, "status": elected_status, "note": extra_note}
text_no_cands = "\n".join(l for l in row.split("\n") if not STRIPE.search(l))
advanced = [c for c in cands if c[4]]
runoff = advanced or re.search(r"run-?off", plain(text_no_cands), re.I)
# the two main candidates: those in the runoff, else notable (linked) Democrats and Republicans first
shown = advanced[:2] or sorted(cands, key=lambda c: (c[1] not in ("D", "R"), not c[3]))[:2]
who = ", ".join(f"{c[0]} ({c[1]})" for c in shown)
parties = {c[1] for c in shown}
if runoff:
m = re.search(r"run-?off[^.]*?((?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}(?:, \d{4})?)",
plain(text_no_cands), re.I)
note = "Runoff" + (f" on {m.group(1)}" if m else "") + (f": {who.replace(', ', ' vs ')}" if who else "")
status = "runoff"
else:
note = "Not yet called" + (f"; {who.replace(', ', ' vs ')}" if who else "")
status = "uncalled"
return {"name": None, "party": parties.pop() if len(parties) == 1 and len(shown) > 1 else None,
"termEnd": term_end, "status": status, "note": (note + ". " + extra_note).strip()}
def header_lines(row):
return [l for l in row.split("\n") if l.lstrip().startswith("!")]
# ---------------------------------------------------------------- Senate
def current_senators(client):
title = "List of current United States senators"
text = strip_noise(client.page(title))
start = text.find('id="senators"')
if start < 0:
raise ParseError(f"senator table not found in “{title}”")
start = text.rfind("{|", 0, start)
end = text.find("\n|}", start)
table = text[start:end]
# footnotes saying an independent sits with the Democrats
dem_caucus_efn = set()
for m in re.finditer(r"\{\{efn\|name=([^|}]+)\|(.*?)\}\}\s*$", text, re.M | re.S):
if re.search(r"Democratic Caucus|caucus(es)? with (the )?Democrat", m.group(2), re.I):
dem_caucus_efn.add(m.group(1).strip())
out, state = [], None
for row in split_rows(table)[1:]:
sm = re.search(r"\[\[List of United States senators from ([^|\]]+)\|", row)
if sm:
state = state_code(sm.group(1))
name_line = next((l for l in row.split("\n") if l.startswith("!")), None)
if not name_line or state is None:
continue
name_cell = name_line[1:].split("|", 1)[-1] if re.match(r"!\s*scope", name_line) else name_line[1:]
name = plain(name_cell)
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", row, re.I)
cm = re.search(r"Class (\d)", row)
if not cm:
continue
caucus = any(f"name={n}" in row for n in dem_caucus_efn) or re.search(r"Democratic Caucus", row)
out.append({"state": state, "name": name or None, "party": party_code(pc.group(1)) if pc else None,
"class": int(cm.group(1)), "vacant": not name or name.lower().startswith("vacant"),
"demCaucus": bool(caucus)})
if len(out) < 90:
raise ParseError(f"only {len(out)} senators found in “{title}”")
return out, title
def senate_current(client, day):
sens, title = current_senators(client)
data = {}
for s in sens:
entry = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"],
"termEnd": C.senate_term_end(s["class"], day), "class": s["class"],
"status": "vacant" if s["vacant"] else "continuing",
"note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""}
data.setdefault(s["state"], []).append(entry)
for v in data.values():
v.sort(key=lambda e: e["class"])
return data, [title]
def senate_election(client, year):
title = f"{year} United States Senate elections"
text = strip_noise(client.page(title))
body = section(text, "Race summary")
if body is None:
raise ParseError(f"“Race summary” section not found in “{title}”")
new_congress = dt.date(year + 1, 1, 3)
regular = C.senate_class_up(year)
races = {}
for row in split_rows(body):
head = "\n".join(header_lines(row))
m = re.search(r"\[\[\s*%d United States Senate (special )?elections? in ([^|\]]+)\|" % year, head)
if not m:
continue
code = state_code(m.group(2))
if not code:
continue
special = bool(m.group(1))
cm = re.search(r"Class (\d)", head)
cls = int(cm.group(1)) if cm else (None if special else regular)
if cls is None:
raise ParseError(f"class of the special election in {C.NAMES[code]} not found")
note = f"Won {year} special election for the remainder of the Class {cls} term." if special else ""
e = race_result(row, C.senate_term_end(cls, new_congress), f"elected-{year}", note)
e["class"] = cls
races[(code, cls)] = e
if len(races) < 30:
raise ParseError(f"only {len(races)} Senate races found in “{title}”")
sens, list_title = current_senators(client)
data = {}
for s in sens:
key = (s["state"], s["class"])
if key in races:
e = dict(races.pop(key))
else:
e = {"name": None if s["vacant"] else s["name"], "party": None if s["vacant"] else s["party"],
"termEnd": C.senate_term_end(s["class"], new_congress), "class": s["class"],
"status": "vacant" if s["vacant"] else "continuing",
"note": "Independent, caucuses with Democrats." if s["party"] == "I" and s["demCaucus"] else ""}
data.setdefault(s["state"], []).append({k: e[k] for k in ("name", "party", "termEnd", "class", "status", "note")})
if races:
raise ParseError("races without a matching seat in the senator list: " + ", ".join(f"{c} class {k}" for c, k in races))
for v in data.values():
v.sort(key=lambda e: e["class"])
return data, [title, list_title]
# ---------------------------------------------------------------- House
def ushr(row):
for l in header_lines(row):
m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", l, re.I)
if m:
d = m.group(2)
return m.group(1).upper(), 0 if not d.isdigit() else int(d)
return None
def house_election(client, year):
title = f"{year} United States House of Representatives elections"
text = strip_noise(client.page(title))
term_end = C.house_term_end(dt.date(year + 1, 1, 3))
data = {}
for head, body in sections(text, 2):
code = state_code(head)
if not code:
continue
reps = []
for row in split_rows(body):
u = ushr(row)
if not u:
continue
e = race_result(row, term_end, f"elected-{year}")
reps.append({"district": u[1], **{k: e[k] for k in ("name", "party", "termEnd", "status", "note")}})
data[code] = reps
if len(data) < 45:
raise ParseError(f"only {len(data)} state sections found in “{title}”")
return data, [title]
def house_current(client, day):
title = "List of current members of the United States House of Representatives"
text = strip_noise(client.page(title))
term_end = C.house_term_end(day)
best = None
for t in tables(text):
if not t.rows:
continue
head = " ".join(c.text.lower() for c in t.rows[0])
if "district" in head and "member" in head and "party" in head and len(t.rows) > 400:
best = t
break
if best is None:
raise ParseError(f"member table not found in “{title}”")
head = [c.text.lower() for c in best.rows[0]]
i_dist = next(i for i, h in enumerate(head) if h.startswith("district"))
i_mem = next(i for i, h in enumerate(head) if h.startswith("member"))
i_party = max(i for i, h in enumerate(head) if h.startswith("party"))
data = {}
for row, raw in zip(best.rows[1:], best.row_raw[1:]):
if len(row) <= max(i_dist, i_mem, i_party) or row[i_dist].header and row[i_dist] is best.rows[0][i_dist]:
continue
dist_raw = row[i_dist].raw
m = re.search(r"\{\{\s*ushr\s*\|\s*(\w\w)\s*\|\s*([\w-]+)", dist_raw, re.I)
if m:
code, d = m.group(1).upper(), m.group(2)
else:
dt_txt = row[i_dist].text
m2 = re.match(r"(.+?)\s+(at-large|\d+)", dt_txt, re.I)
if not m2 or not state_code(m2.group(1)):
continue
code, d = state_code(m2.group(1)), m2.group(2)
if code not in C.APPORTIONMENT:
continue
district = int(d) if d.isdigit() else 0
member = row[i_mem].text
vacant = not member or member.lower().startswith("vacant")
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.raw + " " + c.attrs for c in row), re.I)
party = None if vacant else (party_code(row[i_party].text) or (party_code(pc.group(1)) if pc else None))
note = ""
if vacant:
note = plain(re.sub(r"(?i)^vacant", "", member)).strip(" .–-") or ""
entry = {"district": district, "name": None if vacant else member, "party": party,
"termEnd": term_end, "status": "vacant" if vacant else "continuing", "note": note}
lst = data.setdefault(code, [])
if not any(r["district"] == district for r in lst):
lst.append(entry)
return data, [title]
# ---------------------------------------------------------------- Governors
def current_governors(client):
title = "List of current United States governors"
text = strip_noise(client.page(title))
out = {}
for t in tables(text):
if not t.rows:
continue
head = [c.text.lower() for c in t.rows[0]]
if not any(h.startswith("governor") for h in head) or not any("term end" in h for h in head):
continue
i_gov = next(i for i, h in enumerate(head) if h.startswith("governor"))
i_end = next(i for i, h in enumerate(head) if "term end" in h)
i_party = max((i for i, h in enumerate(head) if h.startswith("party")), default=None)
for row in t.rows[1:]:
st = re.search(r"\[\[\s*Governor of ([^|\]]+)\|", row[0].raw) if row else None
code = state_code(st.group(1)) if st else state_code(row[0].text.split("(")[0]) if row else None
if not code or len(row) <= max(i_gov, i_end):
continue
name = row[i_gov].text
party = None
if i_party is not None and len(row) > i_party:
party = party_code(row[i_party].text)
if not party:
pc = re.search(r"\{\{\s*party color\s*\|([^}]*)\}\}", " ".join(c.attrs for c in row), re.I)
party = party_code(pc.group(1)) if pc else None
end_txt = row[i_end].text
ym = re.search(r"(\d{4})", end_txt)
out[code] = {"name": name or None, "party": party, "endYear": int(ym.group(1)) if ym else None,
"termLimited": "term limit" in end_txt.lower(), "endText": end_txt}
if len(out) >= 50:
break
if len(out) < 45:
raise ParseError(f"only {len(out)} governors found in “{title}”")
return out, title
def _gov_entry(code, g, status, powers):
note = "Term-limited." if g.get("termLimited") else ""
term_end = C.governor_term_end(code, g["endYear"]) if g.get("endYear") else None
return {"name": g["name"], "party": g["party"], "termEnd": term_end, "status": status, "note": note,
"powers": powers.get(code, [])}
def governors_current(client, day):
govs, title = current_governors(client)
powers = C.seed_governor_powers()
data = {code: _gov_entry(code, g, "continuing", powers) for code, g in govs.items()}
return data, [title]
def governors_election(client, year):
title = f"{year} United States gubernatorial elections"
text = strip_noise(client.page(title))
body = section(text, "Race summary")
if body is None:
raise ParseError(f"“Race summary” section not found in “{title}”")
powers = C.seed_governor_powers()
races = {}
for row in split_rows(body):
head = "\n".join(header_lines(row))
m = re.search(r"\[\[\s*%d ([A-Z][A-Za-z .]+?) gubernatorial (?:special )?election\|" % year, head)
if not m:
continue
code = state_code(m.group(1))
if not code:
continue
term = C.GOVERNOR_TERM_YEARS.get(code, 4)
starts_december = C.GOVERNOR_INAUGURATION.get(code, "01").startswith("12")
end_year = year + term if starts_december else year + 1 + term
e = race_result(row, C.governor_term_end(code, end_year), f"elected-{year}",
"End of term deduced from the term length.")
e["powers"] = powers.get(code, [])
races[code] = e
if len(races) < 20:
raise ParseError(f"only {len(races)} governor races found in “{title}”")
govs, list_title = current_governors(client)
data = {}
for code in C.NAMES:
if code in races:
data[code] = races[code]
elif code in govs:
data[code] = _gov_entry(code, govs[code], "continuing", powers)
return data, [title, list_title]
# ---------------------------------------------------------------- State legislatures
LOWER_WORDS = re.compile(r"house|assembly|delegates", re.I)
def chamber_titles():
out = []
for code, (up, low) in C.CHAMBERS.items():
out.append((code, "upper", up[0], up[1]))
if low:
out.append((code, "lower", low[0], low[1]))
return out
GROUP_WORDS = re.compile(r"\b(majority|minority|caucus|coalition|total|officially|nonpartisan)\b", re.I)
PARTY_WORDS = re.compile(r"democrat|republican|independent|vacan|libertarian|green|progressive|dfl", re.I)
def infobox_composition(box):
"""{D, R, I, vacant} from political_groups1 of an Infobox legislature.
Lines are read as groups: a heading (bold or ';' line) with an optional count, then bullet lines.
A bullet without its own count takes the heading's count when it is the heading's only bullet.
"""
pg = box.get("political_groups1") or box.get("political_groups") or ""
groups = [] # [heading label, heading count, [[label, count]]]
for line in pg.split("\n"):
s = line.strip()
if not s:
continue
is_bullet = s.startswith("*") or (s.startswith("|") and not s.startswith("|}"))
txt = plain(s.lstrip("*|; "))
m = re.search(r"^(.*?)\(\s*(\d+)\s*\)\s*$", txt)
label, count = (m.group(1).strip(), int(m.group(2))) if m else (txt.strip(), None)
if is_bullet and (count is not None or PARTY_WORDS.search(label)):
if not groups:
groups.append(["", None, []])
groups[-1][2].append([label, count])
elif s.startswith(("''", ";")) or GROUP_WORDS.search(label) or count is not None:
groups.append([label, count, []])
counts = {"D": 0, "R": 0, "I": 0, "vacant": 0}
found = 0
for head, head_count, bullets in groups:
if len(bullets) == 1 and bullets[0][1] is None and head_count is not None:
bullets[0][1] = head_count
if not bullets and head_count is not None and PARTY_WORDS.search(head) and not GROUP_WORDS.search(head):
bullets = [[head, head_count]]
for label, n in bullets:
if n is None:
continue
if re.search(r"vacan", label, re.I):
counts["vacant"] += n
else:
counts[party_code(label) or "I"] += n
found += 1
return counts if found else None
def chamber_leader(box, which, code):
leaders = []
for i in range(1, 12):
if f"leader{i}" in box:
leaders.append((box.get(f"leader{i}_type", ""), box.get(f"leader{i}", "")))
pick, title, note = None, None, ""
if which == "lower" or code == "NE":
for t, n in leaders:
tp = plain(t)
if re.match(r"speaker\b", tp, re.I) and "pro tem" not in tp.lower():
pick, title = (t, n), ("Speaker of the Legislature" if code == "NE" else "Speaker")
break
if not pick and which == "upper":
lg = False
for t, n in leaders:
tp = plain(t).lower()
is_lg = "lieutenant governor" in t.lower()
if re.match(r"speaker\b", tp) and "pro tem" not in tp:
# Tennessee: the Senate elects its Speaker, who is also lieutenant governor
pick, title = (t, n), "Speaker of the Senate"
note = "Also lieutenant governor." if is_lg else ""
break
if re.match(r"president\b", tp) and "pro tem" not in tp:
if is_lg:
lg = True
continue
pick, title = (t, n), "President of the Senate"
break
if not pick:
for t, n in leaders:
tp = plain(t)
if re.search(r"pro tem|temporary president", tp, re.I):
pick = (t, n)
title = "President pro tempore" if "pro tem" in tp.lower() else "Temporary President"
break
if pick and lg:
note = "Lieutenant governor presides."
if not pick:
others = [l for l in leaders if "lieutenant governor" not in l[0].lower()] or leaders
if not others:
return None
pick = others[0]
title = plain(pick[0]) or "Leader"
txt = plain(pick[1])
m = re.match(r"^(.*?)\s*\(([^()]*)\)\s*$", txt)
name, party = (m.group(1), party_code(m.group(2))) if m else (txt, None)
if not name or name.lower().startswith("vacant"):
return {"title": title, "name": None, "party": None, "termEnd": None, "note": "Vacant.", "status": "vacant"}
if not party:
bg = re.search(r"party color\s*\|([^}]*)\}\}", box.get("background_color", ""), re.I)
if bg and party_code(bg.group(1)) in ("D", "R"):
party = party_code(bg.group(1))
note = (note + " Party deduced from the chamber's majority (not given in the infobox).").strip()
return {"title": title, "name": name.strip(), "party": party, "termEnd": None, "note": note}
def summary_tables(client, year):
"""Post-election compositions from the main state legislative elections page: {(code, which): info}."""
title = f"{year} United States state legislative elections"
text = strip_noise(client.page(title))
out = {}
heads = [(m.start(), plain(m.group(1))) for m in re.finditer(r"^===(?!=)\s*(.+?)\s*===\s*$", text, re.M)]
for t in tables(text):
if not t.caption or not t.rows:
continue
hi = next((i for i, r in enumerate(t.rows[:3]) if any(c.text == "Before" for c in r) and any(c.text == "After" for c in r)), None)
if hi is None:
continue
state = None
for pos, h in heads:
if pos < t.start:
state = h
code = state_code(state or "")
if not code:
continue
which = "upper" if code == "NE" or not LOWER_WORDS.search(t.caption) else "lower"
hdr = [c.text for c in t.rows[hi]]
i_b, i_a = hdr.index("Before"), hdr.index("After")
i_p = max(i for i, h in enumerate(hdr) if h == "Party")
before = {"D": 0, "R": 0, "I": 0, "vacant": 0}
after = {"D": 0, "R": 0, "I": 0, "vacant": 0}
after_known = False
after_missing = False # a party that held seats before has no "After" figure yet
seats = None
for r in t.rows[hi + 1:]:
if len(r) <= max(i_b, i_a, i_p):
continue
label = r[i_p].text
if any(c.text.lower() == "total" for c in r):
nb = re.search(r"\d+", r[i_b].text)
seats = int(nb.group(0)) if nb else None
continue
if not label:
continue
key = "vacant" if re.search(r"vacan", label, re.I) else party_code(label)
nb = re.search(r"\d+", r[i_b].text)
na = re.search(r"\d+", r[i_a].text)
if nb:
before[key] += int(nb.group(0))
if na:
after[key] += int(na.group(0))
after_known = True
elif nb and int(nb.group(0)) > 0 and key != "vacant":
after_missing = True
complete = seats is not None and sum(after.values()) == seats
out[(code, which)] = {"caption": t.caption, "seats": seats, "before": before,
"after": after if after_known and (complete or not after_missing) else None}
if len(out) < 60:
raise ParseError(f"only {len(out)} chamber tables found in “{title}”")
return out, title
def majority_party(c):
top = sorted(("D", "R", "I"), key=lambda k: -c.get(k, 0))
if c.get(top[0], 0) == c.get(top[1], 0):
return None
return top[0]
def fmt_counts(c):
return ", ".join(f"{c[k]} {k}" for k in ("D", "R", "I") if c.get(k))
PARTY_WORD = {"D": "Democrats", "R": "Republicans", "I": "Independents"}
def legislatures(client, mode, day, year=None):
"""Returns ({code: {upper, lower}}, sources, per-chamber problems {(code, which): reason})."""
titles = chamber_titles()
articles = client.pages([t for _, _, t, _ in titles], label="state chamber articles")
summary, summary_title = (summary_tables(client, year) if mode == "election" else ({}, None))
data, problems = {}, {}
for code, which, title, size in titles:
text = articles.get(title)
box = infobox(text, "Infobox legislature") if text else {}
name = plain(box.get("name", "")) or title
leader = chamber_leader(box, which, code) if box else None
next_year = C.next_legislative_election(code, day)
if leader:
leader["termEnd"] = str(next_year)
parts = [leader["note"]]
if mode == "election":
parts.append("Current leader; the leader for the new session may not be chosen yet.")
leader.setdefault("status", "current")
parts.append(f"Leaders serve one legislative session; end deduced from the next regular election ({next_year}).")
leader["note"] = " ".join(p for p in parts if p)
ch = {"name": name, "seats": size, "D": 0, "R": 0, "I": 0, "vacant": 0, "uncalled": 0,
"compositionNote": "", "leader": leader}
s = summary.get((code, which))
if mode == "election" and s:
if s["seats"] and s["seats"] != size:
problems[(code, which)] = f"the summary table gives {s['seats']} seats instead of {size}"
elif s["after"]:
a = s["after"]
known = a["D"] + a["R"] + a["I"] + a["vacant"]
if known > size:
problems[(code, which)] = f"post-election seats add up to {known}, more than {size}"
ch.update({k: a[k] for k in ("D", "R", "I", "vacant")})
ch["uncalled"] = max(0, size - known)
notes = []
if ch["uncalled"]:
notes.append(f"{ch['uncalled']} seat(s) not yet called on Wikipedia.")
mb, ma = majority_party(s["before"]), majority_party(a) if not ch["uncalled"] else None
if mb and ma and mb != ma:
notes.append(f"{PARTY_WORD[ma]} flipped the chamber.")
ch["compositionNote"] = " ".join(notes)
else:
ch["uncalled"] = size
ch["compositionNote"] = (f"Wikipedia has not published {year} results for this chamber yet"
+ (f" (before the election: {fmt_counts(s['before'])})." if fmt_counts(s["before"]) else "."))
ch["_unknown"] = True
else:
comp = infobox_composition(box) if box else None
if not text:
problems[(code, which)] = f"article “{title}” not found"
elif comp is None:
problems[(code, which)] = f"no party composition in the infobox of “{title}”"
else:
ch.update(comp)
total = sum(comp.values())
if total != size:
problems[(code, which)] = f"infobox seats add up to {total} instead of {size}"
if mode == "election":
ch["compositionNote"] = f"No regular election for this chamber in {year}; current composition from the chamber article."
if not leader and (code, which) not in problems:
problems[(code, which)] = f"no presiding officer found in the infobox of “{title}”"
if code == "NE":
ch["compositionNote"] = (ch["compositionNote"] + " Unicameral and officially nonpartisan; counts use members' party affiliation as Wikipedia reports it.").strip()
data.setdefault(code, {"upper": None, "lower": None})[which] = ch
sources = ([summary_title] if summary_title else []) + ["each state chamber's article"]
return data, sources, problems