"""

Bill summaries ingest  -  CRS plain-language summaries from GovInfo BILLSTATUS XML.

The BILLSTATUS XML already fetched by bills_bulk contains a <summaries> element

with official CRS (Congressional Research Service) text.  This module backfills

the `summary` and `summary_source` columns on the `bills` table by re-fetching

those XMLs for bills that don't have a summary yet.

Source: GovInfo BILLSTATUS (bulkdata, keyless)

  https://www.govinfo.gov/bulkdata/BILLSTATUS/{congress}/{type}/BILLSTATUS-{congress}{type}{num}.xml

Usage

-----

    python -m ingest.bill_summaries --once

    python -m ingest.bill_summaries --once --congress 119 --limit 500

    python -m ingest.bill_summaries --once --types hr s

The summary text is stored verbatim (US gov CRS text, public domain).  HTML tags

are stripped; HTML entities are decoded.  Attribution: "CRS via GovInfo".

"""

import argparse

import html

import logging

import re

import sqlite3

import sys

import time

import xml.etree.ElementTree as ET

from typing import List, Optional

import requests

from ingest.db import get_connection, init_schema

logger = logging.getLogger(__name__)

SOURCE = "govinfo_billstatus"

SUMMARY_SOURCE = "CRS via GovInfo"

BULKDATA_BASE = "https://www.govinfo.gov/bulkdata/BILLSTATUS"

BILL_TYPES = ["hr", "s", "hjres", "sjres", "hconres", "sconres", "hres", "sres"]

_HTML_TAG_RE = re.compile(r"<[^>]+>")

_BLOCK_TAG_RE = re.compile(r"</?(p|div|br|li|ul|ol|h[1-6]|tr|table)\b[^>]*>", re.I)

_WS_RUN_RE = re.compile(r"[ \t]+")

_BLANKS_RE = re.compile(r"\n{3,}")

def _ensure_columns(conn: sqlite3.Connection) -> None:

    """Add summary / summary_source columns to bills if they don't exist yet."""

    for col, typedef in [("summary", "TEXT"), ("summary_source", "TEXT")]:

        try:

            conn.execute(f"ALTER TABLE bills ADD COLUMN {col} {typedef}")

            conn.commit()

            logger.info("Added column bills.%s", col)

        except sqlite3.OperationalError:

            pass

def _parse_external_id(external_id: str):

    """

    Decompose an external_id like '119-hr-160' or '119-hconres-1' into

    (congress: int, bill_type: str, number: str).

    """

    congress_str, rest = external_id.split("-", 1)

    bill_type, number = rest.rsplit("-", 1)

    return int(congress_str), bill_type, number

def _billstatus_url(congress: int, bill_type: str, number: str) -> str:

    return (

        f"{BULKDATA_BASE}/{congress}/{bill_type}/"

        f"BILLSTATUS-{congress}{bill_type}{number}.xml"

    )

def _strip_html(raw: str) -> str:

    """Decode HTML entities and strip HTML tags from a CRS summary text field.

    Block tags -> newline, inline tags -> space, so sentences and words never

    concatenate ("voters.</p><p>First" -> "voters.\n\nFirst", not "voters.First").

    """

    unescaped = html.unescape(raw)

    text = _BLOCK_TAG_RE.sub("\n", unescaped)

    text = _HTML_TAG_RE.sub(" ", text)

    text = _WS_RUN_RE.sub(" ", text)

    text = "\n".join(line.strip() for line in text.split("\n"))

    text = _BLANKS_RE.sub("\n\n", text)

    return text.strip()

def _extract_summary(xml_bytes: bytes) -> Optional[str]:

    """

    Parse a BILLSTATUS XML byte string and return the best available CRS

    summary text (plain text, HTML-stripped), or None if not present.

    Strategy: take the item whose <updateDate> is latest.  Falls back to the

    last <item> in document order if dates are missing.

    """

    try:

        root = ET.fromstring(xml_bytes)

    except ET.ParseError:

        return None

    bill_el = root.find("bill")

    if bill_el is None:

        if root.tag in ("bill", "Bill"):

            bill_el = root

        else:

            return None

    summaries_el = bill_el.find("summaries")

    if summaries_el is None:

        return None

    items = summaries_el.findall("summary")

    if not items:

        return None

    best_item = None

    best_date = ""

    for item in items:

        upd_el = item.find("updateDate")

        upd = upd_el.text.strip() if upd_el is not None and upd_el.text else ""

        if upd >= best_date:

            best_date = upd

            best_item = item

    if best_item is None:

        return None

    text_el = best_item.find("text")

    if text_el is None or not text_el.text:

        return None

    return _strip_html(text_el.text) or None

def poll(

    congress: int = 119,

    bill_types: Optional[List[str]] = None,

    db_path: Optional[str] = None,

    limit: Optional[int] = 500,

    delay: float = 0.1,

) -> int:

    """

    Backfill CRS summaries for bills that don't have one yet.

    Fetches BILLSTATUS XML from GovInfo (no API key required) and stores the

    best available summary text.  Idempotent: skips bills that already have a

    summary.  Polite: `delay` seconds between requests.

    Parameters

    ----------

    congress   : Congress number to process (default 119).

    bill_types : Bill types to process (default = all types in BILL_TYPES).

    db_path    : Override SQLite path.

    limit      : Maximum bills to process per run (None = unlimited).

    delay      : Seconds between GovInfo requests.

    Returns

    -------

    Number of summaries newly stored.

    """

    if bill_types is None:

        bill_types = BILL_TYPES

    conn = get_connection(db_path) if db_path else get_connection()

    init_schema(conn)

    _ensure_columns(conn)

    session = requests.Session()

    session.headers["User-Agent"] = "Docket-ingest/1.0 (public transparency research)"

    type_set = set(bill_types)

    rows = conn.execute(

        """

        SELECT id, external_id

        FROM bills

        WHERE source = ?

          AND congress = ?

          AND (summary IS NULL OR summary = '')

        ORDER BY id

        """,

        (SOURCE, congress),

    ).fetchall()

    logger.info(

        "bill_summaries: %d congress-%d bills missing summaries (types=%s, limit=%s)",

        len(rows), congress, bill_types, limit,

    )

    stored = 0

    skipped_type = 0

    for row in rows:

        if limit is not None and stored >= limit:

            logger.info("bill_summaries: limit %d reached, stopping.", limit)

            break

        ext_id = row["external_id"]

        bill_id = row["id"]

        try:

            cong, bill_type, number = _parse_external_id(ext_id)

        except Exception:

            logger.debug("Cannot parse external_id %s, skipping.", ext_id)

            continue

        if bill_type not in type_set:

            skipped_type += 1

            continue

        url = _billstatus_url(cong, bill_type, number)

        logger.debug("Fetching BILLSTATUS: %s", url)

        try:

            r = session.get(url, timeout=30, allow_redirects=True)

        except requests.RequestException as exc:

            logger.warning("Fetch failed for %s: %s", ext_id, exc)

            continue

        if r.status_code != 200:

            logger.debug("HTTP %d for %s", r.status_code, ext_id)

            if delay > 0:

                time.sleep(delay)

            continue

        summary_text = _extract_summary(r.content)

        if not summary_text:

            logger.debug("No summary in BILLSTATUS for %s", ext_id)

            if delay > 0:

                time.sleep(delay)

            continue

        conn.execute(

            "UPDATE bills SET summary=?, summary_source=? WHERE id=?",

            (summary_text, SUMMARY_SOURCE, bill_id),

        )

        stored += 1

        if stored % 50 == 0:

            conn.commit()

            logger.info("bill_summaries: %d summaries committed so far", stored)

        if delay > 0:

            time.sleep(delay)

    conn.commit()

    conn.close()

    logger.info(

        "bill_summaries: done - %d summaries stored, %d skipped (type filter)",

        stored, skipped_type,

    )

    return stored

if __name__ == "__main__":

    logging.basicConfig(

        level=logging.INFO,

        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",

        stream=sys.stdout,

    )

    ap = argparse.ArgumentParser(

        description="Backfill CRS summaries from GovInfo BILLSTATUS XML"

    )

    ap.add_argument("--once", action="store_true", help="Run once and exit")

    ap.add_argument("--congress", type=int, default=119, help="Congress number (default 119)")

    ap.add_argument(

        "--types", nargs="+", default=None,

        help="Bill types to process (hr s hjres …); default = all",

    )

    ap.add_argument(

        "--limit", type=int, default=500,

        help="Max summaries to fetch per run (default 500; 0 = unlimited)",

    )

    ap.add_argument(

        "--delay", type=float, default=0.1,

        help="Seconds between GovInfo requests (default 0.1)",

    )

    args = ap.parse_args()

    limit_val = args.limit if args.limit and args.limit > 0 else None

    n = poll(

        congress=args.congress,

        bill_types=args.types,

        limit=limit_val,

        delay=args.delay,

    )

    print(f"Done: {n} summaries stored")