"""

Tests for ingest.categorize  -  unified taxonomy module.

Covers:

  - agency_to_category: deterministic mapping, prefix match, unknown agencies

  - POLICY_AREAS: completeness and slug format

  - _parse_subjects: XML parsing from a synthetic BILLSTATUS snippet

  - Schema helpers: ensure tables and columns are created idempotently

  - _seed_categories: idempotent seeding, extra bill areas picked up

  - _backfill_awards: correct category assignment, NULL handling

  - category_counts: returns expected shape

"""

import sqlite3

import textwrap

import xml.etree.ElementTree as ET

import pytest

from ingest.categorize import (

    AGENCY_CATEGORY_MAP,

    POLICY_AREAS,

    agency_to_category,

    _parse_subjects,

    _ensure_schema,

    _seed_categories,

    _backfill_awards,

    _slug_from_label,

    category_counts,

)

from ingest.db import init_schema

@pytest.fixture

def conn():

    """In-memory SQLite connection with base schema initialised."""

    c = sqlite3.connect(":memory:")

    c.row_factory = sqlite3.Row

    c.execute("PRAGMA foreign_keys=ON")

    init_schema(c)

    _ensure_schema(c)

    yield c

    c.close()

def _insert_award(conn, agency: str, category: str | None = None) -> int:

    conn.execute(

        """

        INSERT INTO awards (source, external_id, agency, recipient, amount,

                            action_date, program, source_url, category)

        VALUES ('usaspending', ?, ?, 'Test Recipient', 1000000,

                '2026-01-01', 'Test Program',

                'https://www.usaspending.gov/award/TEST', ?)

        """,

        (f"award-{agency[:20]}", agency, category),

    )

    conn.commit()

    return conn.execute("SELECT last_insert_rowid()").fetchone()[0]

def _insert_bill(conn, policy_area: str = "", external_id: str = "119-hr-1") -> int:

    conn.execute(

        """

        INSERT INTO bills (source, external_id, congress, number, title,

                           sponsor_id, status, policy_area, updated_at, source_url)

        VALUES ('govinfo_billstatus', ?, 119, 'H.R. 1', 'Test Bill',

                '', '', ?, '2026-01-01', 'https://www.congress.gov/bill/119th-congress/house-bill/1')

        """,

        (external_id, policy_area),

    )

    conn.commit()

    return conn.execute("SELECT last_insert_rowid()").fetchone()[0]

class TestAgencyToCategory:

    """Deterministic agency→policy-area mapping."""

    def test_exact_match_defense(self):

        assert agency_to_category("Department of Defense") == "Armed Forces and National Security"

    def test_exact_match_health(self):

        assert agency_to_category("Department of Health and Human Services") == "Health"

    def test_exact_match_transportation(self):

        assert agency_to_category("Department of Transportation") == "Transportation and Public Works"

    def test_exact_match_education(self):

        assert agency_to_category("Department of Education") == "Education"

    def test_exact_match_energy(self):

        assert agency_to_category("Department of Energy") == "Energy"

    def test_exact_match_agriculture(self):

        assert agency_to_category("Department of Agriculture") == "Agriculture and Food"

    def test_exact_match_labor(self):

        assert agency_to_category("Department of Labor") == "Labor and Employment"

    def test_exact_match_epa(self):

        assert agency_to_category("Environmental Protection Agency") == "Environmental Protection"

    def test_exact_match_doj(self):

        assert agency_to_category("Department of Justice") == "Crime and Law Enforcement"

    def test_exact_match_state(self):

        assert agency_to_category("Department of State") == "International Affairs"

    def test_exact_match_usaid(self):

        assert agency_to_category("Agency for International Development") == "International Affairs"

    def test_exact_match_commerce(self):

        assert agency_to_category("Department of Commerce") == "Commerce"

    def test_exact_match_treasury(self):

        assert agency_to_category("Department of the Treasury") == "Economics and Public Finance"

    def test_exact_match_va(self):

        assert agency_to_category("Department of Veterans Affairs") == "Armed Forces and National Security"

    def test_exact_match_dhs(self):

        assert agency_to_category("Department of Homeland Security") == "Emergency Management"

    def test_exact_match_interior(self):

        assert agency_to_category("Department of the Interior") == "Public Lands and Natural Resources"

    def test_exact_match_hud(self):

        assert agency_to_category("Department of Housing and Urban Development") == "Housing and Community Development"

    def test_exact_match_ssa(self):

        assert agency_to_category("Social Security Administration") == "Social Welfare"

    def test_exact_match_nasa(self):

        assert agency_to_category("National Aeronautics and Space Administration") == "Science, Technology, Communications"

    def test_exact_match_nsf(self):

        assert agency_to_category("National Science Foundation") == "Science, Technology, Communications"

    def test_exact_match_gsa(self):

        assert agency_to_category("General Services Administration") == "Government Operations and Politics"

    def test_prefix_match_dod_suborg(self):

        """Sub-agency starting with 'Department of Defense' should map correctly."""

        result = agency_to_category("Department of Defense - Defense Advanced Research Projects Agency")

        assert result == "Armed Forces and National Security"

    def test_unknown_agency_returns_none(self):

        assert agency_to_category("Some Completely Unknown Agency XYZ") is None

    def test_empty_string_returns_none(self):

        assert agency_to_category("") is None

    def test_none_returns_none(self):

        assert agency_to_category(None) is None

    def test_all_map_keys_are_in_policy_area_labels(self):

        """Every agency map value must correspond to a canonical policy-area label."""

        valid_labels = {label for _, label in POLICY_AREAS}

        for agency, category in AGENCY_CATEGORY_MAP.items():

            assert category in valid_labels, (

                f"Agency '{agency}' maps to '{category}' which is not a canonical policy area"

            )

class TestPolicyAreas:

    def test_has_expected_count(self):

        assert 28 <= len(POLICY_AREAS) <= 45, f"Unexpected POLICY_AREAS count: {len(POLICY_AREAS)}"

    def test_no_duplicate_slugs(self):

        slugs = [slug for slug, _ in POLICY_AREAS]

        assert len(slugs) == len(set(slugs)), "Duplicate slugs in POLICY_AREAS"

    def test_no_duplicate_labels(self):

        labels = [label for _, label in POLICY_AREAS]

        assert len(labels) == len(set(labels)), "Duplicate labels in POLICY_AREAS"

    def test_slugs_are_lowercase_hyphenated(self):

        import re

        for slug, _ in POLICY_AREAS:

            assert re.match(r"^[a-z0-9-]+$", slug), f"Slug '{slug}' has invalid characters"

    def test_key_areas_present(self):

        labels = {label for _, label in POLICY_AREAS}

        required = {

            "Health",

            "Armed Forces and National Security",

            "Education",

            "Energy",

            "Taxation",

            "Transportation and Public Works",

            "Environmental Protection",

            "Agriculture and Food",

            "Crime and Law Enforcement",

            "International Affairs",

        }

        missing = required - labels

        assert not missing, f"Missing required policy areas: {missing}"

    def test_slug_matches_label(self):

        """Generated slug from label should equal the declared slug (or be a close match)."""

        for declared_slug, label in POLICY_AREAS:

            generated = _slug_from_label(label)

            assert declared_slug == generated or declared_slug.replace("-", "") == generated.replace("-", ""), (

                f"Slug mismatch: declared='{declared_slug}' generated='{generated}' for label='{label}'"

            )

_BILLSTATUS_TEMPLATE = """\

<?xml version="1.0" encoding="UTF-8"?>

<billStatus>

  <bill>

    <subjects>

      <legislativeSubjects>

        {items}

      </legislativeSubjects>

    </subjects>

  </bill>

</billStatus>"""

def _make_billstatus(subjects: list[str]) -> bytes:

    items = "".join(f"<item><name>{s}</name></item>" for s in subjects)

    return _BILLSTATUS_TEMPLATE.format(items=items).encode()

class TestParseSubjects:

    def test_parses_single_subject(self):

        xml_bytes = _make_billstatus(["Health care reform"])

        assert _parse_subjects(xml_bytes) == ["Health care reform"]

    def test_parses_multiple_subjects(self):

        subjects = ["Tax exemptions", "Income tax", "Small business"]

        xml_bytes = _make_billstatus(subjects)

        assert _parse_subjects(xml_bytes) == subjects

    def test_empty_legislative_subjects(self):

        xml_bytes = _make_billstatus([])

        assert _parse_subjects(xml_bytes) == []

    def test_missing_subjects_element(self):

        xml_bytes = b"<billStatus><bill><title>Test</title></bill></billStatus>"

        assert _parse_subjects(xml_bytes) == []

    def test_malformed_xml_returns_empty(self):

        assert _parse_subjects(b"not valid xml<<<") == []

    def test_strips_whitespace_from_subject_names(self):

        xml_bytes = _make_billstatus(["  Defense contracts  "])

        result = _parse_subjects(xml_bytes)

        assert result == ["Defense contracts"]

    def test_bill_root_without_billstatus_wrapper(self):

        """Some GovInfo responses have <bill> as the root element."""

        xml_bytes = b"""<bill>

          <subjects>

            <legislativeSubjects>

              <item><name>Veterans</name></item>

            </legislativeSubjects>

          </subjects>

        </bill>"""

        assert _parse_subjects(xml_bytes) == ["Veterans"]

class TestEnsureSchema:

    def test_creates_categories_table(self, conn):

        tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}

        assert "categories" in tables

    def test_creates_bill_subjects_table(self, conn):

        tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}

        assert "bill_subjects" in tables

    def test_adds_category_column_to_awards(self, conn):

        cols = {r[1] for r in conn.execute("PRAGMA table_info(awards)")}

        assert "category" in cols

    def test_idempotent_second_call(self, conn):

        """Calling _ensure_schema twice must not raise."""

        _ensure_schema(conn)

        cols = {r[1] for r in conn.execute("PRAGMA table_info(awards)")}

        assert "category" in cols

class TestSeedCategories:

    def test_seeds_all_policy_areas(self, conn):

        _seed_categories(conn)

        count = conn.execute("SELECT COUNT(*) FROM categories").fetchone()[0]

        assert count >= len(POLICY_AREAS)

    def test_idempotent_double_seed(self, conn):

        _seed_categories(conn)

        count1 = conn.execute("SELECT COUNT(*) FROM categories").fetchone()[0]

        _seed_categories(conn)

        count2 = conn.execute("SELECT COUNT(*) FROM categories").fetchone()[0]

        assert count1 == count2

    def test_picks_up_extra_bill_policy_area(self, conn):

        """A policy area present in bills but not in POLICY_AREAS is added."""

        conn.execute("""

            INSERT INTO bills (source, external_id, congress, number, title,

                               sponsor_id, status, policy_area, updated_at, source_url)

            VALUES ('govinfo_billstatus', 'test-1', 119, 'H.R. 999', 'Test',

                    '', '', 'Futuristic Technologies', '2026-01-01',

                    'https://www.congress.gov/bill/119th-congress/house-bill/999')

        """)

        conn.commit()

        _seed_categories(conn)

        row = conn.execute(

            "SELECT * FROM categories WHERE label = 'Futuristic Technologies'"

        ).fetchone()

        assert row is not None

    def test_health_area_in_categories(self, conn):

        _seed_categories(conn)

        row = conn.execute("SELECT * FROM categories WHERE label = 'Health'").fetchone()

        assert row is not None

        assert row["kind"] == "policy_area"

class TestBackfillAwards:

    def test_categorizes_known_agency(self, conn):

        _insert_award(conn, "Department of Defense")

        _backfill_awards(conn)

        row = conn.execute("SELECT category FROM awards WHERE agency = 'Department of Defense'").fetchone()

        assert row["category"] == "Armed Forces and National Security"

    def test_leaves_unknown_agency_null(self, conn):

        _insert_award(conn, "Unknown Mystery Agency")

        _backfill_awards(conn)

        row = conn.execute("SELECT category FROM awards WHERE agency = 'Unknown Mystery Agency'").fetchone()

        assert row["category"] is None

    def test_does_not_overwrite_existing_category(self, conn):

        """Awards that already have a category should keep it if the agency is unknown."""

        _insert_award(conn, "Unknown Mystery Agency", category="Existing Category")

        _backfill_awards(conn)

        row = conn.execute("SELECT category FROM awards WHERE agency = 'Unknown Mystery Agency'").fetchone()

        assert row["category"] == "Existing Category"

    def test_multiple_agencies(self, conn):

        agencies = [

            ("Department of Health and Human Services", "Health"),

            ("Department of Transportation",            "Transportation and Public Works"),

            ("National Science Foundation",             "Science, Technology, Communications"),

            ("Social Security Administration",          "Social Welfare"),

        ]

        for agency, _ in agencies:

            _insert_award(conn, agency)

        _backfill_awards(conn)

        for agency, expected_cat in agencies:

            row = conn.execute(

                "SELECT category FROM awards WHERE agency = ?", (agency,)

            ).fetchone()

            assert row["category"] == expected_cat, f"Wrong category for {agency}"

    def test_returns_count_of_updated(self, conn):

        _insert_award(conn, "Department of Energy")

        _insert_award(conn, "Department of Labor")

        updated = _backfill_awards(conn)

        assert updated == 2

    def test_idempotent(self, conn):

        _insert_award(conn, "Department of Defense")

        _backfill_awards(conn)

        updated2 = _backfill_awards(conn)

        assert updated2 == 0

class TestCategoryCounts:

    def test_returns_list(self, conn):

        _seed_categories(conn)

        result = category_counts(conn)

        assert isinstance(result, list)

    def test_each_item_has_expected_keys(self, conn):

        _seed_categories(conn)

        result = category_counts(conn)

        assert len(result) > 0

        for item in result:

            assert "label" in item

            assert "bills" in item

            assert "awards" in item

    def test_bill_counted_in_correct_category(self, conn):

        _seed_categories(conn)

        _insert_bill(conn, policy_area="Health")

        result = category_counts(conn)

        health_row = next((r for r in result if r["label"] == "Health"), None)

        assert health_row is not None

        assert health_row["bills"] == 1

    def test_award_counted_in_correct_category(self, conn):

        _seed_categories(conn)

        _insert_award(conn, "Department of Defense")

        _backfill_awards(conn)

        result = category_counts(conn)

        row = next((r for r in result if r["label"] == "Armed Forces and National Security"), None)

        assert row is not None

        assert row["awards"] == 1