"""Connectors for REAL, publicly-available data feeds. Every function returns normalized rows

that the aggregation pipeline ingests. No keys, no scraping, no TOS violations - these are open

government / academic / OSM APIs. Provenance (source + external_id) is preserved on every row.

Fixed assets  -> SensorAsset dicts   (cameras, weather, radar, hydro, seismic)

Moving entities -> LiveEntity dicts   (aircraft via ADS-B, satellites via TLE)

"""

from __future__ import annotations

import json

import os

import re

from datetime import datetime, timezone

from urllib.parse import quote

import httpx

UA = {"User-Agent": "Sightline-OSINT/1.0 (public-data aggregation; research)"}

_OMM_CACHE = os.path.join(os.path.dirname(__file__), "..", "..", "sample_data", "sat_omm.json")

US_BBOX = (24.0, -125.0, 49.5, -66.0)

STATES = ["AL","AK","AZ","AR","CA","CO","CT","DE","FL","GA","HI","ID","IL","IN","IA","KS","KY",

          "LA","ME","MD","MA","MI","MN","MS","MO","MT","NE","NV","NH","NJ","NM","NY","NC","ND",

          "OH","OK","OR","PA","RI","SC","SD","TN","TX","UT","VT","VA","WA","WV","WI","WY"]

def _get(url, **kw):

    return httpx.get(url, headers=UA, timeout=kw.pop("timeout", 40), **kw)

def nyc_traffic_cameras() -> list[dict]:

    """NYC DOT traffic cameras - REAL public live JPEG snapshots, no key. ~1,000+ cameras with

    direct imageUrl that refreshes. The 'live feed' layer the ALPR-location layer can't provide."""

    r = _get("https://webcams.nyctmc.org/api/cameras", timeout=40)

    if r.status_code != 200:

        return []

    try:

        items = r.json()

    except Exception:

        return []

    if isinstance(items, dict):

        items = items.get("items") or items.get("cameras") or []

    out = []

    for c in items:

        cid = c.get("id")

        if not cid or c.get("latitude") is None:

            continue

        img = c.get("imageUrl") or f"https://webcams.nyctmc.org/api/cameras/{cid}/image"

        out.append({"external_id": f"NYCDOT-{cid}", "name": c.get("name"),

                    "operator": "NYC DOT", "state": "NY", "area": c.get("area"),

                    "lat": c.get("latitude"), "lon": c.get("longitude"),

                    "online": bool(c.get("isOnline", True)), "image_url": img,

                    "live_source": "nyc_dot"})

    return out

def caltrans_cameras() -> list[dict]:

    """Caltrans (California DOT) CCTV - REAL public live JPEG snapshots, no key, ~3,500 statewide

    across 12 districts, each with lat/lon, county, route, direction."""

    out = []

    for d in range(1, 13):

        url = f"https://cwwp2.dot.ca.gov/data/d{d}/cctv/cctvStatusD{d:02d}.json"

        try:

            r = _get(url, timeout=25)

            if r.status_code != 200:

                continue

            for rec in r.json().get("data", []):

                c = rec.get("cctv", {})

                loc = c.get("location", {})

                img = (c.get("imageData", {}).get("static", {}) or {}).get("currentImageURL")

                lat, lon = loc.get("latitude"), loc.get("longitude")

                if not img or not lat or not lon:

                    continue

                try:

                    lat, lon = float(lat), float(lon)

                except (TypeError, ValueError):

                    continue

                if lat == 0 or lon == 0:

                    continue

                out.append({"external_id": f"CALTRANS-D{d:02d}-{c.get('index') or loc.get('locationName','')[:20]}",

                            "name": loc.get("locationName") or loc.get("nearbyPlace"),

                            "operator": "Caltrans", "state": "CA",

                            "area": loc.get("county") or f"District {d}",

                            "lat": lat, "lon": lon, "online": True, "image_url": img,

                            "live_source": "caltrans"})

        except Exception:

            continue

    return out

def tripcheck_cameras() -> list[dict]:

    """Oregon DOT TripCheck CCTV - REAL public live JPEG snapshots, no key, ~1,100 statewide.

    Inventory is a public ArcGIS-style JSON; images refresh at /RoadCams/cams/<filename>."""

    try:

        r = _get("https://tripcheck.com/Scripts/map/data/cctvinventory.js", timeout=40)

        if r.status_code != 200:

            return []

        feats = json.loads(r.text).get("features", [])

    except Exception:

        return []

    out = []

    for f in feats:

        a = f.get("attributes", {})

        fn, lat, lon = a.get("filename"), a.get("latitude"), a.get("longitude")

        if not fn or not lat or not lon:

            continue

        try:

            lat, lon = float(lat), float(lon)

        except (TypeError, ValueError):

            continue

        out.append({"external_id": f"TRIPCHECK-{a.get('cameraId')}-{a.get('publishedImageId')}",

                    "name": (a.get("title") or fn).strip(),

                    "operator": "Oregon DOT", "state": "OR",

                    "area": (a.get("route") or "").strip() or "Oregon",

                    "lat": lat, "lon": lon, "online": True,

                    "image_url": "https://tripcheck.com/RoadCams/cams/" + quote(fn),

                    "live_source": "tripcheck"})

    return out

_WKT_POINT = re.compile(r"POINT \((-?[\d.]+) (-?[\d.]+)\)")

_511_STATE = {"Florida": "FL", "Arizona": "AZ", "Vermont": "VT", "New Hampshire": "NH",

              "Maine": "ME", "Massachusetts": "MA", "Connecticut": "CT", "Rhode Island": "RI",

              "Georgia": "GA"}

def _ibi_511_cameras(host: str, prefix: str, operator: str, default_state: str,

                     live_source: str, max_pages: int = 60) -> list[dict]:

    """Shared connector for IBI-platform 511 sites (fl511.com, az511.gov, newengland511.org...).

    Public DataTables endpoint, no key; live snapshots at https://<host>/map/Cctv/<id>."""

    import time

    out, seen, start, page_fails = [], set(), 0, 0

    for _ in range(max_pages):

        rows = None

        for attempt in range(3):

            try:

                r = httpx.post(f"https://{host}/List/GetData/Cameras", headers=UA, timeout=40,

                               data={"draw": "1", "start": str(start), "length": "100"})

                rows = r.json().get("data") or []

                break

            except Exception:

                time.sleep(1 + attempt)

        if rows is None:

            page_fails += 1

            if page_fails >= 2:

                break

            start += 100

            continue

        page_fails = 0

        if not rows:

            break

        for d in rows:

            wkt = ((d.get("latLng") or {}).get("geography") or {}).get("wellKnownText") or ""

            m = _WKT_POINT.search(wkt)

            img = next((i.get("imageUrl") for i in (d.get("images") or [])

                        if i.get("imageUrl") and not i.get("blocked")), None)

            ext = f"{prefix}-{d.get('id')}"

            if not m or not img or d.get("id") is None or ext in seen:

                continue

            seen.add(ext)

            if img.startswith("/"):

                img = f"https://{host}{img}"

            out.append({"external_id": ext,

                        "name": d.get("location") or f"{d.get('roadway')} {d.get('direction') or ''}".strip(),

                        "operator": operator,

                        "state": _511_STATE.get(d.get("state"), default_state),

                        "area": d.get("roadway") or d.get("region") or default_state,

                        "lat": float(m.group(2)), "lon": float(m.group(1)),

                        "online": bool(d.get("visible", True)), "image_url": img,

                        "live_source": live_source})

        start += 100

    return out

def fl511_cameras() -> list[dict]:

    """Florida FL511 CCTV - REAL public live JPEG snapshots, no key, ~4,800 statewide."""

    return _ibi_511_cameras("fl511.com", "FL511", "Florida DOT (FL511)", "FL", "fl511")

def az511_cameras() -> list[dict]:

    """Arizona AZ511 CCTV - REAL public live JPEG snapshots, no key, ~600 statewide."""

    return _ibi_511_cameras("az511.gov", "AZ511", "Arizona DOT (AZ511)", "AZ", "az511")

def ne511_cameras() -> list[dict]:

    """New England 511 CCTV (VT/NH/ME DOTs) - REAL public live JPEG snapshots, no key, ~400."""

    return _ibi_511_cameras("newengland511.org", "NE511", "New England 511", "VT", "ne511")

def nws_stations(state: str, cap: int = 120) -> list[dict]:

    r = _get(f"https://api.weather.gov/stations?state={state}&limit={cap}")

    if r.status_code != 200:

        return []

    out = []

    for f in r.json().get("features", []):

        p = f.get("properties", {})

        c = (f.get("geometry") or {}).get("coordinates") or [None, None]

        out.append({"kind": "weather_station", "category": "weather", "source": "nws",

                    "external_id": p.get("stationIdentifier"), "name": p.get("name"),

                    "operator": "NOAA / NWS", "lat": c[1], "lon": c[0], "state": state,

                    "attributes": {"elevation_m": (p.get("elevation") or {}).get("value"),

                                   "timezone": p.get("timeZone")}})

    return out

def nws_radar() -> list[dict]:

    r = _get("https://api.weather.gov/radar/stations")

    if r.status_code != 200:

        return []

    out = []

    for f in r.json().get("features", []):

        p = f.get("properties", {})

        c = (f.get("geometry") or {}).get("coordinates") or [None, None]

        out.append({"kind": "weather_radar", "category": "weather", "source": "nws",

                    "external_id": p.get("id") or p.get("stationId"),

                    "name": p.get("name"), "operator": "NOAA / NWS",

                    "lat": c[1], "lon": c[0], "state": None,

                    "attributes": {"type": p.get("stationType"), "rda": (p.get("rda") or {})}})

    return out

def usgs_gauges(state: str, cap: int = 200) -> list[dict]:

    r = _get(f"https://waterservices.usgs.gov/nwis/iv/?format=json&stateCd={state}"

             f"&parameterCd=00060&siteStatus=active")

    if r.status_code != 200:

        return []

    out = []

    for ts in r.json().get("value", {}).get("timeSeries", [])[:cap]:

        src = ts.get("sourceInfo", {})

        geo = (src.get("geoLocation") or {}).get("geogLocation") or {}

        vals = (ts.get("values") or [{}])[0].get("value") or []

        latest = vals[-1]["value"] if vals else None

        out.append({"kind": "stream_gauge", "category": "hydro", "source": "usgs",

                    "external_id": (src.get("siteCode") or [{}])[0].get("value"),

                    "name": src.get("siteName"), "operator": "USGS",

                    "lat": geo.get("latitude"), "lon": geo.get("longitude"),

                    "state": state, "attributes": {"discharge_cfs": latest}})

    return out

def usgs_quakes(min_mag: float = 1.0) -> list[dict]:

    r = _get("https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson")

    if r.status_code != 200:

        return []

    out = []

    for f in r.json().get("features", []):

        p, g = f.get("properties", {}), (f.get("geometry") or {}).get("coordinates") or [None, None, None]

        if (p.get("mag") or 0) < min_mag:

            continue

        out.append({"kind": "seismic_event", "category": "seismic", "source": "usgs",

                    "external_id": f.get("id"), "name": p.get("place"), "operator": "USGS",

                    "lat": g[1], "lon": g[0], "state": None,

                    "attributes": {"magnitude": p.get("mag"), "depth_km": g[2],

                                   "time": p.get("time"), "felt": p.get("felt")}})

    return out

def opensky_aircraft(bbox=US_BBOX) -> list[dict]:

    la1, lo1, la2, lo2 = bbox

    r = _get(f"https://opensky-network.org/api/states/all?lamin={la1}&lomin={lo1}"

             f"&lamax={la2}&lomax={lo2}", timeout=60)

    if r.status_code != 200:

        return []

    now = datetime.now(timezone.utc)

    out = []

    for s in (r.json().get("states") or []):

        icao, callsign, country = s[0], (s[1] or "").strip(), s[2]

        lon, lat, alt, on_ground, vel, heading, vrate = s[5], s[6], s[7], s[8], s[9], s[10], s[11]

        if lat is None or lon is None:

            continue

        out.append({"kind": "aircraft", "source": "opensky", "identifier": icao,

                    "label": callsign or icao, "lat": lat, "lon": lon,

                    "altitude_m": alt, "heading": heading, "velocity": vel,

                    "origin_country": country,

                    "attributes": {"on_ground": on_ground, "vertical_rate": vrate,

                                   "geo_altitude": s[13] if len(s) > 13 else None,

                                   "squawk": s[14] if len(s) > 14 else None}})

    return out

def celestrak_satellites(cap: int = 800, group: str = "active") -> list[dict]:

    """Active satellites from CelesTrak (OMM JSON). Positions propagated to 'now' with SGP4 via

    sgp4.omm.initialize - real current sub-points, not placeholders. CelesTrak throttles repeat

    pulls of the same GROUP per IP for ~2h; on throttle this returns [] gracefully."""

    try:

        from sgp4.api import Satrec, jday

        from sgp4 import omm

    except Exception:

        return []

    r = _get(f"https://celestrak.org/NORAD/elements/gp.php?GROUP={group}&FORMAT=json", timeout=60)

    if r.status_code != 200 or not r.headers.get("content-type", "").startswith("application/json"):

        return []

    try:

        elements = r.json()

    except Exception:

        return []

    now = datetime.now(timezone.utc)

    jd, fr = jday(now.year, now.month, now.day, now.hour, now.minute,

                  now.second + now.microsecond / 1e6)

    gmst = _gmst(jd + fr)

    out = []

    for el in elements[:cap]:

        try:

            sat = Satrec()

            omm.initialize(sat, el)

            e, r_eci, _ = sat.sgp4(jd, fr)

            if e != 0:

                continue

            lat, lon, alt = _eci_to_geodetic(r_eci, gmst)

            out.append({"kind": "satellite", "source": "celestrak",

                        "identifier": str(el.get("NORAD_CAT_ID")), "label": el.get("OBJECT_NAME"),

                        "lat": lat, "lon": lon, "altitude_m": alt * 1000.0,

                        "heading": None, "velocity": None, "origin_country": None,

                        "attributes": {"inclination": el.get("INCLINATION"),

                                       "period_min": (1440.0 / float(el["MEAN_MOTION"])) if el.get("MEAN_MOTION") else None,

                                       "omm": el}})

        except Exception:

            continue

    return out

def _fetch_omm(group: str) -> list[dict]:

    r = _get(f"https://celestrak.org/NORAD/elements/gp.php?GROUP={group}&FORMAT=json", timeout=60)

    if r.status_code != 200 or not r.headers.get("content-type", "").startswith("application/json"):

        return []

    try:

        els = r.json()

    except Exception:

        return []

    for e in els:

        e["_group"] = group

    return els

def imaging_satellites(cap_per_group: int = 400) -> list[dict]:

    """REAL imaging / Earth-observation satellites ('cameras in orbit'): Planet SkySat & Dove +

    Earth-resources birds, positions propagated to now via SGP4. CelesTrak throttles repeat pulls

    per IP (~2h); orbital elements are cached so positions keep propagating locally regardless."""

    try:

        from sgp4.api import Satrec, jday

        from sgp4 import omm as _omm

    except Exception:

        return []

    elements: list[dict] = []

    for group in ("planet", "resource"):

        elements += _fetch_omm(group)[:cap_per_group]

    if elements:

        try:

            os.makedirs(os.path.dirname(_OMM_CACHE), exist_ok=True)

            with open(_OMM_CACHE, "w") as fh:

                json.dump(elements, fh)

        except Exception:

            pass

    elif os.path.exists(_OMM_CACHE):

        try:

            with open(_OMM_CACHE) as fh:

                elements = json.load(fh)

        except Exception:

            elements = []

    if not elements:

        return []

    now = datetime.now(timezone.utc)

    jd, fr = jday(now.year, now.month, now.day, now.hour, now.minute,

                  now.second + now.microsecond / 1e6)

    gmst = _gmst(jd + fr)

    seen, out = set(), []

    for el in elements:

        nid = str(el.get("NORAD_CAT_ID"))

        if nid in seen:

            continue

        try:

            sat = Satrec()

            _omm.initialize(sat, el)

            e, r_eci, _ = sat.sgp4(jd, fr)

            if e != 0:

                continue

            lat, lon, alt = _eci_to_geodetic(r_eci, gmst)

            seen.add(nid)

            out.append({"kind": "satellite_camera", "source": "celestrak", "identifier": nid,

                        "label": el.get("OBJECT_NAME"), "lat": lat, "lon": lon,

                        "altitude_m": alt * 1000.0, "heading": None, "velocity": None,

                        "origin_country": None,

                        "attributes": {"constellation": el.get("_group"),

                                       "inclination": el.get("INCLINATION"),

                                       "period_min": (1440.0 / float(el["MEAN_MOTION"])) if el.get("MEAN_MOTION") else None,

                                       "omm": el}})

        except Exception:

            continue

    return out

def _gmst(jd_ut1: float) -> float:

    import math

    t = (jd_ut1 - 2451545.0) / 36525.0

    g = 280.46061837 + 360.98564736629 * (jd_ut1 - 2451545.0) + 0.000387933 * t * t

    return math.radians(g % 360.0)

def _eci_to_geodetic(r_eci, gmst):

    import math

    x, y, z = r_eci

    lon = math.degrees(math.atan2(y, x) - gmst)

    lon = ((lon + 180) % 360) - 180

    r = math.sqrt(x * x + y * y)

    lat = math.degrees(math.atan2(z, r))

    alt = math.sqrt(x * x + y * y + z * z) - 6371.0

    return lat, lon, alt