| 1 | """Real-data-only seeder for the camera-focused platform. |
| 2 | |
| 3 | Ground cameras -> real ALPR/Flock camera network from OSM/DeFlock (sample_data/real_cameras.json) |
| 4 | Satellite cameras -> real imaging satellites (Planet SkySat/Dove + Earth-resources), live SGP4 positions |
| 5 | |
| 6 | NO synthetic plate reads / persons / detections. Every row here is real public data with provenance. |
| 7 | """ |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import json |
| 11 | import os |
| 12 | import uuid |
| 13 | from datetime import datetime, timezone |
| 14 | |
| 15 | from sqlalchemy import delete, func, insert, select |
| 16 | |
| 17 | from app.core.security import hash_password |
| 18 | from app.db.session import SessionLocal |
| 19 | from app.models import ( |
| 20 | Camera, CameraGroup, Detection, EntityPosition, Investigation, LiveEntity, Location, |
| 21 | Person, PlateRead, SensorAsset, Sighting, Source, User, Vehicle, Watchlist, WatchlistEntry, |
| 22 | WatchlistHit, |
| 23 | ) |
| 24 | from app.services.realsources import ( |
| 25 | az511_cameras, caltrans_cameras, fl511_cameras, imaging_satellites, ne511_cameras, |
| 26 | nyc_traffic_cameras, tripcheck_cameras, |
| 27 | ) |
| 28 | |
| 29 | CAMS_JSON = os.path.join(os.path.dirname(__file__), "sample_data", "real_cameras.json") |
| 30 | |
| 31 | |
| 32 | def main(): |
| 33 | if not os.path.exists(CAMS_JSON): |
| 34 | raise SystemExit(f"missing {CAMS_JSON} - run fetch_cameras.py first") |
| 35 | with open(CAMS_JSON) as fh: |
| 36 | real = [c for c in json.load(fh) if c.get("lat") and c.get("lon")] |
| 37 | print(f"loaded {len(real)} real ALPR cameras from OSM/DeFlock") |
| 38 | |
| 39 | db = SessionLocal() |
| 40 | org = "default" |
| 41 | |
| 42 | for tbl in (Detection, WatchlistHit, Sighting, PlateRead, WatchlistEntry, Watchlist, |
| 43 | Investigation, Person, Vehicle, EntityPosition, LiveEntity, SensorAsset, |
| 44 | Camera, Location, CameraGroup): |
| 45 | db.execute(delete(tbl)) |
| 46 | db.commit() |
| 47 | |
| 48 | for email, pw, name, role in [("admin@sightline.app", "changeme123", "Lead Analyst", "admin"), |
| 49 | ("analyst@sightline.app", "analyst123", "Field Analyst", "analyst"), |
| 50 | ("viewer@sightline.app", "viewer1234", "Watch Desk", "viewer")]: |
| 51 | if not db.scalar(select(User).where(User.email == email)): |
| 52 | db.add(User(email=email, hashed_password=hash_password(pw), full_name=name, role=role)) |
| 53 | db.commit() |
| 54 | source = db.scalar(select(Source).where(Source.name == "OSM / DeFlock ALPR Network")) |
| 55 | if not source: |
| 56 | source = Source(name="OSM / DeFlock ALPR Network", system_type="osm", organization=org, |
| 57 | jurisdiction="United States (public)", trust_level=0.85) |
| 58 | db.add(source); db.commit() |
| 59 | |
| 60 | states = sorted({c["state"] for c in real}) |
| 61 | groups = {st: str(uuid.uuid4()) for st in states} |
| 62 | db.execute(insert(CameraGroup), [{"id": g, "name": f"{st} - Statewide", "organization": org} |
| 63 | for st, g in groups.items()]) |
| 64 | loc_rows, cam_rows = [], [] |
| 65 | for c in real: |
| 66 | lid, cid = str(uuid.uuid4()), str(uuid.uuid4()) |
| 67 | loc_rows.append({"id": lid, "label": f"{c['operator']} ALPR ({c['state']})", |
| 68 | "lat": c["lat"], "lon": c["lon"], "jurisdiction": c["state"]}) |
| 69 | nm = f"{c['operator']} - {c['state']}" + (f" · {c['zone']}" if c.get("zone") else "") |
| 70 | cam_rows.append({"id": cid, "source_id": source.id, "external_camera_id": f"OSM-{c['osm_id']}", |
| 71 | "name": nm, "direction_facing": c.get("direction"), "operator": c["operator"], |
| 72 | "manufacturer": c.get("manufacturer"), "zone": c.get("zone"), |
| 73 | "state": c["state"], "osm_id": str(c["osm_id"]), "location_id": lid, |
| 74 | "camera_group_id": groups[c["state"]], "metadata": {"provenance": "osm/deflock"}}) |
| 75 | db.execute(insert(Location), loc_rows) |
| 76 | db.execute(insert(Camera), cam_rows) |
| 77 | db.commit() |
| 78 | by_op = {} |
| 79 | for c in real: |
| 80 | by_op[c["operator"]] = by_op.get(c["operator"], 0) + 1 |
| 81 | print(f"inserted {len(cam_rows)} ground cameras in {len(states)} states") |
| 82 | print(" operators:", sorted(by_op.items(), key=lambda kv: -kv[1])[:6]) |
| 83 | |
| 84 | live_total = 0 |
| 85 | seen_ext: set[str] = set() |
| 86 | for label, fetch in [("NYC DOT - Live Feeds", nyc_traffic_cameras), |
| 87 | ("Caltrans - Live Feeds", caltrans_cameras), |
| 88 | ("Oregon TripCheck - Live Feeds", tripcheck_cameras), |
| 89 | ("Florida FL511 - Live Feeds", fl511_cameras), |
| 90 | ("Arizona AZ511 - Live Feeds", az511_cameras), |
| 91 | ("New England 511 - Live Feeds", ne511_cameras)]: |
| 92 | try: |
| 93 | feed = fetch() |
| 94 | except Exception as e: |
| 95 | print(f" {label}: fetch failed ({e})") |
| 96 | continue |
| 97 | if not feed: |
| 98 | continue |
| 99 | gid = str(uuid.uuid4()) |
| 100 | db.execute(insert(CameraGroup), [{"id": gid, "name": label, "organization": org}]) |
| 101 | lrows, crows = [], [] |
| 102 | for c in feed: |
| 103 | ext = c["external_id"] |
| 104 | if ext in seen_ext: |
| 105 | continue |
| 106 | seen_ext.add(ext) |
| 107 | lid, cid = str(uuid.uuid4()), str(uuid.uuid4()) |
| 108 | lrows.append({"id": lid, "label": c["name"], "lat": c["lat"], "lon": c["lon"], |
| 109 | "jurisdiction": c.get("area")}) |
| 110 | crows.append({"id": cid, "source_id": source.id, "external_camera_id": ext, |
| 111 | "name": c["name"], "operator": c["operator"], "state": c["state"], |
| 112 | "zone": "traffic", "area": c.get("area"), "location_id": lid, |
| 113 | "camera_group_id": gid, "image_url": c["image_url"], |
| 114 | "has_live_feed": True, "live_source": c["live_source"], |
| 115 | "metadata": {"online": c.get("online", True), "provenance": c["live_source"]}}) |
| 116 | db.execute(insert(Location), lrows) |
| 117 | db.execute(insert(Camera), crows) |
| 118 | db.commit() |
| 119 | live_total += len(crows) |
| 120 | print(f"inserted {len(crows)} LIVE cameras from {label}") |
| 121 | print(f" total live feeds: {live_total}") |
| 122 | |
| 123 | try: |
| 124 | sats = imaging_satellites() |
| 125 | except Exception as e: |
| 126 | print(f" satellite fetch skipped ({e}); tracker will repopulate from cache") |
| 127 | sats = [] |
| 128 | now = datetime.now(timezone.utc) |
| 129 | ent_rows, pos_rows = [], [] |
| 130 | for s in sats: |
| 131 | ent_rows.append({"id": str(uuid.uuid4()), "kind": "satellite_camera", "source": s["source"], |
| 132 | "identifier": s["identifier"], "label": s["label"], "lat": s["lat"], |
| 133 | "lon": s["lon"], "altitude_m": s["altitude_m"], "heading": None, |
| 134 | "velocity": None, "origin_country": None, "attributes": s["attributes"], |
| 135 | "observed_at": now}) |
| 136 | pos_rows.append({"identifier": s["identifier"], "kind": "satellite_camera", |
| 137 | "lat": s["lat"], "lon": s["lon"], "altitude_m": s["altitude_m"], "observed_at": now}) |
| 138 | if ent_rows: |
| 139 | db.execute(insert(LiveEntity), ent_rows) |
| 140 | db.execute(insert(EntityPosition), pos_rows) |
| 141 | db.commit() |
| 142 | print(f"inserted {len(ent_rows)} satellite cameras (imaging sats, live SGP4 positions)") |
| 143 | |
| 144 | nc = db.scalar(select(func.count()).select_from(Camera)) |
| 145 | ns = db.scalar(select(func.count()).select_from(LiveEntity)) |
| 146 | print(f"DONE. ground_cameras={nc}, satellite_cameras={ns}, states={len(states)}") |
| 147 | |
| 148 | |
| 149 | if __name__ == "__main__": |
| 150 | main() |