● Zion Boggan
repos/Sightline/backend/seed_cameras.py
zionboggan.com ↗
150 lines · python
History for this file →
1
"""Real-data-only seeder for the camera-focused platform.
2
 
3
  Ground cameras    -> real ALPR/Flock camera network from OSM/DeFlock (sample_data/real_cameras.json)
4
  Satellite cameras -> real imaging satellites (Planet SkySat/Dove + Earth-resources), live SGP4 positions
5
 
6
NO synthetic plate reads / persons / detections. Every row here is real public data with provenance.
7
"""
8
from __future__ import annotations
9
 
10
import json
11
import os
12
import uuid
13
from datetime import datetime, timezone
14
 
15
from sqlalchemy import delete, func, insert, select
16
 
17
from app.core.security import hash_password
18
from app.db.session import SessionLocal
19
from app.models import (
20
    Camera, CameraGroup, Detection, EntityPosition, Investigation, LiveEntity, Location,
21
    Person, PlateRead, SensorAsset, Sighting, Source, User, Vehicle, Watchlist, WatchlistEntry,
22
    WatchlistHit,
23
)
24
from app.services.realsources import (
25
    az511_cameras, caltrans_cameras, fl511_cameras, imaging_satellites, ne511_cameras,
26
    nyc_traffic_cameras, tripcheck_cameras,
27
)
28
 
29
CAMS_JSON = os.path.join(os.path.dirname(__file__), "sample_data", "real_cameras.json")
30
 
31
 
32
def main():
33
    if not os.path.exists(CAMS_JSON):
34
        raise SystemExit(f"missing {CAMS_JSON} - run fetch_cameras.py first")
35
    with open(CAMS_JSON) as fh:
36
        real = [c for c in json.load(fh) if c.get("lat") and c.get("lon")]
37
    print(f"loaded {len(real)} real ALPR cameras from OSM/DeFlock")
38
 
39
    db = SessionLocal()
40
    org = "default"
41
 
42
    for tbl in (Detection, WatchlistHit, Sighting, PlateRead, WatchlistEntry, Watchlist,
43
                Investigation, Person, Vehicle, EntityPosition, LiveEntity, SensorAsset,
44
                Camera, Location, CameraGroup):
45
        db.execute(delete(tbl))
46
    db.commit()
47
 
48
    for email, pw, name, role in [("admin@sightline.app", "changeme123", "Lead Analyst", "admin"),
49
                                  ("analyst@sightline.app", "analyst123", "Field Analyst", "analyst"),
50
                                  ("viewer@sightline.app", "viewer1234", "Watch Desk", "viewer")]:
51
        if not db.scalar(select(User).where(User.email == email)):
52
            db.add(User(email=email, hashed_password=hash_password(pw), full_name=name, role=role))
53
    db.commit()
54
    source = db.scalar(select(Source).where(Source.name == "OSM / DeFlock ALPR Network"))
55
    if not source:
56
        source = Source(name="OSM / DeFlock ALPR Network", system_type="osm", organization=org,
57
                        jurisdiction="United States (public)", trust_level=0.85)
58
        db.add(source); db.commit()
59
 
60
    states = sorted({c["state"] for c in real})
61
    groups = {st: str(uuid.uuid4()) for st in states}
62
    db.execute(insert(CameraGroup), [{"id": g, "name": f"{st} - Statewide", "organization": org}
63
                                     for st, g in groups.items()])
64
    loc_rows, cam_rows = [], []
65
    for c in real:
66
        lid, cid = str(uuid.uuid4()), str(uuid.uuid4())
67
        loc_rows.append({"id": lid, "label": f"{c['operator']} ALPR ({c['state']})",
68
                         "lat": c["lat"], "lon": c["lon"], "jurisdiction": c["state"]})
69
        nm = f"{c['operator']} - {c['state']}" + (f" · {c['zone']}" if c.get("zone") else "")
70
        cam_rows.append({"id": cid, "source_id": source.id, "external_camera_id": f"OSM-{c['osm_id']}",
71
                         "name": nm, "direction_facing": c.get("direction"), "operator": c["operator"],
72
                         "manufacturer": c.get("manufacturer"), "zone": c.get("zone"),
73
                         "state": c["state"], "osm_id": str(c["osm_id"]), "location_id": lid,
74
                         "camera_group_id": groups[c["state"]], "metadata": {"provenance": "osm/deflock"}})
75
    db.execute(insert(Location), loc_rows)
76
    db.execute(insert(Camera), cam_rows)
77
    db.commit()
78
    by_op = {}
79
    for c in real:
80
        by_op[c["operator"]] = by_op.get(c["operator"], 0) + 1
81
    print(f"inserted {len(cam_rows)} ground cameras in {len(states)} states")
82
    print("  operators:", sorted(by_op.items(), key=lambda kv: -kv[1])[:6])
83
 
84
    live_total = 0
85
    seen_ext: set[str] = set()
86
    for label, fetch in [("NYC DOT - Live Feeds", nyc_traffic_cameras),
87
                         ("Caltrans - Live Feeds", caltrans_cameras),
88
                         ("Oregon TripCheck - Live Feeds", tripcheck_cameras),
89
                         ("Florida FL511 - Live Feeds", fl511_cameras),
90
                         ("Arizona AZ511 - Live Feeds", az511_cameras),
91
                         ("New England 511 - Live Feeds", ne511_cameras)]:
92
        try:
93
            feed = fetch()
94
        except Exception as e:
95
            print(f"  {label}: fetch failed ({e})")
96
            continue
97
        if not feed:
98
            continue
99
        gid = str(uuid.uuid4())
100
        db.execute(insert(CameraGroup), [{"id": gid, "name": label, "organization": org}])
101
        lrows, crows = [], []
102
        for c in feed:
103
            ext = c["external_id"]
104
            if ext in seen_ext:
105
                continue
106
            seen_ext.add(ext)
107
            lid, cid = str(uuid.uuid4()), str(uuid.uuid4())
108
            lrows.append({"id": lid, "label": c["name"], "lat": c["lat"], "lon": c["lon"],
109
                          "jurisdiction": c.get("area")})
110
            crows.append({"id": cid, "source_id": source.id, "external_camera_id": ext,
111
                          "name": c["name"], "operator": c["operator"], "state": c["state"],
112
                          "zone": "traffic", "area": c.get("area"), "location_id": lid,
113
                          "camera_group_id": gid, "image_url": c["image_url"],
114
                          "has_live_feed": True, "live_source": c["live_source"],
115
                          "metadata": {"online": c.get("online", True), "provenance": c["live_source"]}})
116
        db.execute(insert(Location), lrows)
117
        db.execute(insert(Camera), crows)
118
        db.commit()
119
        live_total += len(crows)
120
        print(f"inserted {len(crows)} LIVE cameras from {label}")
121
    print(f"  total live feeds: {live_total}")
122
 
123
    try:
124
        sats = imaging_satellites()
125
    except Exception as e:
126
        print(f"  satellite fetch skipped ({e}); tracker will repopulate from cache")
127
        sats = []
128
    now = datetime.now(timezone.utc)
129
    ent_rows, pos_rows = [], []
130
    for s in sats:
131
        ent_rows.append({"id": str(uuid.uuid4()), "kind": "satellite_camera", "source": s["source"],
132
                         "identifier": s["identifier"], "label": s["label"], "lat": s["lat"],
133
                         "lon": s["lon"], "altitude_m": s["altitude_m"], "heading": None,
134
                         "velocity": None, "origin_country": None, "attributes": s["attributes"],
135
                         "observed_at": now})
136
        pos_rows.append({"identifier": s["identifier"], "kind": "satellite_camera",
137
                         "lat": s["lat"], "lon": s["lon"], "altitude_m": s["altitude_m"], "observed_at": now})
138
    if ent_rows:
139
        db.execute(insert(LiveEntity), ent_rows)
140
        db.execute(insert(EntityPosition), pos_rows)
141
    db.commit()
142
    print(f"inserted {len(ent_rows)} satellite cameras (imaging sats, live SGP4 positions)")
143
 
144
    nc = db.scalar(select(func.count()).select_from(Camera))
145
    ns = db.scalar(select(func.count()).select_from(LiveEntity))
146
    print(f"DONE. ground_cameras={nc}, satellite_cameras={ns}, states={len(states)}")
147
 
148
 
149
if __name__ == "__main__":
150
    main()