● Zion Boggan
repos/Sightline/backend/seed_poc.py
zionboggan.com ↗
148 lines · python
History for this file →
1
"""Proof-of-concept seed: stand up the named camera network, ingest the generated dataset
2
through the real pipeline, and build watchlists + investigations around the embedded scenarios.
3
Idempotent where practical. Run after `python generate_data.py`."""
4
from __future__ import annotations
5
 
6
import hashlib
7
import os
8
import time
9
 
10
from sqlalchemy import select
11
 
12
from app.core.security import hash_password
13
from app.db.session import SessionLocal
14
from app.models import (
15
    Camera, CameraGroup, Import, Investigation, LicensePlate, Location, PlateRead,
16
    Sighting, Source, User, Vehicle, Watchlist, WatchlistEntry,
17
)
18
from app.services.ingest import process_import
19
from generate_data import CAMERAS
20
 
21
SAMPLE = os.path.join(os.path.dirname(__file__), "sample_data", "poc_reads.csv")
22
 
23
 
24
def get_or_create(db, model, defaults=None, **flt):
25
    obj = db.scalar(select(model).filter_by(**flt))
26
    if obj:
27
        return obj, False
28
    obj = model(**flt, **(defaults or {}))
29
    db.add(obj)
30
    db.flush()
31
    return obj, True
32
 
33
 
34
def main():
35
    db = SessionLocal()
36
    try:
37
        admin, created = get_or_create(db, User, email="admin@sightline.app",
38
            defaults={"hashed_password": hash_password("changeme123"), "full_name": "Lead Analyst", "role": "admin"})
39
        if created:
40
            print("admin: admin@sightline.app / changeme123")
41
        get_or_create(db, User, email="analyst@sightline.app",
42
            defaults={"hashed_password": hash_password("analyst123"), "full_name": "Field Analyst", "role": "analyst"})
43
        get_or_create(db, User, email="viewer@sightline.app",
44
            defaults={"hashed_password": hash_password("viewer1234"), "full_name": "Watch Desk", "role": "viewer"})
45
 
46
        source, _ = get_or_create(db, Source, name="National ALPR Aggregate",
47
            defaults={"system_type": "flock", "organization": "default",
48
                      "jurisdiction": "Multi-state (US)", "trust_level": 0.92})
49
 
50
        groups: dict[str, CameraGroup] = {}
51
        for cid, name, lat, lon, grp in CAMERAS:
52
            g = groups.get(grp)
53
            if not g:
54
                g, _ = get_or_create(db, CameraGroup, name=grp, defaults={"organization": "default"})
55
                groups[grp] = g
56
            cam = db.scalar(select(Camera).where(Camera.source_id == source.id,
57
                                                 Camera.external_camera_id == cid))
58
            if not cam:
59
                loc = Location(label=name, lat=lat, lon=lon, jurisdiction="Shelby County, TN")
60
                db.add(loc); db.flush()
61
                db.add(Camera(source_id=source.id, external_camera_id=cid, name=name,
62
                              location_id=loc.id, camera_group_id=g.id))
63
        db.commit()
64
        print(f"camera network: {len(CAMERAS)} cameras in {len(groups)} groups")
65
 
66
        with open(SAMPLE, "rb") as fh:
67
            data = fh.read()
68
        sha = hashlib.sha256(data).hexdigest()
69
        imp = db.scalar(select(Import).where(Import.sha256 == sha))
70
        if imp and imp.status == "done":
71
            print(f"dataset already ingested ({imp.row_count} reads)")
72
        else:
73
            if not imp:
74
                imp = Import(source_id=source.id, filename="poc_reads.csv", content_type="text/csv",
75
                             sha256=sha, raw_blob_ref=SAMPLE, status="received", created_by=admin.id)
76
                db.add(imp); db.commit()
77
            print(f"ingesting {SAMPLE} ...")
78
            t0 = time.time()
79
            summary = process_import(db, imp.id)
80
            print(f"ingested in {time.time()-t0:.1f}s: {summary}")
81
 
82
        bolo, _ = get_or_create(db, Watchlist, name="Stolen Vehicles - Active BOLO",
83
            defaults={"kind": "plate", "organization": "default", "created_by": admin.id})
84
        if not db.scalar(select(WatchlistEntry).where(WatchlistEntry.plate_norm == "BNDT447")):
85
            db.add(WatchlistEntry(watchlist_id=bolo.id, plate_norm="BNDT447",
86
                                  reason="Reported stolen - INC-2026-0440 (black Dodge Charger)"))
87
        poi, _ = get_or_create(db, Watchlist, name="Persons of Interest",
88
            defaults={"kind": "plate", "organization": "default", "created_by": admin.id})
89
        if not db.scalar(select(WatchlistEntry).where(WatchlistEntry.partial_plate == "ESC81")):
90
            db.add(WatchlistEntry(watchlist_id=poi.id, partial_plate="ESC81",
91
                                  reason="Associated vehicles - co-travel pattern under review"))
92
        db.commit()
93
 
94
        from app.services.watchlist import WatchEntry, match_all
95
        from app.models import WatchlistHit
96
        entries = db.scalars(select(WatchlistEntry)).all()
97
        we = [WatchEntry(id=e.id, plate_norm=e.plate_norm, partial_plate=e.partial_plate,
98
                         vehicle_desc=e.vehicle_desc) for e in entries]
99
        existing_hits = {(h.watchlist_entry_id, h.plate_read_id)
100
                         for h in db.scalars(select(WatchlistHit)).all()}
101
        hit_count = 0
102
        for r in db.scalars(select(PlateRead)).all():
103
            for m in match_all(we, r.plate_norm, None):
104
                if (m["entry_id"], r.id) in existing_hits:
105
                    continue
106
                s = db.scalar(select(Sighting).where(Sighting.plate_read_id == r.id))
107
                db.add(WatchlistHit(watchlist_entry_id=m["entry_id"], plate_read_id=r.id,
108
                                    sighting_id=s.id if s else None, matched_on="backfill"))
109
                hit_count += 1
110
        db.commit()
111
        print(f"watchlist hits backfilled: {hit_count}")
112
 
113
        def attach(inv, plate):
114
            n = 0
115
            for r in db.scalars(select(PlateRead).where(PlateRead.plate_norm == plate)).all():
116
                s = db.scalar(select(Sighting).where(Sighting.plate_read_id == r.id))
117
                if s and s.investigation_id != inv.id:
118
                    s.investigation_id = inv.id; n += 1
119
            return n
120
 
121
        inv1, c1 = get_or_create(db, Investigation, title="Operation Nightshade - Stolen Charger",
122
            defaults={"summary": "Tracking stolen black Dodge Charger (BNDT447), INC-2026-0440, "
123
                                 "across the Memphis ALPR network to establish pattern of life.",
124
                      "priority": "high", "status": "active", "organization": "default", "lead_user_id": admin.id})
125
        if c1:
126
            print(f"  Operation Nightshade: attached {attach(inv1, 'BNDT447')} sightings")
127
        inv2, c2 = get_or_create(db, Investigation, title="Cloned Plate Review - CLN9921",
128
            defaults={"summary": "Plate CLN9921 shows physically impossible transit times - "
129
                                 "suspected cloned/duplicated plate. Validate via anomaly analytics.",
130
                      "priority": "normal", "status": "open", "organization": "default", "lead_user_id": admin.id})
131
        if c2:
132
            print(f"  Cloned Plate Review: attached {attach(inv2, 'CLN9921')} sightings")
133
        inv3, c3 = get_or_create(db, Investigation, title="Co-Travel Analysis - ESC8120 / ESC8121",
134
            defaults={"summary": "Two vehicles repeatedly observed together along the East Memphis "
135
                                 "→ Downtown corridor. Assess association.",
136
                      "priority": "normal", "status": "active", "organization": "default", "lead_user_id": admin.id})
137
        if c3:
138
            print(f"  Co-Travel Analysis: attached {attach(inv3, 'ESC8120') + attach(inv3, 'ESC8121')} sightings")
139
        db.commit()
140
 
141
        total = db.scalar(select(PlateRead).limit(1))
142
        print("POC seed complete." if total else "POC seed FAILED - no reads.")
143
    finally:
144
        db.close()
145
 
146
 
147
if __name__ == "__main__":
148
    main()