● Zion Boggan
repos/Sightline/backend/app/services/ingest.py
zionboggan.com ↗
211 lines · python
History for this file →
1
"""DB-bound ingest orchestration: parse -> normalize -> dedup -> upsert -> sighting ->
2
watchlist eval -> audit. Row-level resilient: a bad row is recorded, never fatal."""
3
from __future__ import annotations
4
 
5
from datetime import timezone
6
 
7
from sqlalchemy import select
8
from sqlalchemy.orm import Session
9
 
10
from app.core.config import settings
11
from app.core.storage import get_storage
12
from app.models import (
13
    Alert, AlertRule, Camera, Import, LicensePlate, Location, PlateRead, Sighting, Source,
14
    Vehicle, Watchlist, WatchlistEntry, WatchlistHit, AuditLog,
15
)
16
from app.services.normalize import (
17
    NormalizedRead, build_alias_map, dedup_hash, normalize_read,
18
)
19
from app.services.parsers import get_parser
20
from app.services.watchlist import WatchEntry, match_all
21
 
22
 
23
def _get_or_create_camera(db: Session, source_id: str | None, n: NormalizedRead) -> Camera | None:
24
    external_id = n.external_camera_id
25
    if not external_id:
26
        return None
27
    cam = db.scalar(select(Camera).where(
28
        Camera.source_id == source_id, Camera.external_camera_id == external_id))
29
    if cam:
30
        if cam.location_id is None and n.lat is not None:
31
            loc = Location(label=external_id, lat=n.lat, lon=n.lon)
32
            db.add(loc)
33
            db.flush()
34
            cam.location_id = loc.id
35
        return cam
36
    location_id = None
37
    if n.lat is not None:
38
        loc = Location(label=external_id, lat=n.lat, lon=n.lon)
39
        db.add(loc)
40
        db.flush()
41
        location_id = loc.id
42
    cam = Camera(source_id=source_id, external_camera_id=external_id, name=external_id,
43
                 direction_facing=n.direction, location_id=location_id)
44
    db.add(cam)
45
    db.flush()
46
    return cam
47
 
48
 
49
def _get_or_create_plate(db: Session, n: NormalizedRead) -> LicensePlate:
50
    plate = db.scalar(select(LicensePlate).where(
51
        LicensePlate.plate_norm == n.plate_norm, LicensePlate.state == n.state))
52
    if plate:
53
        return plate
54
    plate = LicensePlate(plate_raw=n.plate_raw, plate_norm=n.plate_norm, state=n.state)
55
    db.add(plate)
56
    db.flush()
57
    return plate
58
 
59
 
60
def _get_or_create_vehicle(db: Session, plate: LicensePlate, n: NormalizedRead) -> Vehicle:
61
    veh = db.scalar(select(Vehicle).where(Vehicle.primary_plate_id == plate.id))
62
    if veh:
63
        for attr, val in (("make", n.make), ("model", n.model), ("color", n.color), ("body_type", n.body_type)):
64
            if val and not getattr(veh, attr):
65
                setattr(veh, attr, val)
66
        return veh
67
    veh = Vehicle(primary_plate_id=plate.id, make=n.make, model=n.model,
68
                  color=n.color, body_type=n.body_type)
69
    db.add(veh)
70
    db.flush()
71
    return veh
72
 
73
 
74
def _active_watch_entries(db: Session) -> list[WatchEntry]:
75
    rows = db.scalars(
76
        select(WatchlistEntry).join(Watchlist).where(Watchlist.active.is_(True))
77
    ).all()
78
    return [WatchEntry(id=e.id, plate_norm=e.plate_norm, partial_plate=e.partial_plate,
79
                       vehicle_desc=e.vehicle_desc) for e in rows]
80
 
81
 
82
def process_import(db: Session, import_id: str) -> dict:
83
    imp = db.get(Import, import_id)
84
    if imp is None:
85
        return {"error": "import not found"}
86
    imp.status = "processing"
87
    db.flush()
88
 
89
    org = (db.get(Source, imp.source_id).organization if imp.source_id else None) or "default"
90
    try:
91
        data = get_storage().get(imp.raw_blob_ref)
92
    except Exception as e:
93
        imp.status, imp.error = "error", f"cannot read blob: {e}"
94
        db.commit()
95
        return {"error": imp.error}
96
 
97
    parser = get_parser(imp.filename, imp.content_type)
98
    result = parser.parse(data)
99
    alias = build_alias_map(result.headers)
100
    return _ingest_rows(db, result.rows, alias, imp, org, list(result.errors))
101
 
102
 
103
def _ingest_rows(db: Session, rows, alias: dict, imp: Import, org: str,
104
                 errors: list[str]) -> dict:
105
    """Shared per-row pipeline: normalize -> dedup -> upsert plate/vehicle/camera ->
106
    plate_read + sighting -> watchlist eval. Used by file imports, connectors, and live stream."""
107
    watch = _active_watch_entries(db)
108
    inserted = duplicates = skipped = hits = 0
109
 
110
    for raw in rows:
111
        try:
112
            n = normalize_read(raw, alias, source_tz=timezone.utc)
113
        except Exception as e:
114
            errors.append(f"row {raw.source_row}: normalize error {e}")
115
            skipped += 1
116
            continue
117
        if n is None:
118
            skipped += 1
119
            continue
120
 
121
        cam = _get_or_create_camera(db, imp.source_id, n)
122
        dh = dedup_hash(org, n.plate_norm, cam.id if cam else None, n.read_ts,
123
                        settings.dedup_window_seconds)
124
 
125
        existing = db.scalar(select(PlateRead).where(PlateRead.dedup_hash == dh))
126
        if existing:
127
            existing.burst_count += 1
128
            duplicates += 1
129
            continue
130
 
131
        plate = _get_or_create_plate(db, n)
132
        veh = _get_or_create_vehicle(db, plate, n)
133
 
134
        read = PlateRead(
135
            import_id=imp.id, source_id=imp.source_id, camera_id=cam.id if cam else None,
136
            plate_id=plate.id, plate_norm=n.plate_norm, read_ts=n.read_ts,
137
            direction=n.direction, confidence=n.confidence, lane=n.lane,
138
            lat=n.lat, lon=n.lon, hotlist_hit=n.hotlist_hit, incident_ref=n.incident_ref,
139
            raw_read=n.raw, dedup_hash=dh,
140
        )
141
        db.add(read)
142
        db.flush()
143
 
144
        sighting = Sighting(plate_read_id=read.id, vehicle_id=veh.id,
145
                            camera_id=cam.id if cam else None, read_ts=n.read_ts,
146
                            confidence=n.confidence)
147
        db.add(sighting)
148
        db.flush()
149
        inserted += 1
150
 
151
        vehicle_desc = {"make": veh.make, "color": veh.color, "body_type": veh.body_type}
152
        for m in match_all(watch, n.plate_norm, vehicle_desc):
153
            wh = WatchlistHit(watchlist_entry_id=m["entry_id"], plate_read_id=read.id,
154
                              sighting_id=sighting.id, matched_on=m["matched_on"])
155
            db.add(wh)
156
            db.flush()
157
            hits += 1
158
            _maybe_alert(db, m["entry_id"], wh, n)
159
 
160
    imp.row_count = inserted
161
    imp.status = "done"
162
    imp.error = "\n".join(errors[:50]) if errors else None
163
    db.add(AuditLog(actor_user_id=imp.created_by, action="import.process",
164
                    entity_type="import", entity_id=imp.id,
165
                    meta={"inserted": inserted, "duplicates": duplicates,
166
                          "skipped": skipped, "watchlist_hits": hits}))
167
    db.commit()
168
    return {"import_id": imp.id, "inserted": inserted, "duplicates": duplicates,
169
            "skipped": skipped, "watchlist_hits": hits, "errors": errors[:50]}
170
 
171
 
172
def ingest_live_reads(db: Session, source_id: str | None, reads: list[dict],
173
                      label: str = "live", created_by: str | None = None) -> dict:
174
    """Ingest already-structured read dicts (from a stream POST or an API connector) through the
175
    same normalize/dedup/watchlist pipeline as file imports. Creates a synthetic Import record
176
    so every live read is still traceable to a batch."""
177
    import hashlib
178
    import json as _json
179
    from app.services.normalize import RawRead
180
 
181
    org = (db.get(Source, source_id).organization if source_id else None) or "default"
182
    payload = _json.dumps(reads, sort_keys=True, default=str).encode()
183
    sha = hashlib.sha256(payload + label.encode()).hexdigest()
184
    imp = db.scalar(select(Import).where(Import.sha256 == sha))
185
    if imp:
186
        return {"import_id": imp.id, "deduplicated": True, "row_count": imp.row_count}
187
    imp = Import(source_id=source_id, filename=f"{label}.json", content_type="application/json",
188
                 sha256=sha, raw_blob_ref=None, status="processing", created_by=created_by)
189
    db.add(imp)
190
    db.flush()
191
 
192
    raw_rows = [RawRead(fields=r, source_row=i) for i, r in enumerate(reads, start=1)]
193
    headers = sorted({k for r in reads for k in r.keys()})
194
    alias = build_alias_map(headers)
195
    return _ingest_rows(db, raw_rows, alias, imp, org, [])
196
 
197
 
198
def _maybe_alert(db: Session, entry_id: str, hit: WatchlistHit, n: NormalizedRead) -> None:
199
    entry = db.get(WatchlistEntry, entry_id)
200
    if not entry:
201
        return
202
    rules = db.scalars(select(AlertRule).where(
203
        AlertRule.watchlist_id == entry.watchlist_id, AlertRule.active.is_(True))).all()
204
    for rule in rules or [None]:
205
        cond = (rule.condition if rule else {}) or {}
206
        min_conf = cond.get("min_confidence")
207
        if min_conf is not None and (n.confidence or 0) < float(min_conf):
208
            continue
209
        db.add(Alert(alert_rule_id=rule.id if rule else None, watchlist_hit_id=hit.id,
210
                     status="new", payload={"plate": n.plate_norm, "matched_on": hit.matched_on,
211
                                            "read_ts": n.read_ts.isoformat()}))