● Zion Boggan
repos/Sightline/backend/app/services/normalize.py
zionboggan.com ↗
182 lines · python
History for this file →
1
"""Normalization of raw ALPR reads into canonical form. Pure functions, no DB - unit-testable."""
2
from __future__ import annotations
3
 
4
import hashlib
5
import re
6
from dataclasses import dataclass, field
7
from datetime import datetime, timezone
8
 
9
from dateutil import parser as dtparser
10
 
11
_PLATE_STRIP = re.compile(r"[^A-Z0-9]")
12
_DIRECTIONS = {"N", "NE", "E", "SE", "S", "SW", "W", "NW"}
13
_DIR_WORDS = {
14
    "north": "N", "northeast": "NE", "east": "E", "southeast": "SE",
15
    "south": "S", "southwest": "SW", "west": "W", "northwest": "NW",
16
    "nb": "N", "eb": "E", "sb": "S", "wb": "W",
17
}
18
 
19
COLUMN_ALIASES: dict[str, list[str]] = {
20
    "plate": ["plate", "license_plate", "licenseplate", "lp", "tag", "plate_number", "platenumber"],
21
    "state": ["state", "plate_state", "region", "jurisdiction_state"],
22
    "read_ts": ["timestamp", "read_time", "captured_at", "datetime", "time", "date_time", "observed_at"],
23
    "camera": ["camera", "camera_id", "device_id", "cam", "device", "sensor", "camera_name"],
24
    "lat": ["lat", "latitude", "gps_lat", "y"],
25
    "lon": ["lon", "lng", "longitude", "gps_lon", "x"],
26
    "direction": ["direction", "heading", "travel_direction", "bearing", "dir"],
27
    "confidence": ["confidence", "score", "match_confidence", "ocr_confidence"],
28
    "lane": ["lane", "lane_id"],
29
    "make": ["make", "vehicle_make"],
30
    "model": ["model", "vehicle_model"],
31
    "color": ["color", "colour", "vehicle_color"],
32
    "body_type": ["body", "body_type", "vehicle_type", "type"],
33
    "hotlist": ["hotlist", "hotlist_hit", "hit", "alert", "watchlist"],
34
    "incident": ["incident", "incident_id", "case", "case_id", "incident_ref"],
35
}
36
 
37
 
38
@dataclass
39
class RawRead:
40
    """One row as emitted by a parser, before normalization."""
41
    fields: dict
42
    source_row: int = 0
43
 
44
 
45
@dataclass
46
class NormalizedRead:
47
    plate_raw: str
48
    plate_norm: str
49
    state: str | None
50
    read_ts: datetime
51
    external_camera_id: str | None
52
    lat: float | None
53
    lon: float | None
54
    direction: str | None
55
    confidence: float | None
56
    lane: str | None
57
    make: str | None = None
58
    model: str | None = None
59
    color: str | None = None
60
    body_type: str | None = None
61
    hotlist_hit: bool = False
62
    incident_ref: str | None = None
63
    raw: dict = field(default_factory=dict)
64
 
65
 
66
def build_alias_map(headers: list[str]) -> dict[str, str]:
67
    """Map canonical field -> actual header present in the file (first match wins)."""
68
    lowered = {h.lower().strip(): h for h in headers}
69
    out: dict[str, str] = {}
70
    for canon, aliases in COLUMN_ALIASES.items():
71
        for a in aliases:
72
            if a in lowered:
73
                out[canon] = lowered[a]
74
                break
75
    return out
76
 
77
 
78
def normalize_plate(value: str | None) -> str:
79
    if not value:
80
        return ""
81
    return _PLATE_STRIP.sub("", str(value).upper())
82
 
83
 
84
def normalize_timestamp(value, source_tz: timezone = timezone.utc) -> datetime | None:
85
    if value in (None, ""):
86
        return None
87
    if isinstance(value, datetime):
88
        dt = value
89
    else:
90
        try:
91
            dt = dtparser.parse(str(value))
92
        except (ValueError, OverflowError, TypeError):
93
            return None
94
    if dt.tzinfo is None:
95
        dt = dt.replace(tzinfo=source_tz)
96
    return dt.astimezone(timezone.utc)
97
 
98
 
99
def normalize_direction(value) -> str | None:
100
    if value in (None, ""):
101
        return None
102
    s = str(value).strip().upper()
103
    if s in _DIRECTIONS:
104
        return s
105
    return _DIR_WORDS.get(str(value).strip().lower())
106
 
107
 
108
def normalize_confidence(value) -> float | None:
109
    if value in (None, ""):
110
        return None
111
    try:
112
        f = float(value)
113
    except (ValueError, TypeError):
114
        return None
115
    if f > 1.0:
116
        f = f / 100.0
117
    return max(0.0, min(1.0, f))
118
 
119
 
120
def _to_float(value) -> float | None:
121
    try:
122
        f = float(value)
123
        return f
124
    except (ValueError, TypeError):
125
        return None
126
 
127
 
128
def _valid_coord(lat: float | None, lon: float | None) -> tuple[float | None, float | None]:
129
    if lat is None or lon is None:
130
        return None, None
131
    if -90 <= lat <= 90 and -180 <= lon <= 180 and not (lat == 0 and lon == 0):
132
        return lat, lon
133
    return None, None
134
 
135
 
136
def _truthy(value) -> bool:
137
    return str(value).strip().lower() in {"1", "true", "yes", "y", "hit", "hotlist"}
138
 
139
 
140
def normalize_read(raw: RawRead, alias: dict[str, str], source_tz: timezone = timezone.utc) -> NormalizedRead | None:
141
    """Project a raw row onto canonical fields. Returns None if no usable plate+timestamp."""
142
    f = raw.fields
143
 
144
    def g(canon: str):
145
        col = alias.get(canon)
146
        return f.get(col) if col else None
147
 
148
    plate_raw = str(g("plate") or "").strip()
149
    plate_norm = normalize_plate(plate_raw)
150
    read_ts = normalize_timestamp(g("read_ts"), source_tz)
151
    if not plate_norm or read_ts is None:
152
        return None
153
 
154
    lat, lon = _valid_coord(_to_float(g("lat")), _to_float(g("lon")))
155
    return NormalizedRead(
156
        plate_raw=plate_raw,
157
        plate_norm=plate_norm,
158
        state=(str(g("state")).strip().upper() or None) if g("state") else None,
159
        read_ts=read_ts,
160
        external_camera_id=(str(g("camera")).strip() or None) if g("camera") else None,
161
        lat=lat,
162
        lon=lon,
163
        direction=normalize_direction(g("direction")),
164
        confidence=normalize_confidence(g("confidence")),
165
        lane=(str(g("lane")).strip() or None) if g("lane") else None,
166
        make=(str(g("make")).strip() or None) if g("make") else None,
167
        model=(str(g("model")).strip() or None) if g("model") else None,
168
        color=(str(g("color")).strip() or None) if g("color") else None,
169
        body_type=(str(g("body_type")).strip() or None) if g("body_type") else None,
170
        hotlist_hit=_truthy(g("hotlist")) if g("hotlist") is not None else False,
171
        incident_ref=(str(g("incident")).strip() or None) if g("incident") else None,
172
        raw=f,
173
    )
174
 
175
 
176
def dedup_hash(org: str, plate_norm: str, camera_id: str | None, read_ts: datetime,
177
               window_seconds: int = 1) -> bytes:
178
    """Stable key for idempotent ingest. Timestamp is bucketed so sub-window bursts collapse."""
179
    epoch = int(read_ts.timestamp())
180
    bucket = epoch - (epoch % max(1, window_seconds))
181
    key = f"{org}|{plate_norm}|{camera_id or '-'}|{bucket}"
182
    return hashlib.sha256(key.encode()).digest()