| 1 | """Normalization of raw ALPR reads into canonical form. Pure functions, no DB - unit-testable.""" |
| 2 | from __future__ import annotations |
| 3 | |
| 4 | import hashlib |
| 5 | import re |
| 6 | from dataclasses import dataclass, field |
| 7 | from datetime import datetime, timezone |
| 8 | |
| 9 | from dateutil import parser as dtparser |
| 10 | |
| 11 | _PLATE_STRIP = re.compile(r"[^A-Z0-9]") |
| 12 | _DIRECTIONS = {"N", "NE", "E", "SE", "S", "SW", "W", "NW"} |
| 13 | _DIR_WORDS = { |
| 14 | "north": "N", "northeast": "NE", "east": "E", "southeast": "SE", |
| 15 | "south": "S", "southwest": "SW", "west": "W", "northwest": "NW", |
| 16 | "nb": "N", "eb": "E", "sb": "S", "wb": "W", |
| 17 | } |
| 18 | |
| 19 | COLUMN_ALIASES: dict[str, list[str]] = { |
| 20 | "plate": ["plate", "license_plate", "licenseplate", "lp", "tag", "plate_number", "platenumber"], |
| 21 | "state": ["state", "plate_state", "region", "jurisdiction_state"], |
| 22 | "read_ts": ["timestamp", "read_time", "captured_at", "datetime", "time", "date_time", "observed_at"], |
| 23 | "camera": ["camera", "camera_id", "device_id", "cam", "device", "sensor", "camera_name"], |
| 24 | "lat": ["lat", "latitude", "gps_lat", "y"], |
| 25 | "lon": ["lon", "lng", "longitude", "gps_lon", "x"], |
| 26 | "direction": ["direction", "heading", "travel_direction", "bearing", "dir"], |
| 27 | "confidence": ["confidence", "score", "match_confidence", "ocr_confidence"], |
| 28 | "lane": ["lane", "lane_id"], |
| 29 | "make": ["make", "vehicle_make"], |
| 30 | "model": ["model", "vehicle_model"], |
| 31 | "color": ["color", "colour", "vehicle_color"], |
| 32 | "body_type": ["body", "body_type", "vehicle_type", "type"], |
| 33 | "hotlist": ["hotlist", "hotlist_hit", "hit", "alert", "watchlist"], |
| 34 | "incident": ["incident", "incident_id", "case", "case_id", "incident_ref"], |
| 35 | } |
| 36 | |
| 37 | |
| 38 | @dataclass |
| 39 | class RawRead: |
| 40 | """One row as emitted by a parser, before normalization.""" |
| 41 | fields: dict |
| 42 | source_row: int = 0 |
| 43 | |
| 44 | |
| 45 | @dataclass |
| 46 | class NormalizedRead: |
| 47 | plate_raw: str |
| 48 | plate_norm: str |
| 49 | state: str | None |
| 50 | read_ts: datetime |
| 51 | external_camera_id: str | None |
| 52 | lat: float | None |
| 53 | lon: float | None |
| 54 | direction: str | None |
| 55 | confidence: float | None |
| 56 | lane: str | None |
| 57 | make: str | None = None |
| 58 | model: str | None = None |
| 59 | color: str | None = None |
| 60 | body_type: str | None = None |
| 61 | hotlist_hit: bool = False |
| 62 | incident_ref: str | None = None |
| 63 | raw: dict = field(default_factory=dict) |
| 64 | |
| 65 | |
| 66 | def build_alias_map(headers: list[str]) -> dict[str, str]: |
| 67 | """Map canonical field -> actual header present in the file (first match wins).""" |
| 68 | lowered = {h.lower().strip(): h for h in headers} |
| 69 | out: dict[str, str] = {} |
| 70 | for canon, aliases in COLUMN_ALIASES.items(): |
| 71 | for a in aliases: |
| 72 | if a in lowered: |
| 73 | out[canon] = lowered[a] |
| 74 | break |
| 75 | return out |
| 76 | |
| 77 | |
| 78 | def normalize_plate(value: str | None) -> str: |
| 79 | if not value: |
| 80 | return "" |
| 81 | return _PLATE_STRIP.sub("", str(value).upper()) |
| 82 | |
| 83 | |
| 84 | def normalize_timestamp(value, source_tz: timezone = timezone.utc) -> datetime | None: |
| 85 | if value in (None, ""): |
| 86 | return None |
| 87 | if isinstance(value, datetime): |
| 88 | dt = value |
| 89 | else: |
| 90 | try: |
| 91 | dt = dtparser.parse(str(value)) |
| 92 | except (ValueError, OverflowError, TypeError): |
| 93 | return None |
| 94 | if dt.tzinfo is None: |
| 95 | dt = dt.replace(tzinfo=source_tz) |
| 96 | return dt.astimezone(timezone.utc) |
| 97 | |
| 98 | |
| 99 | def normalize_direction(value) -> str | None: |
| 100 | if value in (None, ""): |
| 101 | return None |
| 102 | s = str(value).strip().upper() |
| 103 | if s in _DIRECTIONS: |
| 104 | return s |
| 105 | return _DIR_WORDS.get(str(value).strip().lower()) |
| 106 | |
| 107 | |
| 108 | def normalize_confidence(value) -> float | None: |
| 109 | if value in (None, ""): |
| 110 | return None |
| 111 | try: |
| 112 | f = float(value) |
| 113 | except (ValueError, TypeError): |
| 114 | return None |
| 115 | if f > 1.0: |
| 116 | f = f / 100.0 |
| 117 | return max(0.0, min(1.0, f)) |
| 118 | |
| 119 | |
| 120 | def _to_float(value) -> float | None: |
| 121 | try: |
| 122 | f = float(value) |
| 123 | return f |
| 124 | except (ValueError, TypeError): |
| 125 | return None |
| 126 | |
| 127 | |
| 128 | def _valid_coord(lat: float | None, lon: float | None) -> tuple[float | None, float | None]: |
| 129 | if lat is None or lon is None: |
| 130 | return None, None |
| 131 | if -90 <= lat <= 90 and -180 <= lon <= 180 and not (lat == 0 and lon == 0): |
| 132 | return lat, lon |
| 133 | return None, None |
| 134 | |
| 135 | |
| 136 | def _truthy(value) -> bool: |
| 137 | return str(value).strip().lower() in {"1", "true", "yes", "y", "hit", "hotlist"} |
| 138 | |
| 139 | |
| 140 | def normalize_read(raw: RawRead, alias: dict[str, str], source_tz: timezone = timezone.utc) -> NormalizedRead | None: |
| 141 | """Project a raw row onto canonical fields. Returns None if no usable plate+timestamp.""" |
| 142 | f = raw.fields |
| 143 | |
| 144 | def g(canon: str): |
| 145 | col = alias.get(canon) |
| 146 | return f.get(col) if col else None |
| 147 | |
| 148 | plate_raw = str(g("plate") or "").strip() |
| 149 | plate_norm = normalize_plate(plate_raw) |
| 150 | read_ts = normalize_timestamp(g("read_ts"), source_tz) |
| 151 | if not plate_norm or read_ts is None: |
| 152 | return None |
| 153 | |
| 154 | lat, lon = _valid_coord(_to_float(g("lat")), _to_float(g("lon"))) |
| 155 | return NormalizedRead( |
| 156 | plate_raw=plate_raw, |
| 157 | plate_norm=plate_norm, |
| 158 | state=(str(g("state")).strip().upper() or None) if g("state") else None, |
| 159 | read_ts=read_ts, |
| 160 | external_camera_id=(str(g("camera")).strip() or None) if g("camera") else None, |
| 161 | lat=lat, |
| 162 | lon=lon, |
| 163 | direction=normalize_direction(g("direction")), |
| 164 | confidence=normalize_confidence(g("confidence")), |
| 165 | lane=(str(g("lane")).strip() or None) if g("lane") else None, |
| 166 | make=(str(g("make")).strip() or None) if g("make") else None, |
| 167 | model=(str(g("model")).strip() or None) if g("model") else None, |
| 168 | color=(str(g("color")).strip() or None) if g("color") else None, |
| 169 | body_type=(str(g("body_type")).strip() or None) if g("body_type") else None, |
| 170 | hotlist_hit=_truthy(g("hotlist")) if g("hotlist") is not None else False, |
| 171 | incident_ref=(str(g("incident")).strip() or None) if g("incident") else None, |
| 172 | raw=f, |
| 173 | ) |
| 174 | |
| 175 | |
| 176 | def dedup_hash(org: str, plate_norm: str, camera_id: str | None, read_ts: datetime, |
| 177 | window_seconds: int = 1) -> bytes: |
| 178 | """Stable key for idempotent ingest. Timestamp is bucketed so sub-window bursts collapse.""" |
| 179 | epoch = int(read_ts.timestamp()) |
| 180 | bucket = epoch - (epoch % max(1, window_seconds)) |
| 181 | key = f"{org}|{plate_norm}|{camera_id or '-'}|{bucket}" |
| 182 | return hashlib.sha256(key.encode()).digest() |