● Zion Boggan
repos/Sightline/backend/generate_data.py
zionboggan.com ↗
212 lines · python
History for this file →
1
"""Generate a NATIONWIDE ALPR dataset as a Flock-style CSV - camera networks in a city for
2
every U.S. state, vehicles plated across all 50 states, with realistic local commute patterns,
3
some cross-country travelers, and embedded investigative scenarios:
4
 
5
  * a stolen vehicle on a watchlist, driven across multiple states
6
  * a cloned plate with physically impossible coast-to-coast transit (anomaly detector)
7
  * a co-traveling pair repeatedly seen together
8
  * commuters with stable home/work anchor points per city
9
 
10
Deterministic (seeded). Re-running reproduces the same dataset.
11
"""
12
from __future__ import annotations
13
 
14
import csv
15
import math
16
import os
17
import random
18
import string
19
from datetime import datetime, timedelta, timezone
20
 
21
random.seed(50)
22
OUT = os.path.join(os.path.dirname(__file__), "sample_data", "poc_reads.csv")
23
 
24
STATE_CITIES = {
25
    "AL": ("Birmingham", 33.5186, -86.8104), "AK": ("Anchorage", 61.2181, -149.9003),
26
    "AZ": ("Phoenix", 33.4484, -112.0740), "AR": ("Little Rock", 34.7465, -92.2896),
27
    "CA": ("Los Angeles", 34.0522, -118.2437), "CO": ("Denver", 39.7392, -104.9903),
28
    "CT": ("Hartford", 41.7658, -72.6734), "DE": ("Wilmington", 39.7391, -75.5398),
29
    "FL": ("Miami", 25.7617, -80.1918), "GA": ("Atlanta", 33.7490, -84.3880),
30
    "HI": ("Honolulu", 21.3069, -157.8583), "ID": ("Boise", 43.6150, -116.2023),
31
    "IL": ("Chicago", 41.8781, -87.6298), "IN": ("Indianapolis", 39.7684, -86.1581),
32
    "IA": ("Des Moines", 41.5868, -93.6250), "KS": ("Wichita", 37.6872, -97.3301),
33
    "KY": ("Louisville", 38.2527, -85.7585), "LA": ("New Orleans", 29.9511, -90.0715),
34
    "ME": ("Portland", 43.6591, -70.2568), "MD": ("Baltimore", 39.2904, -76.6122),
35
    "MA": ("Boston", 42.3601, -71.0589), "MI": ("Detroit", 42.3314, -83.0458),
36
    "MN": ("Minneapolis", 44.9778, -93.2650), "MS": ("Jackson", 32.2988, -90.1848),
37
    "MO": ("St. Louis", 38.6270, -90.1994), "MT": ("Billings", 45.7833, -108.5007),
38
    "NE": ("Omaha", 41.2565, -95.9345), "NV": ("Las Vegas", 36.1699, -115.1398),
39
    "NH": ("Manchester", 42.9956, -71.4548), "NJ": ("Newark", 40.7357, -74.1724),
40
    "NM": ("Albuquerque", 35.0844, -106.6504), "NY": ("New York", 40.7128, -74.0060),
41
    "NC": ("Charlotte", 35.2271, -80.8431), "ND": ("Fargo", 46.8772, -96.7898),
42
    "OH": ("Columbus", 39.9612, -82.9988), "OK": ("Oklahoma City", 35.4676, -97.5164),
43
    "OR": ("Portland", 45.5152, -122.6784), "PA": ("Philadelphia", 39.9526, -75.1652),
44
    "RI": ("Providence", 41.8240, -71.4128), "SC": ("Columbia", 34.0007, -81.0348),
45
    "SD": ("Sioux Falls", 43.5446, -96.7311), "TN": ("Memphis", 35.1495, -90.0490),
46
    "TX": ("Houston", 29.7604, -95.3698), "UT": ("Salt Lake City", 40.7608, -111.8910),
47
    "VT": ("Burlington", 44.4759, -73.2121), "VA": ("Richmond", 37.5407, -77.4360),
48
    "WA": ("Seattle", 47.6062, -122.3321), "WV": ("Charleston", 38.3498, -81.6326),
49
    "WI": ("Milwaukee", 43.0389, -87.9065), "WY": ("Cheyenne", 41.1400, -104.8202),
50
}
51
 
52
STREETS = ["Main & 1st", "Central & 5th", "Broadway & Market", "Park & Union", "Highway 90 Ramp"]
53
 
54
CAMERAS: list[tuple] = []
55
CITY_CAMS: dict[str, list[str]] = {}
56
for st, (city, lat, lon) in STATE_CITIES.items():
57
    cams = []
58
    for i in range(3):
59
        cid = f"CAM-{st}-{i+1:02d}"
60
        jlat = lat + random.uniform(-0.05, 0.05)
61
        jlon = lon + random.uniform(-0.05, 0.05)
62
        CAMERAS.append((cid, f"{city} - {STREETS[i % len(STREETS)]}", round(jlat, 4),
63
                        round(jlon, 4), f"{city}, {st}"))
64
        cams.append(cid)
65
    CITY_CAMS[st] = cams
66
 
67
CAM_BY_ID = {c[0]: c for c in CAMERAS}
68
ALL_STATES = list(STATE_CITIES.keys())
69
 
70
MAKES = [("Toyota", ["Camry", "Corolla", "RAV4"]), ("Honda", ["Civic", "Accord", "CR-V"]),
71
         ("Ford", ["F-150", "Escape", "Explorer"]), ("Chevrolet", ["Malibu", "Equinox", "Silverado"]),
72
         ("Nissan", ["Altima", "Rogue", "Sentra"]), ("Jeep", ["Grand Cherokee", "Wrangler"]),
73
         ("Dodge", ["Charger", "Ram"]), ("Hyundai", ["Elantra", "Sonata", "Tucson"]),
74
         ("Tesla", ["Model 3", "Model Y"]), ("Subaru", ["Outback", "Forester"])]
75
COLORS = ["White", "Black", "Silver", "Gray", "Blue", "Red", "Green", "Tan"]
76
BODY = {"Camry": "Sedan", "Corolla": "Sedan", "RAV4": "SUV", "Civic": "Sedan", "Accord": "Sedan",
77
        "CR-V": "SUV", "F-150": "Pickup", "Escape": "SUV", "Explorer": "SUV", "Malibu": "Sedan",
78
        "Equinox": "SUV", "Silverado": "Pickup", "Altima": "Sedan", "Rogue": "SUV", "Sentra": "Sedan",
79
        "Grand Cherokee": "SUV", "Wrangler": "SUV", "Charger": "Sedan", "Ram": "Pickup",
80
        "Elantra": "Sedan", "Sonata": "Sedan", "Tucson": "SUV", "Model 3": "Sedan",
81
        "Model Y": "SUV", "Outback": "Wagon", "Forester": "SUV"}
82
 
83
DAYS = 21
84
END = datetime(2026, 6, 10, tzinfo=timezone.utc)
85
START = END - timedelta(days=DAYS)
86
 
87
 
88
def haversine_km(a, b):
89
    la1, lo1 = CAM_BY_ID[a][2], CAM_BY_ID[a][3]
90
    la2, lo2 = CAM_BY_ID[b][2], CAM_BY_ID[b][3]
91
    r = 6371.0
92
    p1, p2 = math.radians(la1), math.radians(la2)
93
    dphi, dlam = math.radians(la2 - la1), math.radians(lo2 - lo1)
94
    h = math.sin(dphi / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dlam / 2) ** 2
95
    return 2 * r * math.asin(math.sqrt(h))
96
 
97
 
98
def bearing(a, b):
99
    la1, lo1 = map(math.radians, (CAM_BY_ID[a][2], CAM_BY_ID[a][3]))
100
    la2, lo2 = map(math.radians, (CAM_BY_ID[b][2], CAM_BY_ID[b][3]))
101
    dl = lo2 - lo1
102
    y = math.sin(dl) * math.cos(la2)
103
    x = math.cos(la1) * math.sin(la2) - math.sin(la1) * math.cos(la2) * math.cos(dl)
104
    deg = (math.degrees(math.atan2(y, x)) + 360) % 360
105
    return ["N", "NE", "E", "SE", "S", "SW", "W", "NW"][int((deg + 22.5) % 360 // 45)]
106
 
107
 
108
def plate_for(state):
109
    fmt = random.choice([3, 3, 4])
110
    return "".join(random.choices(string.ascii_uppercase, k=fmt)) + "".join(random.choices("0123456789", k=7 - fmt))
111
 
112
 
113
def make_vehicle(state, plate=None):
114
    mk, models = random.choice(MAKES)
115
    model = random.choice(models)
116
    return {"plate": plate or plate_for(state), "state": state, "make": mk,
117
            "color": random.choice(COLORS), "body": BODY[model]}
118
 
119
 
120
def emit(rows, veh, cam, dt, hotlist=False, incident="", prev=None):
121
    c = CAM_BY_ID[cam]
122
    rows.append({
123
        "Plate": veh["plate"], "State": veh["state"],
124
        "Timestamp": dt.strftime("%Y-%m-%dT%H:%M:%SZ"), "Camera": cam,
125
        "Latitude": f"{c[2]:.4f}", "Longitude": f"{c[3]:.4f}",
126
        "Direction": bearing(prev, cam) if prev else random.choice(["N", "E", "S", "W"]),
127
        "Confidence": random.randint(82, 99),
128
        "Make": veh["make"], "Color": veh["color"], "VehicleType": veh["body"],
129
        "Hotlist": "true" if hotlist else "false", "IncidentID": incident,
130
    })
131
 
132
 
133
def emit_trip(rows, veh, seq, depart, hotlist=False, incident=""):
134
    t, prev = depart, None
135
    for cam in seq:
136
        if prev:
137
            km = haversine_km(prev, cam)
138
            t = t + timedelta(hours=km / random.uniform(35, 75)) + timedelta(seconds=random.randint(0, 40))
139
        emit(rows, veh, cam, t, hotlist, incident, prev)
140
        prev = cam
141
 
142
 
143
def main():
144
    rows: list[dict] = []
145
 
146
    for st in ALL_STATES:
147
        cams = CITY_CAMS[st]
148
        for _ in range(4):
149
            veh = make_vehicle(st)
150
            home, work = cams[0], cams[-1]
151
            for day in range(DAYS):
152
                d = START + timedelta(days=day)
153
                if d.weekday() >= 5 or random.random() < 0.15:
154
                    continue
155
                am = d.replace(hour=random.randint(6, 8), minute=random.randint(0, 59), second=0)
156
                emit_trip(rows, veh, [home, cams[1], work], am)
157
                pm = d.replace(hour=random.randint(16, 18), minute=random.randint(0, 59), second=0)
158
                emit_trip(rows, veh, [work, cams[1], home], pm)
159
 
160
    for _ in range(25):
161
        st = random.choice(ALL_STATES)
162
        veh = make_vehicle(st)
163
        visited = random.sample(ALL_STATES, k=random.randint(2, 4))
164
        day = random.randint(0, DAYS - 1)
165
        for st2 in visited:
166
            d = START + timedelta(days=day)
167
            cam = random.choice(CITY_CAMS[st2])
168
            t = d.replace(hour=random.randint(8, 20), minute=random.randint(0, 59), second=0)
169
            emit(rows, veh, cam, t)
170
            day = min(DAYS - 1, day + random.randint(1, 2))
171
 
172
    stolen = {"plate": "BNDT447", "state": "TN", "make": "Dodge", "color": "Black", "body": "Sedan"}
173
    route_states = ["TN", "MS", "AL", "GA", "TN", "AR", "MO"]
174
    day = DAYS - 8
175
    for st in route_states:
176
        d = START + timedelta(days=day)
177
        for cam in random.sample(CITY_CAMS[st], k=2):
178
            t = d.replace(hour=random.randint(0, 23), minute=random.randint(0, 59), second=0)
179
            emit(rows, stolen, cam, t, hotlist=True, incident="INC-2026-0440")
180
        day = min(DAYS - 1, day + 1)
181
 
182
    clone = {"plate": "CLN9921", "state": "CA", "make": "Honda", "color": "White", "body": "Sedan"}
183
    base = (START + timedelta(days=DAYS - 4)).replace(hour=14, minute=0, second=0)
184
    for cam, off in [("CAM-CA-01", 0), ("CAM-NY-01", 120), ("CAM-FL-02", 240)]:
185
        emit(rows, clone, cam, base + timedelta(seconds=off))
186
 
187
    lead = make_vehicle("TX", "ESC8120"); tail = make_vehicle("TX", "ESC8121")
188
    for day in range(DAYS - 10, DAYS):
189
        d = START + timedelta(days=day)
190
        if d.weekday() >= 5:
191
            continue
192
        seq = CITY_CAMS["TX"]
193
        depart = d.replace(hour=7, minute=random.randint(20, 40), second=0)
194
        emit_trip(rows, lead, seq, depart)
195
        emit_trip(rows, tail, seq, depart + timedelta(seconds=random.randint(15, 50)))
196
 
197
    rows.sort(key=lambda r: r["Timestamp"])
198
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
199
    cols = ["Plate", "State", "Timestamp", "Camera", "Latitude", "Longitude", "Direction",
200
            "Confidence", "Make", "Color", "VehicleType", "Hotlist", "IncidentID"]
201
    with open(OUT, "w", newline="") as fh:
202
        w = csv.DictWriter(fh, fieldnames=cols)
203
        w.writeheader()
204
        w.writerows(rows)
205
    states = {r["State"] for r in rows}
206
    plates = {r["Plate"] for r in rows}
207
    print(f"wrote {len(rows)} reads, {len(plates)} plates, {len(states)} states, "
208
          f"{len(CAMERAS)} cameras in {len(STATE_CITIES)} cities -> {OUT}")
209
 
210
 
211
if __name__ == "__main__":
212
    main()