| 1 | """ |
| 2 | Darwin RAG bridge - port 9875. |
| 3 | Fast HTTP service that searches homelab-rag index directly. |
| 4 | |
| 5 | Primary: SQLite FTS5 at /var/tmp/homelab-rag/state.sqlite (immutable read snapshot) |
| 6 | Fallback: streaming keyword scan of /shared/projects/.homelab-rag/index.jsonl |
| 7 | (scans first STREAM_SCAN_BYTES ~80MB at ~0.4s on CIFS) |
| 8 | |
| 9 | Endpoints: |
| 10 | GET /search?q=QUERY&limit=N -> JSON array of results |
| 11 | GET /health -> {"ok": true, "sqlite": bool, "stream_fallback": bool} |
| 12 | GET /sources -> {"ok": bool, "now": iso, "sources":[{source_id, |
| 13 | host,type,last_ingest,staleness_h,files_seen}...]} |
| 14 | """ |
| 15 | from __future__ import annotations |
| 16 | |
| 17 | import json |
| 18 | import os |
| 19 | import re |
| 20 | import sqlite3 |
| 21 | import time |
| 22 | from datetime import datetime, timezone |
| 23 | from http.server import BaseHTTPRequestHandler, HTTPServer |
| 24 | from urllib.parse import parse_qs, urlparse |
| 25 | |
| 26 | STATE_DB = os.environ.get("RAG_STATE_DB", "/var/tmp/homelab-rag/state.sqlite") |
| 27 | INDEX_JSONL = os.environ.get( |
| 28 | "RAG_INDEX_JSONL", "/shared/projects/.homelab-rag/index.jsonl" |
| 29 | ) |
| 30 | PORT = int(os.environ.get("RAG_BRIDGE_PORT", "9875")) |
| 31 | MAX_RESULTS = 20 |
| 32 | DEFAULT_LIMIT = 8 |
| 33 | |
| 34 | STREAM_SCAN_BYTES = 50 * 1024 * 1024 |
| 35 | |
| 36 | STOPWORDS = { |
| 37 | "the", "and", "for", "that", "this", "with", "from", "what", "where", |
| 38 | "how", "into", "about", "your", "you", "are", "is", "was", "has", |
| 39 | "have", "been", "not", "but", "can", "will", "would", "could", "should", |
| 40 | "of", "in", "on", "at", "to", "by", "an", "as", "it", "its", "or", |
| 41 | "all", "any", "be", "do", "up", "so", "my", "we", "us", |
| 42 | } |
| 43 | |
| 44 | |
| 45 | def _fts_terms(query: str) -> list[str]: |
| 46 | raw = re.findall(r"[A-Za-z0-9][A-Za-z0-9._:-]{1,}", query.lower()) |
| 47 | return [t.replace('"', "") for t in raw if t not in STOPWORDS] |
| 48 | |
| 49 | |
| 50 | def _fts_match(terms: list[str]) -> str: |
| 51 | if not terms: |
| 52 | return "" |
| 53 | return " OR ".join(f'"{t}"' for t in terms[:16]) |
| 54 | |
| 55 | |
| 56 | def _norm_score(rank: float, authority: int) -> float: |
| 57 | """Normalise bm25 rank (negative, closer to 0 = better) to 0..1.""" |
| 58 | base = min(1.0, max(0.0, -rank / 15.0)) |
| 59 | return round(min(1.0, base + authority * 0.02), 4) |
| 60 | |
| 61 | |
| 62 | def _sqlite_search(query: str, limit: int) -> list[dict] | None: |
| 63 | """Try SQLite FTS. Returns None if DB is unavailable or FTS is corrupt.""" |
| 64 | terms = _fts_terms(query) |
| 65 | match = _fts_match(terms) |
| 66 | if not match: |
| 67 | return [] |
| 68 | |
| 69 | try: |
| 70 | con = sqlite3.connect( |
| 71 | f"file:{STATE_DB}?immutable=1", |
| 72 | uri=True, |
| 73 | timeout=1.5, |
| 74 | check_same_thread=False, |
| 75 | ) |
| 76 | con.row_factory = sqlite3.Row |
| 77 | except sqlite3.OperationalError: |
| 78 | return None |
| 79 | |
| 80 | try: |
| 81 | candidate_limit = max(limit * 10, 60) |
| 82 | rows = con.execute( |
| 83 | """ |
| 84 | SELECT sc.id, sc.path, sc.rel_path AS relPath, sc.project, sc.kind, |
| 85 | sc.title, NULLIF(sc.symbol, '') AS symbol, |
| 86 | sc.start_line AS startLine, sc.end_line AS endLine, |
| 87 | sc.authority, sc.stale, |
| 88 | sc.secret_finding_count AS secretFindingCount, |
| 89 | sc.mtime_ms AS mtimeMs, sc.text, |
| 90 | bm25(search_chunks_fts) AS ftsRank |
| 91 | FROM search_chunks_fts |
| 92 | JOIN search_chunks sc ON sc.rowid = search_chunks_fts.rowid |
| 93 | WHERE search_chunks_fts MATCH ? |
| 94 | AND sc.stale = 0 |
| 95 | AND sc.secret_finding_count = 0 |
| 96 | ORDER BY ftsRank |
| 97 | LIMIT ? |
| 98 | """, |
| 99 | (match, candidate_limit), |
| 100 | ).fetchall() |
| 101 | |
| 102 | results = [ |
| 103 | { |
| 104 | "id": r["id"], |
| 105 | "score": _norm_score(float(r["ftsRank"] or 0), int(r["authority"] or 0)), |
| 106 | "relPath": r["relPath"] or "", |
| 107 | "path": r["path"] or "", |
| 108 | "project": r["project"] or "", |
| 109 | "kind": r["kind"] or "doc", |
| 110 | "title": r["title"] or "", |
| 111 | "symbol": r["symbol"], |
| 112 | "startLine": r["startLine"] or 0, |
| 113 | "endLine": r["endLine"] or 0, |
| 114 | "authority": r["authority"] or 0, |
| 115 | "stale": bool(r["stale"]), |
| 116 | "secretFindingCount": r["secretFindingCount"] or 0, |
| 117 | "mtimeMs": r["mtimeMs"] or 0, |
| 118 | "text": (r["text"] or "")[:600], |
| 119 | } |
| 120 | for r in rows |
| 121 | ] |
| 122 | results.sort(key=lambda x: -x["score"]) |
| 123 | return results[:limit] |
| 124 | |
| 125 | except sqlite3.DatabaseError: |
| 126 | return None |
| 127 | finally: |
| 128 | con.close() |
| 129 | |
| 130 | |
| 131 | def _stream_search(query: str, limit: int) -> list[dict]: |
| 132 | """Streaming keyword scan of index.jsonl. |
| 133 | |
| 134 | Scans the first STREAM_SCAN_BYTES bytes (~80MB) to keep latency ~0.4s. |
| 135 | |
| 136 | Scoring: |
| 137 | - ALL terms matched in content: 0.7 base + authority bonus (0..0.3) |
| 138 | - ALL terms matched in title: extra +0.2 |
| 139 | - Partial match (some terms): 0.35 base only - returned as fallback |
| 140 | Results requiring all terms are always ranked above partial matches. |
| 141 | """ |
| 142 | terms = _fts_terms(query) |
| 143 | if not terms: |
| 144 | return [] |
| 145 | |
| 146 | full_hits: list[dict] = [] |
| 147 | partial_hits: list[dict] = [] |
| 148 | |
| 149 | try: |
| 150 | with open(INDEX_JSONL, "r", errors="replace") as f: |
| 151 | total_bytes = 0 |
| 152 | for line in f: |
| 153 | total_bytes += len(line) |
| 154 | if total_bytes > STREAM_SCAN_BYTES: |
| 155 | break |
| 156 | line = line.strip() |
| 157 | if not line: |
| 158 | continue |
| 159 | line_lower = line.lower() |
| 160 | primary = terms[-1] |
| 161 | if primary not in line_lower: |
| 162 | continue |
| 163 | matched = [t for t in terms if t in line_lower] |
| 164 | is_full = len(matched) == len(terms) |
| 165 | if not is_full and len(matched) < max(1, len(terms) // 2): |
| 166 | continue |
| 167 | try: |
| 168 | obj = json.loads(line) |
| 169 | except json.JSONDecodeError: |
| 170 | continue |
| 171 | if obj.get("secretFindingCount", 0) > 0 or obj.get("stale"): |
| 172 | continue |
| 173 | authority = int(obj.get("authority", 0)) |
| 174 | title_lower = (obj.get("title") or "").lower() |
| 175 | title_match = sum(1 for t in terms if t in title_lower) |
| 176 | |
| 177 | if is_full: |
| 178 | base = 0.70 + min(0.25, authority * 0.04) |
| 179 | title_bonus = min(0.15, title_match * 0.08) |
| 180 | score = round(min(1.0, base + title_bonus), 4) |
| 181 | text = obj.get("text", "") |
| 182 | full_hits.append({ |
| 183 | "id": obj.get("id", ""), |
| 184 | "score": score, |
| 185 | "relPath": obj.get("relPath", ""), |
| 186 | "path": obj.get("path", ""), |
| 187 | "project": obj.get("project", ""), |
| 188 | "kind": obj.get("kind", "doc"), |
| 189 | "title": obj.get("title", ""), |
| 190 | "symbol": obj.get("symbol"), |
| 191 | "startLine": obj.get("startLine", 0), |
| 192 | "endLine": obj.get("endLine", 0), |
| 193 | "authority": authority, |
| 194 | "stale": False, |
| 195 | "secretFindingCount": 0, |
| 196 | "mtimeMs": obj.get("mtimeMs", 0), |
| 197 | "text": str(text)[:600], |
| 198 | }) |
| 199 | else: |
| 200 | score = round(min(0.50, 0.25 + authority * 0.04), 4) |
| 201 | text = obj.get("text", "") |
| 202 | partial_hits.append({ |
| 203 | "id": obj.get("id", ""), |
| 204 | "score": score, |
| 205 | "relPath": obj.get("relPath", ""), |
| 206 | "path": obj.get("path", ""), |
| 207 | "project": obj.get("project", ""), |
| 208 | "kind": obj.get("kind", "doc"), |
| 209 | "title": obj.get("title", ""), |
| 210 | "symbol": obj.get("symbol"), |
| 211 | "startLine": obj.get("startLine", 0), |
| 212 | "endLine": obj.get("endLine", 0), |
| 213 | "authority": authority, |
| 214 | "stale": False, |
| 215 | "secretFindingCount": 0, |
| 216 | "mtimeMs": obj.get("mtimeMs", 0), |
| 217 | "text": str(text)[:600], |
| 218 | }) |
| 219 | except OSError: |
| 220 | pass |
| 221 | |
| 222 | full_hits.sort(key=lambda x: -x["score"]) |
| 223 | partial_hits.sort(key=lambda x: -x["score"]) |
| 224 | |
| 225 | combined = full_hits + partial_hits |
| 226 | return combined[:limit] |
| 227 | |
| 228 | |
| 229 | def _parse_iso(val: str | None) -> datetime | None: |
| 230 | if not val: |
| 231 | return None |
| 232 | try: |
| 233 | return datetime.fromisoformat(str(val).replace("Z", "+00:00")) |
| 234 | except Exception: |
| 235 | return None |
| 236 | |
| 237 | |
| 238 | def do_sources() -> dict: |
| 239 | """Per-source ingest freshness from homelab-rag state.sqlite. |
| 240 | |
| 241 | Freshness signal = max(sync_runs.finished_at) per source (the last time the |
| 242 | rolling collector completed a run for that endpoint). Falls back to |
| 243 | max(source_files.last_seen_at) if no sync_runs row exists. staleness_h is |
| 244 | hours since that timestamp, relative to now (UTC). |
| 245 | |
| 246 | Read-only immutable snapshot - never writes, never blocks the collector. |
| 247 | """ |
| 248 | now = datetime.now(timezone.utc) |
| 249 | out: list[dict] = [] |
| 250 | try: |
| 251 | con = sqlite3.connect( |
| 252 | f"file:{STATE_DB}?immutable=1", |
| 253 | uri=True, |
| 254 | timeout=1.5, |
| 255 | check_same_thread=False, |
| 256 | ) |
| 257 | con.row_factory = sqlite3.Row |
| 258 | except sqlite3.OperationalError: |
| 259 | return {"ok": False, "now": now.isoformat(), "sources": []} |
| 260 | |
| 261 | try: |
| 262 | rows = con.execute( |
| 263 | """ |
| 264 | SELECT s.id AS source_id, s.host, s.type, s.cadence, s.enabled, |
| 265 | MAX(r.finished_at) AS last_sync, |
| 266 | (SELECT files_seen FROM sync_runs r2 |
| 267 | WHERE r2.source_id = s.id |
| 268 | ORDER BY r2.finished_at DESC LIMIT 1) AS files_seen, |
| 269 | (SELECT MAX(last_seen_at) FROM source_files sf |
| 270 | WHERE sf.source_id = s.id) AS max_seen |
| 271 | FROM sources s |
| 272 | LEFT JOIN sync_runs r ON r.source_id = s.id |
| 273 | GROUP BY s.id |
| 274 | """ |
| 275 | ).fetchall() |
| 276 | except sqlite3.DatabaseError: |
| 277 | con.close() |
| 278 | return {"ok": False, "now": now.isoformat(), "sources": []} |
| 279 | |
| 280 | for r in rows: |
| 281 | last = _parse_iso(r["last_sync"]) or _parse_iso(r["max_seen"]) |
| 282 | staleness_h = None |
| 283 | last_iso = None |
| 284 | if last is not None: |
| 285 | last_iso = last.isoformat() |
| 286 | staleness_h = round((now - last).total_seconds() / 3600.0, 2) |
| 287 | out.append({ |
| 288 | "source_id": r["source_id"], |
| 289 | "host": r["host"] or "", |
| 290 | "type": r["type"] or "", |
| 291 | "cadence": r["cadence"] or "", |
| 292 | "enabled": bool(r["enabled"]), |
| 293 | "last_ingest": last_iso, |
| 294 | "staleness_h": staleness_h, |
| 295 | "files_seen": r["files_seen"] or 0, |
| 296 | }) |
| 297 | con.close() |
| 298 | out.sort(key=lambda x: (x["staleness_h"] is None, x["staleness_h"] or 0)) |
| 299 | return {"ok": True, "now": now.isoformat(), "sources": out} |
| 300 | |
| 301 | |
| 302 | def do_search(query: str, limit: int) -> list[dict]: |
| 303 | """Search: try SQLite FTS first, fall back to streaming scan.""" |
| 304 | limit = max(1, min(limit, MAX_RESULTS)) |
| 305 | result = _sqlite_search(query, limit) |
| 306 | if result is not None: |
| 307 | return result |
| 308 | return _stream_search(query, limit) |
| 309 | |
| 310 | |
| 311 | class Handler(BaseHTTPRequestHandler): |
| 312 | def log_message(self, fmt, *args): |
| 313 | pass |
| 314 | |
| 315 | def send_json(self, code: int, data): |
| 316 | body = json.dumps(data).encode() |
| 317 | self.send_response(code) |
| 318 | self.send_header("Content-Type", "application/json") |
| 319 | self.send_header("Content-Length", str(len(body))) |
| 320 | self.end_headers() |
| 321 | self.wfile.write(body) |
| 322 | |
| 323 | def do_GET(self): |
| 324 | parsed = urlparse(self.path) |
| 325 | path = parsed.path |
| 326 | qs = parse_qs(parsed.query) |
| 327 | |
| 328 | if path == "/health": |
| 329 | sqlite_ok = False |
| 330 | stream_ok = os.path.exists(INDEX_JSONL) |
| 331 | try: |
| 332 | con = sqlite3.connect( |
| 333 | f"file:{STATE_DB}?immutable=1", uri=True, timeout=0.5 |
| 334 | ) |
| 335 | con.execute("SELECT 1") |
| 336 | con.close() |
| 337 | sqlite_ok = True |
| 338 | except Exception: |
| 339 | pass |
| 340 | self.send_json(200, { |
| 341 | "ok": True, |
| 342 | "sqlite": sqlite_ok, |
| 343 | "stream_fallback": stream_ok, |
| 344 | "db": STATE_DB, |
| 345 | }) |
| 346 | return |
| 347 | |
| 348 | if path == "/sources": |
| 349 | try: |
| 350 | self.send_json(200, do_sources()) |
| 351 | except Exception as e: |
| 352 | self.send_json(500, {"ok": False, "error": str(e), "sources": []}) |
| 353 | return |
| 354 | |
| 355 | if path == "/search": |
| 356 | query = (qs.get("q") or qs.get("query") or [""])[0].strip() |
| 357 | try: |
| 358 | limit = int((qs.get("limit") or [str(DEFAULT_LIMIT)])[0]) |
| 359 | except ValueError: |
| 360 | limit = DEFAULT_LIMIT |
| 361 | if not query: |
| 362 | self.send_json(400, {"error": "q parameter required"}) |
| 363 | return |
| 364 | t0 = time.time() |
| 365 | try: |
| 366 | results = do_search(query, limit) |
| 367 | elapsed = time.time() - t0 |
| 368 | body = json.dumps(results).encode() |
| 369 | self.send_response(200) |
| 370 | self.send_header("Content-Type", "application/json") |
| 371 | self.send_header("X-Search-Ms", str(int(elapsed * 1000))) |
| 372 | self.send_header("Content-Length", str(len(body))) |
| 373 | self.end_headers() |
| 374 | self.wfile.write(body) |
| 375 | except Exception as e: |
| 376 | self.send_json(500, {"error": str(e)}) |
| 377 | return |
| 378 | |
| 379 | self.send_json(404, {"error": "not found"}) |
| 380 | |
| 381 | |
| 382 | def main(): |
| 383 | server = HTTPServer(("0.0.0.0", PORT), Handler) |
| 384 | print(f"Darwin RAG bridge listening on :{PORT}", flush=True) |
| 385 | print(f"State DB: {STATE_DB}", flush=True) |
| 386 | print(f"Index jsonl: {INDEX_JSONL}", flush=True) |
| 387 | server.serve_forever() |
| 388 | |
| 389 | |
| 390 | if __name__ == "__main__": |
| 391 | main() |