"""

Darwin BUILD-5 - Browser History Collector (standalone, invoked by ingest-all.sh)

Reads browser SQLite history DBs via Python-over-SSH.

Uses python3 (available on Windows via Git/standalone install) to:

  1. Copy the live DB to a temp path (readable even when browser is open).

  2. Query rows newer than the manifest watermark (stored in browser-watermarks.json).

  3. Emit normalized JSON records to stdout for downstream indexing.

Browsers supported:

  edge, chrome, brave - Chromium History SQLite (tables: urls, visits)

  firefox             - places.sqlite (tables: moz_places, moz_historyvisits)

Usage:

  python3 collect-browser-history.py --host HOST --user SSH_USER --browser BROWSER

  python3 collect-browser-history.py --host HOST --user zion --profile-user worky --browser brave

Browser path map (Windows):

  Edge:    C:\\\\Users\\\\<user>\\\\AppData\\\\Local\\\\Microsoft\\\\Edge\\\\User Data\\\\Default\\\\History

  Chrome:  C:\\\\Users\\\\<user>\\\\AppData\\\\Local\\\\Google\\\\Chrome\\\\User Data\\\\Default\\\\History

  Brave:   C:\\\\Users\\\\<user>\\\\AppData\\\\Local\\\\BraveSoftware\\\\Brave-Browser\\\\User Data\\\\Default\\\\History

  Firefox: C:\\\\Users\\\\<user>\\\\AppData\\\\Roaming\\\\Mozilla\\\\Firefox\\\\Profiles\\\\<profile>\\\\places.sqlite

"""

import argparse

import base64

import json

import os

import subprocess

import sys

import time

WATERMARK_PATH = "/var/tmp/homelab-rag/browser-watermarks.json"

OUTPUT_DIR = "/var/tmp/homelab-rag/browser-history"

def load_watermarks():

    if os.path.exists(WATERMARK_PATH):

        with open(WATERMARK_PATH) as f:

            return json.load(f)

    return {}

def save_watermarks(wm):

    os.makedirs(os.path.dirname(WATERMARK_PATH), exist_ok=True)

    with open(WATERMARK_PATH, "w") as f:

        json.dump(wm, f, indent=2)

def run_python_via_ssh(host, ssh_user, py_script, connect_timeout=30, identity=None):

    """Run a Python script on a remote Windows host via SSH + python -c (stdin deploy).

    The script is sent via stdin to avoid command-line length limits.

    """

    target = f"{ssh_user}@{host}" if ssh_user else host

    ssh_args = [

        "ssh",

        "-o", "BatchMode=yes",

        "-o", f"ConnectTimeout={connect_timeout}",

        "-o", "StrictHostKeyChecking=accept-new",

    ]

    if identity:

        ssh_args += ["-i", identity]

    ssh_args += [target, "python -"]

    result = subprocess.run(

        ssh_args,

        input=py_script.encode("utf-8"),

        capture_output=True,

        timeout=120,

    )

    return result

CHROMIUM_PY_TEMPLATE = r"""

import sqlite3, json, shutil, tempfile, os, sys

payload = PAYLOAD_JSON

hist_path = payload["history_path"]

watermark_ms = int(payload["watermark_ms"])

source_id = payload["source_id"]

tmp = tempfile.mktemp(suffix=".db")

try:

    shutil.copy2(hist_path, tmp)

except Exception as e:

    print(json.dumps({"error": "copy_failed", "msg": str(e)}))

    sys.exit(0)

try:

    CHROMIUM_EPOCH_MS = 11644473600000

    watermark_chromium = (watermark_ms + CHROMIUM_EPOCH_MS) * 1000

    conn = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)

    conn.row_factory = sqlite3.Row

    cur = conn.cursor()

    cur.execute(

        "SELECT u.url, u.title, v.visit_time "

        "FROM visits v JOIN urls u ON u.id=v.url "

        "WHERE v.visit_time > ? "

        "ORDER BY v.visit_time DESC LIMIT 50000",

        (watermark_chromium,)

    )

    records = []

    for row in cur.fetchall():

        chromium_us = int(row[2])

        unix_ms = chromium_us // 1000 - CHROMIUM_EPOCH_MS

        records.append({

            "url": row[0] or "",

            "title": row[1] or "",

            "visit_time_ms": unix_ms,

        })

    conn.close()

    print(json.dumps(records))

except Exception as e:

    print(json.dumps({"error": "sqlite_read_failed", "msg": str(e)}))

finally:

    try:

        os.unlink(tmp)

    except Exception:

        pass

"""

FIREFOX_PY_TEMPLATE = r"""

import sqlite3, json, shutil, tempfile, os, sys

payload = PAYLOAD_JSON

db_path = payload["history_path"]

watermark_ms = int(payload["watermark_ms"])

source_id = payload["source_id"]

tmp = tempfile.mktemp(suffix=".sqlite")

try:

    shutil.copy2(db_path, tmp)

except Exception as e:

    print(json.dumps({"error": "copy_failed", "msg": str(e)}))

    sys.exit(0)

try:

    watermark_us = watermark_ms * 1000

    conn = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)

    conn.row_factory = sqlite3.Row

    cur = conn.cursor()

    cur.execute(

        "SELECT p.url, p.title, h.visit_date "

        "FROM moz_historyvisits h JOIN moz_places p ON p.id=h.place_id "

        "WHERE h.visit_date > ? "

        "ORDER BY h.visit_date DESC LIMIT 50000",

        (watermark_us,)

    )

    records = []

    for row in cur.fetchall():

        visit_us = int(row[2])

        unix_ms = visit_us // 1000

        records.append({

            "url": row[0] or "",

            "title": row[1] or "",

            "visit_time_ms": unix_ms,

        })

    conn.close()

    print(json.dumps(records))

except Exception as e:

    print(json.dumps({"error": "sqlite_read_failed", "msg": str(e)}))

finally:

    try:

        os.unlink(tmp)

    except Exception:

        pass

"""

def collect_chromium(host, ssh_user, history_path, source_key, watermarks, identity=None):

    """Collect Chromium-based browser history via Python-over-SSH."""

    wm_ms = watermarks.get(source_key, 0)

    payload = json.dumps({

        "history_path": history_path,

        "watermark_ms": wm_ms,

        "source_id": source_key,

    })

    script = CHROMIUM_PY_TEMPLATE.replace("PAYLOAD_JSON", payload)

    result = run_python_via_ssh(host, ssh_user, script, identity=identity)

    if result.returncode != 0:

        print(f"[browser-history] SSH/python failed for {source_key}: {result.stderr[:300]}", file=sys.stderr)

        return []

    stdout = result.stdout.decode("utf-8", errors="replace").strip()

    if not stdout:

        print(f"[browser-history] Empty output for {source_key}", file=sys.stderr)

        return []

    try:

        rows = json.loads(stdout)

    except json.JSONDecodeError as e:

        print(f"[browser-history] JSON parse error for {source_key}: {e} | raw: {stdout[:200]}", file=sys.stderr)

        return []

    if isinstance(rows, dict) and "error" in rows:

        print(f"[browser-history] Remote error for {source_key}: {rows}", file=sys.stderr)

        return []

    return _process_rows(rows, source_key, wm_ms, watermarks)

def collect_firefox(host, ssh_user, profile_path, source_key, watermarks, identity=None):

    """Collect Firefox places.sqlite via Python-over-SSH."""

    wm_ms = watermarks.get(source_key, 0)

    payload = json.dumps({

        "history_path": profile_path,

        "watermark_ms": wm_ms,

        "source_id": source_key,

    })

    script = FIREFOX_PY_TEMPLATE.replace("PAYLOAD_JSON", payload)

    result = run_python_via_ssh(host, ssh_user, script, identity=identity)

    if result.returncode != 0:

        print(f"[browser-history] SSH/python failed for {source_key}: {result.stderr[:300]}", file=sys.stderr)

        return []

    stdout = result.stdout.decode("utf-8", errors="replace").strip()

    if not stdout:

        print(f"[browser-history] Empty output for {source_key}", file=sys.stderr)

        return []

    try:

        rows = json.loads(stdout)

    except json.JSONDecodeError as e:

        print(f"[browser-history] JSON parse error for {source_key}: {e} | raw: {stdout[:200]}", file=sys.stderr)

        return []

    if isinstance(rows, dict) and "error" in rows:

        print(f"[browser-history] Remote error for {source_key}: {rows}", file=sys.stderr)

        return []

    return _process_rows(rows, source_key, wm_ms, watermarks)

def _process_rows(rows, source_key, wm_ms, watermarks):

    """Normalize rows and update watermark."""

    records = []

    max_ms = wm_ms

    for row in rows:

        if isinstance(row, dict) and "url" in row:

            visit_ms = int(row.get("visit_time_ms", 0))

            if visit_ms > max_ms:

                max_ms = visit_ms

            records.append({

                "source_id": source_key,

                "url": row["url"],

                "title": row.get("title", ""),

                "visit_time_ms": visit_ms,

                "collected_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),

            })

    if max_ms > wm_ms:

        watermarks[source_key] = max_ms

    return records

def main():

    parser = argparse.ArgumentParser(description="Darwin browser history collector")

    parser.add_argument("--host", required=True)

    parser.add_argument("--user", required=True, help="SSH login user")

    parser.add_argument("--profile-user", default=None,

                        help="Windows profile username (if different from SSH user, e.g. 'worky' vs ssh user 'zion')")

    parser.add_argument("--browser", required=True, choices=["edge", "chrome", "brave", "firefox"])

    parser.add_argument("--firefox-profile", default=None,

                        help="Firefox profile dir name (e.g. ojo1adxx.default-release)")

    parser.add_argument("--source-key", default=None,

                        help="Override source key (default: <host>-<profile_user>-<browser>)")

    parser.add_argument("--identity", default=os.path.expanduser("~/.ssh/termius_homelab"))

    parser.add_argument("--output", default=None, help="Output JSONL path (default: auto)")

    args = parser.parse_args()

    profile_user = args.profile_user or args.user

    source_key = args.source_key or f"{args.host}-{profile_user}-{args.browser}"

    watermarks = load_watermarks()

    user_dir = f"C:\\Users\\{profile_user}"

    browser_paths = {

        "edge": f"{user_dir}\\AppData\\Local\\Microsoft\\Edge\\User Data\\Default\\History",

        "chrome": f"{user_dir}\\AppData\\Local\\Google\\Chrome\\User Data\\Default\\History",

        "brave": f"{user_dir}\\AppData\\Local\\BraveSoftware\\Brave-Browser\\User Data\\Default\\History",

    }

    if args.browser in ("edge", "chrome", "brave"):

        history_path = browser_paths[args.browser]

        records = collect_chromium(

            args.host, args.user, history_path, source_key, watermarks,

            identity=args.identity

        )

    elif args.browser == "firefox":

        if args.firefox_profile:

            places_path = f"{user_dir}\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\{args.firefox_profile}\\places.sqlite"

        else:

            places_path = f"{user_dir}\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\ojo1adxx.default-release\\places.sqlite"

        records = collect_firefox(

            args.host, args.user, places_path, source_key, watermarks,

            identity=args.identity

        )

    else:

        print(f"[browser-history] Unsupported browser: {args.browser}", file=sys.stderr)

        records = []

    if records:

        os.makedirs(OUTPUT_DIR, exist_ok=True)

        out_path = args.output or os.path.join(OUTPUT_DIR, f"{source_key}.jsonl")

        with open(out_path, "a", encoding="utf-8") as f:

            for rec in records:

                f.write(json.dumps(rec) + "\n")

        print(f"[browser-history] Wrote {len(records)} records to {out_path}", file=sys.stderr)

        save_watermarks(watermarks)

    else:

        print(f"[browser-history] No new records for {source_key}", file=sys.stderr)

if __name__ == "__main__":

    main()