set -Eeuo pipefail

STATE_DB="${HOMELAB_RAG_STATE_DB:-/var/tmp/homelab-rag/state.sqlite}"

PROJECT_DIR="${HOMELAB_RAG_PROJECT_DIR:-/shared/projects/homelab-rag}"

SSH_KEY="${HOME:-/root}/.ssh/termius_homelab"

HOST="100.76.195.52"

SSH_USER="zion"

WALKER_REMOTE_WIN='C:\Users\zion\AppData\Local\Temp\homelab-rag-walker.ps1'

WALKER_REMOTE_PS='C:\\Users\\zion\\AppData\\Local\\Temp\\homelab-rag-walker.ps1'

DRY_RUN=0

for arg in "$@"; do [ "$arg" = "--dry-run" ] && DRY_RUN=1; done

log() { printf '[%s] %s\n' "$(date -Is)" "$*" >&2; }

SSH_BASE="ssh -o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=accept-new -i ${SSH_KEY} ${SSH_USER}@${HOST}"

WALKER_LOCAL=$(mktemp /tmp/homelab-rag-walker-XXXXXX.ps1)

trap 'rm -f "$WALKER_LOCAL"' EXIT

cat > "$WALKER_LOCAL" <<'WALKER_BODY'

param([string]$PayloadJson)

$p = $PayloadJson | ConvertFrom-Json

$sid = [string]$p.source_id

$roots = @($p.roots)

$excl = @($p.exclude | ForEach-Object { ([string]$_).Replace('\','/').ToLowerInvariant() })

$maxD = [int]$p.max_depth

$maxF = [int]$p.max_files

$maxB = [int64]$p.max_file_bytes

$script:recs = New-Object System.Collections.Generic.List[object]

$script:st = [ordered]@{ filesSeen=0; skipped=0; errors=0; truncated=$false }

$ep = [datetime]'1970-01-01T00:00:00Z'

function cslash([string]$v) { $v.Replace('\','/') }

function is_excl([string]$path, [string]$name) {

  $n = (cslash $path).ToLowerInvariant(); $l = $name.ToLowerInvariant()

  foreach ($x in $excl) { if ($l -eq $x -or $n.EndsWith('/'+$x) -or $n.Contains('/'+$x+'/')) { return $true } }

  return $false

}

function kh([string]$path) {

  $l = $path.ToLowerInvariant()

  if ($l -match '\.(md|txt|rst|docx|pdf|pptx)$') { return 'doc' }

  if ($l -match '\.(json|ya?ml|toml|ini|conf|env|service|ps1)$') { return 'config' }

  if ($l -match '\.(log|out|err)$') { return 'log' }

  if ($l -match '\.(js|ts|py|sh|go|rs|html|css)$') { return 'code' }

  return 'other'

}

function walk([string]$root, [int]$depth) {

  if ($script:st['truncated']) { return }

  if ($maxD -ge 0 -and $depth -gt $maxD) { return }

  try { $entries = Get-ChildItem -LiteralPath $root -Force -ErrorAction Stop } catch { $script:st['errors'] += 1; return }

  foreach ($entry in $entries) {

    if ($script:st['truncated']) { return }

    if (is_excl $entry.FullName $entry.Name) { continue }

    if ($entry.PSIsContainer) { walk ([string]$entry.FullName) ($depth + 1); continue }

    try {

      $sz = [int64]$entry.Length

      $mt = [int64](($entry.LastWriteTimeUtc - $ep).TotalMilliseconds)

      $fp = cslash $entry.FullName

      $sk = if ($sz -gt $maxB) { 'large>' + $maxB } else { '' }

      if ($sk) { $script:st['skipped'] += 1 } else { $script:st['filesSeen'] += 1 }

      $script:recs.Add([ordered]@{

        source_id = $sid; path = $fp; normalized_path = $fp

        size = $sz; mtime_ms = $mt; kind_hint = (kh $entry.FullName)

        seen_at = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')

        skip_reason = $sk

      }) | Out-Null

      if ($maxF -gt 0 -and $script:recs.Count -ge $maxF) { $script:st['truncated'] = $true; return }

    } catch { $script:st['errors'] += 1 }

  }

}

foreach ($r in $roots) { walk ([string]$r) 0 }

$script:st['records'] = $script:recs.ToArray()

$script:st | ConvertTo-Json -Depth 8 -Compress

WALKER_BODY

log "Walker size: $(wc -c < "$WALKER_LOCAL") bytes"

if [ "$DRY_RUN" -eq 1 ]; then

  log "[DRY-RUN] Would deploy walker to $HOST and run sync for all desktop-qi823gr* sources"

  exit 0

fi

log "Deploying walker to $HOST:$WALKER_REMOTE_WIN"

DEPLOY_RESULT=$(cat "$WALKER_LOCAL" | $SSH_BASE \

  "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command \"\$content=[Console]::In.ReadToEnd(); [System.IO.File]::WriteAllText('$WALKER_REMOTE_WIN',\$content,[System.Text.Encoding]::UTF8); Write-Output 'DEPLOYED_OK'\"" 2>&1 || true)

if ! echo "$DEPLOY_RESULT" | grep -q "DEPLOYED_OK"; then

  log "ERROR: Deploy failed. Output: ${DEPLOY_RESULT:0:300}"

  exit 1

fi

log "Walker deployed OK"

run_source() {

  local SOURCE_ID="$1"

  local PAYLOAD="$2"

  local SOURCE_META="$3"

  log "  Running walker for $SOURCE_ID"

  local PAYLOAD_PS="${PAYLOAD//\'/\'\'}"

  local RUN_PS=". '$WALKER_REMOTE_PS' '$PAYLOAD_PS'"

  local RUN_ENC

  RUN_ENC=$(python3 -c "import base64,sys; print(base64.b64encode(sys.argv[1].encode('utf-16-le')).decode())" "$RUN_PS")

  log "  EncodedCommand length: ${#RUN_ENC}"

  local MANIFEST_RAW

  MANIFEST_RAW=$($SSH_BASE "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -EncodedCommand $RUN_ENC" 2>&1 || true)

  local MANIFEST_JSON

  MANIFEST_JSON=$(python3 -c "

import sys, re, json

data = sys.stdin.read()

for cand in re.findall(r'(\{[^\x00]{10,}\})', data, re.DOTALL):

    try:

        obj = json.loads(cand)

        if 'filesSeen' in obj:

            print(cand)

            break

    except Exception:

        pass

" <<< "$MANIFEST_RAW" 2>/dev/null || true)

  if [ -z "$MANIFEST_JSON" ]; then

    log "  ERROR: No JSON in manifest output for $SOURCE_ID. Raw (first 400):"

    echo "${MANIFEST_RAW:0:400}" >&2

    return 1

  fi

  local FILES_SEEN

  FILES_SEEN=$(python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('filesSeen',0))" <<< "$MANIFEST_JSON" 2>/dev/null || echo "?")

  log "  Files seen: $FILES_SEEN"

  local STARTED_AT

  STARTED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")

  local BRIDGE_OUT

  BRIDGE_OUT=$(python3 - <<PYEOF

import json, sys, subprocess, os

state_db    = "$STATE_DB"

source_id   = "$SOURCE_ID"

host        = "$HOST"

started_at  = "$STARTED_AT"

project_dir = "$PROJECT_DIR"

manifest = json.loads(r"""$MANIFEST_JSON""")

records  = manifest.get("records", [])

errors   = manifest.get("errors", 0)

source_meta = json.loads(r"""$SOURCE_META""")

payload = {"action":"sync","statePath":state_db,"source":source_meta,"startedAt":started_at,"records":records,"errors":errors}

bridge = os.path.join(project_dir, "src", "sqlite_bridge.py")

result = subprocess.run(["python3", bridge], input=json.dumps(payload).encode(), capture_output=True, timeout=60)

if result.returncode != 0:

    print("BRIDGE_ERROR:" + result.stderr.decode()[:300], file=sys.stderr)

    sys.exit(1)

print(result.stdout.decode().strip())

PYEOF

)

  log "  Bridge result for $SOURCE_ID: $BRIDGE_OUT"

  local DELTA

  DELTA=$(python3 -c "

import json,sys

d=json.loads(sys.argv[1])

print('new=%d changed=%d unchanged=%d tombstoned=%d' % (d.get('newFiles',0),d.get('changedFiles',0),d.get('unchangedFiles',0),d.get('tombstonedFiles',0)))

" "$BRIDGE_OUT" 2>/dev/null || echo "parse error")

  log "  Change-only delta ($SOURCE_ID): $DELTA"

}

run_source \

  "desktop-qi823gr" \

  '{"source_id":"desktop-qi823gr","roots":["C:\\Users\\zion\\Documents","C:\\Users\\zion\\Desktop"],"exclude":["AppData","node_modules","cache",".git","Videos","Downloads"],"max_depth":4,"max_files":25000,"max_file_bytes":67108864}' \

  '{"id":"desktop-qi823gr","host":"100.76.195.52","type":"windows-desktop","transport":"ssh-powershell","authority":6,"cadence":"2h","canonical":false,"enabled":true,"notes":"White PC over Tailscale. Darwin BUILD-1. SSH stdin deploy workaround.","roots":["C:\\\\Users\\\\zion\\\\Documents","C:\\\\Users\\\\zion\\\\Desktop"],"exclude":["AppData","node_modules","cache",".git","Videos","Downloads"],"maxDepth":4,"maxFiles":25000,"user":"zion"}'

run_source \

  "desktop-qi823gr-worky" \

  '{"source_id":"desktop-qi823gr-worky","roots":["C:\\Users\\worky\\Documents","C:\\Users\\worky\\Desktop","C:\\Users\\worky\\Downloads","C:\\Users\\worky\\AppData\\Roaming\\Microsoft\\Windows\\PowerShell\\PSReadLine"],"exclude":["AppData","node_modules",".git","cache","Videos"],"max_depth":3,"max_files":10000,"max_file_bytes":67108864}' \

  '{"id":"desktop-qi823gr-worky","host":"100.76.195.52","type":"windows-desktop","transport":"ssh-powershell","authority":8,"cadence":"1h","canonical":false,"enabled":true,"notes":"Darwin BUILD-5. White PC worky user files + PSReadLine history.","roots":["C:\\\\Users\\\\worky\\\\Documents","C:\\\\Users\\\\worky\\\\Desktop","C:\\\\Users\\\\worky\\\\Downloads","C:\\\\Users\\\\worky\\\\AppData\\\\Roaming\\\\Microsoft\\\\Windows\\\\PowerShell\\\\PSReadLine"],"exclude":["AppData","node_modules",".git","cache","Videos"],"maxDepth":3,"maxFiles":10000,"user":"zion"}'

run_source \

  "desktop-qi823gr-browser-brave" \

  '{"source_id":"desktop-qi823gr-browser-brave","roots":["C:\\Users\\worky\\AppData\\Local\\BraveSoftware\\Brave-Browser\\User Data\\Default"],"exclude":["Code Cache","GPUCache","ShaderCache","Service Worker","CacheStorage","Cache","cache","IndexedDB","VideoDecodeStats","optimization_guide_hint_cache_store","optimization_guide_model_metadata_store","GCM Store","Sync Data","Extension Rules","Extension Scripts","Platform Notifications"],"max_depth":2,"max_files":5000,"max_file_bytes":67108864}' \

  '{"id":"desktop-qi823gr-browser-brave","host":"100.76.195.52","type":"windows-desktop","transport":"ssh-powershell","authority":9,"cadence":"30m","canonical":false,"enabled":true,"notes":"Darwin BUILD-5. Brave browser profile for White PC worky user. 25.7MB History, active daily.","roots":["C:\\\\Users\\\\worky\\\\AppData\\\\Local\\\\BraveSoftware\\\\Brave-Browser\\\\User Data\\\\Default"],"exclude":["Code Cache","GPUCache","ShaderCache","Service Worker","CacheStorage","Cache","cache","IndexedDB"],"maxDepth":2,"maxFiles":5000,"user":"zion"}'

run_source \

  "desktop-qi823gr-browser-edge" \

  '{"source_id":"desktop-qi823gr-browser-edge","roots":["C:\\Users\\worky\\AppData\\Local\\Microsoft\\Edge\\User Data\\Default"],"exclude":["Code Cache","GPUCache","ShaderCache","Service Worker","CacheStorage","Cache","cache","IndexedDB","VideoDecodeStats","optimization_guide_hint_cache_store","optimization_guide_model_metadata_store","GCM Store","Sync Data","Extension Rules","Extension Scripts","Platform Notifications"],"max_depth":2,"max_files":5000,"max_file_bytes":67108864}' \

  '{"id":"desktop-qi823gr-browser-edge","host":"100.76.195.52","type":"windows-desktop","transport":"ssh-powershell","authority":7,"cadence":"1h","canonical":false,"enabled":true,"notes":"Darwin BUILD-5. Edge browser profile for White PC worky user. 288KB History.","roots":["C:\\\\Users\\\\worky\\\\AppData\\\\Local\\\\Microsoft\\\\Edge\\\\User Data\\\\Default"],"exclude":["Code Cache","GPUCache","ShaderCache","Service Worker","CacheStorage","Cache","cache","IndexedDB"],"maxDepth":2,"maxFiles":5000,"user":"zion"}'

run_source \

  "desktop-qi823gr-browser-firefox" \

  '{"source_id":"desktop-qi823gr-browser-firefox","roots":["C:\\Users\\worky\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\ojo1adxx.default-release"],"exclude":["cache2","startupCache","thumbnails","storage","datareporting","crashes","minidumps"],"max_depth":2,"max_files":3000,"max_file_bytes":67108864}' \

  '{"id":"desktop-qi823gr-browser-firefox","host":"100.76.195.52","type":"windows-desktop","transport":"ssh-powershell","authority":9,"cadence":"30m","canonical":false,"enabled":true,"notes":"Darwin BUILD-5. Firefox places.sqlite for White PC worky user. 5MB, last modified 2026-06-15.","roots":["C:\\\\Users\\\\worky\\\\AppData\\\\Roaming\\\\Mozilla\\\\Firefox\\\\Profiles\\\\ojo1adxx.default-release"],"exclude":["cache2","startupCache","thumbnails","storage","datareporting","crashes","minidumps"],"maxDepth":2,"maxFiles":3000,"user":"zion"}'

log "All desktop-qi823gr* sources complete"