import os

import time

import subprocess

from datetime import datetime, timedelta

from fastapi import FastAPI, Body

from fastapi.responses import JSONResponse

import json

import psycopg2

import requests

import discord

from discord.ext import commands

import asyncio

import threading

import hmac

import hashlib

import base64

import urllib.parse

import uuid

def load_env(path=None):

    if path is None:

        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")

    if os.path.exists(path):

        with open(path) as f:

            for line in f:

                line = line.strip()

                if line and not line.startswith("#") and "=" in line:

                    k, _, v = line.partition("=")

                    os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))

load_env()

OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")

XAI_API_KEY = os.getenv("XAI_API_KEY", "")

BRAVE_API_KEY = os.getenv("BRAVE_API_KEY", "")

DISCORD_TOKEN = os.getenv("DISCORD_TOKEN", "")

DISCORD_CHANNEL_ID = int(os.getenv("DISCORD_CHANNEL_ID", "0"))

OWNER_DISCORD_ID = int(os.getenv("OWNER_DISCORD_ID", "0"))

OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")

OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3:8b")

ACTIVE_BRAIN = "grok"

X_CONSUMER_KEY = os.getenv("X_CONSUMER_KEY", "")

X_CONSUMER_SECRET = os.getenv("X_CONSUMER_SECRET", "")

X_ACCESS_TOKEN = os.getenv("X_ACCESS_TOKEN", "")

X_ACCESS_TOKEN_SECRET = os.getenv("X_ACCESS_TOKEN_SECRET", "")

ANALYTICS_API_KEY = os.getenv("ANALYTICS_API_KEY", "")

ANALYTICS_BASE = os.getenv("ANALYTICS_BASE_URL", "https://api.example.com")

MODEL_COSTS = {

    "gpt-4o-mini": {"input_per_1m": 0.15, "output_per_1m": 0.60},

    "grok-4-1-fast-reasoning": {"input_per_1m": 2.00, "output_per_1m": 10.00},

    "ollama": {"input_per_1m": 0.0, "output_per_1m": 0.0},

}

MONTHLY_BUDGET_CEILING = float(os.getenv("MONTHLY_BUDGET", "10.00"))

RATE_LIMIT = {}

RATE_LIMIT_SECONDS = 5

CHANNEL_LAST_RESULT = {}

STARTUP_TIME = datetime.utcnow()

PENDING_POSTS = []

app = FastAPI(title="Perseus")

DB_CONFIG = {

    "dbname": os.getenv("PG_DBNAME", "perseus"),

    "user": os.getenv("PG_USER", "perseus"),

    "password": os.getenv("PG_PASSWORD", ""),

    "host": os.getenv("PG_HOST", "localhost"),

}

INFRA_MAP = {

    "node-1": {"ip": os.getenv("NODE1_IP", "REDACTED-IP"), "type": "proxmox_host", "desc": "Proxmox host 1"},

    "node-2": {"ip": os.getenv("NODE2_IP", "REDACTED-IP"), "type": "proxmox_host", "desc": "Proxmox host 2 (GPU)"},

    "perseus": {"ip": os.getenv("PERSEUS_IP", "REDACTED-IP"), "type": "lxc", "ct": 120, "node": "node-1", "desc": "Perseus main (FastAPI + Discord)"},

    "vpn": {"ip": os.getenv("VPN_IP", "REDACTED-IP"), "type": "lxc", "ct": 200, "node": "node-1", "desc": "WireGuard VPN"},

    "dns": {"ip": os.getenv("DNS_IP", "REDACTED-IP"), "type": "lxc", "ct": 201, "node": "node-1", "desc": "Pi-hole DNS"},

    "backup": {"ip": os.getenv("BACKUP_IP", "REDACTED-IP"), "type": "lxc", "ct": 203, "node": "node-1", "desc": "Proxmox Backup Server"},

    "gpu": {"ip": os.getenv("GPU_IP", "REDACTED-IP"), "type": "lxc", "ct": 205, "node": "node-2", "desc": "Ollama + GPU inference"},

}

HOST_ALIASES = {

    "gpu node": "node-2", "compute": "node-2",

    "pi-hole": "dns", "pihole": "dns", "adblock": "dns",

    "wireguard": "vpn", "wg": "vpn",

    "backup server": "backup", "pbs": "backup",

    "ollama": "gpu", "llm": "gpu",

    "main": "perseus", "self": "perseus", "local": "perseus",

}

BLOCKED_COMMANDS = ["rm -rf /", "rm -rf /*", "mkfs", "dd if=/dev/zero", ":(){ :|:& };:",

    "chmod -R 777 /", "shutdown", "reboot", "poweroff", "init 0", "init 6", "halt"]

AUTO_APPROVE = ["ls", "cat", "grep", "find", "df", "du", "free", "top", "uptime", "hostname",

    "whoami", "id", "ps", "systemctl status", "journalctl", "ip a", "ip addr", "ifconfig",

    "ping", "nvidia-smi", "ollama list", "ollama ps", "pct list", "qm list", "pvesh",

    "pihole", "docker ps", "head", "tail", "wc", "sort", "date", "cal", "echo", "pwd", "env"]

AGENT_REGISTRY = {

    "scraper": {"model": "langgraph", "local": True},

    "content_curator": {"model": "ollama", "local": True},

    "trend_hunter": {"model": "ollama", "local": True},

    "social_publisher": {"model": "ollama", "local": True},

    "compliance_scanner": {"model": "gpt4o-mini", "local": False},

    "ab_testing_orchestrator": {"model": "gpt4o-mini", "local": False},

    "analytics_reporter": {"model": "langgraph", "local": True},

    "backup_manager": {"model": "langgraph", "local": True},

    "competitor_intelligence": {"model": "grok-fast", "local": False},

    "content_rewriter": {"model": "ollama", "local": True},

    "engagement_analyst": {"model": "ollama", "local": True},

    "funnel_optimizer": {"model": "claude", "local": False},

    "traffic_router": {"model": "langgraph", "local": True},

    "scheduler_intelligence": {"model": "langgraph", "local": True},

    "model_router": {"model": "langgraph", "local": True},

    "rate_limit_resilience": {"model": "langgraph", "local": True},

    "infra_monitor": {"model": "langgraph", "local": True},

    "compliance_approval": {"model": "langgraph", "local": True},

    "capability_builder": {"model": "langgraph", "local": True},

    "revenue_tracker": {"model": "grok-fast", "local": False},

}

intents = discord.Intents.default()

intents.message_content = True

bot = commands.Bot(command_prefix='!', intents=intents)

bot_loop = None

def allowed_ctx(ctx):

    """Check if command is from group channel or owner DM."""

    if ctx.channel.id == DISCORD_CHANNEL_ID:

        return True

    if isinstance(ctx.channel, discord.DMChannel) and ctx.author.id == OWNER_DISCORD_ID:

        return True

    return False

@bot.event

async def on_ready():

    global bot_loop

    bot_loop = asyncio.get_event_loop()

    print(f"Discord bot logged in as {bot.user}")

async def send_discord_alert(msg):

    try:

        ch = bot.get_channel(DISCORD_CHANNEL_ID)

        if ch:

            await ch.send(msg)

    except Exception as e:

        print(f"Alert fail: {e}")

def send_alert_sync(msg):

    if bot_loop and bot_loop.is_running():

        asyncio.run_coroutine_threadsafe(send_discord_alert(msg), bot_loop)

@bot.event

async def on_message(message):

    if message.author == bot.user:

        return

    is_group = message.channel.id == DISCORD_CHANNEL_ID

    is_owner_dm = isinstance(message.channel, discord.DMChannel) and message.author.id == OWNER_DISCORD_ID

    if not is_group and not is_owner_dm:

        return

    if message.content.startswith("!"):

        await bot.process_commands(message)

        return

    if not message.content.strip():

        return

    uid = str(message.author.id)

    now = time.time()

    if uid in RATE_LIMIT and (now - RATE_LIMIT[uid]) < RATE_LIMIT_SECONDS:

        await message.reply(f"Rate limited -- wait {RATE_LIMIT_SECONDS - (now - RATE_LIMIT[uid]):.0f}s")

        return

    RATE_LIMIT[uid] = now

    async with message.channel.typing():

        try:

            payload = {"command": message.content}

            last = CHANNEL_LAST_RESULT.get(message.channel.id)

            if last:

                payload["previous_result"] = last

            resp = requests.post("http://localhost:3002/chat", json=payload, timeout=120)

            data = resp.json()

            agent = data.get("agent", "unknown")

            result = data.get("result", "No result")

            model = data.get("model", "")

            CHANNEL_LAST_RESULT[message.channel.id] = {"agent": agent, "result": result}

            model_sig = f" . `{model}`" if model else ""

            header = f"**[{agent}]{model_sig}**\n"

            full = header + result

            chunks = []

            while len(full) > 1990:

                split_at = full.rfind("\n", 0, 1990)

                if split_at < 500:

                    split_at = 1990

                chunks.append(full[:split_at])

                full = full[split_at:].lstrip("\n")

            chunks.append(full)

            for i, chunk in enumerate(chunks):

                if i == 0:

                    await message.reply(chunk)

                else:

                    await message.channel.send(chunk)

        except Exception as e:

            await message.reply(f"[WARN] {str(e)}")

def run_discord_bot():

    asyncio.run(bot.start(DISCORD_TOKEN))

threading.Thread(target=run_discord_bot, daemon=True).start()

def save_to_db(agent, command, result, cost=0.0, model_used=""):

    try:

        conn = psycopg2.connect(**DB_CONFIG)

        cur = conn.cursor()

        cur.execute("INSERT INTO metadata (agent, timestamp, command, result, cost_usd) VALUES (%s, %s, %s, %s, %s)",

            (agent, datetime.utcnow().isoformat(), command, json.dumps(result), cost))

        conn.commit()

        cur.close()

        conn.close()

    except Exception as e:

        print(f"DB: {e}")

def get_cost_summary():

    try:

        conn = psycopg2.connect(**DB_CONFIG)

        cur = conn.cursor()

        today = datetime.utcnow().strftime('%Y-%m-%d')

        month_start = datetime.utcnow().strftime('%Y-%m-01')

        cur.execute("SELECT COALESCE(SUM(cost_usd),0) FROM metadata WHERE timestamp::text >= %s", (today,))

        td = float(cur.fetchone()[0])

        cur.execute("SELECT COALESCE(SUM(cost_usd),0) FROM metadata WHERE timestamp::text >= %s", (month_start,))

        mo = float(cur.fetchone()[0])

        cur.execute("SELECT COALESCE(SUM(cost_usd),0) FROM metadata")

        tot = float(cur.fetchone()[0])

        cur.execute("SELECT agent, COUNT(*) FROM metadata WHERE timestamp::text >= %s GROUP BY agent ORDER BY COUNT(*) DESC LIMIT 8", (today,))

        ac = cur.fetchall()

        cur.close()

        conn.close()

        return {"today": td, "month": mo, "total": tot, "budget_remaining": MONTHLY_BUDGET_CEILING - mo, "agents": ac}

    except Exception as e:

        return {"error": str(e)}

def get_recent_commands(n=5):

    try:

        conn = psycopg2.connect(**DB_CONFIG)

        cur = conn.cursor()

        cur.execute("SELECT agent, command, timestamp FROM metadata ORDER BY timestamp DESC LIMIT %s", (n,))

        rows = cur.fetchall()

        cur.close()

        conn.close()

        return rows

    except:

        return []

def resolve_host(name):

    n = name.lower().strip()

    if n in INFRA_MAP:

        return n

    return HOST_ALIASES.get(n, None)

def is_blocked(cmd):

    for b in BLOCKED_COMMANDS:

        if b in cmd.lower():

            return True

    return False

def is_safe(cmd):

    first = cmd.lower().split()[0] if cmd.split() else ""

    return any(cmd.lower().startswith(s) or first == s for s in AUTO_APPROVE)

def run_ssh(host, cmd, timeout=30):

    try:

        r = subprocess.run(["ssh", "-o", "ConnectTimeout=10", host, cmd],

            capture_output=True, text=True, timeout=timeout)

        return {"ok": r.returncode == 0, "out": (r.stdout.strip() or r.stderr.strip() or "(empty)"), "host": host}

    except subprocess.TimeoutExpired:

        return {"ok": False, "out": f"Timeout ({timeout}s)", "host": host}

    except Exception as e:

        return {"ok": False, "out": str(e), "host": host}

def run_local(cmd, timeout=30):

    try:

        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)

        return {"ok": r.returncode == 0, "out": (r.stdout.strip() or r.stderr.strip() or "(empty)"), "host": "perseus"}

    except subprocess.TimeoutExpired:

        return {"ok": False, "out": f"Timeout ({timeout}s)", "host": "perseus"}

    except Exception as e:

        return {"ok": False, "out": str(e), "host": "perseus"}

def brave_search(query, count=3):

    try:

        resp = requests.get("https://api.search.brave.com/res/v1/web/search",

            headers={"X-Subscription-Token": BRAVE_API_KEY, "Accept": "application/json"},

            params={"q": query, "count": count}, timeout=10)

        if resp.status_code == 200:

            results = resp.json().get("web", {}).get("results", [])

            return [{"title": r.get("title", ""), "url": r.get("url", ""), "desc": r.get("description", "")[:200]} for r in results]

        return []

    except:

        return []

SYSTEM_PROMPT = """You are Perseus, the command AI for a Proxmox homelab infrastructure. You are direct, efficient, and technically competent.

Infrastructure you control (all have SSH access):

- node-1: Proxmox host 1 (primary compute)

- node-2: Proxmox host 2 (GPU compute node)

- CT 120 / perseus: Your home -- FastAPI, PostgreSQL, Discord bot

- CT 200 / vpn: WireGuard VPN

- CT 201 / dns: Pi-hole DNS/ad blocking

- CT 203 / backup: Proxmox Backup Server

- CT 205 / gpu: Ollama + local LLM inference on GPU

When asked to DO something on infrastructure, include commands as:

EXECUTE:hostname:command (one per line)

Examples:

"check disk on dns" -> EXECUTE:dns:df -h

"GPU status" -> EXECUTE:gpu:nvidia-smi

"whitelist google.com on pihole" -> EXECUTE:dns:pihole -w google.com

When asked about weather, news, or current events, you have web search. Include:

SEARCH:query

and the results will be appended.

For general questions, just answer. Keep responses concise."""

def call_grok(prompt, sys=None):

    msgs = []

    if sys:

        msgs.append({"role": "system", "content": sys})

    msgs.append({"role": "user", "content": prompt})

    try:

        r = requests.post("https://api.x.ai/v1/chat/completions",

            headers={"Authorization": f"Bearer {XAI_API_KEY}", "Content-Type": "application/json"},

            json={"model": "grok-4-1-fast-reasoning", "messages": msgs, "temperature": 0.7, "max_tokens": 1500}, timeout=60)

        if r.status_code == 200:

            d = r.json()

            txt = d["choices"][0]["message"]["content"].strip()

            u = d.get("usage", {})

            cost = (u.get("prompt_tokens", 0) / 1e6 * 2.0) + (u.get("completion_tokens", 0) / 1e6 * 10.0)

            return {"ok": True, "result": txt, "cost": cost, "model": "grok-4-1-fast-reasoning"}

        return {"ok": False, "result": f"Grok HTTP {r.status_code}", "cost": 0, "model": "grok"}

    except Exception as e:

        return {"ok": False, "result": f"Grok: {e}", "cost": 0, "model": "grok"}

def call_gpt(prompt, sys=None):

    msgs = []

    if sys:

        msgs.append({"role": "system", "content": sys})

    msgs.append({"role": "user", "content": prompt})

    try:

        r = requests.post("https://api.openai.com/v1/chat/completions",

            headers={"Authorization": f"Bearer {OPENAI_API_KEY}"},

            json={"model": "gpt-4o-mini", "messages": msgs, "temperature": 0.7, "max_tokens": 1500}, timeout=60)

        if r.status_code == 200:

            d = r.json()

            txt = d["choices"][0]["message"]["content"].strip()

            u = d.get("usage", {})

            cost = (u.get("prompt_tokens", 0) / 1e6 * 0.15) + (u.get("completion_tokens", 0) / 1e6 * 0.60)

            return {"ok": True, "result": txt, "cost": cost, "model": "gpt-4o-mini"}

        return {"ok": False, "result": f"GPT HTTP {r.status_code}", "cost": 0, "model": "gpt"}

    except Exception as e:

        return {"ok": False, "result": f"GPT: {e}", "cost": 0, "model": "gpt"}

def call_ollama(prompt):

    try:

        r = requests.post(f"{OLLAMA_URL}/api/chat",

            json={"model": OLLAMA_MODEL, "messages": [{"role": "user", "content": prompt}],

                  "stream": False, "options": {"temperature": 0.7}}, timeout=90)

        return {"ok": True, "result": r.json().get("message", {}).get("content", "").strip(), "cost": 0, "model": OLLAMA_MODEL}

    except Exception as e:

        return {"ok": False, "result": f"Ollama: {e}", "cost": 0, "model": OLLAMA_MODEL}

def call_brain(prompt, sys=None):

    if ACTIVE_BRAIN == "grok":

        return call_grok(prompt, sys)

    elif ACTIVE_BRAIN == "gpt":

        return call_gpt(prompt, sys)

    else:

        full = f"{sys}\n\nUser: {prompt}" if sys else prompt

        return call_ollama(full)

def process_ai_response(text):

    """Handle EXECUTE: and SEARCH: directives in AI responses."""

    lines = text.split("\n")

    out_lines = []

    exec_results = []

    for line in lines:

        s = line.strip()

        if s.startswith("EXECUTE:"):

            parts = s.replace("EXECUTE:", "").split(":", 1)

            if len(parts) == 2:

                host = resolve_host(parts[0].strip())

                cmd = parts[1].strip()

                if not host:

                    exec_results.append(f"[FAIL] Unknown host: `{parts[0]}`")

                elif is_blocked(cmd):

                    exec_results.append(f"[BLOCKED] `{cmd}`")

                elif is_safe(cmd):

                    r = run_local(cmd) if host == "perseus" else run_ssh(host, cmd)

                    st = "[OK]" if r["ok"] else "[FAIL]"

                    exec_results.append(f"{st} **{r['host']}** `{cmd}`\n```\n{r['out'][:800]}\n```")

                else:

                    exec_results.append(f"[PENDING] **Needs approval:** `{cmd}` on `{host}`\nRun: `!exec {host} {cmd}`")

            else:

                out_lines.append(line)

        elif s.startswith("SEARCH:"):

            query = s.replace("SEARCH:", "").strip()

            results = brave_search(query)

            if results:

                search_text = "\n".join(f"- **{r['title']}**: {r['desc']} ([link]({r['url']}))" for r in results)

                exec_results.append(f"**Search: {query}**\n{search_text}")

            else:

                exec_results.append(f"No results for: {query}")

        else:

            out_lines.append(line)

    final = "\n".join(out_lines).strip()

    if exec_results:

        final += "\n\n" + "\n".join(exec_results)

    return final

def x_sign(method, url, params, cs, ts):

    sp = "&".join(f"{urllib.parse.quote(k,safe='')}={urllib.parse.quote(v,safe='')}" for k, v in sorted(params.items()))

    bs = f"{method}&{urllib.parse.quote(url,safe='')}&{urllib.parse.quote(sp,safe='')}"

    sk = f"{urllib.parse.quote(cs,safe='')}&{urllib.parse.quote(ts,safe='')}"

    return base64.b64encode(hmac.new(sk.encode(), bs.encode(), hashlib.sha1).digest()).decode()

def post_tweet(text):

    url = "https://api.x.com/2/tweets"

    op = {"oauth_consumer_key": X_CONSUMER_KEY, "oauth_token": X_ACCESS_TOKEN,

          "oauth_signature_method": "HMAC-SHA1", "oauth_timestamp": str(int(time.time())),

          "oauth_nonce": uuid.uuid4().hex, "oauth_version": "1.0"}

    op["oauth_signature"] = x_sign("POST", url, op, X_CONSUMER_SECRET, X_ACCESS_TOKEN_SECRET)

    ah = "OAuth " + ", ".join(f'{urllib.parse.quote(k,safe="")}="{urllib.parse.quote(v,safe="")}"' for k, v in sorted(op.items()))

    try:

        r = requests.post(url, headers={"Authorization": ah, "Content-Type": "application/json"},

            json={"text": text}, timeout=30)

        if r.status_code in [200, 201]:

            tid = r.json().get("data", {}).get("id", "?")

            return {"ok": True, "url": f"https://x.com/status/{tid}"}

        return {"ok": False, "error": f"HTTP {r.status_code}: {r.text[:300]}"}

    except Exception as e:

        return {"ok": False, "error": str(e)}

def analytics_get(endpoint, extra_params=None):

    """Generic GET for analytics/ad platform API."""

    params = {"key": ANALYTICS_API_KEY}

    if extra_params:

        params.update(extra_params)

    try:

        r = requests.get(f"{ANALYTICS_BASE}{endpoint}", params=params, timeout=15)

        if r.status_code == 200:

            data = r.json()

            if data.get("status") == "success":

                return {"ok": True, "result": data.get("result", {})}

            return {"ok": False, "error": f"API error: {data}"}

        return {"ok": False, "error": f"HTTP {r.status_code}"}

    except Exception as e:

        return {"ok": False, "error": str(e)}

def analytics_balance():

    return analytics_get("/publisher/balance")

def analytics_stats(date=None, date2=None, group="date"):

    params = {"group": group}

    if date:

        params["date"] = date

    if date2:

        params["date2"] = date2

    return analytics_get("/publisher/listStats", params)

def analytics_full_report():

    """Pull balance + yesterday stats + 7-day stats."""

    today = datetime.utcnow().strftime('%Y-%m-%d')

    yesterday = (datetime.utcnow() - timedelta(days=1)).strftime('%Y-%m-%d')

    week_ago = (datetime.utcnow() - timedelta(days=7)).strftime('%Y-%m-%d')

    balance = analytics_balance()

    daily = analytics_stats(date=yesterday)

    weekly = analytics_stats(date=week_ago, date2=yesterday, group="date")

    return {

        "balance": balance.get("result", {}) if balance.get("ok") else {"error": balance.get("error")},

        "yesterday": daily.get("result", {}) if daily.get("ok") else {"error": daily.get("error")},

        "weekly": weekly.get("result", {}) if weekly.get("ok") else {"error": weekly.get("error")},

    }

def format_analytics_report(report):

    """Format analytics data for Discord display."""

    bal = report.get("balance", {})

    bal_str = f"${bal.get('balance', 'N/A')} {bal.get('currency', '')}" if "error" not in bal else f"[FAIL] {bal['error']}"

    yest = report.get("yesterday", {})

    y_revenue = y_impressions = y_clicks = y_cpm = "N/A"

    if "error" not in yest:

        try:

            total_rev = 0.0

            total_imp = 0

            total_clicks = 0

            found = False

            def walk_stats(obj):

                nonlocal total_rev, total_imp, total_clicks, found

                if isinstance(obj, dict):

                    if "revenue" in obj:

                        found = True

                        total_rev += float(obj.get("revenue", 0))

                        total_imp += int(obj.get("impressions", 0))

                        total_clicks += int(obj.get("clicks", 0))

                    else:

                        for v in obj.values():

                            walk_stats(v)

                elif isinstance(obj, list):

                    for item in obj:

                        walk_stats(item)

            walk_stats(yest)

            if found:

                y_revenue = f"${total_rev:.4f}"

                y_impressions = f"{total_imp:,}"

                y_clicks = f"{total_clicks:,}"

                y_cpm = f"${(total_rev / total_imp * 1000):.4f}" if total_imp > 0 else "N/A"

        except:

            pass

    week = report.get("weekly", {})

    w_revenue = "N/A"

    w_days = []

    if "error" not in week:

        try:

            total_rev_w = 0.0

            def walk_weekly(obj, date_key=None):

                nonlocal total_rev_w, w_days

                if isinstance(obj, dict):

                    if "revenue" in obj:

                        rev = float(obj.get("revenue", 0))

                        total_rev_w += rev

                        if date_key:

                            w_days.append((date_key, rev))

                    else:

                        for k, v in obj.items():

                            walk_weekly(v, date_key=k if len(k) == 10 and "-" in k else date_key)

                elif isinstance(obj, list):

                    for item in obj:

                        walk_weekly(item, date_key)

            walk_weekly(week)

            w_revenue = f"${total_rev_w:.4f}"

        except:

            pass

    txt = (

        f"**Revenue Report**\n--------------------\n"

        f"**Balance:** {bal_str}\n--------------------\n"

        f"**Yesterday:**\n"

        f"  Revenue: {y_revenue}\n"

        f"  Impressions: {y_impressions}\n"

        f"  Clicks: {y_clicks}\n"

        f"  CPM: {y_cpm}\n--------------------\n"

        f"**Last 7 Days:** {w_revenue} total\n"

    )

    if w_days:

        w_days.sort()

        for d, r in w_days:

            txt += f"  {d}: ${r:.4f}\n"

    return txt

BACKUP_TARGETS = {

    "perseus_ct": {

        "type": "vzdump",

        "ct": 120,

        "node": "node-1",

        "desc": "Perseus container (FastAPI, bot, configs)",

        "storage": "pbs-main",

        "keep": 7,

    },

    "postgres": {

        "type": "pg_dump",

        "host": "perseus",

        "db": "perseus",

        "dest": os.path.join(os.path.dirname(os.path.abspath(__file__)), "backups"),

        "keep": 7,

    },

}

def run_backup(target_name):

    """Execute a specific backup target. Returns status dict."""

    target = BACKUP_TARGETS.get(target_name)

    if not target:

        return {"ok": False, "error": f"Unknown target: {target_name}"}

    ts = datetime.utcnow().strftime('%Y-%m-%d_%H%M')

    if target["type"] == "pg_dump":

        dest = target["dest"]

        cmd = f"mkdir -p {dest} && pg_dump {target['db']} > {dest}/perseus_{ts}.sql && ls -la {dest}/"

        r = run_local(cmd)

        if r["ok"]:

            run_local(f"cd {dest} && ls -t *.sql | tail -n +{target['keep'] + 1} | xargs -r rm")

        return {"ok": r["ok"], "target": target_name, "type": "pg_dump", "output": r["out"][:500]}

    elif target["type"] == "vzdump":

        ct = target["ct"]

        cmd = f"vzdump {ct} --storage {target['storage']} --compress zstd --mode snapshot --quiet 1"

        r = run_ssh(target["node"], cmd, timeout=300)

        return {"ok": r["ok"], "target": target_name, "type": "vzdump", "output": r["out"][:500]}

    return {"ok": False, "error": "Unknown backup type"}

def run_all_backups():

    results = []

    for name in BACKUP_TARGETS:

        r = run_backup(name)

        results.append(r)

    return results

def format_backup_report(results):

    txt = "**Backup Report**\n--------------------\n"

    for r in results:

        st = "[OK]" if r.get("ok") else "[FAIL]"

        txt += f"{st} **{r.get('target', '?')}** ({r.get('type', '?')})\n"

        if not r.get("ok"):

            txt += f"  Error: {r.get('error', r.get('output', 'unknown'))[:200]}\n"

    txt += f"--------------------\n**Targets:** {len(BACKUP_TARGETS)} | **Time:** {datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')}"

    return txt

INFRA_THRESHOLDS = {

    "cpu_pct": 90,

    "ram_pct": 90,

    "disk_pct": 85,

    "gpu_temp_c": 80,

}

def check_host_health(host_name):

    """Check CPU, RAM, disk on a host. Returns health dict."""

    health = {"host": host_name, "ok": True, "alerts": [], "metrics": {}}

    if host_name == "perseus":

        runner = run_local

    else:

        runner = lambda cmd: run_ssh(host_name, cmd)

    r = runner("nproc && cat /proc/loadavg")

    if r["ok"]:

        try:

            lines = r["out"].split("\n")

            cores = int(lines[0].strip())

            load1 = float(lines[1].split()[0])

            cpu_pct = round((load1 / cores) * 100, 1)

            health["metrics"]["cpu_pct"] = cpu_pct

            health["metrics"]["load"] = load1

            health["metrics"]["cores"] = cores

            if cpu_pct > INFRA_THRESHOLDS["cpu_pct"]:

                health["alerts"].append(f"[CRIT] CPU {cpu_pct}% (load {load1}/{cores} cores)")

                health["ok"] = False

        except:

            pass

    r = runner("free -m | grep Mem")

    if r["ok"]:

        try:

            parts = r["out"].split()

            total = int(parts[1])

            used = int(parts[2])

            ram_pct = round((used / total) * 100, 1)

            health["metrics"]["ram_total_mb"] = total

            health["metrics"]["ram_used_mb"] = used

            health["metrics"]["ram_pct"] = ram_pct

            if ram_pct > INFRA_THRESHOLDS["ram_pct"]:

                health["alerts"].append(f"[CRIT] RAM {ram_pct}% ({used}/{total} MB)")

                health["ok"] = False

        except:

            pass

    r = runner("df -h / | tail -1")

    if r["ok"]:

        try:

            parts = r["out"].split()

            disk_pct = int(parts[4].replace("%", ""))

            health["metrics"]["disk_pct"] = disk_pct

            health["metrics"]["disk_size"] = parts[1]

            health["metrics"]["disk_used"] = parts[2]

            health["metrics"]["disk_avail"] = parts[3]

            if disk_pct > INFRA_THRESHOLDS["disk_pct"]:

                health["alerts"].append(f"[CRIT] Disk {disk_pct}% ({parts[2]}/{parts[1]})")

                health["ok"] = False

        except:

            pass

    return health

def check_gpu_health():

    health = {"host": "gpu", "ok": True, "alerts": [], "metrics": {}}

    r = run_ssh("gpu", "nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv,noheader,nounits")

    if r["ok"]:

        try:

            parts = [x.strip() for x in r["out"].split(",")]

            gpu_util = int(parts[0])

            mem_used = int(parts[1])

            mem_total = int(parts[2])

            temp = int(parts[3])

            health["metrics"]["gpu_util_pct"] = gpu_util

            health["metrics"]["vram_used_mb"] = mem_used

            health["metrics"]["vram_total_mb"] = mem_total

            health["metrics"]["gpu_temp_c"] = temp

            if temp > INFRA_THRESHOLDS["gpu_temp_c"]:

                health["alerts"].append(f"[CRIT] GPU Temp {temp}C")

                health["ok"] = False

        except:

            health["alerts"].append(f"[WARN] GPU parse error: {r['out'][:100]}")

    else:

        health["ok"] = False

        health["alerts"].append(f"[FAIL] Can't reach GPU: {r['out'][:100]}")

    return health

def check_ollama_health():

    health = {"host": "ollama", "ok": True, "alerts": [], "metrics": {}}

    try:

        r = requests.get(f"{OLLAMA_URL}/api/tags", timeout=5)

        models = [m["name"] for m in r.json().get("models", [])]

        health["metrics"]["available_models"] = models

        health["metrics"]["model_count"] = len(models)

    except:

        health["ok"] = False

        health["alerts"].append("[FAIL] Ollama unreachable")

    try:

        r = requests.get(f"{OLLAMA_URL}/api/ps", timeout=5)

        running = [m["name"] for m in r.json().get("models", [])]

        health["metrics"]["loaded_models"] = running

    except:

        pass

    return health

def full_infra_check():

    results = []

    for host in ["node-1", "node-2"]:

        results.append(check_host_health(host))

    for ct in ["perseus", "dns", "gpu"]:

        results.append(check_host_health(ct))

    results.append(check_gpu_health())

    results.append(check_ollama_health())

    return results

def format_infra_health(results):

    all_ok = all(r.get("ok", False) for r in results)

    overall = "[OK] All Clear" if all_ok else "[WARN] Issues Detected"

    txt = f"**Infrastructure Health** -- {overall}\n--------------------\n"

    for r in results:

        host = r["host"]