#!/usr/bin/env bash
# Stage 6.1 — Docker showcase / e2e stand for Monlet.
# Brings up the full stack with demo agents, drives scenarios, asserts all
# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
# incident open/resolved, outbox sent/retry/failed, disabled server notifications,
# agent spool replay), and verifies metrics + web pages.
set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$HERE/.." && pwd)"
TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}"
# Export so every `docker compose` call (up/exec/down) sees the UI token env,
# including the cleanup trap.
export MONLET_AUTH_TOKEN="$TOKEN"
BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
ADMISSION_MODE="${SHOWCASE_ADMISSION_MODE:-pending}"

# PH-review: dedicated project name so showcase does not collide with the
# smoke/dev stack (orphans, shared postgres volume, stale baseline schema).
COMPOSE=(docker compose
  -p monlet-showcase
  -f "$HERE/docker-compose.yml"
  -f "$HERE/docker-compose.showcase.yml")

log()  { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }

AUTH=(-H "Authorization: Bearer $TOKEN")
EXPECTED_AGENTS=8

cleanup() {
  if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
    log "tearing down stack"
    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
  else
    log "SHOWCASE_KEEP=1 → leaving stack running (web: $WEB_BASE)"
  fi
}
trap cleanup EXIT

wait_url() {
  local url="$1" label="$2" tries="${3:-60}"
  for i in $(seq 1 "$tries"); do
    if curl -sf -o /dev/null "$url"; then return 0; fi
    sleep 2
    [[ "$i" == "$tries" ]] && fail "$label not ready: $url"
  done
}

api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
wait_agents_visible() {
  local expected="$1"
  for i in $(seq 1 60); do
    local count
    count="$(api "/api/v1/agents?limit=500" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['items']))" 2>/dev/null || echo 0)"
    [[ "${count:-0}" -ge "$expected" ]] && return 0
    sleep 2
  done
  fail "agents did not register"
}

wait_agents_accepted() {
  local expected="$1"
  for i in $(seq 1 300); do
    local counts accepted pending
    counts="$(api "/api/v1/agents?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
items=d['items']
accepted=sum(1 for a in items if a['admission_state']=='accepted')
pending=sum(1 for a in items if a['admission_state']=='pending')
print(accepted, pending)
" 2>/dev/null || echo "0 0")"
    read -r accepted pending <<<"$counts"
    [[ "${accepted:-0}" -ge "$expected" && "${pending:-0}" -eq 0 ]] && return 0
    sleep 2
  done
  fail "agents were not accepted in time"
}

# Multi-phase startup keeps DB migration explicit, then agents register as
# pending. Full assertions wait read-only until an operator accepts them.
log "pre-clean previous showcase stack (if any)"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true

log "phase 1: postgres"
# --wait blocks until the postgres healthcheck (pg_isready) reports healthy,
# so the alembic step below does not race a still-booting DB.
"${COMPOSE[@]}" up -d --wait --build postgres

log "building current server image for migrations"
"${COMPOSE[@]}" build server >/dev/null || fail "server image build failed"

log "applying alembic migrations (one-off, server not yet running)"
"${COMPOSE[@]}" run --rm --no-deps server sh -c "alembic upgrade head" \
  >/dev/null || fail "alembic upgrade failed"

log "phase 2: server + agents + web"
"${COMPOSE[@]}" up -d --build

log "waiting for server /api/v1/ready"
wait_url "$BASE/api/v1/ready" "server" 90

log "waiting for web /"
wait_url "$WEB_BASE/" "web" 90

log "waiting for agents to register as pending"
wait_agents_visible "$EXPECTED_AGENTS"

case "$ADMISSION_MODE" in
  pending)
    log "agents are pending; automatic accept is disabled"
    log "accept them in the UI/API, or rerun with SHOWCASE_ADMISSION_MODE=wait to continue assertions after manual admission"
    exit 0
    ;;
  wait)
    log "waiting for operator to accept agents (read-only polling)"
    wait_agents_accepted "$EXPECTED_AGENTS"
    ;;
  *)
    fail "unknown SHOWCASE_ADMISSION_MODE=$ADMISSION_MODE (expected pending or wait)"
    ;;
esac

# --- Phase A: let agents run a few cycles ---
log "letting demo agents push events for ~25s"
sleep 25

log "asserting agent-01 labels include scenario and monlet_agent_version"
api "/api/v1/agents/agent-01" | python3 -c "
import json,sys
d=json.load(sys.stdin)
labels=d.get('labels') or {}
ver=labels.get('monlet_agent_version')
ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
print('agent-01 labels:', labels)
sys.exit(0 if ok else 1)" \
  || fail "agent-01 labels missing scenario or numeric monlet_agent_version"

log "asserting agent-01 has 4 distinct check statuses (ok/warning/critical/unknown)"
api "/api/v1/checks?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
want={'ok','warning','critical','unknown'}
have={c['status'] for c in d['items'] if c['agent_id']=='agent-01'}
missing=want-have
print('agent-01 statuses:', sorted(have))
sys.exit(0 if not missing else 1)" \
  || fail "agent-01 missing one of ok/warning/critical/unknown"

log "asserting timeout check is explicit critical"
api "/api/v1/checks?agent_id=agent-04&limit=50" | python3 -c "
import json,sys
d=json.load(sys.stdin)
slow=next((c for c in d['items'] if c['check_id']=='slow_probe'), None)
print('slow_probe:', slow)
ok=slow and slow['status']=='critical' and slow['exit_code']==2 and 'timed out' in (slow.get('summary') or '')
sys.exit(0 if ok else 1)" \
  || fail "agent-04/slow_probe is not explicit critical timeout"

# --- Phase B: trigger resolve scenario ---
log "triggering resolve: touch /state/resolve.flag in agent-resolve"
"${COMPOSE[@]}" exec -T agent-resolve sh -c 'touch /state/resolve.flag' \
  || fail "could not touch resolve flag"

log "waiting for resolve event to be processed (~15s)"
sleep 15

log "asserting resolved incident exists for agent-02/flapping"
api "/api/v1/incidents?state=resolved&limit=500" \
  | python3 -c "
import json,sys
d=json.load(sys.stdin)
ok=any(i['agent_id']=='agent-02' and i['check_id']=='flapping' for i in d['items'])
sys.exit(0 if ok else 1)" \
  || fail "no resolved incident for agent-02/flapping"

# --- Phase C: disabled server notifications ---
log "asserting notifications-disabled incident exists but no outbox row for it"
PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
for i in d['items']:
    if i['agent_id']=='agent-05' and i['check_id']=='promcheck':
        print(i['id']); break")"
[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
  && fail "outbox unexpectedly contains row for prometheus-owned incident $PROM_INC"
log "no outbox row for notifications-disabled incident"

# --- Phase D: stop stale/dead agents to drive lifecycle transitions ---
log "stopping agent-stale and agent-dead to age their heartbeats"
"${COMPOSE[@]}" stop agent-stale agent-dead >/dev/null

log "waiting ~25s for stale transition"
sleep 25
api "/api/v1/agents/agent-07" | grep -q '"status":"stale"' \
  || api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
  || fail "agent-07 did not transition to stale/dead"

log "waiting another ~25s for dead transition"
sleep 25
api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
  || fail "agent-07 did not transition to dead"
api "/api/v1/agents/agent-08"  | grep -q '"status":"dead"' \
  || fail "agent-08 did not transition to dead"

log "asserting dead agents have open liveness incidents"
api "/api/v1/incidents?state=open&limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
want={'agent-07','agent-08'}
have={
    i['agent_id']
    for i in d['items']
    if i['check_id']=='agent_liveness' and i['state']=='open' and i['severity']=='critical'
}
missing=want-have
print('liveness incidents:', sorted(have))
sys.exit(0 if not missing else 1)" \
  || fail "missing critical agent_liveness incident for dead agents"

# At least one agent must still be alive (agent-01 keeps pushing).
api "/api/v1/agents/agent-01" | grep -q '"status":"alive"' \
  || fail "agent-01 not alive"

# --- Phase E: outbox state coverage (live via webhook + debug) ---
log "waiting for notifier worker cycles (~10s)"
sleep 10

OUTBOX="$(api "/api/v1/notifiers/outbox?limit=500")"
echo "$OUTBOX" | python3 -c "
import json,sys
d=json.load(sys.stdin)
states={i['state'] for i in d['items']}
need={'sent','retry','failed'}
missing=need-states
print('outbox states:', sorted(states))
sys.exit(0 if not missing else 2)" \
  || fail "outbox missing one of sent/retry/failed"

# --- Phase G: spool replay assertion ---
log "forcing server outage so agent-spool-replay writes to local spool"
"${COMPOSE[@]}" stop server >/dev/null
spool_count=0
for i in $(seq 1 30); do
  spool_count=$("${COMPOSE[@]}" exec -T agent-spool-replay \
    sh -c 'ls /var/lib/monlet-agent/spool 2>/dev/null | wc -l' 2>/dev/null \
    | tr -d '[:space:]' || echo 0)
  [[ "${spool_count:-0}" -gt 0 ]] && break
  sleep 2
done
[[ "${spool_count:-0}" -gt 0 ]] || fail "agent-spool-replay did not spool any events"
log "spool populated with $spool_count file(s)"

log "restarting server and waiting for replay"
"${COMPOSE[@]}" up -d --no-deps server >/dev/null
wait_url "$BASE/api/v1/ready" "server after replay outage" 90
sleep 15

log "asserting agent-06 spool replay events arrived without duplicates"
api "/api/v1/events/query?agent_id=agent-06&check_id=replay_check&limit=500" \
  | python3 -c "
import json,sys
d=json.load(sys.stdin)
items=d['items']
ids=[e['event_id'] for e in items]
if len(items) < 2:
    print(f'too few events: {len(items)}'); sys.exit(1)
if len(set(ids)) != len(ids):
    print('duplicate event_ids'); sys.exit(2)
print(f'spool replay events: {len(items)} unique')" \
  || fail "spool replay assertion failed"

# --- Phase H: metrics ---
log "server /metrics contains monlet_server_*"
curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
  || fail "server metrics missing monlet_server_*"

log "agent-01 /metrics contains monlet_agent_check_status"
curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
  || fail "agent metrics missing monlet_agent_check_status"

# --- Phase I: web UI ---
log "web pages render with expected content"
curl -sfL "$WEB_BASE/"         | grep -q "Agents"  || fail "web / did not redirect to Agents"
curl -sf "$WEB_BASE/agents"    | grep -q "mixed-status" || fail "web /agents missing mixed-status label"
curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
INCIDENTS_HTML="$(curl -sfL "$WEB_BASE/incidents?state=open&sort=status&direction=desc")"
echo "$INCIDENTS_HTML" | grep -q "agent-07" || fail "web /incidents missing agent-07"
echo "$INCIDENTS_HTML" | grep -q "agent-08" || fail "web /incidents missing agent-08"
curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"

if [[ "${SHOWCASE_KEEP:-0}" == "1" ]]; then
  log "refreshing transient containers before leaving stack"
  "${COMPOSE[@]}" stop \
    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
    agent-spool-replay >/dev/null || true
  "${COMPOSE[@]}" stop mock-webhook server >/dev/null || true
  "${COMPOSE[@]}" rm -f \
    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
    agent-spool-replay agent-stale agent-dead server mock-webhook >/dev/null || true
  "${COMPOSE[@]}" up -d --no-deps mock-webhook server >/dev/null \
    || fail "could not restart runtime services"
  wait_url "$BASE/api/v1/ready" "server after refresh" 90
  "${COMPOSE[@]}" up -d --no-deps \
    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
    agent-spool-replay >/dev/null \
    || fail "could not restart live showcase agents"
  wait_agents_visible "$EXPECTED_AGENTS"
  wait_agents_accepted "$EXPECTED_AGENTS"
  "${COMPOSE[@]}" create agent-stale agent-dead >/dev/null \
    || fail "could not recreate transient containers"
fi

log "ALL SHOWCASE ASSERTIONS PASSED"