Implement Monlet MVP stack and UI updates

deploy/e2e-showcase.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# Stage 6.1 — Docker showcase / e2e stand for Monlet.
+# Brings up the full stack with demo agents, drives scenarios, asserts all
+# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
+# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications,
+# agent spool replay), and verifies metrics + web pages.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
+
+COMPOSE=(docker compose
+  -f "$HERE/docker-compose.yml"
+  -f "$HERE/docker-compose.showcase.yml")
+
+log()  { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }
+
+AUTH=(-H "Authorization: Bearer $TOKEN")
+
+cleanup() {
+  if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
+    log "tearing down stack"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  else
+    log "SHOWCASE_KEEP=1 → leaving stack running (web: $WEB_BASE)"
+  fi
+}
+trap cleanup EXIT
+
+wait_url() {
+  local url="$1" label="$2" tries="${3:-60}"
+  for i in $(seq 1 "$tries"); do
+    if curl -sf -o /dev/null "$url"; then return 0; fi
+    sleep 2
+    [[ "$i" == "$tries" ]] && fail "$label not ready: $url"
+  done
+}
+
+api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
+
+log "starting full showcase stack"
+MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build
+
+log "waiting for server /api/v1/ready"
+wait_url "$BASE/api/v1/ready" "server" 90
+
+log "waiting for web /"
+wait_url "$WEB_BASE/" "web" 90
+
+# --- Phase A: let agents run a few cycles ---
+log "letting demo agents push events for ~25s"
+sleep 25
+
+log "asserting agent labels include scenario and monlet_agent_version"
+api "/api/v1/agents/agent-mixed" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+labels=d.get('labels') or {}
+ver=labels.get('monlet_agent_version')
+ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
+print('agent-mixed labels:', labels)
+sys.exit(0 if ok else 1)" \
+  || fail "agent-mixed labels missing scenario or numeric monlet_agent_version"
+
+log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)"
+api "/api/v1/checks?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+want={'ok','warning','critical','unknown'}
+have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'}
+missing=want-have
+print('agent-mixed statuses:', sorted(have))
+sys.exit(0 if not missing else 1)" \
+  || fail "agent-mixed missing one of ok/warning/critical/unknown"
+
+# --- Phase B: trigger resolve scenario ---
+log "triggering resolve: touch /state/resolve.flag in agent-resolve"
+"${COMPOSE[@]}" exec -T agent-resolve sh -c 'touch /state/resolve.flag' \
+  || fail "could not touch resolve flag"
+
+log "waiting for resolve event to be processed (~15s)"
+sleep 15
+
+log "asserting resolved incident exists for agent-resolve/flapping"
+api "/api/v1/incidents?state=resolved&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items'])
+sys.exit(0 if ok else 1)" \
+  || fail "no resolved incident for agent-resolve/flapping"
+
+# --- Phase C: disabled server notifications ---
+log "asserting notifications-disabled incident exists but no outbox row for it"
+PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+for i in d['items']:
+    if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck':
+        print(i['id']); break")"
+[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
+api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
+  && fail "outbox unexpectedly contains row for prometheus-owned incident $PROM_INC"
+log "no outbox row for notifications-disabled incident"
+
+# --- Phase D: stop stale/dead agents to drive lifecycle transitions ---
+log "stopping agent-stale and agent-dead to age their heartbeats"
+"${COMPOSE[@]}" stop agent-stale agent-dead >/dev/null
+
+log "waiting ~25s for stale transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \
+  || api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to stale/dead"
+
+log "waiting another ~25s for dead transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to dead"
+api "/api/v1/agents/agent-dead"  | grep -q '"status":"dead"' \
+  || fail "agent-dead did not transition to dead"
+
+# At least one agent must still be alive (agent-mixed keeps pushing).
+api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \
+  || fail "agent-mixed not alive"
+
+# --- Phase E: outbox state coverage (live via webhook + debug) ---
+log "waiting for notifier worker cycles (~10s)"
+sleep 10
+
+OUTBOX="$(api "/api/v1/notifiers/outbox?limit=500")"
+echo "$OUTBOX" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+states={i['state'] for i in d['items']}
+need={'sent','retry','failed'}
+missing=need-states
+print('outbox states:', sorted(states))
+sys.exit(0 if not missing else 2)" \
+  || fail "outbox missing one of sent/retry/failed"
+
+# --- Phase F: seed discarded row, wait for worker ---
+log "seeding 'discarded' outbox row via SQL"
+"${COMPOSE[@]}" exec -T postgres \
+  psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \
+  < "$HERE/showcase/seed_discarded.sql" >/dev/null \
+  || fail "seed_discarded.sql failed"
+
+log "waiting for worker to mark unknown-notifier row as discarded (~10s)"
+sleep 10
+api "/api/v1/notifiers/outbox?state=discarded&limit=10" \
+  | grep -q '"state":"discarded"' \
+  || fail "no outbox row in state=discarded"
+
+# --- Phase G: spool replay assertion ---
+log "asserting agent-spool-replay events arrived without duplicates"
+api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+items=d['items']
+ids=[e['event_id'] for e in items]
+if len(items) < 2:
+    print(f'too few events: {len(items)}'); sys.exit(1)
+if len(set(ids)) != len(ids):
+    print('duplicate event_ids'); sys.exit(2)
+print(f'spool replay events: {len(items)} unique')" \
+  || fail "spool replay assertion failed"
+
+# --- Phase H: metrics ---
+log "server /metrics contains monlet_server_*"
+curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
+  || fail "server metrics missing monlet_server_*"
+
+log "agent-mixed /metrics contains monlet_agent_check_status"
+curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
+  || fail "agent metrics missing monlet_agent_check_status"
+
+# --- Phase I: web UI ---
+log "web pages render with expected content"
+curl -sf "$WEB_BASE/"          | grep -q "Overview"  || fail "web / missing Overview"
+curl -sf "$WEB_BASE/agents"    | grep -q "agent-mixed" || fail "web /agents missing agent-mixed"
+curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
+curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve"
+curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
+curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
+
+log "ALL SHOWCASE ASSERTIONS PASSED"