Harden production workflows and agent admission

2026-05-28 14:19:27 +04:00
parent a2e88b4e76
commit 37b1a1d6d6
109 changed files with 4927 additions and 894 deletions
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -41,9 +41,9 @@ Boots the stack, hits `/health`, `/ready`, `/metrics`, ingests example payloads

 Full-fat demo stack with several dockerized agents driving the system through
 every witness state: agent alive/stale/dead, check ok/warning/critical/unknown,
-incident open/resolved, outbox sent/retry/failed/discarded,
+incident open/resolved, outbox sent/retry/failed,
 anti-double-alerting with `notifications_enabled=false`, and agent spool
-replay against a server that becomes available after the agent starts.
+replay after a controlled server outage.

 ```sh
 bash deploy/e2e-showcase.sh
@@ -52,7 +52,7 @@ bash deploy/e2e-showcase.sh
 Internally:

 ```sh
-MONLET_AUTH_TOKEN=monlet-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
+MONLET_AUTH_TOKEN=monlet-ui-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
 ```

 `SHOWCASE_KEEP=1 bash deploy/e2e-showcase.sh` leaves the stack up so you can
@@ -71,7 +71,7 @@ for demo/e2e only — production deploys the Go binary under systemd.
 - `smoke.sh` — local stack smoke entry point.
 - `docker-compose.showcase.yml` — override that adds demo agents + mock webhook.
 - `docker/agent.Dockerfile` — demo/e2e agent image (not for production).
- `showcase/` — demo agent configs, check scripts, mock webhook, seed SQL.
+- `showcase/` — demo agent configs, check scripts, and mock webhook.
 - `e2e-showcase.sh` — full showcase runner (Stage 6.1).

 Docker images install Python/Node project dependencies inside the image; host venvs are never mounted (see `AGENTS.md`).
--- a/deploy/docker-compose.showcase.yml
+++ b/deploy/docker-compose.showcase.yml
@@ -7,7 +7,7 @@ services:
  server:
    environment:
      MONLET_STALE_AFTER_SEC: "20"
-      MONLET_DEAD_AFTER_SEC: "40"
+      MONLET_DEAD_AFTER_SEC: "30"
      MONLET_DETECTOR_TICK_SEC: "5"
      MONLET_EVENTS_RETENTION_MONTHS: "1"
      MONLET_OUTBOX_RETENTION_MAX_ROWS: "1000"
@@ -28,6 +28,7 @@ services:

  agent-mixed:
    image: monlet-showcase-agent
+    hostname: agent-01
    build:
      context: ..
      dockerfile: deploy/docker/agent.Dockerfile
@@ -36,6 +37,7 @@ services:
    volumes:
      - ./showcase/agents/mixed.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - mixed_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -44,11 +46,13 @@ services:

  agent-resolve:
    image: monlet-showcase-agent
+    hostname: agent-02
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/resolve.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - resolve_agent_state:/var/lib/monlet-agent
      - resolve_state:/state
    depends_on:
      server:
@@ -58,11 +62,13 @@ services:

  agent-flap:
    image: monlet-showcase-agent
+    hostname: agent-03
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/flap.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - flap_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -71,11 +77,13 @@ services:

  agent-timeout:
    image: monlet-showcase-agent
+    hostname: agent-04
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/timeout.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - timeout_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -84,11 +92,13 @@ services:

  agent-prometheus-owned:
    image: monlet-showcase-agent
+    hostname: agent-05
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/prometheus_owned.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - prometheus_owned_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -97,11 +107,13 @@ services:

  agent-stale:
    image: monlet-showcase-agent
+    hostname: agent-07
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/stale.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - stale_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -110,11 +122,13 @@ services:

  agent-dead:
    image: monlet-showcase-agent
+    hostname: agent-08
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
      - ./showcase/agents/dead.toml:/etc/monlet/agent.toml:ro
      - ./showcase/checks:/opt/monlet/checks:ro
+      - dead_agent_state:/var/lib/monlet-agent
    depends_on:
      server:
        condition: service_healthy
@@ -124,6 +138,7 @@ services:
  # No depends_on:server — starts in parallel so it must spool events until server is ready.
  agent-spool-replay:
    image: monlet-showcase-agent
+    hostname: agent-06
    environment:
      MONLET_CONFIG: /etc/monlet/agent.toml
    volumes:
@@ -135,5 +150,12 @@ services:
        condition: service_healthy

 volumes:
+  mixed_agent_state:
+  resolve_agent_state:
  resolve_state:
+  flap_agent_state:
+  timeout_agent_state:
+  prometheus_owned_agent_state:
+  stale_agent_state:
+  dead_agent_state:
  spool_replay_state:
--- a/deploy/docker-compose.yml
+++ b/deploy/docker-compose.yml
@@ -1,10 +1,11 @@
 services:
  postgres:
-    image: postgres:16-alpine
+    image: postgres:16
    environment:
      POSTGRES_USER: monlet
      POSTGRES_PASSWORD: monlet
      POSTGRES_DB: monlet
+      POSTGRES_INITDB_ARGS: "--auth-local=scram-sha-256 --auth-host=scram-sha-256"
      TZ: UTC
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U monlet -d monlet"]
@@ -14,7 +15,7 @@ services:
    volumes:
      - monlet_pg:/var/lib/postgresql/data
    ports:
-      - "5432:5432"
+      - "${MONLET_POSTGRES_HOST_PORT:-5432}:5432"

  server:
    build:
@@ -36,12 +37,21 @@ services:
      - -c
      - "alembic upgrade head && uvicorn monlet_server.main:app --host 0.0.0.0 --port 8000"
    ports:
-      - "8000:8000"
+      - "${MONLET_SERVER_HOST_PORT:-8000}:8000"
    healthcheck:
      test: ["CMD-SHELL", "python -c 'import urllib.request,sys;sys.exit(0) if urllib.request.urlopen(\"http://localhost:8000/api/v1/ready\").status==200 else sys.exit(1)'"]
      interval: 5s
      timeout: 3s
      retries: 30
+    # PH-015: production hardening for runtime containers.
+    read_only: true
+    tmpfs:
+      - /tmp:size=64M
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    stop_grace_period: 15s

  web:
    build:
@@ -53,9 +63,29 @@ services:
      MONLET_API_BASE_URL: http://server:8000
      MONLET_API_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required}
      MONLET_WEB_TIME_ZONE: ${MONLET_WEB_TIME_ZONE:-UTC}
+      MONLET_WEB_AUTH_USERNAME: ${MONLET_WEB_AUTH_USERNAME:-}
+      MONLET_WEB_AUTH_PASSWORD: ${MONLET_WEB_AUTH_PASSWORD:-}
+      MONLET_WEB_TRUST_PROXY_AUTH: ${MONLET_WEB_TRUST_PROXY_AUTH:-false}
      TZ: ${MONLET_WEB_TIME_ZONE:-UTC}
    ports:
-      - "3000:3000"
+      - "${MONLET_WEB_HOST_PORT:-3000}:3000"
+    # PH-015: production hardening for the Next.js runtime. Next writes a small
+    # cache under .next/cache so we keep that path writable via tmpfs.
+    read_only: true
+    tmpfs:
+      - /tmp:size=64M
+      - /app/.next/cache:size=64M
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O- http://localhost:3000/api/health >/dev/null 2>&1 || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
+    stop_grace_period: 15s

  prometheus:
    image: prom/prometheus:v2.55.0
--- a/deploy/e2e-showcase.sh
+++ b/deploy/e2e-showcase.sh
@@ -2,18 +2,25 @@
 # Stage 6.1 — Docker showcase / e2e stand for Monlet.
 # Brings up the full stack with demo agents, drives scenarios, asserts all
 # witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
-# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications,
+# incident open/resolved, outbox sent/retry/failed, disabled server notifications,
 # agent spool replay), and verifies metrics + web pages.
 set -euo pipefail

 HERE="$(cd "$(dirname "$0")" && pwd)"
 ROOT="$(cd "$HERE/.." && pwd)"
-TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}"
+# Export so every `docker compose` call (up/exec/down) sees the UI token env,
+# including the cleanup trap.
+export MONLET_AUTH_TOKEN="$TOKEN"
 BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
 WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
 AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
+ADMISSION_MODE="${SHOWCASE_ADMISSION_MODE:-pending}"

+# PH-review: dedicated project name so showcase does not collide with the
+# smoke/dev stack (orphans, shared postgres volume, stale baseline schema).
 COMPOSE=(docker compose
+  -p monlet-showcase
  -f "$HERE/docker-compose.yml"
  -f "$HERE/docker-compose.showcase.yml")

@@ -21,6 +28,7 @@ log()  { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
 fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }

 AUTH=(-H "Authorization: Bearer $TOKEN")
+EXPECTED_AGENTS=8

 cleanup() {
  if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
@@ -42,9 +50,55 @@ wait_url() {
 }

 api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
+wait_agents_visible() {
+  local expected="$1"
+  for i in $(seq 1 60); do
+    local count
+    count="$(api "/api/v1/agents?limit=500" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['items']))" 2>/dev/null || echo 0)"
+    [[ "${count:-0}" -ge "$expected" ]] && return 0
+    sleep 2
+  done
+  fail "agents did not register"
+}

-log "starting full showcase stack"
-MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build
+wait_agents_accepted() {
+  local expected="$1"
+  for i in $(seq 1 300); do
+    local counts accepted pending
+    counts="$(api "/api/v1/agents?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+items=d['items']
+accepted=sum(1 for a in items if a['admission_state']=='accepted')
+pending=sum(1 for a in items if a['admission_state']=='pending')
+print(accepted, pending)
+" 2>/dev/null || echo "0 0")"
+    read -r accepted pending <<<"$counts"
+    [[ "${accepted:-0}" -ge "$expected" && "${pending:-0}" -eq 0 ]] && return 0
+    sleep 2
+  done
+  fail "agents were not accepted in time"
+}
+
+# Multi-phase startup keeps DB migration explicit, then agents register as
+# pending. Full assertions wait read-only until an operator accepts them.
+log "pre-clean previous showcase stack (if any)"
+"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+
+log "phase 1: postgres"
+# --wait blocks until the postgres healthcheck (pg_isready) reports healthy,
+# so the alembic step below does not race a still-booting DB.
+"${COMPOSE[@]}" up -d --wait --build postgres
+
+log "building current server image for migrations"
+"${COMPOSE[@]}" build server >/dev/null || fail "server image build failed"
+
+log "applying alembic migrations (one-off, server not yet running)"
+"${COMPOSE[@]}" run --rm --no-deps server sh -c "alembic upgrade head" \
+  >/dev/null || fail "alembic upgrade failed"
+
+log "phase 2: server + agents + web"
+"${COMPOSE[@]}" up -d --build

 log "waiting for server /api/v1/ready"
 wait_url "$BASE/api/v1/ready" "server" 90
@@ -52,31 +106,59 @@ wait_url "$BASE/api/v1/ready" "server" 90
 log "waiting for web /"
 wait_url "$WEB_BASE/" "web" 90

+log "waiting for agents to register as pending"
+wait_agents_visible "$EXPECTED_AGENTS"
+
+case "$ADMISSION_MODE" in
+  pending)
+    log "agents are pending; automatic accept is disabled"
+    log "accept them in the UI/API, or rerun with SHOWCASE_ADMISSION_MODE=wait to continue assertions after manual admission"
+    exit 0
+    ;;
+  wait)
+    log "waiting for operator to accept agents (read-only polling)"
+    wait_agents_accepted "$EXPECTED_AGENTS"
+    ;;
+  *)
+    fail "unknown SHOWCASE_ADMISSION_MODE=$ADMISSION_MODE (expected pending or wait)"
+    ;;
+esac
+
 # --- Phase A: let agents run a few cycles ---
 log "letting demo agents push events for ~25s"
 sleep 25

-log "asserting agent labels include scenario and monlet_agent_version"
-api "/api/v1/agents/agent-mixed" | python3 -c "
+log "asserting agent-01 labels include scenario and monlet_agent_version"
+api "/api/v1/agents/agent-01" | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
 labels=d.get('labels') or {}
 ver=labels.get('monlet_agent_version')
 ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
-print('agent-mixed labels:', labels)
+print('agent-01 labels:', labels)
 sys.exit(0 if ok else 1)" \
-  || fail "agent-mixed labels missing scenario or numeric monlet_agent_version"
+  || fail "agent-01 labels missing scenario or numeric monlet_agent_version"

-log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)"
+log "asserting agent-01 has 4 distinct check statuses (ok/warning/critical/unknown)"
 api "/api/v1/checks?limit=500" | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
 want={'ok','warning','critical','unknown'}
-have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'}
+have={c['status'] for c in d['items'] if c['agent_id']=='agent-01'}
 missing=want-have
-print('agent-mixed statuses:', sorted(have))
+print('agent-01 statuses:', sorted(have))
 sys.exit(0 if not missing else 1)" \
-  || fail "agent-mixed missing one of ok/warning/critical/unknown"
+  || fail "agent-01 missing one of ok/warning/critical/unknown"
+
+log "asserting timeout check is explicit critical"
+api "/api/v1/checks?agent_id=agent-04&limit=50" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+slow=next((c for c in d['items'] if c['check_id']=='slow_probe'), None)
+print('slow_probe:', slow)
+ok=slow and slow['status']=='critical' and slow['exit_code']==2 and 'timed out' in (slow.get('summary') or '')
+sys.exit(0 if ok else 1)" \
+  || fail "agent-04/slow_probe is not explicit critical timeout"

 # --- Phase B: trigger resolve scenario ---
 log "triggering resolve: touch /state/resolve.flag in agent-resolve"
@@ -86,14 +168,14 @@ log "triggering resolve: touch /state/resolve.flag in agent-resolve"
 log "waiting for resolve event to be processed (~15s)"
 sleep 15

-log "asserting resolved incident exists for agent-resolve/flapping"
+log "asserting resolved incident exists for agent-02/flapping"
 api "/api/v1/incidents?state=resolved&limit=500" \
  | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
-ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items'])
+ok=any(i['agent_id']=='agent-02' and i['check_id']=='flapping' for i in d['items'])
 sys.exit(0 if ok else 1)" \
-  || fail "no resolved incident for agent-resolve/flapping"
+  || fail "no resolved incident for agent-02/flapping"

 # --- Phase C: disabled server notifications ---
 log "asserting notifications-disabled incident exists but no outbox row for it"
@@ -101,7 +183,7 @@ PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
 for i in d['items']:
-    if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck':
+    if i['agent_id']=='agent-05' and i['check_id']=='promcheck':
        print(i['id']); break")"
 [[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
 api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
@@ -114,22 +196,22 @@ log "stopping agent-stale and agent-dead to age their heartbeats"

 log "waiting ~25s for stale transition"
 sleep 25
-api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \
-  || api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
-  || fail "agent-stale did not transition to stale/dead"
+api "/api/v1/agents/agent-07" | grep -q '"status":"stale"' \
+  || api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
+  || fail "agent-07 did not transition to stale/dead"

 log "waiting another ~25s for dead transition"
 sleep 25
-api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
-  || fail "agent-stale did not transition to dead"
-api "/api/v1/agents/agent-dead"  | grep -q '"status":"dead"' \
-  || fail "agent-dead did not transition to dead"
+api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
+  || fail "agent-07 did not transition to dead"
+api "/api/v1/agents/agent-08"  | grep -q '"status":"dead"' \
+  || fail "agent-08 did not transition to dead"

 log "asserting dead agents have open liveness incidents"
 api "/api/v1/incidents?state=open&limit=500" | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
-want={'agent-stale','agent-dead'}
+want={'agent-07','agent-08'}
 have={
    i['agent_id']
    for i in d['items']
@@ -140,9 +222,9 @@ print('liveness incidents:', sorted(have))
 sys.exit(0 if not missing else 1)" \
  || fail "missing critical agent_liveness incident for dead agents"

-# At least one agent must still be alive (agent-mixed keeps pushing).
-api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \
-  || fail "agent-mixed not alive"
+# At least one agent must still be alive (agent-01 keeps pushing).
+api "/api/v1/agents/agent-01" | grep -q '"status":"alive"' \
+  || fail "agent-01 not alive"

 # --- Phase E: outbox state coverage (live via webhook + debug) ---
 log "waiting for notifier worker cycles (~10s)"
@@ -159,22 +241,27 @@ print('outbox states:', sorted(states))
 sys.exit(0 if not missing else 2)" \
  || fail "outbox missing one of sent/retry/failed"

-# --- Phase F: seed discarded row, wait for worker ---
-log "seeding 'discarded' outbox row via SQL"
-"${COMPOSE[@]}" exec -T postgres \
-  psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \
-  < "$HERE/showcase/seed_discarded.sql" >/dev/null \
-  || fail "seed_discarded.sql failed"
-
-log "waiting for worker to mark unknown-notifier row as discarded (~10s)"
-sleep 10
-api "/api/v1/notifiers/outbox?state=discarded&limit=10" \
-  | grep -q '"state":"discarded"' \
-  || fail "no outbox row in state=discarded"
-
 # --- Phase G: spool replay assertion ---
-log "asserting agent-spool-replay events arrived without duplicates"
-api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \
+log "forcing server outage so agent-spool-replay writes to local spool"
+"${COMPOSE[@]}" stop server >/dev/null
+spool_count=0
+for i in $(seq 1 30); do
+  spool_count=$("${COMPOSE[@]}" exec -T agent-spool-replay \
+    sh -c 'ls /var/lib/monlet-agent/spool 2>/dev/null | wc -l' 2>/dev/null \
+    | tr -d '[:space:]' || echo 0)
+  [[ "${spool_count:-0}" -gt 0 ]] && break
+  sleep 2
+done
+[[ "${spool_count:-0}" -gt 0 ]] || fail "agent-spool-replay did not spool any events"
+log "spool populated with $spool_count file(s)"
+
+log "restarting server and waiting for replay"
+"${COMPOSE[@]}" up -d --no-deps server >/dev/null
+wait_url "$BASE/api/v1/ready" "server after replay outage" 90
+sleep 15
+
+log "asserting agent-06 spool replay events arrived without duplicates"
+api "/api/v1/events/query?agent_id=agent-06&check_id=replay_check&limit=500" \
  | python3 -c "
 import json,sys
 d=json.load(sys.stdin)
@@ -192,17 +279,40 @@ log "server /metrics contains monlet_server_*"
 curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
  || fail "server metrics missing monlet_server_*"

-log "agent-mixed /metrics contains monlet_agent_check_status"
+log "agent-01 /metrics contains monlet_agent_check_status"
 curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
  || fail "agent metrics missing monlet_agent_check_status"

 # --- Phase I: web UI ---
 log "web pages render with expected content"
-curl -sf "$WEB_BASE/"          | grep -q "Overview"  || fail "web / missing Overview"
-curl -sf "$WEB_BASE/agents"    | grep -q "agent-mixed" || fail "web /agents missing agent-mixed"
+curl -sfL "$WEB_BASE/"         | grep -q "Agents"  || fail "web / did not redirect to Agents"
+curl -sf "$WEB_BASE/agents"    | grep -q "mixed-status" || fail "web /agents missing mixed-status label"
 curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
-curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve"
+curl -sf "$WEB_BASE/incidents?state=open&limit=100&sort=status&direction=desc" \
+  | grep -q "agent-08" || fail "web /incidents missing agent-08"
 curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
 curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"

+if [[ "${SHOWCASE_KEEP:-0}" == "1" ]]; then
+  log "refreshing transient containers before leaving stack"
+  "${COMPOSE[@]}" stop \
+    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
+    agent-spool-replay >/dev/null || true
+  "${COMPOSE[@]}" stop mock-webhook server >/dev/null || true
+  "${COMPOSE[@]}" rm -f \
+    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
+    agent-spool-replay agent-stale agent-dead server mock-webhook >/dev/null || true
+  "${COMPOSE[@]}" up -d --no-deps mock-webhook server >/dev/null \
+    || fail "could not restart runtime services"
+  wait_url "$BASE/api/v1/ready" "server after refresh" 90
+  "${COMPOSE[@]}" up -d --no-deps \
+    agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
+    agent-spool-replay >/dev/null \
+    || fail "could not restart live showcase agents"
+  wait_agents_visible "$EXPECTED_AGENTS"
+  wait_agents_accepted "$EXPECTED_AGENTS"
+  "${COMPOSE[@]}" create agent-stale agent-dead >/dev/null \
+    || fail "could not recreate transient containers"
+fi
+
 log "ALL SHOWCASE ASSERTIONS PASSED"
--- a/deploy/showcase/agents/dead.toml
+++ b/deploy/showcase/agents/dead.toml
@@ -1,4 +1,4 @@
-hostname = "agent-dead"
+agent_id = "agent-08"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

--- a/deploy/showcase/agents/flap.toml
+++ b/deploy/showcase/agents/flap.toml
@@ -1,4 +1,4 @@
-hostname = "agent-flap"
+agent_id = "agent-03"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

@@ -22,4 +21,4 @@ name = "Flapping check (oscillates ok/critical)"
 command = "/opt/monlet/checks/flap.sh"
 interval = "10s"
 timeout = "3s"
-dedupe_key = "agent-flap:flap"
+dedupe_key = "agent-03:flap"
--- a/deploy/showcase/agents/mixed.toml
+++ b/deploy/showcase/agents/mixed.toml
@@ -1,4 +1,4 @@
-hostname = "agent-mixed"
+agent_id = "agent-01"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

@@ -36,7 +35,7 @@ name = "Showcase critical (permanent webhook fail)"
 command = "/opt/monlet/checks/exit_2.sh"
 interval = "5s"
 timeout = "3s"
-dedupe_key = "agent-mixed:critical:fail"
+dedupe_key = "agent-01:critical:fail"

 [[checks]]
 id = "unknown_check"
--- a/deploy/showcase/agents/prometheus_owned.toml
+++ b/deploy/showcase/agents/prometheus_owned.toml
@@ -1,4 +1,4 @@
-hostname = "agent-prometheus-owned"
+agent_id = "agent-05"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

--- a/deploy/showcase/agents/resolve.toml
+++ b/deploy/showcase/agents/resolve.toml
@@ -1,4 +1,4 @@
-hostname = "agent-resolve"
+agent_id = "agent-02"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

@@ -22,4 +21,4 @@ name = "Flapping check (critical then ok)"
 command = "/opt/monlet/checks/flapping.sh"
 interval = "5s"
 timeout = "3s"
-dedupe_key = "agent-resolve:flapping:retry"
+dedupe_key = "agent-02:flapping:retry"
--- a/deploy/showcase/agents/spool_replay.toml
+++ b/deploy/showcase/agents/spool_replay.toml
@@ -1,4 +1,4 @@
-hostname = "agent-spool-replay"
+agent_id = "agent-06"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

--- a/deploy/showcase/agents/stale.toml
+++ b/deploy/showcase/agents/stale.toml
@@ -1,4 +1,4 @@
-hostname = "agent-stale"
+agent_id = "agent-07"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

--- a/deploy/showcase/agents/timeout.toml
+++ b/deploy/showcase/agents/timeout.toml
@@ -1,4 +1,4 @@
-hostname = "agent-timeout"
+agent_id = "agent-04"
 state_dir = "/var/lib/monlet-agent"

 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"

@@ -22,7 +21,7 @@ name = "Healthy probe"
 command = "/opt/monlet/checks/exit_0.sh"
 interval = "10s"
 timeout = "3s"
-dedupe_key = "agent-timeout:ok_probe"
+dedupe_key = "agent-04:ok_probe"

 [[checks]]
 id = "slow_probe"
@@ -30,4 +29,4 @@ name = "Probe that exceeds the timeout"
 command = "/opt/monlet/checks/sleep_long.sh"
 interval = "10s"
 timeout = "2s"
-dedupe_key = "agent-timeout:slow_probe"
+dedupe_key = "agent-04:slow_probe"
--- a/deploy/showcase/checks/replay_check.sh
+++ b/deploy/showcase/checks/replay_check.sh
@@ -1,3 +1,15 @@
 #!/bin/sh
-echo "ok: replay tick"
+# PH-review: spool-replay scenario needs many DISTINCT events because the
+# server only persists an event row when the check status TRANSITIONS
+# (server/monlet_server/services/ingestion.py). Alternate ok/warning on each
+# run so every tick produces a fresh event for the spool to replay.
+STATE_FILE=/var/lib/monlet-agent/replay_tick
+COUNT=$(cat "$STATE_FILE" 2>/dev/null || echo 0)
+COUNT=$((COUNT + 1))
+echo "$COUNT" > "$STATE_FILE"
+if [ $((COUNT % 2)) -eq 0 ]; then
+  echo "warning: replay tick $COUNT"
+  exit 1
+fi
+echo "ok: replay tick $COUNT"
 exit 0
--- a/deploy/showcase/seed_discarded.sql
+++ b/deploy/showcase/seed_discarded.sql
@@ -1,16 +0,0 @@
-- Seed one outbox row with an unknown notifier name so the worker marks it 'discarded'.
-- Attached to any existing incident; if none exists yet the script is a no-op.
-INSERT INTO notification_outbox
-    (id, incident_id, notifier, event_type, state, attempts, next_attempt_at, payload)
-SELECT
-    gen_random_uuid(),
-    i.id,
-    'showcase-unknown',
-    'firing',
-    'pending',
-    0,
-    now(),
-    '{"showcase":"discarded"}'::jsonb
-FROM incidents i
-ORDER BY i.opened_at DESC
-LIMIT 1;
--- a/deploy/smoke.sh
+++ b/deploy/smoke.sh
@@ -6,9 +6,22 @@ set -euo pipefail

 HERE="$(cd "$(dirname "$0")" && pwd)"
 ROOT="$(cd "$HERE/.." && pwd)"
-TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
-BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
-COMPOSE=(docker compose -f "$HERE/docker-compose.yml")
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}"
+AGENT_KEY="${MONLET_AGENT_KEY:-monlet-agent-dev-key-0000000000000000}"
+POSTGRES_HOST_PORT="${MONLET_POSTGRES_HOST_PORT:-15432}"
+SERVER_HOST_PORT="${MONLET_SERVER_HOST_PORT:-18000}"
+WEB_HOST_PORT="${MONLET_WEB_HOST_PORT:-13000}"
+# Export so every `docker compose` call (up/exec/down) sees the UI token env,
+# including the cleanup trap.
+export MONLET_AUTH_TOKEN="$TOKEN"
+export MONLET_POSTGRES_HOST_PORT="$POSTGRES_HOST_PORT"
+export MONLET_SERVER_HOST_PORT="$SERVER_HOST_PORT"
+export MONLET_WEB_HOST_PORT="$WEB_HOST_PORT"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:$SERVER_HOST_PORT}"
+# PH-review: dedicated project name so smoke does not collide with the
+# showcase/dev stack (orphan containers, shared postgres volume with a stale
+# baseline schema, etc.).
+COMPOSE=(docker compose -p monlet-smoke -f "$HERE/docker-compose.yml")

 log() { printf "\033[1;36m[smoke]\033[0m %s\n" "$*"; }
 fail() { printf "\033[1;31m[smoke]\033[0m %s\n" "$*" >&2; exit 1; }
@@ -21,8 +34,11 @@ cleanup() {
 }
 trap cleanup EXIT

-log "starting stack"
-MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build postgres server web
+log "pre-clean previous smoke stack (if any)"
+"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+
+log "starting stack (phase 1: postgres+server)"
+"${COMPOSE[@]}" up -d --build postgres server

 log "waiting for /api/v1/ready"
 for i in $(seq 1 60); do
@@ -37,33 +53,41 @@ curl -sf "$BASE/api/v1/health" | grep -q '"status":"ok"' || fail "health wrong b
 log "GET /metrics"
 curl -sf "$BASE/metrics" | grep -q '^monlet_server_' || fail "metrics missing"

-AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json")
+log "starting web (phase 2)"
+"${COMPOSE[@]}" up -d --build web
+
+AGENT_AUTH=(-H "Authorization: Bearer $AGENT_KEY" -H "Content-Type: application/json")
+UI_AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json")

 log "POST heartbeat (examples/heartbeat.json)"
-curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \
+curl -sf "${AGENT_AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \
  | grep -q '"accepted":true' || fail "heartbeat not accepted"

+log "accept pending agent"
+curl -sf "${UI_AUTH[@]}" -X POST "$BASE/api/v1/agents/host-01.prod/accept" \
+  | grep -q '"count":1' || fail "agent not accepted"
+
 log "POST events (examples/event-batch.json)"
-curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \
+curl -sf "${AGENT_AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \
  | grep -q '"accepted":' || fail "event batch not accepted"

 log "GET /agents has host-01.prod"
-curl -sf "${AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \
+curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \
  || fail "host-01.prod missing"

 log "GET /checks not empty"
-curl -sf "${AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \
+curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \
  || fail "checks empty"

 log "GET /incidents not empty (event-batch.json contains a critical event)"
-curl -sf "${AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \
+curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \
  || fail "no open incident"

 log "GET /notifiers/outbox not empty"
-curl -sf "${AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \
+curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \
  || fail "outbox empty"

-WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:$WEB_HOST_PORT}"
 log "waiting for web $WEB_BASE"
 for i in $(seq 1 60); do
  if curl -sf -o /dev/null "$WEB_BASE/"; then break; fi
@@ -71,8 +95,8 @@ for i in $(seq 1 60); do
  [[ "$i" == "60" ]] && fail "web did not become ready"
 done

-log "GET / (web overview)"
-curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web overview missing 'Overview'"
+log "GET / (web root follows redirect to /agents)"
+curl -sfL "$WEB_BASE/" | grep -q "host-01.prod" || fail "web root did not render agents list"

 log "GET /agents (web list)"
 curl -sf "$WEB_BASE/agents" | grep -q "host-01.prod" || fail "web /agents missing host-01.prod"