320 lines
12 KiB
Bash
Executable File
320 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Stage 6.1 — Docker showcase / e2e stand for Monlet.
|
|
# Brings up the full stack with demo agents, drives scenarios, asserts all
|
|
# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
|
|
# incident open/resolved, outbox sent/retry/failed, disabled server notifications,
|
|
# agent spool replay), and verifies metrics + web pages.
|
|
set -euo pipefail
|
|
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
ROOT="$(cd "$HERE/.." && pwd)"
|
|
TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}"
|
|
# Export so every `docker compose` call (up/exec/down) sees the UI token env,
|
|
# including the cleanup trap.
|
|
export MONLET_AUTH_TOKEN="$TOKEN"
|
|
BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
|
|
WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
|
|
AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
|
|
ADMISSION_MODE="${SHOWCASE_ADMISSION_MODE:-pending}"
|
|
|
|
# PH-review: dedicated project name so showcase does not collide with the
|
|
# smoke/dev stack (orphans, shared postgres volume, stale baseline schema).
|
|
COMPOSE=(docker compose
|
|
-p monlet-showcase
|
|
-f "$HERE/docker-compose.yml"
|
|
-f "$HERE/docker-compose.showcase.yml")
|
|
|
|
log() { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
|
|
fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }
|
|
|
|
AUTH=(-H "Authorization: Bearer $TOKEN")
|
|
EXPECTED_AGENTS=8
|
|
|
|
cleanup() {
|
|
if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
|
|
log "tearing down stack"
|
|
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
else
|
|
log "SHOWCASE_KEEP=1 → leaving stack running (web: $WEB_BASE)"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
wait_url() {
|
|
local url="$1" label="$2" tries="${3:-60}"
|
|
for i in $(seq 1 "$tries"); do
|
|
if curl -sf -o /dev/null "$url"; then return 0; fi
|
|
sleep 2
|
|
[[ "$i" == "$tries" ]] && fail "$label not ready: $url"
|
|
done
|
|
}
|
|
|
|
api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
|
|
wait_agents_visible() {
|
|
local expected="$1"
|
|
for i in $(seq 1 60); do
|
|
local count
|
|
count="$(api "/api/v1/agents?limit=500" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['items']))" 2>/dev/null || echo 0)"
|
|
[[ "${count:-0}" -ge "$expected" ]] && return 0
|
|
sleep 2
|
|
done
|
|
fail "agents did not register"
|
|
}
|
|
|
|
wait_agents_accepted() {
|
|
local expected="$1"
|
|
for i in $(seq 1 300); do
|
|
local counts accepted pending
|
|
counts="$(api "/api/v1/agents?limit=500" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
items=d['items']
|
|
accepted=sum(1 for a in items if a['admission_state']=='accepted')
|
|
pending=sum(1 for a in items if a['admission_state']=='pending')
|
|
print(accepted, pending)
|
|
" 2>/dev/null || echo "0 0")"
|
|
read -r accepted pending <<<"$counts"
|
|
[[ "${accepted:-0}" -ge "$expected" && "${pending:-0}" -eq 0 ]] && return 0
|
|
sleep 2
|
|
done
|
|
fail "agents were not accepted in time"
|
|
}
|
|
|
|
# Multi-phase startup keeps DB migration explicit, then agents register as
|
|
# pending. Full assertions wait read-only until an operator accepts them.
|
|
log "pre-clean previous showcase stack (if any)"
|
|
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
|
|
|
|
log "phase 1: postgres"
|
|
# --wait blocks until the postgres healthcheck (pg_isready) reports healthy,
|
|
# so the alembic step below does not race a still-booting DB.
|
|
"${COMPOSE[@]}" up -d --wait --build postgres
|
|
|
|
log "building current server image for migrations"
|
|
"${COMPOSE[@]}" build server >/dev/null || fail "server image build failed"
|
|
|
|
log "applying alembic migrations (one-off, server not yet running)"
|
|
"${COMPOSE[@]}" run --rm --no-deps server sh -c "alembic upgrade head" \
|
|
>/dev/null || fail "alembic upgrade failed"
|
|
|
|
log "phase 2: server + agents + web"
|
|
"${COMPOSE[@]}" up -d --build
|
|
|
|
log "waiting for server /api/v1/ready"
|
|
wait_url "$BASE/api/v1/ready" "server" 90
|
|
|
|
log "waiting for web /"
|
|
wait_url "$WEB_BASE/" "web" 90
|
|
|
|
log "waiting for agents to register as pending"
|
|
wait_agents_visible "$EXPECTED_AGENTS"
|
|
|
|
case "$ADMISSION_MODE" in
|
|
pending)
|
|
log "agents are pending; automatic accept is disabled"
|
|
log "accept them in the UI/API, or rerun with SHOWCASE_ADMISSION_MODE=wait to continue assertions after manual admission"
|
|
exit 0
|
|
;;
|
|
wait)
|
|
log "waiting for operator to accept agents (read-only polling)"
|
|
wait_agents_accepted "$EXPECTED_AGENTS"
|
|
;;
|
|
*)
|
|
fail "unknown SHOWCASE_ADMISSION_MODE=$ADMISSION_MODE (expected pending or wait)"
|
|
;;
|
|
esac
|
|
|
|
# --- Phase A: let agents run a few cycles ---
|
|
log "letting demo agents push events for ~25s"
|
|
sleep 25
|
|
|
|
log "asserting agent-01 labels include scenario and monlet_agent_version"
|
|
api "/api/v1/agents/agent-01" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
labels=d.get('labels') or {}
|
|
ver=labels.get('monlet_agent_version')
|
|
ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
|
|
print('agent-01 labels:', labels)
|
|
sys.exit(0 if ok else 1)" \
|
|
|| fail "agent-01 labels missing scenario or numeric monlet_agent_version"
|
|
|
|
log "asserting agent-01 has 4 distinct check statuses (ok/warning/critical/unknown)"
|
|
api "/api/v1/checks?limit=500" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
want={'ok','warning','critical','unknown'}
|
|
have={c['status'] for c in d['items'] if c['agent_id']=='agent-01'}
|
|
missing=want-have
|
|
print('agent-01 statuses:', sorted(have))
|
|
sys.exit(0 if not missing else 1)" \
|
|
|| fail "agent-01 missing one of ok/warning/critical/unknown"
|
|
|
|
log "asserting timeout check is explicit critical"
|
|
api "/api/v1/checks?agent_id=agent-04&limit=50" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
slow=next((c for c in d['items'] if c['check_id']=='slow_probe'), None)
|
|
print('slow_probe:', slow)
|
|
ok=slow and slow['status']=='critical' and slow['exit_code']==2 and 'timed out' in (slow.get('summary') or '')
|
|
sys.exit(0 if ok else 1)" \
|
|
|| fail "agent-04/slow_probe is not explicit critical timeout"
|
|
|
|
# --- Phase B: trigger resolve scenario ---
|
|
log "triggering resolve: touch /state/resolve.flag in agent-resolve"
|
|
"${COMPOSE[@]}" exec -T agent-resolve sh -c 'touch /state/resolve.flag' \
|
|
|| fail "could not touch resolve flag"
|
|
|
|
log "waiting for resolve event to be processed (~15s)"
|
|
sleep 15
|
|
|
|
log "asserting resolved incident exists for agent-02/flapping"
|
|
api "/api/v1/incidents?state=resolved&limit=500" \
|
|
| python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
ok=any(i['agent_id']=='agent-02' and i['check_id']=='flapping' for i in d['items'])
|
|
sys.exit(0 if ok else 1)" \
|
|
|| fail "no resolved incident for agent-02/flapping"
|
|
|
|
# --- Phase C: disabled server notifications ---
|
|
log "asserting notifications-disabled incident exists but no outbox row for it"
|
|
PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
for i in d['items']:
|
|
if i['agent_id']=='agent-05' and i['check_id']=='promcheck':
|
|
print(i['id']); break")"
|
|
[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
|
|
api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
|
|
&& fail "outbox unexpectedly contains row for prometheus-owned incident $PROM_INC"
|
|
log "no outbox row for notifications-disabled incident"
|
|
|
|
# --- Phase D: stop stale/dead agents to drive lifecycle transitions ---
|
|
log "stopping agent-stale and agent-dead to age their heartbeats"
|
|
"${COMPOSE[@]}" stop agent-stale agent-dead >/dev/null
|
|
|
|
log "waiting ~25s for stale transition"
|
|
sleep 25
|
|
api "/api/v1/agents/agent-07" | grep -q '"status":"stale"' \
|
|
|| api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
|
|
|| fail "agent-07 did not transition to stale/dead"
|
|
|
|
log "waiting another ~25s for dead transition"
|
|
sleep 25
|
|
api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
|
|
|| fail "agent-07 did not transition to dead"
|
|
api "/api/v1/agents/agent-08" | grep -q '"status":"dead"' \
|
|
|| fail "agent-08 did not transition to dead"
|
|
|
|
log "asserting dead agents have open liveness incidents"
|
|
api "/api/v1/incidents?state=open&limit=500" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
want={'agent-07','agent-08'}
|
|
have={
|
|
i['agent_id']
|
|
for i in d['items']
|
|
if i['check_id']=='agent_liveness' and i['state']=='open' and i['severity']=='critical'
|
|
}
|
|
missing=want-have
|
|
print('liveness incidents:', sorted(have))
|
|
sys.exit(0 if not missing else 1)" \
|
|
|| fail "missing critical agent_liveness incident for dead agents"
|
|
|
|
# At least one agent must still be alive (agent-01 keeps pushing).
|
|
api "/api/v1/agents/agent-01" | grep -q '"status":"alive"' \
|
|
|| fail "agent-01 not alive"
|
|
|
|
# --- Phase E: outbox state coverage (live via webhook + debug) ---
|
|
log "waiting for notifier worker cycles (~10s)"
|
|
sleep 10
|
|
|
|
OUTBOX="$(api "/api/v1/notifiers/outbox?limit=500")"
|
|
echo "$OUTBOX" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
states={i['state'] for i in d['items']}
|
|
need={'sent','retry','failed'}
|
|
missing=need-states
|
|
print('outbox states:', sorted(states))
|
|
sys.exit(0 if not missing else 2)" \
|
|
|| fail "outbox missing one of sent/retry/failed"
|
|
|
|
# --- Phase G: spool replay assertion ---
|
|
log "forcing server outage so agent-spool-replay writes to local spool"
|
|
"${COMPOSE[@]}" stop server >/dev/null
|
|
spool_count=0
|
|
for i in $(seq 1 30); do
|
|
spool_count=$("${COMPOSE[@]}" exec -T agent-spool-replay \
|
|
sh -c 'ls /var/lib/monlet-agent/spool 2>/dev/null | wc -l' 2>/dev/null \
|
|
| tr -d '[:space:]' || echo 0)
|
|
[[ "${spool_count:-0}" -gt 0 ]] && break
|
|
sleep 2
|
|
done
|
|
[[ "${spool_count:-0}" -gt 0 ]] || fail "agent-spool-replay did not spool any events"
|
|
log "spool populated with $spool_count file(s)"
|
|
|
|
log "restarting server and waiting for replay"
|
|
"${COMPOSE[@]}" up -d --no-deps server >/dev/null
|
|
wait_url "$BASE/api/v1/ready" "server after replay outage" 90
|
|
sleep 15
|
|
|
|
log "asserting agent-06 spool replay events arrived without duplicates"
|
|
api "/api/v1/events/query?agent_id=agent-06&check_id=replay_check&limit=500" \
|
|
| python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
items=d['items']
|
|
ids=[e['event_id'] for e in items]
|
|
if len(items) < 2:
|
|
print(f'too few events: {len(items)}'); sys.exit(1)
|
|
if len(set(ids)) != len(ids):
|
|
print('duplicate event_ids'); sys.exit(2)
|
|
print(f'spool replay events: {len(items)} unique')" \
|
|
|| fail "spool replay assertion failed"
|
|
|
|
# --- Phase H: metrics ---
|
|
log "server /metrics contains monlet_server_*"
|
|
curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
|
|
|| fail "server metrics missing monlet_server_*"
|
|
|
|
log "agent-01 /metrics contains monlet_agent_check_status"
|
|
curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
|
|
|| fail "agent metrics missing monlet_agent_check_status"
|
|
|
|
# --- Phase I: web UI ---
|
|
log "web pages render with expected content"
|
|
curl -sfL "$WEB_BASE/" | grep -q "Agents" || fail "web / did not redirect to Agents"
|
|
curl -sf "$WEB_BASE/agents" | grep -q "mixed-status" || fail "web /agents missing mixed-status label"
|
|
curl -sf "$WEB_BASE/checks" | grep -q "critical_check" || fail "web /checks missing critical_check"
|
|
INCIDENTS_HTML="$(curl -sfL "$WEB_BASE/incidents?state=open&sort=status&direction=desc")"
|
|
echo "$INCIDENTS_HTML" | grep -q "agent-07" || fail "web /incidents missing agent-07"
|
|
echo "$INCIDENTS_HTML" | grep -q "agent-08" || fail "web /incidents missing agent-08"
|
|
curl -sf "$WEB_BASE/outbox" | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
|
|
curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
|
|
|
|
if [[ "${SHOWCASE_KEEP:-0}" == "1" ]]; then
|
|
log "refreshing transient containers before leaving stack"
|
|
"${COMPOSE[@]}" stop \
|
|
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
|
|
agent-spool-replay >/dev/null || true
|
|
"${COMPOSE[@]}" stop mock-webhook server >/dev/null || true
|
|
"${COMPOSE[@]}" rm -f \
|
|
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
|
|
agent-spool-replay agent-stale agent-dead server mock-webhook >/dev/null || true
|
|
"${COMPOSE[@]}" up -d --no-deps mock-webhook server >/dev/null \
|
|
|| fail "could not restart runtime services"
|
|
wait_url "$BASE/api/v1/ready" "server after refresh" 90
|
|
"${COMPOSE[@]}" up -d --no-deps \
|
|
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
|
|
agent-spool-replay >/dev/null \
|
|
|| fail "could not restart live showcase agents"
|
|
wait_agents_visible "$EXPECTED_AGENTS"
|
|
wait_agents_accepted "$EXPECTED_AGENTS"
|
|
"${COMPOSE[@]}" create agent-stale agent-dead >/dev/null \
|
|
|| fail "could not recreate transient containers"
|
|
fi
|
|
|
|
log "ALL SHOWCASE ASSERTIONS PASSED"
|