creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden production workflows and agent admission

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-28 14:19:27 +04:00
parent a2e88b4e76
commit 37b1a1d6d6
109 changed files with 4927 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
deploy/e2e-showcase.sh
View File

@@ -2,18 +2,25 @@
# Stage 6.1 — Docker showcase / e2e stand for Monlet.
# Brings up the full stack with demo agents, drives scenarios, asserts all
# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications,
# incident open/resolved, outbox sent/retry/failed, disabled server notifications,
# agent spool replay), and verifies metrics + web pages.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$HERE/.." && pwd)"
TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}"
# Export so every `docker compose` call (up/exec/down) sees the UI token env,
# including the cleanup trap.
export MONLET_AUTH_TOKEN="$TOKEN"
BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
ADMISSION_MODE="${SHOWCASE_ADMISSION_MODE:-pending}"
# PH-review: dedicated project name so showcase does not collide with the
# smoke/dev stack (orphans, shared postgres volume, stale baseline schema).
COMPOSE=(docker compose
-p monlet-showcase
-f "$HERE/docker-compose.yml"
-f "$HERE/docker-compose.showcase.yml")
@@ -21,6 +28,7 @@ log() { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }
AUTH=(-H "Authorization: Bearer $TOKEN")
EXPECTED_AGENTS=8
cleanup() {
if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
@@ -42,9 +50,55 @@ wait_url() {
}
api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
wait_agents_visible() {
local expected="$1"
for i in $(seq 1 60); do
local count
count="$(api "/api/v1/agents?limit=500" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['items']))" 2>/dev/null || echo 0)"
[[ "${count:-0}" -ge "$expected" ]] && return 0
sleep 2
done
fail "agents did not register"
}
log "starting full showcase stack"
MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build
wait_agents_accepted() {
local expected="$1"
for i in $(seq 1 300); do
local counts accepted pending
counts="$(api "/api/v1/agents?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
items=d['items']
accepted=sum(1 for a in items if a['admission_state']=='accepted')
pending=sum(1 for a in items if a['admission_state']=='pending')
print(accepted, pending)
" 2>/dev/null || echo "0 0")"
read -r accepted pending <<<"$counts"
[[ "${accepted:-0}" -ge "$expected" && "${pending:-0}" -eq 0 ]] && return 0
sleep 2
done
fail "agents were not accepted in time"
}
# Multi-phase startup keeps DB migration explicit, then agents register as
# pending. Full assertions wait read-only until an operator accepts them.
log "pre-clean previous showcase stack (if any)"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
log "phase 1: postgres"
# --wait blocks until the postgres healthcheck (pg_isready) reports healthy,
# so the alembic step below does not race a still-booting DB.
"${COMPOSE[@]}" up -d --wait --build postgres
log "building current server image for migrations"
"${COMPOSE[@]}" build server >/dev/null || fail "server image build failed"
log "applying alembic migrations (one-off, server not yet running)"
"${COMPOSE[@]}" run --rm --no-deps server sh -c "alembic upgrade head" \
>/dev/null || fail "alembic upgrade failed"
log "phase 2: server + agents + web"
"${COMPOSE[@]}" up -d --build
log "waiting for server /api/v1/ready"
wait_url "$BASE/api/v1/ready" "server" 90
@@ -52,31 +106,59 @@ wait_url "$BASE/api/v1/ready" "server" 90
log "waiting for web /"
wait_url "$WEB_BASE/" "web" 90
log "waiting for agents to register as pending"
wait_agents_visible "$EXPECTED_AGENTS"
case "$ADMISSION_MODE" in
pending)
log "agents are pending; automatic accept is disabled"
log "accept them in the UI/API, or rerun with SHOWCASE_ADMISSION_MODE=wait to continue assertions after manual admission"
exit 0
;;
wait)
log "waiting for operator to accept agents (read-only polling)"
wait_agents_accepted "$EXPECTED_AGENTS"
;;
*)
fail "unknown SHOWCASE_ADMISSION_MODE=$ADMISSION_MODE (expected pending or wait)"
;;
esac
# --- Phase A: let agents run a few cycles ---
log "letting demo agents push events for ~25s"
sleep 25
log "asserting agent labels include scenario and monlet_agent_version"
api "/api/v1/agents/agent-mixed" | python3 -c "
log "asserting agent-01 labels include scenario and monlet_agent_version"
api "/api/v1/agents/agent-01" | python3 -c "
import json,sys
d=json.load(sys.stdin)
labels=d.get('labels') or {}
ver=labels.get('monlet_agent_version')
ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
print('agent-mixed labels:', labels)
print('agent-01 labels:', labels)
sys.exit(0 if ok else 1)" \
|| fail "agent-mixed labels missing scenario or numeric monlet_agent_version"
|| fail "agent-01 labels missing scenario or numeric monlet_agent_version"
log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)"
log "asserting agent-01 has 4 distinct check statuses (ok/warning/critical/unknown)"
api "/api/v1/checks?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
want={'ok','warning','critical','unknown'}
have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'}
have={c['status'] for c in d['items'] if c['agent_id']=='agent-01'}
missing=want-have
print('agent-mixed statuses:', sorted(have))
print('agent-01 statuses:', sorted(have))
sys.exit(0 if not missing else 1)" \
|| fail "agent-mixed missing one of ok/warning/critical/unknown"
|| fail "agent-01 missing one of ok/warning/critical/unknown"
log "asserting timeout check is explicit critical"
api "/api/v1/checks?agent_id=agent-04&limit=50" | python3 -c "
import json,sys
d=json.load(sys.stdin)
slow=next((c for c in d['items'] if c['check_id']=='slow_probe'), None)
print('slow_probe:', slow)
ok=slow and slow['status']=='critical' and slow['exit_code']==2 and 'timed out' in (slow.get('summary') or '')
sys.exit(0 if ok else 1)" \
|| fail "agent-04/slow_probe is not explicit critical timeout"
# --- Phase B: trigger resolve scenario ---
log "triggering resolve: touch /state/resolve.flag in agent-resolve"
@@ -86,14 +168,14 @@ log "triggering resolve: touch /state/resolve.flag in agent-resolve"
log "waiting for resolve event to be processed (~15s)"
sleep 15
log "asserting resolved incident exists for agent-resolve/flapping"
log "asserting resolved incident exists for agent-02/flapping"
api "/api/v1/incidents?state=resolved&limit=500" \
| python3 -c "
import json,sys
d=json.load(sys.stdin)
ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items'])
ok=any(i['agent_id']=='agent-02' and i['check_id']=='flapping' for i in d['items'])
sys.exit(0 if ok else 1)" \
|| fail "no resolved incident for agent-resolve/flapping"
|| fail "no resolved incident for agent-02/flapping"
# --- Phase C: disabled server notifications ---
log "asserting notifications-disabled incident exists but no outbox row for it"
@@ -101,7 +183,7 @@ PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
for i in d['items']:
if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck':
if i['agent_id']=='agent-05' and i['check_id']=='promcheck':
print(i['id']); break")"
[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
@@ -114,22 +196,22 @@ log "stopping agent-stale and agent-dead to age their heartbeats"
log "waiting ~25s for stale transition"
sleep 25
api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \
|| api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
|| fail "agent-stale did not transition to stale/dead"
api "/api/v1/agents/agent-07" | grep -q '"status":"stale"' \
|| api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
|| fail "agent-07 did not transition to stale/dead"
log "waiting another ~25s for dead transition"
sleep 25
api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
|| fail "agent-stale did not transition to dead"
api "/api/v1/agents/agent-dead" | grep -q '"status":"dead"' \
|| fail "agent-dead did not transition to dead"
api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \
|| fail "agent-07 did not transition to dead"
api "/api/v1/agents/agent-08" | grep -q '"status":"dead"' \
|| fail "agent-08 did not transition to dead"
log "asserting dead agents have open liveness incidents"
api "/api/v1/incidents?state=open&limit=500" | python3 -c "
import json,sys
d=json.load(sys.stdin)
want={'agent-stale','agent-dead'}
want={'agent-07','agent-08'}
have={
i['agent_id']
for i in d['items']
@@ -140,9 +222,9 @@ print('liveness incidents:', sorted(have))
sys.exit(0 if not missing else 1)" \
|| fail "missing critical agent_liveness incident for dead agents"
# At least one agent must still be alive (agent-mixed keeps pushing).
api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \
|| fail "agent-mixed not alive"
# At least one agent must still be alive (agent-01 keeps pushing).
api "/api/v1/agents/agent-01" | grep -q '"status":"alive"' \
|| fail "agent-01 not alive"
# --- Phase E: outbox state coverage (live via webhook + debug) ---
log "waiting for notifier worker cycles (~10s)"
@@ -159,22 +241,27 @@ print('outbox states:', sorted(states))
sys.exit(0 if not missing else 2)" \
|| fail "outbox missing one of sent/retry/failed"
# --- Phase F: seed discarded row, wait for worker ---
log "seeding 'discarded' outbox row via SQL"
"${COMPOSE[@]}" exec -T postgres \
psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \
< "$HERE/showcase/seed_discarded.sql" >/dev/null \
|| fail "seed_discarded.sql failed"
log "waiting for worker to mark unknown-notifier row as discarded (~10s)"
sleep 10
api "/api/v1/notifiers/outbox?state=discarded&limit=10" \
| grep -q '"state":"discarded"' \
|| fail "no outbox row in state=discarded"
# --- Phase G: spool replay assertion ---
log "asserting agent-spool-replay events arrived without duplicates"
api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \
log "forcing server outage so agent-spool-replay writes to local spool"
"${COMPOSE[@]}" stop server >/dev/null
spool_count=0
for i in $(seq 1 30); do
spool_count=$("${COMPOSE[@]}" exec -T agent-spool-replay \
sh -c 'ls /var/lib/monlet-agent/spool 2>/dev/null | wc -l' 2>/dev/null \
| tr -d '[:space:]' || echo 0)
[[ "${spool_count:-0}" -gt 0 ]] && break
sleep 2
done
[[ "${spool_count:-0}" -gt 0 ]] || fail "agent-spool-replay did not spool any events"
log "spool populated with $spool_count file(s)"
log "restarting server and waiting for replay"
"${COMPOSE[@]}" up -d --no-deps server >/dev/null
wait_url "$BASE/api/v1/ready" "server after replay outage" 90
sleep 15
log "asserting agent-06 spool replay events arrived without duplicates"
api "/api/v1/events/query?agent_id=agent-06&check_id=replay_check&limit=500" \
| python3 -c "
import json,sys
d=json.load(sys.stdin)
@@ -192,17 +279,40 @@ log "server /metrics contains monlet_server_*"
curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
|| fail "server metrics missing monlet_server_*"
log "agent-mixed /metrics contains monlet_agent_check_status"
log "agent-01 /metrics contains monlet_agent_check_status"
curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
|| fail "agent metrics missing monlet_agent_check_status"
# --- Phase I: web UI ---
log "web pages render with expected content"
curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web / missing Overview"
curl -sf "$WEB_BASE/agents" | grep -q "agent-mixed" || fail "web /agents missing agent-mixed"
curl -sfL "$WEB_BASE/" | grep -q "Agents" || fail "web / did not redirect to Agents"
curl -sf "$WEB_BASE/agents" | grep -q "mixed-status" || fail "web /agents missing mixed-status label"
curl -sf "$WEB_BASE/checks" | grep -q "critical_check" || fail "web /checks missing critical_check"
curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve"
curl -sf "$WEB_BASE/incidents?state=open&limit=100&sort=status&direction=desc" \
| grep -q "agent-08" || fail "web /incidents missing agent-08"
curl -sf "$WEB_BASE/outbox" | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
if [[ "${SHOWCASE_KEEP:-0}" == "1" ]]; then
log "refreshing transient containers before leaving stack"
"${COMPOSE[@]}" stop \
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
agent-spool-replay >/dev/null || true
"${COMPOSE[@]}" stop mock-webhook server >/dev/null || true
"${COMPOSE[@]}" rm -f \
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
agent-spool-replay agent-stale agent-dead server mock-webhook >/dev/null || true
"${COMPOSE[@]}" up -d --no-deps mock-webhook server >/dev/null \
|| fail "could not restart runtime services"
wait_url "$BASE/api/v1/ready" "server after refresh" 90
"${COMPOSE[@]}" up -d --no-deps \
agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \
agent-spool-replay >/dev/null \
|| fail "could not restart live showcase agents"
wait_agents_visible "$EXPECTED_AGENTS"
wait_agents_accepted "$EXPECTED_AGENTS"
"${COMPOSE[@]}" create agent-stale agent-dead >/dev/null \
|| fail "could not recreate transient containers"
fi
log "ALL SHOWCASE ASSERTIONS PASSED"