Implement Monlet MVP stack and UI updates

2026-05-27 10:01:59 +04:00
parent dcea096327
commit edc51e9c59
145 changed files with 15618 additions and 248 deletions
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -1,15 +1,75 @@
 # Deploy

-Deployment examples live here.
+Local stack and example operator configs.

-Planned files:
+## Quick start

- Docker Compose for local development;
- PostgreSQL service;
- optional Prometheus scrape config;
- optional Alertmanager config;
- optional Grafana dashboard stub.
+```sh
+cd deploy
+MONLET_AUTH_TOKEN=$(openssl rand -hex 16) docker compose up --build
+```

-No production-ready deployment files are implemented in Stage 0.
+Brings up: PostgreSQL 16, Monlet server (with Alembic upgrade on start), Web UI.

-Docker examples must not install Python, Node, or Go project dependencies on the host. Dependency installation belongs inside images/containers or repo-local development directories.
+- Server: http://127.0.0.1:8000 (`/api/v1/health`, `/api/v1/ready`, `/metrics`)
+- Web UI: http://127.0.0.1:3000
+- PostgreSQL: 127.0.0.1:5432, db `monlet`, user `monlet`
+
+Add Prometheus + Alertmanager + Grafana:
+
+```sh
+MONLET_AUTH_TOKEN=$(openssl rand -hex 16) docker compose --profile observability up
+```
+
+- Prometheus: http://127.0.0.1:9090
+- Alertmanager: http://127.0.0.1:9093
+- Grafana: http://127.0.0.1:3001 (admin / `$GF_ADMIN_PASSWORD`, default `admin`)
+
+## Smoke test
+
+```sh
+bash deploy/smoke.sh
+```
+
+Boots the stack, hits `/health`, `/ready`, `/metrics`, ingests example payloads from `examples/`, and verifies the API exposes resulting agent/check/incident/outbox state.
+
+`SMOKE_KEEP=1 bash deploy/smoke.sh` leaves the stack up after the run.
+
+## Showcase / E2E stand (Stage 6.1)
+
+Full-fat demo stack with several dockerized agents driving the system through
+every witness state: agent alive/stale/dead, check ok/warning/critical/unknown,
+incident open/resolved, outbox sent/retry/failed/discarded,
+anti-double-alerting with `notifications_enabled=false`, and agent spool
+replay against a server that becomes available after the agent starts.
+
+```sh
+bash deploy/e2e-showcase.sh
+```
+
+Internally:
+
+```sh
+MONLET_AUTH_TOKEN=monlet-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
+```
+
+`SHOWCASE_KEEP=1 bash deploy/e2e-showcase.sh` leaves the stack up so you can
+inspect the web UI at http://127.0.0.1:3000.
+
+Dockerized agents (`deploy/docker/agent.Dockerfile`, `deploy/showcase/`) exist
+for demo/e2e only — production deploys the Go binary under systemd.
+
+## Files
+
+- `docker-compose.yml` — full local stack (server / postgres / web / optional observability profile).
+- `prometheus/prometheus.yml` — example scrape config (Monlet server + agent target).
+- `alertmanager/alertmanager.yml` — example route + receiver.
+- `grafana/provisioning/` — datasource + dashboard providers.
+- `grafana/dashboards/monlet-overview.json` — starter dashboard.
+- `smoke.sh` — local stack smoke entry point.
+- `docker-compose.showcase.yml` — override that adds demo agents + mock webhook.
+- `docker/agent.Dockerfile` — demo/e2e agent image (not for production).
+- `showcase/` — demo agent configs, check scripts, mock webhook, seed SQL.
+- `e2e-showcase.sh` — full showcase runner (Stage 6.1).
+
+Docker images install Python/Node project dependencies inside the image; host venvs are never mounted (see `AGENTS.md`).
--- a/deploy/alertmanager/alertmanager.yml
+++ b/deploy/alertmanager/alertmanager.yml
@@ -0,0 +1,15 @@
+# Example Alertmanager config. REPLACE the receiver before using in any real
+# environment — the default 'null' receiver silently drops alerts.
+
+route:
+  group_by: ["alertname", "agent_id", "check_id"]
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+  receiver: "null"
+
+receivers:
+  - name: "null"
+    # Intentionally empty: drops everything routed here. Add your real
+    # receiver (email_configs, telegram_configs, webhook_configs, ...) and
+    # repoint `route.receiver` at it.
--- a/deploy/docker-compose.showcase.yml
+++ b/deploy/docker-compose.showcase.yml
@@ -0,0 +1,113 @@
+# Showcase / e2e override on top of deploy/docker-compose.yml.
+# Usage:
+#   docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
+# Demo agents only — production uses systemd-managed Go binary.
+
+services:
+  server:
+    environment:
+      MONLET_STALE_AFTER_SEC: "20"
+      MONLET_DEAD_AFTER_SEC: "40"
+      MONLET_DETECTOR_TICK_SEC: "5"
+      MONLET_EVENTS_RETENTION_MAX_ROWS: "5000"
+      MONLET_OUTBOX_RETENTION_MAX_ROWS: "1000"
+      MONLET_NOTIFIER_DEBUG_ENABLED: "true"
+      MONLET_NOTIFIER_WEBHOOK_ENABLED: "true"
+      MONLET_NOTIFIER_WEBHOOK_URL: "http://mock-webhook:8080/"
+
+  mock-webhook:
+    image: python:3.14-alpine
+    command: ["python3", "-u", "/opt/mock_webhook.py"]
+    volumes:
+      - ./showcase/mock_webhook.py:/opt/mock_webhook.py:ro
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- --post-data='{}' http://localhost:8080/ >/dev/null 2>&1 || true"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  agent-mixed:
+    image: monlet-showcase-agent
+    build:
+      context: ..
+      dockerfile: deploy/docker/agent.Dockerfile
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/mixed.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+    ports:
+      - "9465:9465"
+
+  agent-resolve:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/resolve.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+      - resolve_state:/state
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-prometheus-owned:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/prometheus_owned.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-stale:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/stale.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-dead:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/dead.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  # No depends_on:server — starts in parallel so it must spool events until server is ready.
+  agent-spool-replay:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/spool_replay.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+      - spool_replay_state:/var/lib/monlet-agent
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+volumes:
+  resolve_state:
+  spool_replay_state:
--- a/deploy/docker-compose.yml
+++ b/deploy/docker-compose.yml
@@ -0,0 +1,85 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: monlet
+      POSTGRES_PASSWORD: monlet
+      POSTGRES_DB: monlet
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U monlet -d monlet"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    volumes:
+      - monlet_pg:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  server:
+    build:
+      context: ../server
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      MONLET_AUTH_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required}
+      MONLET_DATABASE_URL: postgresql+asyncpg://monlet:monlet@postgres:5432/monlet
+      MONLET_LOG_LEVEL: ${MONLET_LOG_LEVEL:-INFO}
+      MONLET_ENABLE_DETECTOR: "true"
+      MONLET_ENABLE_NOTIFIER_WORKER: "true"
+      MONLET_NOTIFIER_DEBUG_ENABLED: "true"
+    command:
+      - sh
+      - -c
+      - "alembic upgrade head && uvicorn monlet_server.main:app --host 0.0.0.0 --port 8000"
+    ports:
+      - "8000:8000"
+    healthcheck:
+      test: ["CMD-SHELL", "python -c 'import urllib.request,sys;sys.exit(0) if urllib.request.urlopen(\"http://localhost:8000/api/v1/ready\").status==200 else sys.exit(1)'"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+
+  web:
+    build:
+      context: ../web
+    depends_on:
+      server:
+        condition: service_healthy
+    environment:
+      MONLET_API_BASE_URL: http://server:8000
+      MONLET_API_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required}
+    ports:
+      - "3000:3000"
+
+  prometheus:
+    image: prom/prometheus:v2.55.0
+    profiles: ["observability"]
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    ports:
+      - "9090:9090"
+    depends_on:
+      - server
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    profiles: ["observability"]
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+    ports:
+      - "9093:9093"
+
+  grafana:
+    image: grafana/grafana:11.3.0
+    profiles: ["observability"]
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASSWORD:-admin}
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    ports:
+      - "3001:3000"
+
+volumes:
+  monlet_pg:
--- a/deploy/docker/agent.Dockerfile
+++ b/deploy/docker/agent.Dockerfile
@@ -0,0 +1,21 @@
+# Dockerized monlet-agent for the showcase / e2e stand only.
+# Production deployment uses a systemd-managed Go binary; do not use this image in prod.
+# Build context: repo root (we need ../agent).
+
+FROM golang:1.25-alpine AS builder
+WORKDIR /src
+ENV CGO_ENABLED=0 GOTOOLCHAIN=auto
+ARG MONLET_AGENT_VERSION=0.1.0
+COPY agent/go.mod agent/go.sum ./
+RUN go mod download
+COPY agent/ ./
+RUN go build -trimpath -ldflags="-s -w -X main.version=${MONLET_AGENT_VERSION}" -o /out/monlet-agent ./cmd/monlet-agent
+
+FROM ubuntu:24.04
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /out/monlet-agent /usr/local/bin/monlet-agent
+RUN mkdir -p /var/lib/monlet-agent /etc/monlet /opt/monlet/checks
+ENV MONLET_CONFIG=/etc/monlet/agent.toml
+ENTRYPOINT ["/bin/sh","-c","exec /usr/local/bin/monlet-agent -config \"$MONLET_CONFIG\""]
--- a/deploy/e2e-showcase.sh
+++ b/deploy/e2e-showcase.sh
@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# Stage 6.1 — Docker showcase / e2e stand for Monlet.
+# Brings up the full stack with demo agents, drives scenarios, asserts all
+# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
+# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications,
+# agent spool replay), and verifies metrics + web pages.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
+
+COMPOSE=(docker compose
+  -f "$HERE/docker-compose.yml"
+  -f "$HERE/docker-compose.showcase.yml")
+
+log()  { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }
+
+AUTH=(-H "Authorization: Bearer $TOKEN")
+
+cleanup() {
+  if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
+    log "tearing down stack"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  else
+    log "SHOWCASE_KEEP=1 → leaving stack running (web: $WEB_BASE)"
+  fi
+}
+trap cleanup EXIT
+
+wait_url() {
+  local url="$1" label="$2" tries="${3:-60}"
+  for i in $(seq 1 "$tries"); do
+    if curl -sf -o /dev/null "$url"; then return 0; fi
+    sleep 2
+    [[ "$i" == "$tries" ]] && fail "$label not ready: $url"
+  done
+}
+
+api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
+
+log "starting full showcase stack"
+MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build
+
+log "waiting for server /api/v1/ready"
+wait_url "$BASE/api/v1/ready" "server" 90
+
+log "waiting for web /"
+wait_url "$WEB_BASE/" "web" 90
+
+# --- Phase A: let agents run a few cycles ---
+log "letting demo agents push events for ~25s"
+sleep 25
+
+log "asserting agent labels include scenario and monlet_agent_version"
+api "/api/v1/agents/agent-mixed" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+labels=d.get('labels') or {}
+ver=labels.get('monlet_agent_version')
+ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
+print('agent-mixed labels:', labels)
+sys.exit(0 if ok else 1)" \
+  || fail "agent-mixed labels missing scenario or numeric monlet_agent_version"
+
+log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)"
+api "/api/v1/checks?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+want={'ok','warning','critical','unknown'}
+have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'}
+missing=want-have
+print('agent-mixed statuses:', sorted(have))
+sys.exit(0 if not missing else 1)" \
+  || fail "agent-mixed missing one of ok/warning/critical/unknown"
+
+# --- Phase B: trigger resolve scenario ---
+log "triggering resolve: touch /state/resolve.flag in agent-resolve"
+"${COMPOSE[@]}" exec -T agent-resolve sh -c 'touch /state/resolve.flag' \
+  || fail "could not touch resolve flag"
+
+log "waiting for resolve event to be processed (~15s)"
+sleep 15
+
+log "asserting resolved incident exists for agent-resolve/flapping"
+api "/api/v1/incidents?state=resolved&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items'])
+sys.exit(0 if ok else 1)" \
+  || fail "no resolved incident for agent-resolve/flapping"
+
+# --- Phase C: disabled server notifications ---
+log "asserting notifications-disabled incident exists but no outbox row for it"
+PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+for i in d['items']:
+    if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck':
+        print(i['id']); break")"
+[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
+api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
+  && fail "outbox unexpectedly contains row for prometheus-owned incident $PROM_INC"
+log "no outbox row for notifications-disabled incident"
+
+# --- Phase D: stop stale/dead agents to drive lifecycle transitions ---
+log "stopping agent-stale and agent-dead to age their heartbeats"
+"${COMPOSE[@]}" stop agent-stale agent-dead >/dev/null
+
+log "waiting ~25s for stale transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \
+  || api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to stale/dead"
+
+log "waiting another ~25s for dead transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to dead"
+api "/api/v1/agents/agent-dead"  | grep -q '"status":"dead"' \
+  || fail "agent-dead did not transition to dead"
+
+# At least one agent must still be alive (agent-mixed keeps pushing).
+api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \
+  || fail "agent-mixed not alive"
+
+# --- Phase E: outbox state coverage (live via webhook + debug) ---
+log "waiting for notifier worker cycles (~10s)"
+sleep 10
+
+OUTBOX="$(api "/api/v1/notifiers/outbox?limit=500")"
+echo "$OUTBOX" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+states={i['state'] for i in d['items']}
+need={'sent','retry','failed'}
+missing=need-states
+print('outbox states:', sorted(states))
+sys.exit(0 if not missing else 2)" \
+  || fail "outbox missing one of sent/retry/failed"
+
+# --- Phase F: seed discarded row, wait for worker ---
+log "seeding 'discarded' outbox row via SQL"
+"${COMPOSE[@]}" exec -T postgres \
+  psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \
+  < "$HERE/showcase/seed_discarded.sql" >/dev/null \
+  || fail "seed_discarded.sql failed"
+
+log "waiting for worker to mark unknown-notifier row as discarded (~10s)"
+sleep 10
+api "/api/v1/notifiers/outbox?state=discarded&limit=10" \
+  | grep -q '"state":"discarded"' \
+  || fail "no outbox row in state=discarded"
+
+# --- Phase G: spool replay assertion ---
+log "asserting agent-spool-replay events arrived without duplicates"
+api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+items=d['items']
+ids=[e['event_id'] for e in items]
+if len(items) < 2:
+    print(f'too few events: {len(items)}'); sys.exit(1)
+if len(set(ids)) != len(ids):
+    print('duplicate event_ids'); sys.exit(2)
+print(f'spool replay events: {len(items)} unique')" \
+  || fail "spool replay assertion failed"
+
+# --- Phase H: metrics ---
+log "server /metrics contains monlet_server_*"
+curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
+  || fail "server metrics missing monlet_server_*"
+
+log "agent-mixed /metrics contains monlet_agent_check_status"
+curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
+  || fail "agent metrics missing monlet_agent_check_status"
+
+# --- Phase I: web UI ---
+log "web pages render with expected content"
+curl -sf "$WEB_BASE/"          | grep -q "Overview"  || fail "web / missing Overview"
+curl -sf "$WEB_BASE/agents"    | grep -q "agent-mixed" || fail "web /agents missing agent-mixed"
+curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
+curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve"
+curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
+curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
+
+log "ALL SHOWCASE ASSERTIONS PASSED"
--- a/deploy/grafana/dashboards/monlet-overview.json
+++ b/deploy/grafana/dashboards/monlet-overview.json
@@ -0,0 +1,42 @@
+{
+  "title": "Monlet Overview",
+  "uid": "monlet-overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Agents by status",
+      "targets": [{ "expr": "monlet_server_agents", "legendFormat": "{{status}}" }],
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Open incidents",
+      "targets": [{ "expr": "monlet_server_open_incidents" }],
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Events/sec by result",
+      "targets": [
+        { "expr": "rate(monlet_server_events_total[1m])", "legendFormat": "{{result}}" }
+      ],
+      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 6 }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Notifications/sec by notifier+result",
+      "targets": [
+        { "expr": "rate(monlet_server_notifications_total[1m])", "legendFormat": "{{notifier}} {{result}}" }
+      ],
+      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 14 }
+    }
+  ]
+}
--- a/deploy/grafana/provisioning/dashboards/monlet.yml
+++ b/deploy/grafana/provisioning/dashboards/monlet.yml
@@ -0,0 +1,7 @@
+apiVersion: 1
+providers:
+  - name: monlet
+    folder: ""
+    type: file
+    options:
+      path: /var/lib/grafana/dashboards
--- a/deploy/grafana/provisioning/datasources/prometheus.yml
+++ b/deploy/grafana/provisioning/datasources/prometheus.yml
@@ -0,0 +1,7 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
--- a/deploy/prometheus/README.md
+++ b/deploy/prometheus/README.md
@@ -6,4 +6,4 @@ Planned examples:

 - scrape Monlet agent `/metrics`;
 - scrape Monlet server `/metrics`;
- sample rules that avoid duplicate alerting when `notification_owner = server`.
+- sample rules for checks that use `notifications_enabled = false`.
--- a/deploy/prometheus/prometheus.yml
+++ b/deploy/prometheus/prometheus.yml
@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: monlet-server
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["server:8000"]
+
+  - job_name: monlet-agent
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["host.docker.internal:9100"]
--- a/deploy/showcase/agents/dead.toml
+++ b/deploy/showcase/agents/dead.toml
@@ -0,0 +1,24 @@
+hostname = "agent-dead"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "dead"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Dead agent placeholder check"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "10s"
+timeout = "3s"
--- a/deploy/showcase/agents/mixed.toml
+++ b/deploy/showcase/agents/mixed.toml
@@ -0,0 +1,46 @@
+hostname = "agent-mixed"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "mixed-status"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = true
+listen = "0.0.0.0:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Showcase ok"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "5s"
+timeout = "3s"
+
+[[checks]]
+id = "warning_check"
+name = "Showcase warning"
+command = "/opt/monlet/checks/exit_1.sh"
+interval = "5s"
+timeout = "3s"
+
+[[checks]]
+id = "critical_check"
+name = "Showcase critical (permanent webhook fail)"
+command = "/opt/monlet/checks/exit_2.sh"
+interval = "5s"
+timeout = "3s"
+dedupe_key = "agent-mixed:critical:fail"
+
+[[checks]]
+id = "unknown_check"
+name = "Showcase unknown"
+command = "/opt/monlet/checks/exit_3.sh"
+interval = "5s"
+timeout = "3s"
--- a/deploy/showcase/agents/prometheus_owned.toml
+++ b/deploy/showcase/agents/prometheus_owned.toml
@@ -0,0 +1,25 @@
+hostname = "agent-prometheus-owned"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "prometheus-owned"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "promcheck"
+name = "Prometheus-owned critical (no outbox expected)"
+command = "/opt/monlet/checks/exit_2.sh"
+interval = "5s"
+timeout = "3s"
+notifications_enabled = false
--- a/deploy/showcase/agents/resolve.toml
+++ b/deploy/showcase/agents/resolve.toml
@@ -0,0 +1,25 @@
+hostname = "agent-resolve"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "resolve"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "flapping"
+name = "Flapping check (critical then ok)"
+command = "/opt/monlet/checks/flapping.sh"
+interval = "5s"
+timeout = "3s"
+dedupe_key = "agent-resolve:flapping:retry"
--- a/deploy/showcase/agents/spool_replay.toml
+++ b/deploy/showcase/agents/spool_replay.toml
@@ -0,0 +1,24 @@
+hostname = "agent-spool-replay"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "spool-replay"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "replay_check"
+name = "Spool replay check"
+command = "/opt/monlet/checks/replay_check.sh"
+interval = "3s"
+timeout = "2s"
--- a/deploy/showcase/agents/stale.toml
+++ b/deploy/showcase/agents/stale.toml
@@ -0,0 +1,24 @@
+hostname = "agent-stale"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "stale"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Stale agent placeholder check"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "10s"
+timeout = "3s"
--- a/deploy/showcase/checks/exit_0.sh
+++ b/deploy/showcase/checks/exit_0.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "ok: $(date -u +%FT%TZ)"
+exit 0
--- a/deploy/showcase/checks/exit_1.sh
+++ b/deploy/showcase/checks/exit_1.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "warning: degraded"
+exit 1
--- a/deploy/showcase/checks/exit_2.sh
+++ b/deploy/showcase/checks/exit_2.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "critical: probe failed"
+exit 2
--- a/deploy/showcase/checks/exit_3.sh
+++ b/deploy/showcase/checks/exit_3.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "unknown: probe returned unexpected exit code"
+exit 3
--- a/deploy/showcase/checks/flapping.sh
+++ b/deploy/showcase/checks/flapping.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# critical until /state/resolve.flag exists, then ok.
+if [ -f /state/resolve.flag ]; then
+  echo "ok: resolve flag present"
+  exit 0
+fi
+echo "critical: waiting for resolve flag"
+exit 2
--- a/deploy/showcase/checks/replay_check.sh
+++ b/deploy/showcase/checks/replay_check.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "ok: replay tick"
+exit 0
--- a/deploy/showcase/mock_webhook.py
+++ b/deploy/showcase/mock_webhook.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Mock webhook for showcase/e2e.
+
+Decides response status by substring in incident.incident_key from request body:
+- contains 'retry' -> 503 (retryable failure)
+- contains 'fail'  -> 400 (permanent failure)
+- otherwise        -> 200 (success)
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+def _classify(body: bytes) -> int:
+    try:
+        data = json.loads(body or b"{}")
+    except Exception:
+        return 200
+    incident = data.get("incident") or {}
+    key = str(incident.get("incident_key") or "")
+    if "retry" in key:
+        return 503
+    if "fail" in key:
+        return 400
+    return 200
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_POST(self) -> None:  # noqa: N802
+        length = int(self.headers.get("Content-Length") or 0)
+        body = self.rfile.read(length) if length else b""
+        code = _classify(body)
+        self.send_response(code)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+        try:
+            data = json.loads(body or b"{}")
+            inc = (data.get("incident") or {})
+            print(
+                f"webhook {code} event_type={data.get('event_type')!r} "
+                f"incident_key={inc.get('incident_key')!r}",
+                flush=True,
+            )
+        except Exception:
+            print(f"webhook {code} (unparsable body, {length}B)", flush=True)
+
+    def log_message(self, *_args: object) -> None:
+        return
+
+
+def main() -> int:
+    addr = ("0.0.0.0", 8080)
+    srv = ThreadingHTTPServer(addr, Handler)
+    print(f"mock_webhook listening on {addr[0]}:{addr[1]}", flush=True)
+    try:
+        srv.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        srv.server_close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/deploy/showcase/seed_discarded.sql
+++ b/deploy/showcase/seed_discarded.sql
@@ -0,0 +1,16 @@
+-- Seed one outbox row with an unknown notifier name so the worker marks it 'discarded'.
+-- Attached to any existing incident; if none exists yet the script is a no-op.
+INSERT INTO notification_outbox
+    (id, incident_id, notifier, event_type, state, attempts, next_attempt_at, payload)
+SELECT
+    gen_random_uuid(),
+    i.id,
+    'showcase-unknown',
+    'firing',
+    'pending',
+    0,
+    now(),
+    '{"showcase":"discarded"}'::jsonb
+FROM incidents i
+ORDER BY i.opened_at DESC
+LIMIT 1;
--- a/deploy/smoke.sh
+++ b/deploy/smoke.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Local stack smoke test for Stage 6.
+# Brings the stack up, hits /health /ready /metrics, ingests sample heartbeat + events,
+# verifies the API exposes agent / check / incident state.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
+COMPOSE=(docker compose -f "$HERE/docker-compose.yml")
+
+log() { printf "\033[1;36m[smoke]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[smoke]\033[0m %s\n" "$*" >&2; exit 1; }
+
+cleanup() {
+  if [[ "${SMOKE_KEEP:-0}" != "1" ]]; then
+    log "tearing down stack"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+log "starting stack"
+MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build postgres server web
+
+log "waiting for /api/v1/ready"
+for i in $(seq 1 60); do
+  if curl -sf "$BASE/api/v1/ready" >/dev/null; then break; fi
+  sleep 2
+  [[ "$i" == "60" ]] && fail "server did not become ready"
+done
+
+log "GET /api/v1/health"
+curl -sf "$BASE/api/v1/health" | grep -q '"status":"ok"' || fail "health wrong body"
+
+log "GET /metrics"
+curl -sf "$BASE/metrics" | grep -q '^monlet_server_' || fail "metrics missing"
+
+AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json")
+
+log "POST heartbeat (examples/heartbeat.json)"
+curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \
+  | grep -q '"accepted":true' || fail "heartbeat not accepted"
+
+log "POST events (examples/event-batch.json)"
+curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \
+  | grep -q '"accepted":' || fail "event batch not accepted"
+
+log "GET /agents has host-01.prod"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \
+  || fail "host-01.prod missing"
+
+log "GET /checks not empty"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \
+  || fail "checks empty"
+
+log "GET /incidents not empty (event-batch.json contains a critical event)"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \
+  || fail "no open incident"
+
+log "GET /notifiers/outbox not empty"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \
+  || fail "outbox empty"
+
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+log "waiting for web $WEB_BASE"
+for i in $(seq 1 60); do
+  if curl -sf -o /dev/null "$WEB_BASE/"; then break; fi
+  sleep 2
+  [[ "$i" == "60" ]] && fail "web did not become ready"
+done
+
+log "GET / (web overview)"
+curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web overview missing 'Overview'"
+
+log "GET /agents (web list)"
+curl -sf "$WEB_BASE/agents" | grep -q "host-01.prod" || fail "web /agents missing host-01.prod"
+
+log "ALL SMOKE TESTS PASSED"