From 37b1a1d6d6f74d329bb357499364f4b4de91251b Mon Sep 17 00:00:00 2001 From: Stanislav Rossovskii Date: Thu, 28 May 2026 14:19:27 +0400 Subject: [PATCH] Harden production workflows and agent admission --- .github/workflows/ci.yml | 10 +- README.md | 16 +- agent/README.md | 53 +- agent/cmd/monlet-agent/main.go | 12 +- agent/config.example.toml | 13 +- agent/internal/agentkey/agentkey.go | 75 +++ agent/internal/agentkey/agentkey_test.go | 75 +++ agent/internal/app/app.go | 37 +- agent/internal/app/app_test.go | 3 +- agent/internal/client/client.go | 5 +- agent/internal/config/config.go | 37 +- agent/internal/config/config_test.go | 83 ++- agent/internal/ids/ids.go | 2 +- agent/internal/metrics/metrics.go | 7 +- agent/internal/runner/runner.go | 5 +- agent/internal/runner/runner_test.go | 5 +- agent/internal/spool/spool.go | 35 +- agent/internal/spool/spool_test.go | 11 + api/openapi.yaml | 384 +++++++++++++- deploy/README.md | 8 +- deploy/docker-compose.showcase.yml | 24 +- deploy/docker-compose.yml | 38 +- deploy/e2e-showcase.sh | 202 +++++-- deploy/showcase/agents/dead.toml | 3 +- deploy/showcase/agents/flap.toml | 5 +- deploy/showcase/agents/mixed.toml | 5 +- deploy/showcase/agents/prometheus_owned.toml | 3 +- deploy/showcase/agents/resolve.toml | 5 +- deploy/showcase/agents/spool_replay.toml | 3 +- deploy/showcase/agents/stale.toml | 3 +- deploy/showcase/agents/timeout.toml | 7 +- deploy/showcase/checks/replay_check.sh | 14 +- deploy/showcase/seed_discarded.sql | 16 - deploy/smoke.sh | 54 +- docs/operations/deployment.md | 2 +- docs/ops/backup-restore.md | 16 +- docs/ops/deployment-checklist.md | 5 +- docs/ops/token-rotation.md | 58 +- server/.env.example | 9 +- server/Dockerfile | 9 + server/README.md | 16 +- server/alembic/versions/0001_baseline.py | 89 +++- server/monlet_server/agent_auth.py | 30 ++ server/monlet_server/api/agents.py | 330 +++++++++++- server/monlet_server/api/checks.py | 47 +- server/monlet_server/api/events.py | 69 ++- server/monlet_server/api/heartbeat.py | 7 +- server/monlet_server/api/incidents.py | 129 ++++- server/monlet_server/api/outbox.py | 74 ++- server/monlet_server/api/system_summary.py | 43 ++ server/monlet_server/db.py | 12 +- server/monlet_server/errors.py | 8 +- server/monlet_server/main.py | 3 +- server/monlet_server/middleware/auth.py | 28 +- server/monlet_server/models.py | 88 ++- server/monlet_server/schemas.py | 36 ++ server/monlet_server/services/detector.py | 109 +++- server/monlet_server/services/ingestion.py | 168 +++++- .../monlet_server/services/notifier_worker.py | 41 +- server/monlet_server/settings.py | 41 +- server/tests/_helpers.py | 23 +- server/tests/conftest.py | 13 +- server/tests/test_agent_admission.py | 354 ++++++++++++ server/tests/test_auth.py | 67 ++- server/tests/test_detector.py | 145 ++++- server/tests/test_events_ingestion.py | 44 +- server/tests/test_heartbeat.py | 6 +- server/tests/test_incident_key.py | 24 +- server/tests/test_incidents.py | 53 +- server/tests/test_integration_agent.py | 4 +- server/tests/test_notifiers.py | 8 +- server/tests/test_pagination.py | 176 +++++- server/tests/test_partitioning.py | 6 +- server/tests/test_system_summary.py | 35 ++ web/Dockerfile | 4 + web/README.md | 10 +- web/src/app/agents/[agent_id]/page.tsx | 13 +- web/src/app/agents/blacklist/page.tsx | 15 + web/src/app/api/agents/action/route.ts | 56 ++ web/src/app/api/health/route.ts | 7 + web/src/app/api/poll/events/route.ts | 1 + web/src/app/api/poll/incidents/route.ts | 23 +- web/src/app/api/poll/outbox/route.ts | 32 +- .../{overview => system-summary}/route.ts | 4 +- web/src/app/checks/[check_id]/page.tsx | 19 +- web/src/app/events/page.tsx | 11 +- web/src/app/incidents/page.tsx | 38 +- web/src/app/layout.tsx | 52 +- web/src/app/outbox/page.tsx | 39 +- web/src/app/page.tsx | 14 +- web/src/components/AgentBlacklistBrowser.tsx | 104 ++++ web/src/components/AgentDetailBrowser.tsx | 20 +- web/src/components/AgentsBrowser.tsx | 302 +++++++++-- web/src/components/CheckDetailBrowser.tsx | 18 +- web/src/components/EventsBrowser.tsx | 16 +- web/src/components/IncidentsBrowser.tsx | 217 ++++---- web/src/components/OutboxBrowser.tsx | 94 +++- web/src/components/OverviewDashboard.tsx | 56 -- web/src/components/Pager.tsx | 42 +- web/src/components/TopNav.tsx | 93 ++++ web/src/lib/api-types.ts | 502 +++++++++++++++++- web/src/lib/api.ts | 54 +- web/src/lib/loaders.ts | 124 +++-- web/src/lib/pagination.ts | 9 + web/src/lib/usePolling.ts | 61 ++- web/src/lib/view-types.ts | 15 +- web/src/proxy.ts | 48 ++ web/tests/mock-server.mjs | 15 + web/tests/smoke.spec.ts | 10 +- 109 files changed, 4927 insertions(+), 894 deletions(-) create mode 100644 agent/internal/agentkey/agentkey.go create mode 100644 agent/internal/agentkey/agentkey_test.go delete mode 100644 deploy/showcase/seed_discarded.sql create mode 100644 server/monlet_server/agent_auth.py create mode 100644 server/monlet_server/api/system_summary.py create mode 100644 server/tests/test_agent_admission.py create mode 100644 server/tests/test_system_summary.py create mode 100644 web/src/app/agents/blacklist/page.tsx create mode 100644 web/src/app/api/agents/action/route.ts create mode 100644 web/src/app/api/health/route.ts rename web/src/app/api/poll/{overview => system-summary}/route.ts (67%) create mode 100644 web/src/components/AgentBlacklistBrowser.tsx delete mode 100644 web/src/components/OverviewDashboard.tsx create mode 100644 web/src/components/TopNav.tsx create mode 100644 web/src/proxy.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d28e3d..957c2d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,16 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "22" + # PH-021: cache npm so the pinned @redocly/cli version below is fetched + # once per cache hit; the version is pinned by exact tag in this file + # so the same artifact is used across runs. + cache: npm + cache-dependency-path: web/package-lock.json - name: Lint OpenAPI - run: npx --yes @redocly/cli@1.34.6 lint api/openapi.yaml + # PH-021: @redocly/cli is not in package.json because it is a lint-only + # tool that does not ship with the web bundle. The exact version below + # is the reproducibility pin; do not float to "latest" or a range. + run: npx --yes @redocly/cli@2.11.1 lint api/openapi.yaml agent: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 6549190..1a3366f 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ It is split into three independent applications: - `agent/` - a small Go binary installed on monitored hosts. - `server/` - a Python/FastAPI backend that owns inventory, state, incidents, and notifications. -- `web/` - a read-only Next.js dashboard. +- `web/` - a Next.js dashboard with agent admission controls. -The first project phase is documentation and planning bootstrap only. Core business logic is intentionally not implemented yet. +All three applications are implemented and run together via Docker Compose. The current release covers agent ingestion with operator admission, server-owned incident lifecycle and notifier outbox, a Next.js dashboard, and partitioned event retention. -## MVP +## Scope -Monlet v1 focuses on local checks, central state, and simple read-only visibility: +Monlet v1 focuses on local checks, central state, and simple visibility: - agents run local checks from TOML config; - agents expose Prometheus metrics and/or push facts to the server; @@ -39,9 +39,9 @@ Monlet v1 focuses on local checks, central state, and simple read-only visibilit - `docs/` - architecture, ADRs, development, and operations docs. - `plan/` - stage workflow and exit checkpoints. - `api/` - versioned OpenAPI contract. -- `agent/` - Go agent placeholder and deployment notes. -- `server/` - Python backend placeholder. -- `web/` - Next.js frontend placeholder. +- `agent/` - Go agent: scheduler, runner, spool, metrics. +- `server/` - Python/FastAPI backend: ingestion, detector, notifier outbox. +- `web/` - Next.js dashboard and agent admission UI. - `deploy/` - local and observability deployment examples. - `examples/` - example checks and configs. @@ -59,4 +59,4 @@ Read in this order: ## Current Status -Stage 0 is the only active stage: documentation bootstrap. Do not start agent, server, or web core logic until Stage 0 exits and the API contract is reviewed. +Active development. Stages 0–6 are implemented. Run the stack via `docker compose up --build` from `deploy/`. See `docs/operations/deployment.md` and `docs/ops/` runbooks. diff --git a/agent/README.md b/agent/README.md index fd71dbb..a7a3f83 100644 --- a/agent/README.md +++ b/agent/README.md @@ -1,6 +1,6 @@ # Monlet Agent -Go agent (Stage 2 MVP). Reads `agent.toml`, runs configured commands on schedule, captures status/output, pushes contract-valid `heartbeat` and `events` batches to the Monlet server with Bearer auth, and survives outages via a local on-disk spool. +Go agent. Reads `agent.toml`, runs configured commands on schedule, captures status/output, pushes contract-valid `heartbeat` and `events` batches to the Monlet server with Bearer auth, and survives outages via a local on-disk spool. ## Build and run @@ -14,22 +14,28 @@ Do not use `go install` or `go env -w`. Module cache, build cache, and tool bina ## Configuration -See `config.example.toml`. Required fields: top-level `state_dir`, at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`), and at least one output enabled: `server.enabled = true` or `metrics.enabled = true`. `agent_id` is optional; if omitted, it defaults to `hostname`. +See `config.example.toml`. Required fields: top-level `state_dir`, at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`), and at least one output enabled: `server.enabled = true` or `metrics.enabled = true`. `agent_id` is optional; if omitted, it defaults to the OS hostname. `hostname` is not a config key. | Key | Default | Notes | |---|---|---| -| `server.enabled` | `false` | Enables heartbeat/event push. Requires `server.url` and `server.token` when true. | +| `server.enabled` | `false` | Enables heartbeat/event push. Requires `server.url` when true. The agent generates and stores its own Bearer key in `state_dir/agent.key`. | | `metrics.enabled` | `false` | Enables `/metrics`. | | `labels` | `{}` | Optional heartbeat labels. Max 31 custom labels because `monlet_agent_version` is generated by the agent. Secret-like keys are rejected. | -| `server.heartbeat_interval` | `30s` | ADR-0005 | +| `server.heartbeat_interval` | `10s` | ADR-0005 | | `server.batch_interval` | `10s` | events flush cadence | | `metrics.listen` | `127.0.0.1:9465` | low-cardinality only | -| `checks[].command` | required | Shell string executed as `/bin/sh -c`; TOML multi-line strings are supported. | +| `checks[].command` | required | Shell string executed as `/bin/sh -c`. Both forms accepted: `command = "check_disk --warn 80 --crit 90"` and triple-quoted `command = '''...'''`. Argv arrays are rejected. See [Command security contract](#command-security-contract). | | `checks[].notifications_enabled` | `true` | Set `false` for checks that must not create server notifications. | -Example: +Examples — one-line and multi-line forms: ```toml +[[checks]] +id = "uptime" +command = "test $(cut -d. -f1 /proc/uptime) -gt 60" +interval = "60s" +timeout = "5s" + [[checks]] id = "disk_root" command = ''' @@ -40,23 +46,47 @@ interval = "60s" timeout = "10s" ``` +## Command security contract + +`checks[].command` is the only public field for what a check runs. The agent +executes it as `/bin/sh -c ` on the host where the agent runs. + +- Local-config only. Commands are loaded from the agent's TOML file on disk. + The server does not push config and does not execute remote commands; see + `docs/architecture/security.md#no-remote-execution`. Anyone able to edit the + agent config or replace the agent binary already has local root-equivalent + power, so shell execution adds no new attack surface beyond that file. +- Argv arrays (`command = ["..."]`) are intentionally rejected by config + validation. Use a shell string for both short and long commands. +- Quoting and escaping are the operator's responsibility. Triple-quoted + `'''...'''` is recommended for multi-line scripts and avoids most escaping. +- Do not put secrets in `command`. Pass them via the systemd unit + `Environment=` / `EnvironmentFile=` and reference them from the script. + Label keys containing `token`, `secret`, `password`, `credential`, + `authorization`, `cookie`, `api_key`/`apikey` are rejected by config + validation to keep secrets out of heartbeats. +- Each run uses `exec.CommandContext` with `Setpgid`. On timeout the whole + process group is killed and the check result becomes `critical`. +- Combined stdout+stderr is truncated by UTF-8 byte length to 8 KiB with marker + `...[truncated N bytes]` before being persisted, exposed, or notified. + ## Behaviour - Per-check scheduler with no-overlap: a tick is skipped if the previous run is still in flight (`monlet_agent_check_skipped_total`). - Check commands run through `/bin/sh -c`; the old argv-array form is intentionally unsupported. -- Timeout via `exec.CommandContext` with `Setpgid` so the process group is killed on expiry; timeout → `unknown`. +- Timeout via `exec.CommandContext` with `Setpgid` so the process group is killed on expiry; timeout → `critical`. - Exit code mapping: 0=ok, 1=warning, 2=critical, 3+=unknown (ADR-0005). - `output` (stdout+stderr) is truncated by **UTF-8 byte length** to 8 KiB with marker `...[truncated N bytes]`. - `event_id` is UUIDv7 generated on check completion and preserved across spool replay (ADR-0006 idempotency). -- Spool: per-event JSON files in `state_dir/spool/`, FIFO, limits 10 000 events / 50 MiB, drop oldest on overflow (ADR-0005). -- Retry backoff for heartbeat and events: 1s → 30s, exp ×2, jitter ±20%. 4xx are dropped (not retried) to avoid hot loop. -- Bearer auth header on both endpoints (ADR-0007). +- Spool: per-event JSON files in `state_dir/spool/`, FIFO, limits 10 000 events / 50 MiB, drop oldest on overflow (ADR-0005). Drops are counted in `monlet_agent_events_dropped_total{reason}` with bounded reasons; one warning is logged per ~30s drop burst, not per event. +- Retry backoff for heartbeat and events: 1s → 30s, exp ×2, jitter ±20%. 4xx are dropped (not retried) to avoid hot loop; dropped batches are counted as `reason=send_non_retryable` and logged once per batch. +- Bearer auth header on both endpoints uses the local `state_dir/agent.key` value (ADR-0007). - Heartbeats include configured labels plus reserved `monlet_agent_version=`. - Heartbeats include explicit feature flags: `push` from `server.enabled`, `metrics` from `metrics.enabled`. ## Prometheus metrics -Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `status`, `kind`): +Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `status`, `kind`). The agent caps `checks` at 256 (PH-019) and each `id` must match `[A-Za-z0-9._:-]+`; do not embed dynamic values (timestamps, sequence numbers, hostnames) in `id` because metric cardinality scales with it on Prometheus. - `monlet_agent_check_runs_total{check_id,status}` - `monlet_agent_check_skipped_total{check_id}` @@ -70,6 +100,7 @@ Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `statu - `monlet_agent_spool_events`, `monlet_agent_spool_bytes` - `monlet_agent_send_attempts_total{kind}`, `monlet_agent_send_failures_total{kind}` (`kind` ∈ `heartbeat`, `events`) - `monlet_agent_events_accepted_total`, `monlet_agent_events_deduplicated_total` +- `monlet_agent_events_dropped_total{reason}` — `reason ∈ spool_overflow_events | spool_overflow_bytes | send_non_retryable` - `monlet_agent_last_heartbeat_timestamp_seconds` - `monlet_agent_build_info{version,push,metrics}` (constant 1) diff --git a/agent/cmd/monlet-agent/main.go b/agent/cmd/monlet-agent/main.go index a9d397c..657acaa 100644 --- a/agent/cmd/monlet-agent/main.go +++ b/agent/cmd/monlet-agent/main.go @@ -9,6 +9,7 @@ import ( "path/filepath" "syscall" + "github.com/monlet/agent/internal/agentkey" "github.com/monlet/agent/internal/app" "github.com/monlet/agent/internal/client" "github.com/monlet/agent/internal/config" @@ -43,11 +44,20 @@ func main() { os.Exit(1) } + key := "" + if cfg.PushesToServer() { + key, err = agentkey.LoadOrCreate(cfg.StateDir) + if err != nil { + log.Error("agent key", "err", err) + os.Exit(1) + } + } + a := &app.App{ Cfg: cfg, AgentID: agentID, Version: version, - Client: client.New(cfg.Server.URL, cfg.Server.Token, agentID), + Client: client.New(cfg.Server.URL, key, agentID), Spool: sp, Metrics: metrics.New(), Log: log, diff --git a/agent/config.example.toml b/agent/config.example.toml index 3e94c05..262815d 100644 --- a/agent/config.example.toml +++ b/agent/config.example.toml @@ -1,4 +1,5 @@ -hostname = "example-host-01" +# Optional. If omitted, the OS hostname is used as agent_id. +# agent_id = "example-host-01" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,8 +9,7 @@ role = "api" [server] enabled = true url = "http://127.0.0.1:8000" -token = "replace-me" -heartbeat_interval = "30s" +heartbeat_interval = "10s" batch_interval = "10s" [metrics] @@ -24,3 +24,10 @@ command = ''' ''' interval = "60s" timeout = "10s" + +[[checks]] +id = "uptime" +name = "Uptime probe" +command = "test $(cut -d. -f1 /proc/uptime) -gt 60" +interval = "60s" +timeout = "5s" diff --git a/agent/internal/agentkey/agentkey.go b/agent/internal/agentkey/agentkey.go new file mode 100644 index 0000000..9328d90 --- /dev/null +++ b/agent/internal/agentkey/agentkey.go @@ -0,0 +1,75 @@ +package agentkey + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + fileName = "agent.key" + keyBytes = 32 + keyLen = keyBytes * 2 +) + +func LoadOrCreate(stateDir string) (string, error) { + if err := os.MkdirAll(stateDir, 0o700); err != nil { + return "", err + } + if err := os.Chmod(stateDir, 0o700); err != nil { + return "", err + } + path := filepath.Join(stateDir, fileName) + if info, err := os.Stat(path); err == nil { + if !info.Mode().IsRegular() { + return "", fmt.Errorf("agent key is not a regular file: %s", path) + } + if info.Mode().Perm() != 0o600 { + if err := os.Chmod(path, 0o600); err != nil { + return "", err + } + } + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return validate(strings.TrimSpace(string(b)), path) + } else if !os.IsNotExist(err) { + return "", err + } + + key, err := generate() + if err != nil { + return "", err + } + tmp := fmt.Sprintf("%s.%d.tmp", path, os.Getpid()) + defer os.Remove(tmp) + if err := os.WriteFile(tmp, []byte(key+"\n"), 0o600); err != nil { + return "", err + } + if err := os.Rename(tmp, path); err != nil { + return "", err + } + return key, nil +} + +func validate(key string, path string) (string, error) { + if len(key) != keyLen { + return "", fmt.Errorf("invalid agent key in %s", path) + } + if _, err := hex.DecodeString(key); err != nil { + return "", fmt.Errorf("invalid agent key in %s", path) + } + return strings.ToLower(key), nil +} + +func generate() (string, error) { + var b [keyBytes]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} diff --git a/agent/internal/agentkey/agentkey_test.go b/agent/internal/agentkey/agentkey_test.go new file mode 100644 index 0000000..f01c5b0 --- /dev/null +++ b/agent/internal/agentkey/agentkey_test.go @@ -0,0 +1,75 @@ +package agentkey + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadOrCreatePersistsKey(t *testing.T) { + dir := t.TempDir() + k1, err := LoadOrCreate(dir) + if err != nil { + t.Fatal(err) + } + k2, err := LoadOrCreate(dir) + if err != nil { + t.Fatal(err) + } + if k1 != k2 { + t.Fatal("key was not persisted") + } + if len(k1) != keyLen { + t.Fatalf("unexpected key length %d", len(k1)) + } + info, err := os.Stat(filepath.Join(dir, fileName)) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("unexpected key mode %o", got) + } +} + +func TestLoadOrCreateRejectsInvalidExistingKey(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, fileName), []byte("bad\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadOrCreate(dir); err == nil { + t.Fatal("expected invalid key error") + } +} + +func TestLoadOrCreateRejectsInvalidHexExistingKey(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, fileName), []byte("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadOrCreate(dir); err == nil { + t.Fatal("expected invalid key error") + } +} + +func TestLoadOrCreateRepairsKeyPermissions(t *testing.T) { + dir := t.TempDir() + key := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + path := filepath.Join(dir, fileName) + if err := os.WriteFile(path, []byte(key+"\n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := LoadOrCreate(dir) + if err != nil { + t.Fatal(err) + } + if got != key { + t.Fatalf("unexpected key %q", got) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("unexpected key mode %o", got) + } +} diff --git a/agent/internal/app/app.go b/agent/internal/app/app.go index 6237529..6535c72 100644 --- a/agent/internal/app/app.go +++ b/agent/internal/app/app.go @@ -22,11 +22,23 @@ type App struct { Spool *spool.Spool Metrics *metrics.Metrics Log *slog.Logger + + lastDropLog time.Time } +const ( + dropReasonOverflowEvents = "spool_overflow_events" + dropReasonOverflowBytes = "spool_overflow_bytes" + dropReasonSendRejected = "send_non_retryable" + + dropLogThrottle = 30 * time.Second +) + func (a *App) Run(ctx context.Context) error { events := make(chan scheduler.Event, 256) + // Account drops accumulated during spool Open (e.g. shrunk limits on restart). + a.recordSpoolDrops(a.Spool.Drops()) a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks))) a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(a.Cfg.PushesToServer()), boolLabel(a.Cfg.ExposesMetrics())).Set(1) for _, c := range a.Cfg.Checks { @@ -108,6 +120,7 @@ func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) { if err := a.Spool.Enqueue(ev); err != nil { a.Log.Error("spool enqueue", "err", err) } + a.recordSpoolDrops(a.Spool.Drops()) a.Metrics.SpoolDepth.Set(float64(a.Spool.Len())) a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes())) } @@ -188,7 +201,9 @@ func (a *App) sendLoop(ctx context.Context) { } continue } - // non-retryable: drop to avoid hot loop + // non-retryable: drop to avoid hot loop. + a.Metrics.EventsDropped.WithLabelValues(dropReasonSendRejected).Add(float64(len(seqs))) + a.Log.Warn("events dropped: server rejected batch", "count", len(seqs), "reason", dropReasonSendRejected) a.Spool.Commit(seqs) attempt = 0 continue @@ -214,6 +229,26 @@ func sleep(ctx context.Context, d time.Duration) bool { } } +func (a *App) recordSpoolDrops(d spool.DropStats) { + if d.OverflowEvents == 0 && d.OverflowBytes == 0 { + return + } + if d.OverflowEvents > 0 { + a.Metrics.EventsDropped.WithLabelValues(dropReasonOverflowEvents).Add(float64(d.OverflowEvents)) + } + if d.OverflowBytes > 0 { + a.Metrics.EventsDropped.WithLabelValues(dropReasonOverflowBytes).Add(float64(d.OverflowBytes)) + } + now := time.Now() + if now.Sub(a.lastDropLog) >= dropLogThrottle { + a.Log.Warn("spool drops", + "events_overflow", d.OverflowEvents, + "bytes_overflow", d.OverflowBytes, + ) + a.lastDropLog = now + } +} + func boolLabel(v bool) string { if v { return "true" diff --git a/agent/internal/app/app_test.go b/agent/internal/app/app_test.go index 5081800..cd70e30 100644 --- a/agent/internal/app/app_test.go +++ b/agent/internal/app/app_test.go @@ -31,7 +31,7 @@ func TestPrometheusOnlyDoesNotPush(t *testing.T) { cfg := &config.Config{ AgentID: "a1", StateDir: dir, - Server: config.ServerConfig{Enabled: boolPtr(false), URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}}, + Server: config.ServerConfig{Enabled: boolPtr(false), URL: srv.URL, HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}}, Metrics: config.MetricsConfig{Enabled: true, Listen: "127.0.0.1:0"}, Checks: []config.CheckConfig{{ ID: "c1", @@ -106,7 +106,6 @@ func TestEndToEndPushAndReplay(t *testing.T) { Server: config.ServerConfig{ Enabled: boolPtr(true), URL: srv.URL, - Token: "t", HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond}, BatchInterval: config.Duration{Duration: 100 * time.Millisecond}, }, diff --git a/agent/internal/client/client.go b/agent/internal/client/client.go index 4cd8c68..518ddec 100644 --- a/agent/internal/client/client.go +++ b/agent/internal/client/client.go @@ -29,7 +29,10 @@ func New(baseURL, token, agentID string) *Client { baseURL: strings.TrimRight(baseURL, "/"), token: token, agentID: agentID, - http: &http.Client{Timeout: 15 * time.Second}, + http: &http.Client{ + Timeout: 15 * time.Second, + Transport: &http.Transport{DisableKeepAlives: true}, + }, } } diff --git a/agent/internal/config/config.go b/agent/internal/config/config.go index d5a404e..8c884a8 100644 --- a/agent/internal/config/config.go +++ b/agent/internal/config/config.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" "time" "unicode/utf8" @@ -14,19 +15,24 @@ import ( var idPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]+$`) const ( - maxIDLen = 128 - maxLabelKeyLen = 64 - maxLabelValueLen = 256 - maxLabels = 32 + maxIDLen = 128 + maxLabelKeyLen = 64 + maxLabelValueLen = 256 + maxLabels = 32 + // PH-019: bounded check inventory keeps {check_id} metric cardinality + // predictable. ValidateID already restricts each check_id to a stable + // identifier alphabet; this cap bounds the total. Override via + // MONLET_AGENT_MAX_CHECKS for fleets that genuinely need more. + defaultMaxChecks = 256 AgentVersionLabel = "monlet_agent_version" - defaultHeartbeat = 30 * time.Second + defaultHeartbeat = 10 * time.Second defaultBatch = 10 * time.Second defaultMetricsAddr = "127.0.0.1:9465" ) type Config struct { AgentID string `toml:"agent_id"` - Hostname string `toml:"hostname"` + Hostname string `toml:"-"` StateDir string `toml:"state_dir"` Labels map[string]string `toml:"labels"` Server ServerConfig `toml:"server"` @@ -37,7 +43,6 @@ type Config struct { type ServerConfig struct { Enabled *bool `toml:"enabled"` URL string `toml:"url"` - Token string `toml:"token"` HeartbeatInterval Duration `toml:"heartbeat_interval"` BatchInterval Duration `toml:"batch_interval"` } @@ -124,11 +129,8 @@ func (c *Config) Validate() error { if c.Server.URL == "" { return fmt.Errorf("server.url is required when server.enabled = true") } - if c.Server.Token == "" { - return fmt.Errorf("server.token is required when server.enabled = true") - } - } else if c.Server.URL != "" || c.Server.Token != "" { - return fmt.Errorf("server.url/token require server.enabled = true") + } else if c.Server.URL != "" { + return fmt.Errorf("server.url requires server.enabled = true") } if !c.PushesToServer() && !c.ExposesMetrics() { return fmt.Errorf("server.enabled or metrics.enabled must be true") @@ -136,6 +138,17 @@ func (c *Config) Validate() error { if len(c.Checks) == 0 { return fmt.Errorf("at least one check is required") } + maxChecks := defaultMaxChecks + if v, ok := os.LookupEnv("MONLET_AGENT_MAX_CHECKS"); ok { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return fmt.Errorf("MONLET_AGENT_MAX_CHECKS must be a positive integer, got %q", v) + } + maxChecks = n + } + if len(c.Checks) > maxChecks { + return fmt.Errorf("too many checks: %d (max %d) — bounded for metric cardinality (PH-019)", len(c.Checks), maxChecks) + } seen := make(map[string]struct{}, len(c.Checks)) for i := range c.Checks { ch := &c.Checks[i] diff --git a/agent/internal/config/config_test.go b/agent/internal/config/config_test.go index ee75244..1ce3eb0 100644 --- a/agent/internal/config/config_test.go +++ b/agent/internal/config/config_test.go @@ -24,7 +24,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" @@ -57,7 +56,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" @@ -87,7 +85,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" @@ -100,6 +97,36 @@ timeout = "5s" } } +func TestEmptyCommandRejected(t *testing.T) { + for name, cmd := range map[string]string{ + "empty": `""`, + "whitespace": `" \t "`, + } { + t.Run(name, func(t *testing.T) { + _, err := Load(writeTmp(t, ` +agent_id = "host-1" +state_dir = "/tmp/x" + +[server] +enabled = true +url = "http://localhost" + +[[checks]] +id = "c1" +command = `+cmd+` +interval = "10s" +timeout = "5s" +`)) + if err == nil { + t.Fatal("expected empty command to be rejected") + } + if !strings.Contains(err.Error(), "command is required") { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + func TestLoadLabels(t *testing.T) { c, err := Load(writeTmp(t, ` agent_id = "host-1" @@ -112,7 +139,6 @@ role = "api" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" @@ -148,7 +174,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" command = "true" @@ -167,7 +192,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" command = "true" @@ -188,7 +212,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" command = "true" @@ -268,7 +291,28 @@ interval = "10s" timeout = "5s" `)) if err == nil { - t.Fatal("push_only must require server.url/token") + t.Fatal("push_only must require server.url") + } +} + +func TestServerTokenRejected(t *testing.T) { + _, err := Load(writeTmp(t, ` +state_dir = "/tmp/x" +[server] +enabled = true +url = "http://localhost" +token = "obsolete" +[[checks]] +id = "c1" +command = "true" +interval = "10s" +timeout = "5s" +`)) + if err == nil { + t.Fatal("expected obsolete server.token to be rejected") + } + if !strings.Contains(err.Error(), "server.token") { + t.Fatalf("unexpected error: %v", err) } } @@ -279,7 +323,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" command = "true" @@ -291,6 +334,27 @@ timeout = "5s" } } +func TestHostnameConfigKeyRejected(t *testing.T) { + _, err := Load(writeTmp(t, ` +hostname = "host-1" +state_dir = "/tmp/x" +[server] +enabled = true +url = "http://localhost" +[[checks]] +id = "c1" +command = "true" +interval = "10s" +timeout = "5s" +`)) + if err == nil { + t.Fatal("expected hostname key to be rejected") + } + if !strings.Contains(err.Error(), "hostname") { + t.Fatalf("unexpected error: %v", err) + } +} + func TestNotificationsEnabledCanDisable(t *testing.T) { c, err := Load(writeTmp(t, minimal+` notifications_enabled = false @@ -329,7 +393,6 @@ state_dir = "/tmp/x" [server] enabled = true url = "http://localhost" -token = "t" [[checks]] id = "c1" command = "true" diff --git a/agent/internal/ids/ids.go b/agent/internal/ids/ids.go index bc81f3f..47d4fee 100644 --- a/agent/internal/ids/ids.go +++ b/agent/internal/ids/ids.go @@ -19,7 +19,7 @@ func ResolveAgentID(configured, hostname string) (string, error) { if id == "" { return "", fmt.Errorf("hostname is empty") } - if err := config.ValidateID("hostname as agent_id", id); err != nil { + if err := config.ValidateID("OS hostname as agent_id", id); err != nil { return "", err } return id, nil diff --git a/agent/internal/metrics/metrics.go b/agent/internal/metrics/metrics.go index 60af019..c8e80b7 100644 --- a/agent/internal/metrics/metrics.go +++ b/agent/internal/metrics/metrics.go @@ -28,6 +28,7 @@ type Metrics struct { SendFailures *prometheus.CounterVec EventsAccepted prometheus.Counter EventsDedup prometheus.Counter + EventsDropped *prometheus.CounterVec LastHeartbeat prometheus.Gauge BuildInfo *prometheus.GaugeVec } @@ -86,6 +87,10 @@ func New() *Metrics { Name: "monlet_agent_events_deduplicated_total", Help: "Events the server reported as duplicates.", }) + m.EventsDropped = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "monlet_agent_events_dropped_total", + Help: "Events dropped by the agent before server acceptance, by reason (low-cardinality enum).", + }, []string{"reason"}) m.CheckStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "monlet_agent_check_status", Help: "Last observed status per check: 0=ok,1=warning,2=critical,3=unknown.", @@ -123,7 +128,7 @@ func New() *Metrics { m.CheckInterval, m.ChecksConfigured, m.SpoolDepth, m.SpoolBytes, m.SendAttempts, m.SendFailures, - m.EventsAccepted, m.EventsDedup, + m.EventsAccepted, m.EventsDedup, m.EventsDropped, m.LastHeartbeat, m.BuildInfo) return m } diff --git a/agent/internal/runner/runner.go b/agent/internal/runner/runner.go index dafaf26..2ec851d 100644 --- a/agent/internal/runner/runner.go +++ b/agent/internal/runner/runner.go @@ -62,8 +62,9 @@ func Run(ctx context.Context, argv []string, timeout time.Duration) Result { } if cctx.Err() == context.DeadlineExceeded { - r.Status = StatusUnknown - r.ExitCode = -1 + r.Status = StatusCritical + r.ExitCode = 2 + r.Output = fmt.Sprintf("critical: check timed out after %s", timeout) return r } if err != nil { diff --git a/agent/internal/runner/runner_test.go b/agent/internal/runner/runner_test.go index 582f6b8..50bdde5 100644 --- a/agent/internal/runner/runner_test.go +++ b/agent/internal/runner/runner_test.go @@ -97,9 +97,12 @@ func TestRunUnknownExit(t *testing.T) { func TestRunTimeout(t *testing.T) { r := Run(context.Background(), []string{"sh", "-c", "sleep 5"}, 200*time.Millisecond) - if r.Status != StatusUnknown { + if r.Status != StatusCritical || r.ExitCode != 2 { t.Fatalf("got %+v", r) } + if !strings.Contains(r.Output, "critical: check timed out after 200ms") { + t.Fatalf("output: %q", r.Output) + } if r.DurationMs > 2000 { t.Fatalf("duration too long: %d", r.DurationMs) } diff --git a/agent/internal/spool/spool.go b/agent/internal/spool/spool.go index 3751495..007cb38 100644 --- a/agent/internal/spool/spool.go +++ b/agent/internal/spool/spool.go @@ -24,10 +24,19 @@ type Spool struct { maxEvents int maxBytes int64 - mu sync.Mutex - nextSeq uint64 - files []entry // sorted ascending by seq - bytes int64 + mu sync.Mutex + nextSeq uint64 + files []entry // sorted ascending by seq + bytes int64 + dropEvents int + dropBytes int +} + +// DropStats reports events dropped due to bounded-spool overflow since the +// previous call to Drops. Counters are reset on read. +type DropStats struct { + OverflowEvents int + OverflowBytes int } type entry struct { @@ -135,13 +144,31 @@ func (s *Spool) dropOldestLocked() { if len(s.files) == 0 { return } + byCount := len(s.files) > s.maxEvents old := s.files[0] _ = os.Remove(filepath.Join(s.dir, old.name)) s.bytes -= old.size s.files = s.files[1:] + if byCount { + s.dropEvents++ + } else { + s.dropBytes++ + } } } +// Drops returns and resets the count of events dropped due to overflow since +// the previous call. Callers use this to feed bounded-cardinality metrics and +// throttled logs without spamming on every Enqueue. +func (s *Spool) Drops() DropStats { + s.mu.Lock() + defer s.mu.Unlock() + d := DropStats{OverflowEvents: s.dropEvents, OverflowBytes: s.dropBytes} + s.dropEvents = 0 + s.dropBytes = 0 + return d +} + // Peek returns up to limit oldest items. Does not remove them. func (s *Spool) Peek(limit int) ([]Item, error) { s.mu.Lock() diff --git a/agent/internal/spool/spool_test.go b/agent/internal/spool/spool_test.go index 2b1817b..43fccfc 100644 --- a/agent/internal/spool/spool_test.go +++ b/agent/internal/spool/spool_test.go @@ -55,6 +55,13 @@ func TestDropOldestByCount(t *testing.T) { if items[0].Event.EventID != "e2" { t.Fatalf("expected oldest dropped, got %q", items[0].Event.EventID) } + d := s.Drops() + if d.OverflowEvents != 2 || d.OverflowBytes != 0 { + t.Fatalf("drop stats: %+v", d) + } + if again := s.Drops(); again.OverflowEvents != 0 || again.OverflowBytes != 0 { + t.Fatalf("Drops must reset, got %+v", again) + } } func TestDropOldestByBytes(t *testing.T) { @@ -71,6 +78,10 @@ func TestDropOldestByBytes(t *testing.T) { if s.Bytes() > 3000 { t.Fatalf("bytes=%d > 3000", s.Bytes()) } + d := s.Drops() + if d.OverflowBytes == 0 { + t.Fatalf("expected bytes-overflow drops to be counted, got %+v", d) + } } func TestReopenAppliesLimits(t *testing.T) { diff --git a/api/openapi.yaml b/api/openapi.yaml index 4891722..6d8ee2e 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -5,11 +5,16 @@ info: description: | Monlet v1 API contract. Stage 1 frozen. - All endpoints under `/api/v1`. Authentication uses a single shared Bearer token (see ADR-0007). Agent-facing ingestion endpoints (`/heartbeat`, `/events`) and read-only UI-facing endpoints share the same security scheme in v1. + All endpoints under `/api/v1`. Authentication is split (ADR-0007): + + - `agentToken` — self-generated agent Bearer key stored locally by the agent. Used by ingestion paths `POST /api/v1/heartbeat` and `POST /api/v1/events`. + - `uiToken` — operator/UI shared Bearer token from `MONLET_AUTH_TOKEN`. Used by every other authenticated endpoint. Supports overlap rotation via space/comma-separated list (PH-008). + + Both are presented as `Authorization: Bearer `. UI token comparison is constant-time. Agent keys are stored server-side only as hashes/fingerprints and never appear in logs, metrics, or notifications. servers: - url: http://127.0.0.1:8000 security: - - bearerAuth: [] + - uiToken: [] paths: /api/v1/health: get: @@ -45,6 +50,8 @@ paths: post: summary: Ingest agent heartbeat operationId: ingestHeartbeat + security: + - agentToken: [] requestBody: required: true content: @@ -62,14 +69,26 @@ paths: $ref: "#/components/responses/ValidationError" "401": $ref: "#/components/responses/Unauthorized" + "403": + description: Agent key is blocked. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "409": + $ref: "#/components/responses/Conflict" "413": $ref: "#/components/responses/PayloadTooLarge" + "429": + $ref: "#/components/responses/RateLimited" "500": $ref: "#/components/responses/InternalError" /api/v1/events: post: summary: Ingest agent check result events operationId: ingestEvents + security: + - agentToken: [] requestBody: required: true content: @@ -87,6 +106,12 @@ paths: $ref: "#/components/responses/ValidationError" "401": $ref: "#/components/responses/Unauthorized" + "403": + description: Agent key is blocked. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" "413": $ref: "#/components/responses/PayloadTooLarge" "500": @@ -97,6 +122,7 @@ paths: operationId: listAgents parameters: - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Back" - $ref: "#/components/parameters/Limit" responses: "200": @@ -115,6 +141,109 @@ paths: $ref: "#/components/schemas/Agent" "401": $ref: "#/components/responses/Unauthorized" + /api/v1/agents/admission/blacklist: + get: + summary: List blocked agent keys + operationId: listAgentBlacklist + parameters: + - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Limit" + responses: + "200": + description: Agent blacklist. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/PageEnvelope" + - type: object + required: [items] + properties: + items: + type: array + items: + $ref: "#/components/schemas/AgentBlacklistItem" + "401": + $ref: "#/components/responses/Unauthorized" + delete: + summary: Clear blocked agent keys + operationId: clearAgentBlacklist + responses: + "200": + description: Number of removed blacklist rows. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + /api/v1/agents/admission/blacklist/{key_id}: + delete: + summary: Remove one blocked agent key + operationId: deleteAgentBlacklistItem + parameters: + - name: key_id + in: path + required: true + schema: + type: string + minLength: 64 + maxLength: 64 + responses: + "200": + description: Number of removed blacklist rows. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + /api/v1/agents/admission/accept-all: + post: + summary: Accept all pending agents + operationId: acceptAllPendingAgents + parameters: + - name: include_rotation + in: query + schema: + type: boolean + default: false + description: When false, accepted agents with pending replacement keys are skipped. + responses: + "200": + description: Number of accepted agents. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + /api/v1/agents/admission/block-all: + post: + summary: Block all pending agents + operationId: blockAllPendingAgents + responses: + "200": + description: Number of blocked agent keys. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + /api/v1/agents/admission/remove-all: + post: + summary: Remove all pending agents + operationId: removeAllPendingAgents + responses: + "200": + description: Number of removed agents. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" /api/v1/agents/{agent_id}: get: summary: Get agent detail @@ -132,6 +261,56 @@ paths: $ref: "#/components/responses/Unauthorized" "404": $ref: "#/components/responses/NotFound" + delete: + summary: Remove agent and all owned state + operationId: deleteAgent + parameters: + - $ref: "#/components/parameters/AgentId" + responses: + "200": + description: Number of removed agents. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "404": + $ref: "#/components/responses/NotFound" + /api/v1/agents/{agent_id}/accept: + post: + summary: Accept pending agent key + operationId: acceptAgent + parameters: + - $ref: "#/components/parameters/AgentId" + responses: + "200": + description: Number of accepted agents. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "404": + $ref: "#/components/responses/NotFound" + /api/v1/agents/{agent_id}/block: + post: + summary: Block pending agent key + operationId: blockAgent + parameters: + - $ref: "#/components/parameters/AgentId" + responses: + "200": + description: Number of blocked agent keys. + content: + application/json: + schema: + $ref: "#/components/schemas/ActionCountResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "404": + $ref: "#/components/responses/NotFound" /api/v1/checks: get: summary: List current check states @@ -145,6 +324,7 @@ paths: maxLength: 128 pattern: "^[A-Za-z0-9._:-]+$" - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Back" - $ref: "#/components/parameters/Limit" responses: "200": @@ -174,7 +354,37 @@ paths: schema: type: string enum: [open, resolved] + - name: severity + in: query + schema: + type: string + enum: [warning, critical, unknown] + - name: agent_id + in: query + schema: + type: string + maxLength: 128 + pattern: "^[A-Za-z0-9._:-]+$" + - name: check_id + in: query + schema: + type: string + maxLength: 128 + pattern: "^[A-Za-z0-9._:-]+$" + - name: sort + in: query + schema: + type: string + enum: [status, opened_at, resolved_at] + default: status + - name: direction + in: query + schema: + type: string + enum: [asc, desc] + default: desc - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Back" - $ref: "#/components/parameters/Limit" responses: "200": @@ -213,6 +423,7 @@ paths: maxLength: 128 pattern: "^[A-Za-z0-9._:-]+$" - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Back" - $ref: "#/components/parameters/Limit" responses: "200": @@ -231,12 +442,56 @@ paths: $ref: "#/components/schemas/StoredCheckResultEvent" "401": $ref: "#/components/responses/Unauthorized" + /api/v1/system-summary: + get: + summary: System navigation summary + operationId: getSystemSummary + description: | + Bounded aggregate counts for the top navigation. Safe to poll from many + open dashboards because response size is independent of row counts. + responses: + "200": + description: System summary snapshot. + content: + application/json: + schema: + $ref: "#/components/schemas/SystemSummaryResponse" + "401": + $ref: "#/components/responses/Unauthorized" /api/v1/notifiers/outbox: get: summary: List notification outbox items operationId: listOutbox parameters: + - name: state + in: query + schema: + type: string + enum: [pending, sending, sent, retry, failed, discarded] + - name: notifier + in: query + schema: + type: string + enum: [debug, telegram, webhook, alertmanager] + - name: event_type + in: query + schema: + type: string + enum: [firing, resolved] + - name: sort + in: query + schema: + type: string + enum: [created_at, updated_at] + default: updated_at + - name: direction + in: query + schema: + type: string + enum: [asc, desc] + default: desc - $ref: "#/components/parameters/Cursor" + - $ref: "#/components/parameters/Back" - $ref: "#/components/parameters/Limit" responses: "200": @@ -257,10 +512,21 @@ paths: $ref: "#/components/responses/Unauthorized" components: securitySchemes: - bearerAuth: + agentToken: type: http scheme: bearer - description: Shared static Bearer token in v1. See ADR-0007. + description: | + Self-generated agent Bearer key. The agent creates and persists it in + local state; the server stores only hash/fingerprint and requires UI + acceptance before events affect state. Used only by `/heartbeat` and + `/events`. See ADR-0007. + uiToken: + type: http + scheme: bearer + description: | + Operator/UI Bearer token from `MONLET_AUTH_TOKEN`. Multiple tokens may + be active simultaneously (space/comma separated) for zero-loss rotation. + See PH-008 and ADR-0007. parameters: AgentId: name: agent_id @@ -274,10 +540,18 @@ components: name: cursor in: query required: false - description: Opaque pagination cursor returned in `next_cursor`. + description: Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). schema: type: string maxLength: 512 + Back: + name: back + in: query + required: false + description: When true, treat the `cursor` as a `prev_cursor` and return the previous page. + schema: + type: boolean + default: false Limit: name: limit in: query @@ -300,12 +574,24 @@ components: application/json: schema: $ref: "#/components/schemas/ErrorResponse" + Conflict: + description: Request conflicts with current server state. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" PayloadTooLarge: description: Payload exceeds configured limit (1 MiB). content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" + RateLimited: + description: Request rejected by configured rate/volume limits. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" ValidationError: description: Request body or parameters failed validation. content: @@ -338,6 +624,9 @@ components: type: string enum: - unauthorized + - forbidden + - conflict + - agent_blocked - not_found - validation - payload_too_large @@ -355,6 +644,9 @@ components: next_cursor: type: [string, "null"] description: Pass back as `cursor` to fetch the next page. Null/absent if no more results. + prev_cursor: + type: [string, "null"] + description: Pass back as `cursor` with `back=true` to fetch the previous page. Null on the first page. AcceptedResponse: type: object required: [accepted] @@ -438,7 +730,7 @@ components: description: | Event body used inside `EventBatchRequest.events`. The owning `agent_id` is supplied once at the batch envelope; per-event `agent_id` is intentionally - absent so requests cannot mix or spoof other agents under a shared token. + absent so one HTTP request cannot mix multiple agents. Server response endpoints (`GET /events/query`) include `agent_id` separately in their item schema. required: @@ -510,7 +802,7 @@ components: format: date-time Agent: type: object - required: [agent_id, hostname, status, last_seen_at, features] + required: [agent_id, hostname, status, admission_state, last_seen_at, features] properties: agent_id: type: string @@ -519,6 +811,17 @@ components: status: type: string enum: [alive, stale, dead] + admission_state: + type: string + enum: [pending, accepted] + rotation_pending: + type: boolean + default: false + description: True when a pending key would replace an already accepted key for this agent. + key_fingerprint: + type: [string, "null"] + minLength: 16 + maxLength: 16 last_seen_at: type: string format: date-time @@ -528,6 +831,36 @@ components: type: object additionalProperties: type: string + AgentBlacklistItem: + type: object + required: [id, key_fingerprint, agent_id, hostname, created_at, last_seen_at] + properties: + id: + type: string + minLength: 64 + maxLength: 64 + description: Opaque blacklist row id. Use for delete operations; display `key_fingerprint` to humans. + key_fingerprint: + type: string + minLength: 16 + maxLength: 16 + agent_id: + type: string + hostname: + type: string + created_at: + type: string + format: date-time + last_seen_at: + type: string + format: date-time + ActionCountResponse: + type: object + required: [count] + properties: + count: + type: integer + minimum: 0 CheckState: type: object required: [agent_id, check_id, status, last_observed_at] @@ -602,3 +935,40 @@ components: updated_at: type: string format: date-time + SystemSummaryResponse: + type: object + description: Bounded aggregate counts for the web navigation. + required: + - agents_online + - agents_offline + - checks_ok + - checks_warning + - checks_critical + - checks_unknown + - open_incidents + - generated_at + properties: + agents_online: + type: integer + minimum: 0 + agents_offline: + type: integer + minimum: 0 + checks_ok: + type: integer + minimum: 0 + checks_warning: + type: integer + minimum: 0 + checks_critical: + type: integer + minimum: 0 + checks_unknown: + type: integer + minimum: 0 + open_incidents: + type: integer + minimum: 0 + generated_at: + type: string + format: date-time diff --git a/deploy/README.md b/deploy/README.md index 8d66116..5efc062 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -41,9 +41,9 @@ Boots the stack, hits `/health`, `/ready`, `/metrics`, ingests example payloads Full-fat demo stack with several dockerized agents driving the system through every witness state: agent alive/stale/dead, check ok/warning/critical/unknown, -incident open/resolved, outbox sent/retry/failed/discarded, +incident open/resolved, outbox sent/retry/failed, anti-double-alerting with `notifications_enabled=false`, and agent spool -replay against a server that becomes available after the agent starts. +replay after a controlled server outage. ```sh bash deploy/e2e-showcase.sh @@ -52,7 +52,7 @@ bash deploy/e2e-showcase.sh Internally: ```sh -MONLET_AUTH_TOKEN=monlet-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build +MONLET_AUTH_TOKEN=monlet-ui-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build ``` `SHOWCASE_KEEP=1 bash deploy/e2e-showcase.sh` leaves the stack up so you can @@ -71,7 +71,7 @@ for demo/e2e only — production deploys the Go binary under systemd. - `smoke.sh` — local stack smoke entry point. - `docker-compose.showcase.yml` — override that adds demo agents + mock webhook. - `docker/agent.Dockerfile` — demo/e2e agent image (not for production). -- `showcase/` — demo agent configs, check scripts, mock webhook, seed SQL. +- `showcase/` — demo agent configs, check scripts, and mock webhook. - `e2e-showcase.sh` — full showcase runner (Stage 6.1). Docker images install Python/Node project dependencies inside the image; host venvs are never mounted (see `AGENTS.md`). diff --git a/deploy/docker-compose.showcase.yml b/deploy/docker-compose.showcase.yml index adc0241..6af6dea 100644 --- a/deploy/docker-compose.showcase.yml +++ b/deploy/docker-compose.showcase.yml @@ -7,7 +7,7 @@ services: server: environment: MONLET_STALE_AFTER_SEC: "20" - MONLET_DEAD_AFTER_SEC: "40" + MONLET_DEAD_AFTER_SEC: "30" MONLET_DETECTOR_TICK_SEC: "5" MONLET_EVENTS_RETENTION_MONTHS: "1" MONLET_OUTBOX_RETENTION_MAX_ROWS: "1000" @@ -28,6 +28,7 @@ services: agent-mixed: image: monlet-showcase-agent + hostname: agent-01 build: context: .. dockerfile: deploy/docker/agent.Dockerfile @@ -36,6 +37,7 @@ services: volumes: - ./showcase/agents/mixed.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - mixed_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -44,11 +46,13 @@ services: agent-resolve: image: monlet-showcase-agent + hostname: agent-02 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/resolve.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - resolve_agent_state:/var/lib/monlet-agent - resolve_state:/state depends_on: server: @@ -58,11 +62,13 @@ services: agent-flap: image: monlet-showcase-agent + hostname: agent-03 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/flap.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - flap_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -71,11 +77,13 @@ services: agent-timeout: image: monlet-showcase-agent + hostname: agent-04 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/timeout.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - timeout_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -84,11 +92,13 @@ services: agent-prometheus-owned: image: monlet-showcase-agent + hostname: agent-05 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/prometheus_owned.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - prometheus_owned_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -97,11 +107,13 @@ services: agent-stale: image: monlet-showcase-agent + hostname: agent-07 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/stale.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - stale_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -110,11 +122,13 @@ services: agent-dead: image: monlet-showcase-agent + hostname: agent-08 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: - ./showcase/agents/dead.toml:/etc/monlet/agent.toml:ro - ./showcase/checks:/opt/monlet/checks:ro + - dead_agent_state:/var/lib/monlet-agent depends_on: server: condition: service_healthy @@ -124,6 +138,7 @@ services: # No depends_on:server — starts in parallel so it must spool events until server is ready. agent-spool-replay: image: monlet-showcase-agent + hostname: agent-06 environment: MONLET_CONFIG: /etc/monlet/agent.toml volumes: @@ -135,5 +150,12 @@ services: condition: service_healthy volumes: + mixed_agent_state: + resolve_agent_state: resolve_state: + flap_agent_state: + timeout_agent_state: + prometheus_owned_agent_state: + stale_agent_state: + dead_agent_state: spool_replay_state: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f802f01..fd02c2f 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,10 +1,11 @@ services: postgres: - image: postgres:16-alpine + image: postgres:16 environment: POSTGRES_USER: monlet POSTGRES_PASSWORD: monlet POSTGRES_DB: monlet + POSTGRES_INITDB_ARGS: "--auth-local=scram-sha-256 --auth-host=scram-sha-256" TZ: UTC healthcheck: test: ["CMD-SHELL", "pg_isready -U monlet -d monlet"] @@ -14,7 +15,7 @@ services: volumes: - monlet_pg:/var/lib/postgresql/data ports: - - "5432:5432" + - "${MONLET_POSTGRES_HOST_PORT:-5432}:5432" server: build: @@ -36,12 +37,21 @@ services: - -c - "alembic upgrade head && uvicorn monlet_server.main:app --host 0.0.0.0 --port 8000" ports: - - "8000:8000" + - "${MONLET_SERVER_HOST_PORT:-8000}:8000" healthcheck: test: ["CMD-SHELL", "python -c 'import urllib.request,sys;sys.exit(0) if urllib.request.urlopen(\"http://localhost:8000/api/v1/ready\").status==200 else sys.exit(1)'"] interval: 5s timeout: 3s retries: 30 + # PH-015: production hardening for runtime containers. + read_only: true + tmpfs: + - /tmp:size=64M + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + stop_grace_period: 15s web: build: @@ -53,9 +63,29 @@ services: MONLET_API_BASE_URL: http://server:8000 MONLET_API_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required} MONLET_WEB_TIME_ZONE: ${MONLET_WEB_TIME_ZONE:-UTC} + MONLET_WEB_AUTH_USERNAME: ${MONLET_WEB_AUTH_USERNAME:-} + MONLET_WEB_AUTH_PASSWORD: ${MONLET_WEB_AUTH_PASSWORD:-} + MONLET_WEB_TRUST_PROXY_AUTH: ${MONLET_WEB_TRUST_PROXY_AUTH:-false} TZ: ${MONLET_WEB_TIME_ZONE:-UTC} ports: - - "3000:3000" + - "${MONLET_WEB_HOST_PORT:-3000}:3000" + # PH-015: production hardening for the Next.js runtime. Next writes a small + # cache under .next/cache so we keep that path writable via tmpfs. + read_only: true + tmpfs: + - /tmp:size=64M + - /app/.next/cache:size=64M + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://localhost:3000/api/health >/dev/null 2>&1 || exit 1"] + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s + stop_grace_period: 15s prometheus: image: prom/prometheus:v2.55.0 diff --git a/deploy/e2e-showcase.sh b/deploy/e2e-showcase.sh index 643593f..d75fe6c 100755 --- a/deploy/e2e-showcase.sh +++ b/deploy/e2e-showcase.sh @@ -2,18 +2,25 @@ # Stage 6.1 — Docker showcase / e2e stand for Monlet. # Brings up the full stack with demo agents, drives scenarios, asserts all # witness states (agent alive/stale/dead, check ok/warning/critical/unknown, -# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications, +# incident open/resolved, outbox sent/retry/failed, disabled server notifications, # agent spool replay), and verifies metrics + web pages. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" ROOT="$(cd "$HERE/.." && pwd)" -TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}" +TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}" +# Export so every `docker compose` call (up/exec/down) sees the UI token env, +# including the cleanup trap. +export MONLET_AUTH_TOKEN="$TOKEN" BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}" WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}" AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}" +ADMISSION_MODE="${SHOWCASE_ADMISSION_MODE:-pending}" +# PH-review: dedicated project name so showcase does not collide with the +# smoke/dev stack (orphans, shared postgres volume, stale baseline schema). COMPOSE=(docker compose + -p monlet-showcase -f "$HERE/docker-compose.yml" -f "$HERE/docker-compose.showcase.yml") @@ -21,6 +28,7 @@ log() { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; } fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; } AUTH=(-H "Authorization: Bearer $TOKEN") +EXPECTED_AGENTS=8 cleanup() { if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then @@ -42,9 +50,55 @@ wait_url() { } api() { curl -sf "${AUTH[@]}" "$BASE$1"; } +wait_agents_visible() { + local expected="$1" + for i in $(seq 1 60); do + local count + count="$(api "/api/v1/agents?limit=500" | python3 -c "import json,sys; print(len(json.load(sys.stdin)['items']))" 2>/dev/null || echo 0)" + [[ "${count:-0}" -ge "$expected" ]] && return 0 + sleep 2 + done + fail "agents did not register" +} -log "starting full showcase stack" -MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build +wait_agents_accepted() { + local expected="$1" + for i in $(seq 1 300); do + local counts accepted pending + counts="$(api "/api/v1/agents?limit=500" | python3 -c " +import json,sys +d=json.load(sys.stdin) +items=d['items'] +accepted=sum(1 for a in items if a['admission_state']=='accepted') +pending=sum(1 for a in items if a['admission_state']=='pending') +print(accepted, pending) +" 2>/dev/null || echo "0 0")" + read -r accepted pending <<<"$counts" + [[ "${accepted:-0}" -ge "$expected" && "${pending:-0}" -eq 0 ]] && return 0 + sleep 2 + done + fail "agents were not accepted in time" +} + +# Multi-phase startup keeps DB migration explicit, then agents register as +# pending. Full assertions wait read-only until an operator accepts them. +log "pre-clean previous showcase stack (if any)" +"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true + +log "phase 1: postgres" +# --wait blocks until the postgres healthcheck (pg_isready) reports healthy, +# so the alembic step below does not race a still-booting DB. +"${COMPOSE[@]}" up -d --wait --build postgres + +log "building current server image for migrations" +"${COMPOSE[@]}" build server >/dev/null || fail "server image build failed" + +log "applying alembic migrations (one-off, server not yet running)" +"${COMPOSE[@]}" run --rm --no-deps server sh -c "alembic upgrade head" \ + >/dev/null || fail "alembic upgrade failed" + +log "phase 2: server + agents + web" +"${COMPOSE[@]}" up -d --build log "waiting for server /api/v1/ready" wait_url "$BASE/api/v1/ready" "server" 90 @@ -52,31 +106,59 @@ wait_url "$BASE/api/v1/ready" "server" 90 log "waiting for web /" wait_url "$WEB_BASE/" "web" 90 +log "waiting for agents to register as pending" +wait_agents_visible "$EXPECTED_AGENTS" + +case "$ADMISSION_MODE" in + pending) + log "agents are pending; automatic accept is disabled" + log "accept them in the UI/API, or rerun with SHOWCASE_ADMISSION_MODE=wait to continue assertions after manual admission" + exit 0 + ;; + wait) + log "waiting for operator to accept agents (read-only polling)" + wait_agents_accepted "$EXPECTED_AGENTS" + ;; + *) + fail "unknown SHOWCASE_ADMISSION_MODE=$ADMISSION_MODE (expected pending or wait)" + ;; +esac + # --- Phase A: let agents run a few cycles --- log "letting demo agents push events for ~25s" sleep 25 -log "asserting agent labels include scenario and monlet_agent_version" -api "/api/v1/agents/agent-mixed" | python3 -c " +log "asserting agent-01 labels include scenario and monlet_agent_version" +api "/api/v1/agents/agent-01" | python3 -c " import json,sys d=json.load(sys.stdin) labels=d.get('labels') or {} ver=labels.get('monlet_agent_version') ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev' -print('agent-mixed labels:', labels) +print('agent-01 labels:', labels) sys.exit(0 if ok else 1)" \ - || fail "agent-mixed labels missing scenario or numeric monlet_agent_version" + || fail "agent-01 labels missing scenario or numeric monlet_agent_version" -log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)" +log "asserting agent-01 has 4 distinct check statuses (ok/warning/critical/unknown)" api "/api/v1/checks?limit=500" | python3 -c " import json,sys d=json.load(sys.stdin) want={'ok','warning','critical','unknown'} -have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'} +have={c['status'] for c in d['items'] if c['agent_id']=='agent-01'} missing=want-have -print('agent-mixed statuses:', sorted(have)) +print('agent-01 statuses:', sorted(have)) sys.exit(0 if not missing else 1)" \ - || fail "agent-mixed missing one of ok/warning/critical/unknown" + || fail "agent-01 missing one of ok/warning/critical/unknown" + +log "asserting timeout check is explicit critical" +api "/api/v1/checks?agent_id=agent-04&limit=50" | python3 -c " +import json,sys +d=json.load(sys.stdin) +slow=next((c for c in d['items'] if c['check_id']=='slow_probe'), None) +print('slow_probe:', slow) +ok=slow and slow['status']=='critical' and slow['exit_code']==2 and 'timed out' in (slow.get('summary') or '') +sys.exit(0 if ok else 1)" \ + || fail "agent-04/slow_probe is not explicit critical timeout" # --- Phase B: trigger resolve scenario --- log "triggering resolve: touch /state/resolve.flag in agent-resolve" @@ -86,14 +168,14 @@ log "triggering resolve: touch /state/resolve.flag in agent-resolve" log "waiting for resolve event to be processed (~15s)" sleep 15 -log "asserting resolved incident exists for agent-resolve/flapping" +log "asserting resolved incident exists for agent-02/flapping" api "/api/v1/incidents?state=resolved&limit=500" \ | python3 -c " import json,sys d=json.load(sys.stdin) -ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items']) +ok=any(i['agent_id']=='agent-02' and i['check_id']=='flapping' for i in d['items']) sys.exit(0 if ok else 1)" \ - || fail "no resolved incident for agent-resolve/flapping" + || fail "no resolved incident for agent-02/flapping" # --- Phase C: disabled server notifications --- log "asserting notifications-disabled incident exists but no outbox row for it" @@ -101,7 +183,7 @@ PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c " import json,sys d=json.load(sys.stdin) for i in d['items']: - if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck': + if i['agent_id']=='agent-05' and i['check_id']=='promcheck': print(i['id']); break")" [[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent" api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \ @@ -114,22 +196,22 @@ log "stopping agent-stale and agent-dead to age their heartbeats" log "waiting ~25s for stale transition" sleep 25 -api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \ - || api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \ - || fail "agent-stale did not transition to stale/dead" +api "/api/v1/agents/agent-07" | grep -q '"status":"stale"' \ + || api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \ + || fail "agent-07 did not transition to stale/dead" log "waiting another ~25s for dead transition" sleep 25 -api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \ - || fail "agent-stale did not transition to dead" -api "/api/v1/agents/agent-dead" | grep -q '"status":"dead"' \ - || fail "agent-dead did not transition to dead" +api "/api/v1/agents/agent-07" | grep -q '"status":"dead"' \ + || fail "agent-07 did not transition to dead" +api "/api/v1/agents/agent-08" | grep -q '"status":"dead"' \ + || fail "agent-08 did not transition to dead" log "asserting dead agents have open liveness incidents" api "/api/v1/incidents?state=open&limit=500" | python3 -c " import json,sys d=json.load(sys.stdin) -want={'agent-stale','agent-dead'} +want={'agent-07','agent-08'} have={ i['agent_id'] for i in d['items'] @@ -140,9 +222,9 @@ print('liveness incidents:', sorted(have)) sys.exit(0 if not missing else 1)" \ || fail "missing critical agent_liveness incident for dead agents" -# At least one agent must still be alive (agent-mixed keeps pushing). -api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \ - || fail "agent-mixed not alive" +# At least one agent must still be alive (agent-01 keeps pushing). +api "/api/v1/agents/agent-01" | grep -q '"status":"alive"' \ + || fail "agent-01 not alive" # --- Phase E: outbox state coverage (live via webhook + debug) --- log "waiting for notifier worker cycles (~10s)" @@ -159,22 +241,27 @@ print('outbox states:', sorted(states)) sys.exit(0 if not missing else 2)" \ || fail "outbox missing one of sent/retry/failed" -# --- Phase F: seed discarded row, wait for worker --- -log "seeding 'discarded' outbox row via SQL" -"${COMPOSE[@]}" exec -T postgres \ - psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \ - < "$HERE/showcase/seed_discarded.sql" >/dev/null \ - || fail "seed_discarded.sql failed" - -log "waiting for worker to mark unknown-notifier row as discarded (~10s)" -sleep 10 -api "/api/v1/notifiers/outbox?state=discarded&limit=10" \ - | grep -q '"state":"discarded"' \ - || fail "no outbox row in state=discarded" - # --- Phase G: spool replay assertion --- -log "asserting agent-spool-replay events arrived without duplicates" -api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \ +log "forcing server outage so agent-spool-replay writes to local spool" +"${COMPOSE[@]}" stop server >/dev/null +spool_count=0 +for i in $(seq 1 30); do + spool_count=$("${COMPOSE[@]}" exec -T agent-spool-replay \ + sh -c 'ls /var/lib/monlet-agent/spool 2>/dev/null | wc -l' 2>/dev/null \ + | tr -d '[:space:]' || echo 0) + [[ "${spool_count:-0}" -gt 0 ]] && break + sleep 2 +done +[[ "${spool_count:-0}" -gt 0 ]] || fail "agent-spool-replay did not spool any events" +log "spool populated with $spool_count file(s)" + +log "restarting server and waiting for replay" +"${COMPOSE[@]}" up -d --no-deps server >/dev/null +wait_url "$BASE/api/v1/ready" "server after replay outage" 90 +sleep 15 + +log "asserting agent-06 spool replay events arrived without duplicates" +api "/api/v1/events/query?agent_id=agent-06&check_id=replay_check&limit=500" \ | python3 -c " import json,sys d=json.load(sys.stdin) @@ -192,17 +279,40 @@ log "server /metrics contains monlet_server_*" curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \ || fail "server metrics missing monlet_server_*" -log "agent-mixed /metrics contains monlet_agent_check_status" +log "agent-01 /metrics contains monlet_agent_check_status" curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \ || fail "agent metrics missing monlet_agent_check_status" # --- Phase I: web UI --- log "web pages render with expected content" -curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web / missing Overview" -curl -sf "$WEB_BASE/agents" | grep -q "agent-mixed" || fail "web /agents missing agent-mixed" +curl -sfL "$WEB_BASE/" | grep -q "Agents" || fail "web / did not redirect to Agents" +curl -sf "$WEB_BASE/agents" | grep -q "mixed-status" || fail "web /agents missing mixed-status label" curl -sf "$WEB_BASE/checks" | grep -q "critical_check" || fail "web /checks missing critical_check" -curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve" +curl -sf "$WEB_BASE/incidents?state=open&limit=100&sort=status&direction=desc" \ + | grep -q "agent-08" || fail "web /incidents missing agent-08" curl -sf "$WEB_BASE/outbox" | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names" curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200" +if [[ "${SHOWCASE_KEEP:-0}" == "1" ]]; then + log "refreshing transient containers before leaving stack" + "${COMPOSE[@]}" stop \ + agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \ + agent-spool-replay >/dev/null || true + "${COMPOSE[@]}" stop mock-webhook server >/dev/null || true + "${COMPOSE[@]}" rm -f \ + agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \ + agent-spool-replay agent-stale agent-dead server mock-webhook >/dev/null || true + "${COMPOSE[@]}" up -d --no-deps mock-webhook server >/dev/null \ + || fail "could not restart runtime services" + wait_url "$BASE/api/v1/ready" "server after refresh" 90 + "${COMPOSE[@]}" up -d --no-deps \ + agent-mixed agent-resolve agent-flap agent-timeout agent-prometheus-owned \ + agent-spool-replay >/dev/null \ + || fail "could not restart live showcase agents" + wait_agents_visible "$EXPECTED_AGENTS" + wait_agents_accepted "$EXPECTED_AGENTS" + "${COMPOSE[@]}" create agent-stale agent-dead >/dev/null \ + || fail "could not recreate transient containers" +fi + log "ALL SHOWCASE ASSERTIONS PASSED" diff --git a/deploy/showcase/agents/dead.toml b/deploy/showcase/agents/dead.toml index 9aa2a7f..5b5fb67 100644 --- a/deploy/showcase/agents/dead.toml +++ b/deploy/showcase/agents/dead.toml @@ -1,4 +1,4 @@ -hostname = "agent-dead" +agent_id = "agent-08" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" diff --git a/deploy/showcase/agents/flap.toml b/deploy/showcase/agents/flap.toml index 946dc2a..8a6c77b 100644 --- a/deploy/showcase/agents/flap.toml +++ b/deploy/showcase/agents/flap.toml @@ -1,4 +1,4 @@ -hostname = "agent-flap" +agent_id = "agent-03" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" @@ -22,4 +21,4 @@ name = "Flapping check (oscillates ok/critical)" command = "/opt/monlet/checks/flap.sh" interval = "10s" timeout = "3s" -dedupe_key = "agent-flap:flap" +dedupe_key = "agent-03:flap" diff --git a/deploy/showcase/agents/mixed.toml b/deploy/showcase/agents/mixed.toml index 3a12b34..4592481 100644 --- a/deploy/showcase/agents/mixed.toml +++ b/deploy/showcase/agents/mixed.toml @@ -1,4 +1,4 @@ -hostname = "agent-mixed" +agent_id = "agent-01" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" @@ -36,7 +35,7 @@ name = "Showcase critical (permanent webhook fail)" command = "/opt/monlet/checks/exit_2.sh" interval = "5s" timeout = "3s" -dedupe_key = "agent-mixed:critical:fail" +dedupe_key = "agent-01:critical:fail" [[checks]] id = "unknown_check" diff --git a/deploy/showcase/agents/prometheus_owned.toml b/deploy/showcase/agents/prometheus_owned.toml index c89f61f..035389e 100644 --- a/deploy/showcase/agents/prometheus_owned.toml +++ b/deploy/showcase/agents/prometheus_owned.toml @@ -1,4 +1,4 @@ -hostname = "agent-prometheus-owned" +agent_id = "agent-05" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" diff --git a/deploy/showcase/agents/resolve.toml b/deploy/showcase/agents/resolve.toml index 7624548..4c8c1b7 100644 --- a/deploy/showcase/agents/resolve.toml +++ b/deploy/showcase/agents/resolve.toml @@ -1,4 +1,4 @@ -hostname = "agent-resolve" +agent_id = "agent-02" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" @@ -22,4 +21,4 @@ name = "Flapping check (critical then ok)" command = "/opt/monlet/checks/flapping.sh" interval = "5s" timeout = "3s" -dedupe_key = "agent-resolve:flapping:retry" +dedupe_key = "agent-02:flapping:retry" diff --git a/deploy/showcase/agents/spool_replay.toml b/deploy/showcase/agents/spool_replay.toml index 9bb326a..b2e3ad4 100644 --- a/deploy/showcase/agents/spool_replay.toml +++ b/deploy/showcase/agents/spool_replay.toml @@ -1,4 +1,4 @@ -hostname = "agent-spool-replay" +agent_id = "agent-06" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" diff --git a/deploy/showcase/agents/stale.toml b/deploy/showcase/agents/stale.toml index 07f24cc..af161d6 100644 --- a/deploy/showcase/agents/stale.toml +++ b/deploy/showcase/agents/stale.toml @@ -1,4 +1,4 @@ -hostname = "agent-stale" +agent_id = "agent-07" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" diff --git a/deploy/showcase/agents/timeout.toml b/deploy/showcase/agents/timeout.toml index ca14822..0e33604 100644 --- a/deploy/showcase/agents/timeout.toml +++ b/deploy/showcase/agents/timeout.toml @@ -1,4 +1,4 @@ -hostname = "agent-timeout" +agent_id = "agent-04" state_dir = "/var/lib/monlet-agent" [labels] @@ -8,7 +8,6 @@ role = "showcase" [server] enabled = true url = "http://server:8000" -token = "monlet-dev-token" heartbeat_interval = "5s" batch_interval = "3s" @@ -22,7 +21,7 @@ name = "Healthy probe" command = "/opt/monlet/checks/exit_0.sh" interval = "10s" timeout = "3s" -dedupe_key = "agent-timeout:ok_probe" +dedupe_key = "agent-04:ok_probe" [[checks]] id = "slow_probe" @@ -30,4 +29,4 @@ name = "Probe that exceeds the timeout" command = "/opt/monlet/checks/sleep_long.sh" interval = "10s" timeout = "2s" -dedupe_key = "agent-timeout:slow_probe" +dedupe_key = "agent-04:slow_probe" diff --git a/deploy/showcase/checks/replay_check.sh b/deploy/showcase/checks/replay_check.sh index 9d254df..9f3034c 100755 --- a/deploy/showcase/checks/replay_check.sh +++ b/deploy/showcase/checks/replay_check.sh @@ -1,3 +1,15 @@ #!/bin/sh -echo "ok: replay tick" +# PH-review: spool-replay scenario needs many DISTINCT events because the +# server only persists an event row when the check status TRANSITIONS +# (server/monlet_server/services/ingestion.py). Alternate ok/warning on each +# run so every tick produces a fresh event for the spool to replay. +STATE_FILE=/var/lib/monlet-agent/replay_tick +COUNT=$(cat "$STATE_FILE" 2>/dev/null || echo 0) +COUNT=$((COUNT + 1)) +echo "$COUNT" > "$STATE_FILE" +if [ $((COUNT % 2)) -eq 0 ]; then + echo "warning: replay tick $COUNT" + exit 1 +fi +echo "ok: replay tick $COUNT" exit 0 diff --git a/deploy/showcase/seed_discarded.sql b/deploy/showcase/seed_discarded.sql deleted file mode 100644 index ad1bb4b..0000000 --- a/deploy/showcase/seed_discarded.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Seed one outbox row with an unknown notifier name so the worker marks it 'discarded'. --- Attached to any existing incident; if none exists yet the script is a no-op. -INSERT INTO notification_outbox - (id, incident_id, notifier, event_type, state, attempts, next_attempt_at, payload) -SELECT - gen_random_uuid(), - i.id, - 'showcase-unknown', - 'firing', - 'pending', - 0, - now(), - '{"showcase":"discarded"}'::jsonb -FROM incidents i -ORDER BY i.opened_at DESC -LIMIT 1; diff --git a/deploy/smoke.sh b/deploy/smoke.sh index eba8279..56cae53 100755 --- a/deploy/smoke.sh +++ b/deploy/smoke.sh @@ -6,9 +6,22 @@ set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" ROOT="$(cd "$HERE/.." && pwd)" -TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}" -BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}" -COMPOSE=(docker compose -f "$HERE/docker-compose.yml") +TOKEN="${MONLET_AUTH_TOKEN:-monlet-ui-dev-token}" +AGENT_KEY="${MONLET_AGENT_KEY:-monlet-agent-dev-key-0000000000000000}" +POSTGRES_HOST_PORT="${MONLET_POSTGRES_HOST_PORT:-15432}" +SERVER_HOST_PORT="${MONLET_SERVER_HOST_PORT:-18000}" +WEB_HOST_PORT="${MONLET_WEB_HOST_PORT:-13000}" +# Export so every `docker compose` call (up/exec/down) sees the UI token env, +# including the cleanup trap. +export MONLET_AUTH_TOKEN="$TOKEN" +export MONLET_POSTGRES_HOST_PORT="$POSTGRES_HOST_PORT" +export MONLET_SERVER_HOST_PORT="$SERVER_HOST_PORT" +export MONLET_WEB_HOST_PORT="$WEB_HOST_PORT" +BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:$SERVER_HOST_PORT}" +# PH-review: dedicated project name so smoke does not collide with the +# showcase/dev stack (orphan containers, shared postgres volume with a stale +# baseline schema, etc.). +COMPOSE=(docker compose -p monlet-smoke -f "$HERE/docker-compose.yml") log() { printf "\033[1;36m[smoke]\033[0m %s\n" "$*"; } fail() { printf "\033[1;31m[smoke]\033[0m %s\n" "$*" >&2; exit 1; } @@ -21,8 +34,11 @@ cleanup() { } trap cleanup EXIT -log "starting stack" -MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build postgres server web +log "pre-clean previous smoke stack (if any)" +"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true + +log "starting stack (phase 1: postgres+server)" +"${COMPOSE[@]}" up -d --build postgres server log "waiting for /api/v1/ready" for i in $(seq 1 60); do @@ -37,33 +53,41 @@ curl -sf "$BASE/api/v1/health" | grep -q '"status":"ok"' || fail "health wrong b log "GET /metrics" curl -sf "$BASE/metrics" | grep -q '^monlet_server_' || fail "metrics missing" -AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json") +log "starting web (phase 2)" +"${COMPOSE[@]}" up -d --build web + +AGENT_AUTH=(-H "Authorization: Bearer $AGENT_KEY" -H "Content-Type: application/json") +UI_AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json") log "POST heartbeat (examples/heartbeat.json)" -curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \ +curl -sf "${AGENT_AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \ | grep -q '"accepted":true' || fail "heartbeat not accepted" +log "accept pending agent" +curl -sf "${UI_AUTH[@]}" -X POST "$BASE/api/v1/agents/host-01.prod/accept" \ + | grep -q '"count":1' || fail "agent not accepted" + log "POST events (examples/event-batch.json)" -curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \ +curl -sf "${AGENT_AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \ | grep -q '"accepted":' || fail "event batch not accepted" log "GET /agents has host-01.prod" -curl -sf "${AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \ +curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \ || fail "host-01.prod missing" log "GET /checks not empty" -curl -sf "${AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \ +curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \ || fail "checks empty" log "GET /incidents not empty (event-batch.json contains a critical event)" -curl -sf "${AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \ +curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \ || fail "no open incident" log "GET /notifiers/outbox not empty" -curl -sf "${AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \ +curl -sf "${UI_AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \ || fail "outbox empty" -WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}" +WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:$WEB_HOST_PORT}" log "waiting for web $WEB_BASE" for i in $(seq 1 60); do if curl -sf -o /dev/null "$WEB_BASE/"; then break; fi @@ -71,8 +95,8 @@ for i in $(seq 1 60); do [[ "$i" == "60" ]] && fail "web did not become ready" done -log "GET / (web overview)" -curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web overview missing 'Overview'" +log "GET / (web root follows redirect to /agents)" +curl -sfL "$WEB_BASE/" | grep -q "host-01.prod" || fail "web root did not render agents list" log "GET /agents (web list)" curl -sf "$WEB_BASE/agents" | grep -q "host-01.prod" || fail "web /agents missing host-01.prod" diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index ed3fa1c..898464f 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -46,7 +46,7 @@ Docker image rule: ## Web -Web is read-only in v1. +Web is read-only for monitoring state in v1. Agent admission and blacklist controls are the explicit write exception. Auth plan: diff --git a/docs/ops/backup-restore.md b/docs/ops/backup-restore.md index 64826a2..9a7da58 100644 --- a/docs/ops/backup-restore.md +++ b/docs/ops/backup-restore.md @@ -6,7 +6,7 @@ All persistent state lives in PostgreSQL (`agents`, `checks`, `events`, `inciden - The Monlet PostgreSQL database (logical or physical). - The server `MONLET_AUTH_TOKEN` and `.env` (kept in your secret store, not in backups of the DB). -- The agent `config.toml` per host (kept in your config management). +- The agent `config.toml` per host and local `state_dir/agent.key` if you need to preserve accepted identity without re-acceptance. ## Logical backup (recommended for small deployments) @@ -31,19 +31,15 @@ The schema is owned by Alembic. Restoring a logical dump from the same Monlet ve ## Retention -- `events` grows unbounded in v1. Decide a retention policy that matches your storage budget (typical: 7–30 days). Stage 6+ may add server-side trimming; until then, run a manual cron: - - ```sql - DELETE FROM events WHERE observed_at < now() - interval '14 days'; - ``` - - `checks.last_event_id` is not a FK, so trimming `events` does not break current state. - -- `incidents` and `notification_outbox` are small. Trim outbox `state IN ('sent','failed','discarded')` rows older than 7 days if storage is tight. +- `events` is partitioned by `received_at` (one monthly partition per range). Retention is controlled by `MONLET_EVENTS_RETENTION_MONTHS` (default 36). The partition maintenance loop drops old partitions on `MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC` — disk is reclaimed by `DROP PARTITION`, not by row-level `DELETE`, so reclaim is bulk-fast and lock-light. Do NOT run row-level `DELETE FROM events` — it bloats the table and conflicts with partition rotation. +- `event_ingest_dedup` is the idempotency table; TTL `MONLET_EVENT_DEDUP_RETENTION_DAYS` (default 30) prunes rows in bounded batches each detector tick. After dedup TTL elapses an agent replaying a very old spooled event will be accepted again (no double-side effects because the corresponding `events` partition is already dropped on the same horizon). +- `notification_outbox` retention is bounded by `MONLET_OUTBOX_RETENTION_MAX_ROWS` (default 50000) and pruned in batches of `MONLET_OUTBOX_PRUNE_BATCH_SIZE` per tick (PH-012). Only terminal states (`sent`, `failed`, `discarded`) are eligible — active rows (`pending`, `sending`, `retry`) are never deleted. +- `incidents` is small and not pruned automatically; size scales with operator activity rather than ingestion rate. ## Disaster recovery - Restore the database. - Restart the server. Agents will resume sending; spooled events on each agent are replayed and de-duplicated by `event_id` (ADR-0006). - The agent spool is bounded (10 000 events / 50 MiB FIFO, ADR-0005). During a long outage the oldest events get dropped to make room for new ones, so observations from the start of the outage are lost first. Plan recovery time accordingly — for an agent producing ~1 event/s per check, ~10 000 events is roughly 2–3 hours of buffering for a handful of checks. +- Spool overflow and non-retryable (4xx) server rejections are observable via `monlet_agent_events_dropped_total{reason}` with `reason ∈ spool_overflow_events | spool_overflow_bytes | send_non_retryable`. Alert on any non-zero rate. Operator action: investigate the agent log (one warning per ~30s drop burst), the server (for 4xx), and whether spool limits need raising in `state_dir`. - If agents were also lost, only events that were not yet persisted to the spool file are lost in addition. diff --git a/docs/ops/deployment-checklist.md b/docs/ops/deployment-checklist.md index ea97f53..3a5959c 100644 --- a/docs/ops/deployment-checklist.md +++ b/docs/ops/deployment-checklist.md @@ -28,8 +28,8 @@ A pragmatic list for the first production-ish Monlet stand-up. Adjust per enviro ## 4. Agent (per host) - [ ] Install binary (see `agent/systemd/`). -- [ ] Render `config.toml` from `agent/config.example.toml` with hostname or explicit `agent_id`, and `[server].token`. -- [ ] Start under systemd; verify a heartbeat reaches the server (`GET /api/v1/agents`). +- [ ] Render `config.toml` from `agent/config.example.toml`; omit `agent_id` to use the OS hostname, or set explicit `agent_id` when the host name is not a stable Monlet identifier. +- [ ] Start under systemd; verify the host appears as `pending` in `GET /api/v1/agents`, then accept it in the UI/API. - [ ] Run one configured check and verify a row appears in `/api/v1/checks` and an event in `/api/v1/events/query`. ## 5. Notifications @@ -41,6 +41,7 @@ A pragmatic list for the first production-ish Monlet stand-up. Adjust per enviro ## 6. Web UI - [ ] Set `MONLET_API_BASE_URL`, `MONLET_API_TOKEN`, and optional `MONLET_WEB_TIME_ZONE` for the web container. +- [ ] Protect the web UI with either `MONLET_WEB_AUTH_USERNAME`/`MONLET_WEB_AUTH_PASSWORD` or a trusted reverse proxy that sets `X-Forwarded-User` plus `MONLET_WEB_TRUST_PROXY_AUTH=true`. - [ ] Set optional `MONLET_NOTIFICATION_TIME_ZONE` for human-readable notifier timestamps. - [ ] Browse `/`, `/agents`, `/checks`, `/incidents`, `/events`, `/outbox`. - [ ] Verify the web image does not log the token (`docker logs` greps clean). diff --git a/docs/ops/token-rotation.md b/docs/ops/token-rotation.md index 777ed55..7829bc5 100644 --- a/docs/ops/token-rotation.md +++ b/docs/ops/token-rotation.md @@ -1,20 +1,33 @@ # Token rotation -Monlet v1 uses a single shared Bearer token (`MONLET_AUTH_TOKEN`) for all agent and UI endpoints (ADR-0007). Per-agent tokens and dual-token acceptance are out of scope for v1. +Monlet uses split Bearer-token auth (ADR-0007): -## Why rotation is disruptive in v1 +- **UI token** — `MONLET_AUTH_TOKEN` env. Accepted by every UI/operator endpoint. The variable holds one or more space/comma-separated tokens; the server accepts any of them, so rotation is zero-loss. +- **Agent key** — generated locally by each agent and stored in `state_dir/agent.key`. It is not configured on the server; operators accept, block, remove, or blacklist keys through the Agents UI/API. -The server validates exactly one token at a time. Agents that present the old token after the server has switched receive `401`, which the agent treats as a **non-retryable** failure and drops the affected batch from its local spool (`agent/internal/client/client.go`, `agent/internal/app/app.go`). Events flushed during the mismatch window are lost — they are not re-spooled and not re-sent. +Rotate UI tokens independently from agent admission. Changing `MONLET_AUTH_TOKEN` does not affect accepted agents. -Plan rotations during a low-traffic window and either stop the agents first (so spooled events persist on disk until they restart with the new token) or accept the event-loss window. +## How multi-token overlap works + +`MONLET_AUTH_TOKEN` is a list: tokens separated by spaces and/or commas. Examples: + +```sh +MONLET_AUTH_TOKEN="single-token" +MONLET_AUTH_TOKEN="old-token new-token" +MONLET_AUTH_TOKEN="old-token,new-token" +``` + +All listed tokens are accepted. The middleware compares in constant time across the full list, so timing-side-channels do not leak which token matched. + +Logs and metrics never include token values. ## When to rotate -- A suspected leak (token committed, log exposure, lost laptop with `config.toml`). +- Suspected UI token leak. - Operator turnover. - Routine schedule (e.g., quarterly). -## Procedure +## Zero-loss procedure 1. **Generate a new token**: @@ -22,24 +35,39 @@ Plan rotations during a low-traffic window and either stop the agents first (so openssl rand -hex 32 ``` -2. **Stop or pause agents** (recommended). Either: - - `systemctl stop monlet-agent` on each host; spooled events remain on disk and replay after restart, or - - skip this step and accept that any events sent during steps 3–4 are lost on the `401`. +2. **Add the new token alongside the old one** in the server environment: -3. **Restart server** with the new `MONLET_AUTH_TOKEN`. Verify: + ```sh + MONLET_AUTH_TOKEN="$OLD $NEW" + ``` + + Restart (or reload) the server. The old token still works, the new one is now accepted as well. + +3. **Move clients to the new token**: + - Update Web UI `MONLET_API_TOKEN` and restart. + - Update any operator scripts or curl examples to use the new value. + +4. **Verify everything uses the new token**: ```sh curl -sf -H "Authorization: Bearer $NEW" http://server:8000/api/v1/agents | jq . + # Optional: tail server logs for unauthorized attempts during the window. ``` -4. **Update agent `config.toml`** (`[server].token`) on each host and start agents. +5. **Remove the old token** from the server environment: -5. **Update Web UI** (`MONLET_API_TOKEN`) and restart. + ```sh + MONLET_AUTH_TOKEN="$NEW" + ``` -6. **Revoke the old token**: remove the old value from secret stores, CI variables, and operator notes. + Restart the server. From this point the old token is revoked. + +6. **Scrub the old value** from secret stores, CI variables, and operator notes. ## Notes -- The token is the only auth in v1; protect it like a database password. -- The token never appears in metrics labels, incident keys, or notification labels (ADR-0007, redaction policy). +- Tokens are the only auth in v1; protect them like a database password. +- Tokens never appear in metrics labels, incident keys, or notification labels (ADR-0007, redaction policy). - The server redacts `Authorization`/`Cookie`-style headers before logging (`monlet_server/redaction.py`). +- The middleware uses `hmac.compare_digest` against every accepted token; rotation does not introduce a timing oracle. +- To rotate an individual agent key, delete `state_dir/agent.key` on that host and restart the agent. The server will show a pending replacement for the same hostname; accept it after verifying the host. diff --git a/server/.env.example b/server/.env.example index e690ba2..9a3367a 100644 --- a/server/.env.example +++ b/server/.env.example @@ -3,11 +3,14 @@ MONLET_DATABASE_URL=postgresql+asyncpg://monlet:monlet@localhost:5432/monlet MONLET_LOG_LEVEL=INFO MONLET_BODY_LIMIT_BYTES=1048576 MONLET_OUTPUT_MAX_BYTES=8192 -MONLET_STALE_AFTER_SEC=90 -MONLET_DEAD_AFTER_SEC=300 -MONLET_DETECTOR_TICK_SEC=15 +MONLET_STALE_AFTER_SEC=20 +MONLET_DEAD_AFTER_SEC=30 +MONLET_DETECTOR_TICK_SEC=5 MONLET_ENABLE_DETECTOR=true MONLET_EVENTS_RETENTION_MONTHS=36 +MONLET_MAX_PENDING_AGENTS=10000 +MONLET_PENDING_AGENT_TTL_DAYS=7 +MONLET_PENDING_AGENT_PRUNE_BATCH_SIZE=1000 MONLET_OUTBOX_RETENTION_MAX_ROWS=50000 MONLET_ENABLE_NOTIFIER_WORKER=true diff --git a/server/Dockerfile b/server/Dockerfile index c8e4c62..40dea36 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -20,8 +20,17 @@ RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev FROM python:3.14-slim ENV PATH="/app/.venv/bin:$PATH" \ PYTHONUNBUFFERED=1 + +# PH-015: non-root runtime user. UID/GID are stable so bind-mounts can be +# pre-chowned by operators if they need writable host paths. +RUN groupadd --system --gid 1000 monlet \ + && useradd --system --uid 1000 --gid monlet --home-dir /app --shell /usr/sbin/nologin monlet + WORKDIR /app COPY --from=builder /app /app +RUN chown -R monlet:monlet /app +USER monlet:monlet EXPOSE 8000 +STOPSIGNAL SIGTERM CMD ["uvicorn", "monlet_server.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/server/README.md b/server/README.md index 1332049..1c91e9f 100644 --- a/server/README.md +++ b/server/README.md @@ -30,18 +30,21 @@ UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run fastapi dev monlet | Var | Default | Purpose | |---|---|---| -| `MONLET_AUTH_TOKEN` | required | Shared Bearer token (ADR-0007); `changeme` is rejected | +| `MONLET_AUTH_TOKEN` | required | UI/operator Bearer token. Space/comma-separated list; any accepted. `changeme` is rejected. | | `MONLET_DATABASE_URL` | `postgresql+asyncpg://monlet:monlet@127.0.0.1:5432/monlet` | Async DSN | | `MONLET_LOG_LEVEL` | `INFO` | Log level | | `MONLET_BODY_LIMIT_BYTES` | `1048576` | Request body limit (ADR-0005) | | `MONLET_OUTPUT_MAX_BYTES` | `8192` | Per-event output limit | -| `MONLET_STALE_AFTER_SEC` | `90` | Stale threshold | -| `MONLET_DEAD_AFTER_SEC` | `300` | Dead threshold | -| `MONLET_DETECTOR_TICK_SEC` | `15` | Detector tick | +| `MONLET_STALE_AFTER_SEC` | `20` | Stale threshold | +| `MONLET_DEAD_AFTER_SEC` | `30` | Dead threshold | +| `MONLET_DETECTOR_TICK_SEC` | `5` | Detector tick | | `MONLET_ENABLE_DETECTOR` | `true` | Toggle detector task | | `MONLET_EVENTS_RETENTION_MONTHS` | `36` | Months of `events` partitions to keep; `0` disables drop | | `MONLET_EVENTS_FUTURE_PARTITIONS` | `3` | Future month partitions to pre-create | | `MONLET_EVENT_DEDUP_RETENTION_DAYS` | `30` | Idempotency window for `event_id` | +| `MONLET_MAX_PENDING_AGENTS` | `10000` | Maximum pending agent keys before heartbeat rejects with 429 | +| `MONLET_PENDING_AGENT_TTL_DAYS` | `7` | Pending key cleanup TTL; `0` disables pruning | +| `MONLET_PENDING_AGENT_PRUNE_BATCH_SIZE` | `1000` | Pending cleanup rows per detector tick | | `MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC` | `3600` | Partition create/drop worker interval | | `MONLET_OUTBOX_RETENTION_MAX_ROWS` | `50000` | Maximum retained terminal outbox rows; `0` disables pruning | | `MONLET_ENABLE_NOTIFIER_WORKER` | `true` | Toggle outbox worker | @@ -79,4 +82,7 @@ docker build -t monlet-server . ## Endpoints -All routes under `/api/v1` (per `api/openapi.yaml`). `/metrics` and `/api/v1/health`, `/api/v1/ready` are open; everything else requires `Authorization: Bearer `. +All routes under `/api/v1` (per `api/openapi.yaml`). `/metrics` and `/api/v1/health`, `/api/v1/ready` are open. + +- UI/operator endpoints require `Authorization: Bearer `. +- Agent ingestion endpoints accept the per-agent key generated and stored by the agent. A new key creates a pending agent; events are ignored until an operator accepts it. diff --git a/server/alembic/versions/0001_baseline.py b/server/alembic/versions/0001_baseline.py index f745e0e..5c888f6 100644 --- a/server/alembic/versions/0001_baseline.py +++ b/server/alembic/versions/0001_baseline.py @@ -20,18 +20,53 @@ def upgrade() -> None: op.execute( """ CREATE TABLE agents ( - agent_id TEXT PRIMARY KEY, - hostname TEXT NOT NULL, - status TEXT NOT NULL CHECK (status IN ('alive','stale','dead')) DEFAULT 'alive', - last_seen_at TIMESTAMPTZ NOT NULL, - features JSONB NOT NULL DEFAULT jsonb_build_object('push', false, 'metrics', false), - labels JSONB NOT NULL DEFAULT '{}'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + agent_id TEXT PRIMARY KEY, + hostname TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('alive','stale','dead')) DEFAULT 'alive', + last_seen_at TIMESTAMPTZ NOT NULL, + features JSONB NOT NULL DEFAULT jsonb_build_object('push', false, 'metrics', false), + labels JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + accepted_key_hash TEXT, + accepted_key_fingerprint TEXT, + accepted_at TIMESTAMPTZ, + pending_key_hash TEXT, + pending_key_fingerprint TEXT, + pending_hostname TEXT, + pending_seen_at TIMESTAMPTZ, + pending_features JSONB, + pending_labels JSONB, + admission_rank INTEGER NOT NULL GENERATED ALWAYS AS ( + CASE WHEN pending_key_hash IS NOT NULL THEN 0 ELSE 1 END + ) STORED ); """ ) op.execute("CREATE INDEX agents_status_idx ON agents (status);") op.execute("CREATE INDEX agents_last_seen_idx ON agents (last_seen_at);") + op.execute("CREATE INDEX agents_admission_rank_idx ON agents (admission_rank, agent_id);") + op.execute("CREATE UNIQUE INDEX agents_hostname_idx ON agents (hostname);") + op.execute( + "CREATE UNIQUE INDEX agents_accepted_key_hash_idx ON agents (accepted_key_hash) WHERE accepted_key_hash IS NOT NULL;" + ) + op.execute( + "CREATE UNIQUE INDEX agents_pending_key_hash_idx ON agents (pending_key_hash) WHERE pending_key_hash IS NOT NULL;" + ) + + op.execute( + """ + CREATE TABLE agent_blacklist ( + key_hash TEXT PRIMARY KEY, + key_fingerprint TEXT NOT NULL, + agent_id TEXT NOT NULL, + hostname TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL + ); + """ + ) + op.execute("CREATE INDEX agent_blacklist_fingerprint_idx ON agent_blacklist (key_fingerprint);") + op.execute("CREATE INDEX agent_blacklist_last_seen_idx ON agent_blacklist (last_seen_at);") op.execute( """ @@ -76,6 +111,18 @@ def upgrade() -> None: ) op.execute("CREATE INDEX events_observed_at_idx ON events (observed_at DESC);") op.execute("CREATE INDEX events_received_at_idx ON events (received_at DESC);") + # PH-010: pagination uses ORDER BY (received_at DESC, event_id DESC). Compound + # indexes match the unfiltered and per-agent/per-check filtered queries so the + # planner does not fall back to seq-scan + sort on hot partitions. + op.execute( + "CREATE INDEX events_received_event_idx ON events (received_at DESC, event_id DESC);" + ) + op.execute( + "CREATE INDEX events_agent_received_event_idx ON events (agent_id, received_at DESC, event_id DESC);" + ) + op.execute( + "CREATE INDEX events_check_received_event_idx ON events (check_id, received_at DESC, event_id DESC);" + ) # Global idempotency table: server enforces event_id uniqueness here, not on the # partitioned events table. Pruned by retention TTL (event_dedup_retention_days). @@ -91,6 +138,10 @@ def upgrade() -> None: "CREATE INDEX event_ingest_dedup_received_at_idx ON event_ingest_dedup (received_at);" ) + # PH-review: status_rank is a STORED generated column so the default UI sort + # (status_rank, opened_at, id) maps to a trivial B-tree index. Keep the + # CASE expression in sync with `_STATUS_RANK` in + # server/monlet_server/api/incidents.py. op.execute( """ CREATE TABLE incidents ( @@ -103,14 +154,26 @@ def upgrade() -> None: opened_at TIMESTAMPTZ NOT NULL, resolved_at TIMESTAMPTZ, summary TEXT, - last_event_id UUID NOT NULL + last_event_id UUID NOT NULL, + status_rank INTEGER NOT NULL GENERATED ALWAYS AS ( + CASE + WHEN state = 'open' AND severity = 'critical' THEN 5 + WHEN state = 'open' AND severity = 'warning' THEN 4 + WHEN state = 'open' AND severity = 'unknown' THEN 3 + WHEN state = 'resolved' AND severity = 'critical' THEN 2 + WHEN state = 'resolved' AND severity = 'warning' THEN 1 + WHEN state = 'resolved' AND severity = 'unknown' THEN 0 + END + ) STORED ); """ ) op.execute( "CREATE UNIQUE INDEX incidents_open_key_idx ON incidents (incident_key) WHERE state = 'open';" ) - op.execute("CREATE INDEX incidents_state_opened_idx ON incidents (state, opened_at DESC);") + op.execute( + "CREATE INDEX incidents_status_rank_idx ON incidents (status_rank DESC, opened_at DESC, id DESC);" + ) op.execute( """ @@ -132,6 +195,13 @@ def upgrade() -> None: op.execute( "CREATE INDEX outbox_pending_idx ON notification_outbox (state, next_attempt_at) WHERE state IN ('pending','retry');" ) + # PH-review: UI lists outbox with cursor pagination ordered by created_at/updated_at. + op.execute( + "CREATE INDEX outbox_created_at_idx ON notification_outbox (created_at DESC, id DESC);" + ) + op.execute( + "CREATE INDEX outbox_updated_at_idx ON notification_outbox (updated_at DESC, id DESC);" + ) def downgrade() -> None: @@ -140,4 +210,5 @@ def downgrade() -> None: op.execute("DROP TABLE IF EXISTS event_ingest_dedup CASCADE;") op.execute("DROP TABLE IF EXISTS events CASCADE;") op.execute("DROP TABLE IF EXISTS checks CASCADE;") + op.execute("DROP TABLE IF EXISTS agent_blacklist CASCADE;") op.execute("DROP TABLE IF EXISTS agents CASCADE;") diff --git a/server/monlet_server/agent_auth.py b/server/monlet_server/agent_auth.py new file mode 100644 index 0000000..a228211 --- /dev/null +++ b/server/monlet_server/agent_auth.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from hashlib import sha256 + +from fastapi import Request + + +@dataclass(frozen=True) +class AgentCredential: + key_hash: str + key_fingerprint: str + + +def hash_agent_key(token: str) -> str: + return sha256(token.encode("utf-8")).hexdigest() + + +def fingerprint_from_hash(key_hash: str) -> str: + return key_hash[:16] + + +def credential_from_token(token: str) -> AgentCredential: + key_hash = hash_agent_key(token) + return AgentCredential(key_hash=key_hash, key_fingerprint=fingerprint_from_hash(key_hash)) + + +def get_agent_credential(request: Request) -> AgentCredential: + cred = getattr(request.state, "agent_credential", None) + if not isinstance(cred, AgentCredential): + raise RuntimeError("agent credential missing") + return cred diff --git a/server/monlet_server/api/agents.py b/server/monlet_server/api/agents.py index 26a3bf9..c0ded70 100644 --- a/server/monlet_server/api/agents.py +++ b/server/monlet_server/api/agents.py @@ -1,49 +1,341 @@ +from datetime import UTC, datetime + from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import select +from sqlalchemy import and_, delete, or_, select +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from ..cursors import decode, encode from ..db import get_session from ..models import Agent as AgentModel -from ..schemas import Agent, AgentsPage +from ..models import AgentBlacklist, Event, Incident, NotificationOutbox +from ..schemas import ( + ActionCountResponse, + Agent, + AgentBlacklistItem, + AgentBlacklistPage, + AgentsPage, +) from ..timeutil import to_utc router = APIRouter() +def _is_pending(a: AgentModel) -> bool: + return a.pending_key_hash is not None + + def _to_schema(a: AgentModel) -> Agent: + pending = _is_pending(a) + hostname = a.pending_hostname if pending and a.pending_hostname is not None else a.hostname + last_seen_at = ( + a.pending_seen_at if pending and a.pending_seen_at is not None else a.last_seen_at + ) + features = a.pending_features if pending and a.pending_features is not None else a.features + labels = a.pending_labels if pending and a.pending_labels is not None else a.labels return Agent( agent_id=a.agent_id, - hostname=a.hostname, + hostname=hostname, status=a.status, - last_seen_at=to_utc(a.last_seen_at), - features=a.features or {"push": False, "metrics": False}, - labels=a.labels or {}, + admission_state="pending" if pending or a.accepted_key_hash is None else "accepted", + rotation_pending=pending and a.accepted_key_hash is not None, + key_fingerprint=(a.pending_key_fingerprint if pending else a.accepted_key_fingerprint), + last_seen_at=to_utc(last_seen_at), + features=features or {"push": False, "metrics": False}, + labels=labels or {}, ) +def _blacklist_schema(row: AgentBlacklist) -> AgentBlacklistItem: + return AgentBlacklistItem( + id=row.key_hash, + key_fingerprint=row.key_fingerprint, + agent_id=row.agent_id, + hostname=row.hostname, + created_at=to_utc(row.created_at), + last_seen_at=to_utc(row.last_seen_at), + ) + + +async def _get_agent(session: AsyncSession, agent_id: str) -> AgentModel: + res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id)) + a = res.scalar_one_or_none() + if a is None: + raise HTTPException(status_code=404, detail="agent not found") + return a + + +def _clear_pending(a: AgentModel) -> None: + a.pending_key_hash = None + a.pending_key_fingerprint = None + a.pending_hostname = None + a.pending_seen_at = None + a.pending_features = None + a.pending_labels = None + + +def _accept_pending(a: AgentModel) -> bool: + if not _is_pending(a): + return False + now = datetime.now(UTC) + a.accepted_key_hash = a.pending_key_hash + a.accepted_key_fingerprint = a.pending_key_fingerprint + a.accepted_at = now + a.hostname = a.pending_hostname or a.hostname + a.last_seen_at = a.pending_seen_at or a.last_seen_at + if a.pending_features is not None: + a.features = a.pending_features + if a.pending_labels is not None: + a.labels = a.pending_labels + a.status = "alive" + _clear_pending(a) + return True + + +async def _block_pending(session: AsyncSession, a: AgentModel) -> bool: + if not _is_pending(a): + return False + now = datetime.now(UTC) + stmt = pg_insert(AgentBlacklist).values( + key_hash=a.pending_key_hash, + key_fingerprint=a.pending_key_fingerprint, + agent_id=a.agent_id, + hostname=a.pending_hostname or a.hostname, + last_seen_at=a.pending_seen_at or now, + ) + stmt = stmt.on_conflict_do_update( + index_elements=[AgentBlacklist.key_hash], + set_={ + "agent_id": stmt.excluded.agent_id, + "hostname": stmt.excluded.hostname, + "last_seen_at": stmt.excluded.last_seen_at, + }, + ) + await session.execute(stmt) + _clear_pending(a) + if a.accepted_key_hash is None: + await _delete_agent_data(session, a.agent_id) + return True + + +async def _delete_agent_data(session: AsyncSession, agent_id: str) -> None: + incident_ids = select(Incident.id).where(Incident.agent_id == agent_id) + await session.execute( + delete(NotificationOutbox).where(NotificationOutbox.incident_id.in_(incident_ids)) + ) + await session.execute(delete(Incident).where(Incident.agent_id == agent_id)) + await session.execute(delete(Event).where(Event.agent_id == agent_id)) + await session.execute(delete(AgentModel).where(AgentModel.agent_id == agent_id)) + + +async def _remove_pending_or_agent(session: AsyncSession, a: AgentModel) -> None: + if _is_pending(a) and a.accepted_key_hash is not None: + _clear_pending(a) + return + await _delete_agent_data(session, a.agent_id) + + +def _agent_rank(a: AgentModel) -> int: + return a.admission_rank + + +def _decode_agent_cursor(cursor: str) -> tuple[int, str]: + rank_raw, agent_id = decode(cursor, 2) + try: + rank = int(rank_raw) + except ValueError as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + if rank not in (0, 1): + raise HTTPException(status_code=400, detail="invalid cursor") + return rank, agent_id + + @router.get("/agents", response_model=AgentsPage) async def list_agents( cursor: str | None = Query(default=None, max_length=512), + back: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), session: AsyncSession = Depends(get_session), ) -> AgentsPage: - stmt = select(AgentModel).order_by(AgentModel.agent_id) + if back: + stmt = select(AgentModel).order_by( + AgentModel.admission_rank.desc(), AgentModel.agent_id.desc() + ) + else: + stmt = select(AgentModel).order_by( + AgentModel.admission_rank.asc(), AgentModel.agent_id.asc() + ) if cursor is not None: - (last_id,) = decode(cursor, 1) - stmt = stmt.where(AgentModel.agent_id > last_id) + last_rank, last_id = _decode_agent_cursor(cursor) + if back: + stmt = stmt.where( + or_( + AgentModel.admission_rank < last_rank, + and_( + AgentModel.admission_rank == last_rank, + AgentModel.agent_id < last_id, + ), + ) + ) + else: + stmt = stmt.where( + or_( + AgentModel.admission_rank > last_rank, + and_( + AgentModel.admission_rank == last_rank, + AgentModel.agent_id > last_id, + ), + ) + ) rows = (await session.execute(stmt.limit(limit + 1))).scalars().all() - next_cursor = None - if len(rows) > limit: - rows = rows[:limit] - next_cursor = encode(rows[-1].agent_id) - return AgentsPage(items=[_to_schema(a) for a in rows], next_cursor=next_cursor) + has_more = len(rows) > limit + rows = rows[:limit] + if back: + rows = list(reversed(rows)) + next_cursor: str | None = None + prev_cursor: str | None = None + if rows: + if back: + next_cursor = encode(_agent_rank(rows[-1]), rows[-1].agent_id) + if has_more: + prev_cursor = encode(_agent_rank(rows[0]), rows[0].agent_id) + else: + if has_more: + next_cursor = encode(_agent_rank(rows[-1]), rows[-1].agent_id) + if cursor is not None: + prev_cursor = encode(_agent_rank(rows[0]), rows[0].agent_id) + return AgentsPage( + items=[_to_schema(a) for a in rows], + next_cursor=next_cursor, + prev_cursor=prev_cursor, + ) + + +@router.get("/agents/admission/blacklist", response_model=AgentBlacklistPage) +async def list_agent_blacklist( + cursor: str | None = Query(default=None, max_length=512), + limit: int = Query(default=100, ge=1, le=500), + session: AsyncSession = Depends(get_session), +) -> AgentBlacklistPage: + stmt = select(AgentBlacklist).order_by(AgentBlacklist.key_fingerprint, AgentBlacklist.key_hash) + if cursor is not None: + last_fingerprint, last_hash = decode(cursor, 2) + stmt = stmt.where( + or_( + AgentBlacklist.key_fingerprint > last_fingerprint, + and_( + AgentBlacklist.key_fingerprint == last_fingerprint, + AgentBlacklist.key_hash > last_hash, + ), + ) + ) + rows = (await session.execute(stmt.limit(limit + 1))).scalars().all() + has_more = len(rows) > limit + rows = rows[:limit] + return AgentBlacklistPage( + items=[_blacklist_schema(r) for r in rows], + next_cursor=encode(rows[-1].key_fingerprint, rows[-1].key_hash) + if rows and has_more + else None, + prev_cursor=None, + ) + + +@router.delete("/agents/admission/blacklist", response_model=ActionCountResponse) +async def clear_agent_blacklist( + session: AsyncSession = Depends(get_session), +) -> ActionCountResponse: + res = await session.execute(delete(AgentBlacklist)) + await session.commit() + return ActionCountResponse(count=res.rowcount or 0) + + +@router.delete("/agents/admission/blacklist/{key_id}", response_model=ActionCountResponse) +async def delete_agent_blacklist_item( + key_id: str, + session: AsyncSession = Depends(get_session), +) -> ActionCountResponse: + res = await session.execute(delete(AgentBlacklist).where(AgentBlacklist.key_hash == key_id)) + await session.commit() + return ActionCountResponse(count=res.rowcount or 0) + + +@router.post("/agents/admission/accept-all", response_model=ActionCountResponse) +async def accept_all_pending_agents( + include_rotation: bool = Query(default=False), + session: AsyncSession = Depends(get_session), +) -> ActionCountResponse: + stmt = select(AgentModel).where(AgentModel.pending_key_hash.is_not(None)) + if not include_rotation: + stmt = stmt.where(AgentModel.accepted_key_hash.is_(None)) + rows = (await session.execute(stmt)).scalars().all() + count = sum(1 for a in rows if _accept_pending(a)) + await session.commit() + return ActionCountResponse(count=count) + + +@router.post("/agents/admission/block-all", response_model=ActionCountResponse) +async def block_all_pending_agents( + session: AsyncSession = Depends(get_session), +) -> ActionCountResponse: + rows = ( + (await session.execute(select(AgentModel).where(AgentModel.pending_key_hash.is_not(None)))) + .scalars() + .all() + ) + count = 0 + for a in rows: + if await _block_pending(session, a): + count += 1 + await session.commit() + return ActionCountResponse(count=count) + + +@router.post("/agents/admission/remove-all", response_model=ActionCountResponse) +async def remove_all_pending_agents( + session: AsyncSession = Depends(get_session), +) -> ActionCountResponse: + rows = ( + (await session.execute(select(AgentModel).where(AgentModel.pending_key_hash.is_not(None)))) + .scalars() + .all() + ) + for agent in rows: + await _remove_pending_or_agent(session, agent) + await session.commit() + return ActionCountResponse(count=len(rows)) + + +@router.post("/agents/{agent_id}/accept", response_model=ActionCountResponse) +async def accept_agent( + agent_id: str, session: AsyncSession = Depends(get_session) +) -> ActionCountResponse: + agent = await _get_agent(session, agent_id) + count = 1 if _accept_pending(agent) else 0 + await session.commit() + return ActionCountResponse(count=count) + + +@router.post("/agents/{agent_id}/block", response_model=ActionCountResponse) +async def block_agent( + agent_id: str, session: AsyncSession = Depends(get_session) +) -> ActionCountResponse: + agent = await _get_agent(session, agent_id) + count = 1 if await _block_pending(session, agent) else 0 + await session.commit() + return ActionCountResponse(count=count) + + +@router.delete("/agents/{agent_id}", response_model=ActionCountResponse) +async def delete_agent( + agent_id: str, session: AsyncSession = Depends(get_session) +) -> ActionCountResponse: + agent = await _get_agent(session, agent_id) + await _remove_pending_or_agent(session, agent) + await session.commit() + return ActionCountResponse(count=1) @router.get("/agents/{agent_id}", response_model=Agent) async def get_agent(agent_id: str, session: AsyncSession = Depends(get_session)) -> Agent: - res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id)) - a = res.scalar_one_or_none() - if a is None: - raise HTTPException(status_code=404, detail="agent not found") - return _to_schema(a) + return _to_schema(await _get_agent(session, agent_id)) diff --git a/server/monlet_server/api/checks.py b/server/monlet_server/api/checks.py index 1ccc4bb..42d6d71 100644 --- a/server/monlet_server/api/checks.py +++ b/server/monlet_server/api/checks.py @@ -15,25 +15,50 @@ router = APIRouter() async def list_checks( agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN), cursor: str | None = Query(default=None, max_length=512), + back: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), session: AsyncSession = Depends(get_session), ) -> ChecksPage: - stmt = select(Check).order_by(Check.agent_id, Check.check_id) + # PH-review: bidirectional cursor. Natural sort is ASC (agent_id, check_id). + if back: + stmt = select(Check).order_by(Check.agent_id.desc(), Check.check_id.desc()) + else: + stmt = select(Check).order_by(Check.agent_id, Check.check_id) if agent_id is not None: stmt = stmt.where(Check.agent_id == agent_id) if cursor is not None: last_agent, last_check = decode(cursor, 2) - stmt = stmt.where( - or_( - Check.agent_id > last_agent, - and_(Check.agent_id == last_agent, Check.check_id > last_check), + if back: + stmt = stmt.where( + or_( + Check.agent_id < last_agent, + and_(Check.agent_id == last_agent, Check.check_id < last_check), + ) + ) + else: + stmt = stmt.where( + or_( + Check.agent_id > last_agent, + and_(Check.agent_id == last_agent, Check.check_id > last_check), + ) ) - ) rows = (await session.execute(stmt.limit(limit + 1))).scalars().all() - next_cursor = None - if len(rows) > limit: - rows = rows[:limit] - next_cursor = encode(rows[-1].agent_id, rows[-1].check_id) + has_more = len(rows) > limit + rows = rows[:limit] + if back: + rows = list(reversed(rows)) + next_cursor: str | None = None + prev_cursor: str | None = None + if rows: + if back: + next_cursor = encode(rows[-1].agent_id, rows[-1].check_id) + if has_more: + prev_cursor = encode(rows[0].agent_id, rows[0].check_id) + else: + if has_more: + next_cursor = encode(rows[-1].agent_id, rows[-1].check_id) + if cursor is not None: + prev_cursor = encode(rows[0].agent_id, rows[0].check_id) items = [ CheckState( agent_id=r.agent_id, @@ -46,4 +71,4 @@ async def list_checks( ) for r in rows ] - return ChecksPage(items=items, next_cursor=next_cursor) + return ChecksPage(items=items, next_cursor=next_cursor, prev_cursor=prev_cursor) diff --git a/server/monlet_server/api/events.py b/server/monlet_server/api/events.py index d42f192..2b91503 100644 --- a/server/monlet_server/api/events.py +++ b/server/monlet_server/api/events.py @@ -1,11 +1,13 @@ import base64 import binascii from datetime import datetime +from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy import and_, or_, select from sqlalchemy.ext.asyncio import AsyncSession +from ..agent_auth import AgentCredential, get_agent_credential from ..db import get_session from ..models import Event from ..schemas import ( @@ -15,7 +17,7 @@ from ..schemas import ( EventsPage, StoredCheckResultEvent, ) -from ..services.ingestion import ingest_event +from ..services.ingestion import accepted_agent_for_key, ingest_event from ..timeutil import to_utc router = APIRouter() @@ -26,12 +28,14 @@ def _encode_cursor(received_at: datetime, event_id) -> str: return base64.urlsafe_b64encode(raw).decode().rstrip("=") -def _decode_cursor(cursor: str) -> tuple[datetime, str]: +def _decode_cursor(cursor: str) -> tuple[datetime, UUID]: pad = "=" * (-len(cursor) % 4) try: raw = base64.urlsafe_b64decode(cursor + pad).decode() ts_str, eid = raw.split("|", 1) - return to_utc(datetime.fromisoformat(ts_str)), eid + # PH-010: decode event_id to UUID before comparing to the UUID column + # to avoid silent type-coercion behavior across DB drivers. + return to_utc(datetime.fromisoformat(ts_str)), UUID(eid) except (binascii.Error, ValueError) as exc: raise HTTPException(status_code=400, detail="invalid cursor") from exc @@ -42,12 +46,18 @@ def _decode_cursor(cursor: str) -> tuple[datetime, str]: status_code=status.HTTP_202_ACCEPTED, ) async def events( - body: EventBatchRequest, session: AsyncSession = Depends(get_session) + body: EventBatchRequest, + credential: AgentCredential = Depends(get_agent_credential), + session: AsyncSession = Depends(get_session), ) -> EventBatchAcceptedResponse: + agent = await accepted_agent_for_key(session, credential) + if agent is None: + await session.commit() + return EventBatchAcceptedResponse(accepted=0, deduplicated=0) accepted = 0 deduplicated = 0 for ev in body.events: - ok = await ingest_event(session, body.agent_id, ev) + ok = await ingest_event(session, agent.agent_id, ev) if ok: accepted += 1 else: @@ -61,28 +71,55 @@ async def query_events( agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN), check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN), cursor: str | None = Query(default=None, max_length=512), + back: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), session: AsyncSession = Depends(get_session), ) -> EventsPage: - stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc()) + # PH-review: bidirectional cursor. The natural sort is DESC on + # (received_at, event_id); `back=True` reverses the query so we can return + # the previous page without an unbounded cursor stack in the URL. + if back: + stmt = select(Event).order_by(Event.received_at.asc(), Event.event_id.asc()) + else: + stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc()) if agent_id is not None: stmt = stmt.where(Event.agent_id == agent_id) if check_id is not None: stmt = stmt.where(Event.check_id == check_id) if cursor is not None: ts, eid = _decode_cursor(cursor) - stmt = stmt.where( - or_( - Event.received_at < ts, - and_(Event.received_at == ts, Event.event_id < eid), + if back: + stmt = stmt.where( + or_( + Event.received_at > ts, + and_(Event.received_at == ts, Event.event_id > eid), + ) + ) + else: + stmt = stmt.where( + or_( + Event.received_at < ts, + and_(Event.received_at == ts, Event.event_id < eid), + ) ) - ) stmt = stmt.limit(limit + 1) rows = (await session.execute(stmt)).scalars().all() - next_cursor = None - if len(rows) > limit: - rows = rows[:limit] - next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id) + has_more = len(rows) > limit + rows = rows[:limit] + if back: + rows = list(reversed(rows)) + next_cursor: str | None = None + prev_cursor: str | None = None + if rows: + if back: + next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id) + if has_more: + prev_cursor = _encode_cursor(rows[0].received_at, rows[0].event_id) + else: + if has_more: + next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id) + if cursor is not None: + prev_cursor = _encode_cursor(rows[0].received_at, rows[0].event_id) items = [ StoredCheckResultEvent( event_id=str(r.event_id), @@ -100,4 +137,4 @@ async def query_events( ) for r in rows ] - return EventsPage(items=items, next_cursor=next_cursor) + return EventsPage(items=items, next_cursor=next_cursor, prev_cursor=prev_cursor) diff --git a/server/monlet_server/api/heartbeat.py b/server/monlet_server/api/heartbeat.py index edfef28..bd30a14 100644 --- a/server/monlet_server/api/heartbeat.py +++ b/server/monlet_server/api/heartbeat.py @@ -1,6 +1,7 @@ from fastapi import APIRouter, Depends, status from sqlalchemy.ext.asyncio import AsyncSession +from ..agent_auth import AgentCredential, get_agent_credential from ..db import get_session from ..schemas import AcceptedResponse, HeartbeatRequest from ..services.ingestion import upsert_agent_heartbeat @@ -14,8 +15,10 @@ router = APIRouter() status_code=status.HTTP_202_ACCEPTED, ) async def heartbeat( - body: HeartbeatRequest, session: AsyncSession = Depends(get_session) + body: HeartbeatRequest, + credential: AgentCredential = Depends(get_agent_credential), + session: AsyncSession = Depends(get_session), ) -> AcceptedResponse: - await upsert_agent_heartbeat(session, body) + await upsert_agent_heartbeat(session, body, credential) await session.commit() return AcceptedResponse(accepted=True) diff --git a/server/monlet_server/api/incidents.py b/server/monlet_server/api/incidents.py index befcdf9..6d9c4e3 100644 --- a/server/monlet_server/api/incidents.py +++ b/server/monlet_server/api/incidents.py @@ -1,3 +1,5 @@ +from typing import Literal + from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import and_, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -5,11 +7,29 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..cursors import decode, decode_datetime, encode from ..db import get_session from ..models import Incident as IncidentModel -from ..schemas import Incident, IncidentsPage +from ..schemas import ID_PATTERN, Incident, IncidentsPage from ..timeutil import to_utc router = APIRouter() +_SORT_COLUMNS = { + "opened_at": IncidentModel.opened_at, + "resolved_at": IncidentModel.resolved_at, +} +SortField = Literal["status", "opened_at", "resolved_at"] + +# PH-review: the rank is materialized as a STORED generated column +# `incidents.status_rank` (see models.py / 0001_baseline.py). This map mirrors +# the SQL CASE for Python-side cursor encoding only. +_STATUS_RANK: dict[tuple[str, str], int] = { + ("open", "critical"): 5, + ("open", "warning"): 4, + ("open", "unknown"): 3, + ("resolved", "critical"): 2, + ("resolved", "warning"): 1, + ("resolved", "unknown"): 0, +} + def _to_schema(i: IncidentModel) -> Incident: return Incident( @@ -25,30 +45,105 @@ def _to_schema(i: IncidentModel) -> Incident: ) +def _encode_incident_cursor(i: IncidentModel, sort: SortField) -> str: + if sort == "status": + return encode(i.status_rank, to_utc(i.opened_at).isoformat(), i.id) + return encode(to_utc(getattr(i, sort)).isoformat(), i.id) + + @router.get("/incidents", response_model=IncidentsPage) async def list_incidents( - state: str | None = Query(default=None), + state: Literal["open", "resolved"] | None = Query(default=None), + severity: Literal["warning", "critical", "unknown"] | None = Query(default=None), + agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN), + check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN), + sort: SortField = Query(default="status"), + direction: Literal["asc", "desc"] = Query(default="desc"), cursor: str | None = Query(default=None, max_length=512), + back: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), session: AsyncSession = Depends(get_session), ) -> IncidentsPage: - if state is not None and state not in ("open", "resolved"): - raise HTTPException(status_code=400, detail="invalid state") - stmt = select(IncidentModel).order_by(IncidentModel.opened_at.desc(), IncidentModel.id.desc()) + # PH-001 finding-4: sort=resolved_at must skip open incidents — their + # resolved_at is NULL and would otherwise break to_utc() on the cursor. + if sort == "resolved_at" and state is None: + state = "resolved" + if sort == "resolved_at" and state == "open": + raise HTTPException(status_code=400, detail="sort=resolved_at requires state=resolved") + # PH-review: `back=True` flips the effective sort to walk to the previous + # page. We reverse the rows before returning so the response order matches + # the canonical (user-requested) direction. + effective_desc = (direction == "desc") ^ back + if sort == "status": + rank_col = IncidentModel.status_rank + rank_order = rank_col.desc() if effective_desc else rank_col.asc() + opened_order = ( + IncidentModel.opened_at.desc() if effective_desc else IncidentModel.opened_at.asc() + ) + id_order = IncidentModel.id.desc() if effective_desc else IncidentModel.id.asc() + stmt = select(IncidentModel).order_by(rank_order, opened_order, id_order) + else: + col = _SORT_COLUMNS[sort] + order_col = col.desc() if effective_desc else col.asc() + tiebreak = IncidentModel.id.desc() if effective_desc else IncidentModel.id.asc() + stmt = select(IncidentModel).order_by(order_col, tiebreak) + if sort == "resolved_at": + stmt = stmt.where(IncidentModel.resolved_at.is_not(None)) if state is not None: stmt = stmt.where(IncidentModel.state == state) + if severity is not None: + stmt = stmt.where(IncidentModel.severity == severity) + if agent_id is not None: + stmt = stmt.where(IncidentModel.agent_id == agent_id) + if check_id is not None: + stmt = stmt.where(IncidentModel.check_id == check_id) if cursor is not None: - ts_str, last_id = decode(cursor, 2) - ts = decode_datetime(ts_str) - stmt = stmt.where( - or_( - IncidentModel.opened_at < ts, - and_(IncidentModel.opened_at == ts, IncidentModel.id < last_id), + if sort == "status": + rank_str, ts_str, last_id = decode(cursor, 3) + try: + rank = int(rank_str) + except ValueError as exc: + raise HTTPException(status_code=400, detail="invalid cursor") from exc + ts = decode_datetime(ts_str) + rank_col = IncidentModel.status_rank + rank_cmp = rank_col < rank if effective_desc else rank_col > rank + opened_cmp = ( + IncidentModel.opened_at < ts if effective_desc else IncidentModel.opened_at > ts ) - ) + id_cmp = IncidentModel.id < last_id if effective_desc else IncidentModel.id > last_id + stmt = stmt.where( + or_( + rank_cmp, + and_(rank_col == rank, opened_cmp), + and_(rank_col == rank, IncidentModel.opened_at == ts, id_cmp), + ) + ) + else: + col = _SORT_COLUMNS[sort] + ts_str, last_id = decode(cursor, 2) + ts = decode_datetime(ts_str) + cmp = col < ts if effective_desc else col > ts + tie = IncidentModel.id < last_id if effective_desc else IncidentModel.id > last_id + stmt = stmt.where(or_(cmp, and_(col == ts, tie))) rows = (await session.execute(stmt.limit(limit + 1))).scalars().all() - next_cursor = None - if len(rows) > limit: - rows = rows[:limit] - next_cursor = encode(to_utc(rows[-1].opened_at).isoformat(), rows[-1].id) - return IncidentsPage(items=[_to_schema(r) for r in rows], next_cursor=next_cursor) + has_more = len(rows) > limit + rows = rows[:limit] + if back: + rows = list(reversed(rows)) + next_cursor: str | None = None + prev_cursor: str | None = None + if rows: + if back: + next_cursor = _encode_incident_cursor(rows[-1], sort) + if has_more: + prev_cursor = _encode_incident_cursor(rows[0], sort) + else: + if has_more: + next_cursor = _encode_incident_cursor(rows[-1], sort) + if cursor is not None: + prev_cursor = _encode_incident_cursor(rows[0], sort) + return IncidentsPage( + items=[_to_schema(r) for r in rows], + next_cursor=next_cursor, + prev_cursor=prev_cursor, + ) diff --git a/server/monlet_server/api/outbox.py b/server/monlet_server/api/outbox.py index f4720d0..f101738 100644 --- a/server/monlet_server/api/outbox.py +++ b/server/monlet_server/api/outbox.py @@ -1,3 +1,5 @@ +from typing import Literal + from fastapi import APIRouter, Depends, Query from sqlalchemy import and_, or_, select from sqlalchemy.ext.asyncio import AsyncSession @@ -10,36 +12,73 @@ from ..timeutil import to_utc router = APIRouter() +# PH-002/PH-011/PH-018: explicit sort + filter contracts. +OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"] +OutboxEventType = Literal["firing", "resolved"] +OutboxNotifier = Literal["debug", "telegram", "webhook", "alertmanager"] +_SORT_COLUMNS = { + "created_at": NotificationOutbox.created_at, + "updated_at": NotificationOutbox.updated_at, +} +SortField = Literal["created_at", "updated_at"] + + +def _encode_outbox_cursor(row: NotificationOutbox, sort: SortField) -> str: + return encode(to_utc(getattr(row, sort)).isoformat(), row.id) + @router.get("/notifiers/outbox", response_model=OutboxPage) async def list_outbox( - state: str | None = Query(default=None), + state: OutboxState | None = Query(default=None), + notifier: OutboxNotifier | None = Query(default=None), + event_type: OutboxEventType | None = Query(default=None), + sort: SortField = Query(default="updated_at"), + direction: Literal["asc", "desc"] = Query(default="desc"), cursor: str | None = Query(default=None, max_length=512), + back: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), session: AsyncSession = Depends(get_session), ) -> OutboxPage: - stmt = select(NotificationOutbox).order_by( - NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc() - ) + col = _SORT_COLUMNS[sort] + # PH-review: bidirectional cursor — for `back=True` we run the query in the + # opposite direction relative to the requested sort and reverse the result + # client-side. This avoids the unbounded cursor_stack in the URL. + effective_desc = (direction == "desc") ^ back + order_col = col.desc() if effective_desc else col.asc() + tiebreak = NotificationOutbox.id.desc() if effective_desc else NotificationOutbox.id.asc() + stmt = select(NotificationOutbox).order_by(order_col, tiebreak) if state is not None: stmt = stmt.where(NotificationOutbox.state == state) + if notifier is not None: + stmt = stmt.where(NotificationOutbox.notifier == notifier) + if event_type is not None: + stmt = stmt.where(NotificationOutbox.event_type == event_type) if cursor is not None: ts_str, last_id = decode(cursor, 2) ts = decode_datetime(ts_str) - stmt = stmt.where( - or_( - NotificationOutbox.created_at < ts, - and_( - NotificationOutbox.created_at == ts, - NotificationOutbox.id < last_id, - ), - ) - ) + cmp = col < ts if effective_desc else col > ts + tie = NotificationOutbox.id < last_id if effective_desc else NotificationOutbox.id > last_id + stmt = stmt.where(or_(cmp, and_(col == ts, tie))) rows = (await session.execute(stmt.limit(limit + 1))).scalars().all() - next_cursor = None - if len(rows) > limit: - rows = rows[:limit] - next_cursor = encode(to_utc(rows[-1].created_at).isoformat(), rows[-1].id) + has_more = len(rows) > limit + rows = rows[:limit] + if back: + rows = list(reversed(rows)) + next_cursor: str | None = None + prev_cursor: str | None = None + if rows: + if back: + # Came from a forward page → there is always a next page (the one + # we just came from). `has_more` here means there are still earlier + # pages remaining. + next_cursor = _encode_outbox_cursor(rows[-1], sort) + if has_more: + prev_cursor = _encode_outbox_cursor(rows[0], sort) + else: + if has_more: + next_cursor = _encode_outbox_cursor(rows[-1], sort) + if cursor is not None: + prev_cursor = _encode_outbox_cursor(rows[0], sort) return OutboxPage( items=[ NotificationOutboxItem( @@ -56,4 +95,5 @@ async def list_outbox( for r in rows ], next_cursor=next_cursor, + prev_cursor=prev_cursor, ) diff --git a/server/monlet_server/api/system_summary.py b/server/monlet_server/api/system_summary.py new file mode 100644 index 0000000..b5e9e16 --- /dev/null +++ b/server/monlet_server/api/system_summary.py @@ -0,0 +1,43 @@ +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..db import get_session +from ..models import Agent, Check, Incident +from ..schemas import SystemSummaryResponse + +router = APIRouter() + + +@router.get("/system-summary", response_model=SystemSummaryResponse) +async def system_summary(session: AsyncSession = Depends(get_session)) -> SystemSummaryResponse: + agents_rows = ( + await session.execute( + select(Agent.status, func.count()) + .where(Agent.accepted_key_hash.is_not(None)) + .group_by(Agent.status) + ) + ).all() + checks_rows = ( + await session.execute(select(Check.status, func.count()).group_by(Check.status)) + ).all() + open_incidents = ( + await session.execute( + select(func.count()).select_from(Incident).where(Incident.state == "open") + ) + ).scalar_one() + + agents = {status: count for status, count in agents_rows} + checks = {status: count for status, count in checks_rows} + return SystemSummaryResponse( + agents_online=agents.get("alive", 0), + agents_offline=agents.get("stale", 0) + agents.get("dead", 0), + checks_ok=checks.get("ok", 0), + checks_warning=checks.get("warning", 0), + checks_critical=checks.get("critical", 0), + checks_unknown=checks.get("unknown", 0), + open_incidents=open_incidents, + generated_at=datetime.now(UTC), + ) diff --git a/server/monlet_server/db.py b/server/monlet_server/db.py index 86ec333..f582633 100644 --- a/server/monlet_server/db.py +++ b/server/monlet_server/db.py @@ -17,7 +17,17 @@ def get_engine() -> AsyncEngine: global _engine, _sessionmaker if _engine is None: s = get_settings() - _engine = create_async_engine(s.database_url, pool_pre_ping=True, future=True) + # PH-016: explicit pool sizing. pool_pre_ping protects against stale + # connections after PG restarts; pool_recycle bounds connection age. + _engine = create_async_engine( + s.database_url, + pool_pre_ping=True, + pool_size=s.db_pool_size, + max_overflow=s.db_max_overflow, + pool_timeout=s.db_pool_timeout_sec, + pool_recycle=s.db_pool_recycle_sec, + future=True, + ) _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False) return _engine diff --git a/server/monlet_server/errors.py b/server/monlet_server/errors.py index 4f14618..6ab9995 100644 --- a/server/monlet_server/errors.py +++ b/server/monlet_server/errors.py @@ -28,6 +28,8 @@ def error_response( _STATUS_TO_CODE: dict[int, ErrorCode] = { 400: "validation", 401: "unauthorized", + 403: "forbidden", + 409: "conflict", 404: "not_found", 413: "payload_too_large", 429: "rate_limited", @@ -44,7 +46,11 @@ def install_error_handlers(app: FastAPI) -> None: @app.exception_handler(StarletteHTTPException) async def _http(request: Request, exc: StarletteHTTPException): - code = _STATUS_TO_CODE.get(exc.status_code, "internal") + code = ( + "agent_blocked" + if exc.status_code == 403 and exc.detail == "agent blocked" + else _STATUS_TO_CODE.get(exc.status_code, "internal") + ) msg = exc.detail if isinstance(exc.detail, str) else code return error_response(request, exc.status_code, code, msg) diff --git a/server/monlet_server/main.py b/server/monlet_server/main.py index e06347c..2f3bb57 100644 --- a/server/monlet_server/main.py +++ b/server/monlet_server/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI from fastapi.responses import Response from . import metrics -from .api import agents, checks, events, health, heartbeat, incidents, outbox +from .api import agents, checks, events, health, heartbeat, incidents, outbox, system_summary from .db import dispose_engine, get_engine, get_sessionmaker from .errors import install_error_handlers from .logging_config import configure_logging, get_logger @@ -86,6 +86,7 @@ def create_app() -> FastAPI: app.include_router(checks.router, prefix="/api/v1") app.include_router(incidents.router, prefix="/api/v1") app.include_router(outbox.router, prefix="/api/v1") + app.include_router(system_summary.router, prefix="/api/v1") @app.get("/metrics") async def metrics_endpoint() -> Response: diff --git a/server/monlet_server/middleware/auth.py b/server/monlet_server/middleware/auth.py index e2e689f..2470d79 100644 --- a/server/monlet_server/middleware/auth.py +++ b/server/monlet_server/middleware/auth.py @@ -1,22 +1,42 @@ +import hmac + from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request +from ..agent_auth import credential_from_token from ..errors import error_response from ..settings import get_settings NO_AUTH_PATHS = {"/api/v1/health", "/api/v1/ready", "/metrics"} +AGENT_PATHS = {"/api/v1/heartbeat", "/api/v1/events"} + + +def _match_token(presented: str, accepted: list[str]) -> bool: + matched = False + # Constant-time across all accepted tokens to avoid timing oracle. + for t in accepted: + if hmac.compare_digest(presented, t): + matched = True + return matched + class BearerAuthMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): - if request.url.path in NO_AUTH_PATHS: + path = request.url.path + if path in NO_AUTH_PATHS: return await call_next(request) - expected = get_settings().auth_token header = request.headers.get("authorization") if not header or not header.lower().startswith("bearer "): return error_response(request, 401, "unauthorized", "authentication required") token = header.split(" ", 1)[1].strip() - if token != expected: - return error_response(request, 401, "unauthorized", "authentication required") + + if path in AGENT_PATHS: + if len(token) < 32 or len(token) > 512: + return error_response(request, 401, "unauthorized", "authentication required") + request.state.agent_credential = credential_from_token(token) + else: + if not _match_token(token, get_settings().auth_tokens): + return error_response(request, 401, "unauthorized", "authentication required") return await call_next(request) diff --git a/server/monlet_server/models.py b/server/monlet_server/models.py index acb5b3d..2b89427 100644 --- a/server/monlet_server/models.py +++ b/server/monlet_server/models.py @@ -5,6 +5,7 @@ from sqlalchemy import ( JSON, Boolean, CheckConstraint, + Computed, DateTime, ForeignKey, Index, @@ -51,11 +52,57 @@ class Agent(Base): created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) + accepted_key_hash: Mapped[str | None] = mapped_column(Text, nullable=True) + accepted_key_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True) + accepted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + pending_key_hash: Mapped[str | None] = mapped_column(Text, nullable=True) + pending_key_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True) + pending_hostname: Mapped[str | None] = mapped_column(Text, nullable=True) + pending_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + pending_features: Mapped[dict | None] = mapped_column(_jsonb(), nullable=True) + pending_labels: Mapped[dict | None] = mapped_column(_jsonb(), nullable=True) + admission_rank: Mapped[int] = mapped_column( + Integer, + Computed("CASE WHEN pending_key_hash IS NOT NULL THEN 0 ELSE 1 END", persisted=True), + nullable=False, + ) __table_args__ = ( CheckConstraint("status IN ('alive','stale','dead')", name="agents_status_chk"), Index("agents_status_idx", "status"), Index("agents_last_seen_idx", "last_seen_at"), + Index("agents_admission_rank_idx", "admission_rank", "agent_id"), + Index("agents_hostname_idx", "hostname", unique=True), + Index( + "agents_accepted_key_hash_idx", + "accepted_key_hash", + unique=True, + postgresql_where=text("accepted_key_hash IS NOT NULL"), + ), + Index( + "agents_pending_key_hash_idx", + "pending_key_hash", + unique=True, + postgresql_where=text("pending_key_hash IS NOT NULL"), + ), + ) + + +class AgentBlacklist(Base): + __tablename__ = "agent_blacklist" + + key_hash: Mapped[str] = mapped_column(Text, primary_key=True) + key_fingerprint: Mapped[str] = mapped_column(Text, nullable=False) + agent_id: Mapped[str] = mapped_column(Text, nullable=False) + hostname: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + __table_args__ = ( + Index("agent_blacklist_fingerprint_idx", "key_fingerprint"), + Index("agent_blacklist_last_seen_idx", "last_seen_at"), ) @@ -117,6 +164,19 @@ class Event(Base): ), Index("events_observed_at_idx", text("observed_at DESC")), Index("events_received_at_idx", text("received_at DESC")), + Index("events_received_event_idx", text("received_at DESC"), text("event_id DESC")), + Index( + "events_agent_received_event_idx", + "agent_id", + text("received_at DESC"), + text("event_id DESC"), + ), + Index( + "events_check_received_event_idx", + "check_id", + text("received_at DESC"), + text("event_id DESC"), + ), ) @@ -144,6 +204,25 @@ class Incident(Base): resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) summary: Mapped[str | None] = mapped_column(Text, nullable=True) last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False) + # PH-review: stored generated column so the default UI sort + # (status_rank, opened_at, id) is served by a plain B-tree index without + # planner mismatches between query-time and index-time CASE expressions. + # Keep aligned with `_STATUS_RANK` in api/incidents.py. + status_rank: Mapped[int] = mapped_column( + Integer, + Computed( + "CASE " + "WHEN state = 'open' AND severity = 'critical' THEN 5 " + "WHEN state = 'open' AND severity = 'warning' THEN 4 " + "WHEN state = 'open' AND severity = 'unknown' THEN 3 " + "WHEN state = 'resolved' AND severity = 'critical' THEN 2 " + "WHEN state = 'resolved' AND severity = 'warning' THEN 1 " + "WHEN state = 'resolved' AND severity = 'unknown' THEN 0 " + "END", + persisted=True, + ), + nullable=False, + ) __table_args__ = ( CheckConstraint("state IN ('open','resolved')", name="incidents_state_chk"), @@ -156,7 +235,12 @@ class Incident(Base): unique=True, postgresql_where=text("state = 'open'"), ), - Index("incidents_state_opened_idx", "state", text("opened_at DESC")), + Index( + "incidents_status_rank_idx", + text("status_rank DESC"), + text("opened_at DESC"), + text("id DESC"), + ), ) @@ -193,4 +277,6 @@ class NotificationOutbox(Base): "next_attempt_at", postgresql_where=text("state IN ('pending','retry')"), ), + Index("outbox_created_at_idx", text("created_at DESC"), text("id DESC")), + Index("outbox_updated_at_idx", text("updated_at DESC"), text("id DESC")), ) diff --git a/server/monlet_server/schemas.py b/server/monlet_server/schemas.py index 74733c8..1930128 100644 --- a/server/monlet_server/schemas.py +++ b/server/monlet_server/schemas.py @@ -23,18 +23,22 @@ SENSITIVE_LABEL_WORDS = ( Status = Literal["ok", "warning", "critical", "unknown"] AgentStatus = Literal["alive", "stale", "dead"] +AgentAdmissionState = Literal["pending", "accepted"] IncidentState = Literal["open", "resolved"] Severity = Literal["warning", "critical", "unknown"] OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"] OutboxEventType = Literal["firing", "resolved"] ErrorCode = Literal[ "unauthorized", + "forbidden", + "conflict", "not_found", "validation", "payload_too_large", "rate_limited", "internal", "not_ready", + "agent_blocked", ] @@ -140,6 +144,9 @@ class Agent(BaseModel): agent_id: str hostname: str status: AgentStatus + admission_state: AgentAdmissionState + rotation_pending: bool = False + key_fingerprint: str | None = None last_seen_at: datetime features: AgentFeatures labels: dict[str, str] = Field(default_factory=dict) @@ -186,12 +193,30 @@ class NotificationOutboxItem(BaseModel): class PageEnvelope(BaseModel): next_cursor: str | None = None + prev_cursor: str | None = None class AgentsPage(PageEnvelope): items: list[Agent] +class AgentBlacklistItem(BaseModel): + id: str + key_fingerprint: str + agent_id: str + hostname: str + created_at: datetime + last_seen_at: datetime + + +class AgentBlacklistPage(PageEnvelope): + items: list[AgentBlacklistItem] + + +class ActionCountResponse(BaseModel): + count: int = Field(ge=0) + + class ChecksPage(PageEnvelope): items: list[CheckState] @@ -212,3 +237,14 @@ class UUIDStr(BaseModel): """Helper to validate generated UUIDs.""" value: UUID + + +class SystemSummaryResponse(BaseModel): + agents_online: int + agents_offline: int + checks_ok: int + checks_warning: int + checks_critical: int + checks_unknown: int + open_incidents: int + generated_at: datetime diff --git a/server/monlet_server/services/detector.py b/server/monlet_server/services/detector.py index 6c3c410..14f3d3f 100644 --- a/server/monlet_server/services/detector.py +++ b/server/monlet_server/services/detector.py @@ -202,14 +202,16 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime summary=summary, ) ) - else: + elif changed: + # PH-009: update liveness check row only on status transitions to avoid + # O(N) per-tick rewrites of every agent_liveness row. last_observed_at + # then reflects the meaningful transition timestamp, not detector ticks. + cur.status = liveness_status + cur.exit_code = exit_code + cur.last_event_id = event_id + cur.incident_key = incident_key cur.last_observed_at = now cur.summary = summary - if changed: - cur.status = liveness_status - cur.exit_code = exit_code - cur.last_event_id = event_id - cur.incident_key = incident_key if next_status in ("stale", "dead"): await _open_liveness_incident(session, agent, next_status, now, event_id) @@ -219,6 +221,34 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime async def _prune_history(session) -> None: s = get_settings() + if s.pending_agent_ttl_days > 0: + cutoff = datetime.now(UTC) - timedelta(days=s.pending_agent_ttl_days) + rows = ( + ( + await session.execute( + select(Agent) + .where( + Agent.pending_key_hash.is_not(None), + Agent.pending_seen_at < cutoff, + ) + .order_by(Agent.pending_seen_at) + .limit(max(s.pending_agent_prune_batch_size, 1)) + ) + ) + .scalars() + .all() + ) + for agent in rows: + if agent.accepted_key_hash is None: + await session.delete(agent) + else: + agent.pending_key_hash = None + agent.pending_key_fingerprint = None + agent.pending_hostname = None + agent.pending_seen_at = None + agent.pending_features = None + agent.pending_labels = None + if s.event_dedup_retention_days > 0: cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days) batch = max(s.event_dedup_prune_batch_size, 1) @@ -233,17 +263,29 @@ async def _prune_history(session) -> None: await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub))) if s.outbox_retention_max_rows > 0: + # PH-012: keep the N newest terminal rows. Delete in a bounded batch to + # avoid one huge DELETE under failure storms. Active rows ('pending', + # 'sending', 'retry') are never eligible — terminal-only filter + # protects them. keep_outbox = ( select(NotificationOutbox.id) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()) .limit(s.outbox_retention_max_rows) ) - await session.execute( - delete(NotificationOutbox) + prune_ids = ( + select(NotificationOutbox.id) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .where(NotificationOutbox.id.not_in(keep_outbox)) + .limit(max(s.outbox_prune_batch_size, 1)) + .scalar_subquery() ) + await session.execute( + delete(NotificationOutbox).where(NotificationOutbox.id.in_(prune_ids)) + ) + + +DETECTOR_BATCH_SIZE = 500 async def _tick(sm: async_sessionmaker) -> None: @@ -252,23 +294,50 @@ async def _tick(sm: async_sessionmaker) -> None: stale_cut = now - timedelta(seconds=s.stale_after_sec) dead_cut = now - timedelta(seconds=s.dead_after_sec) async with sm() as session: - agents = (await session.execute(select(Agent))).scalars().all() - for agent in agents: - if agent.last_seen_at <= dead_cut: - next_status = "dead" - elif agent.last_seen_at <= stale_cut: - next_status = "stale" - else: - next_status = "alive" - if agent.status != next_status: - agent.status = next_status - await _apply_liveness(session, agent, next_status, now) + # PH-009: keyset-paginate agents by agent_id to bound per-tick memory + # and per-batch transaction work. Each batch is committed separately so + # large inventories do not produce one huge transaction. + # Race trade-off: a new agent inserted with agent_id < last_seen_id + # mid-tick is skipped this tick and processed next tick. That delay is + # bounded by detector_tick_sec (5s by default) and is acceptable — + # liveness detection has a stale_after_sec / dead_after_sec horizon + # measured in tens of seconds, so a one-tick lag is invisible. + last_seen_id: str | None = None + while True: + q = ( + select(Agent) + .where(Agent.accepted_key_hash.is_not(None)) + .order_by(Agent.agent_id) + .limit(DETECTOR_BATCH_SIZE) + ) + if last_seen_id is not None: + q = q.where(Agent.agent_id > last_seen_id) + agents = (await session.execute(q)).scalars().all() + if not agents: + break + for agent in agents: + if agent.last_seen_at <= dead_cut: + next_status = "dead" + elif agent.last_seen_at <= stale_cut: + next_status = "stale" + else: + next_status = "alive" + if agent.status != next_status: + agent.status = next_status + await _apply_liveness(session, agent, next_status, now) + last_seen_id = agents[-1].agent_id + await session.commit() + if len(agents) < DETECTOR_BATCH_SIZE: + break + await _prune_history(session) await session.commit() for status in ("alive", "stale", "dead"): r = await session.execute( - select(func.count()).select_from(Agent).where(Agent.status == status) + select(func.count()) + .select_from(Agent) + .where(Agent.accepted_key_hash.is_not(None), Agent.status == status) ) metrics.agents_gauge.labels(status=status).set(r.scalar_one()) r = await session.execute( diff --git a/server/monlet_server/services/ingestion.py b/server/monlet_server/services/ingestion.py index a62e456..2e859e7 100644 --- a/server/monlet_server/services/ingestion.py +++ b/server/monlet_server/services/ingestion.py @@ -2,13 +2,22 @@ from datetime import UTC, datetime from uuid import UUID, uuid4 from fastapi import HTTPException -from sqlalchemy import select, text +from sqlalchemy import func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from .. import metrics +from ..agent_auth import AgentCredential from ..logging_config import get_logger -from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox +from ..models import ( + Agent, + AgentBlacklist, + Check, + Event, + EventIngestDedup, + Incident, + NotificationOutbox, +) from ..redaction import redact from ..schemas import CheckResultEvent, HeartbeatRequest from ..settings import get_settings @@ -37,28 +46,143 @@ def _incident_key(agent_id: str, ev: CheckResultEvent) -> str: return key -async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None: +async def _blacklist_row(session: AsyncSession, cred: AgentCredential) -> AgentBlacklist | None: + res = await session.execute( + select(AgentBlacklist).where(AgentBlacklist.key_hash == cred.key_hash) + ) + return res.scalar_one_or_none() + + +async def reject_if_blocked( + session: AsyncSession, + cred: AgentCredential, + agent_id: str, + hostname: str, + observed: datetime | None = None, +) -> None: + row = await _blacklist_row(session, cred) + if row is None: + return + raise HTTPException(status_code=403, detail="agent blocked") + + +async def _ensure_pending_capacity(session: AsyncSession) -> None: + max_pending = get_settings().max_pending_agents + if max_pending <= 0: + return + count = ( + await session.execute( + select(func.count()).select_from(Agent).where(Agent.pending_key_hash.is_not(None)) + ) + ).scalar_one() + if count >= max_pending: + raise HTTPException(status_code=429, detail="too many pending agents") + + +async def _reject_hostname_collision(session: AsyncSession, agent_id: str, hostname: str) -> None: + row = ( + await session.execute( + select(Agent).where(Agent.hostname == hostname, Agent.agent_id != agent_id) + ) + ).scalar_one_or_none() + if row is not None: + raise HTTPException(status_code=409, detail="hostname already belongs to another agent") + + +def _set_pending( + agent: Agent, + hb: HeartbeatRequest, + cred: AgentCredential, + observed: datetime, + features: dict, +) -> None: + agent.pending_key_hash = cred.key_hash + agent.pending_key_fingerprint = cred.key_fingerprint + agent.pending_hostname = hb.hostname + agent.pending_seen_at = observed + agent.pending_features = features + agent.pending_labels = hb.labels + + +async def upsert_agent_heartbeat( + session: AsyncSession, hb: HeartbeatRequest, cred: AgentCredential +) -> str: observed = to_utc(hb.observed_at) - stmt = pg_insert(Agent).values( - agent_id=hb.agent_id, - hostname=hb.hostname, - status="alive", - last_seen_at=observed, - features=hb.features.model_dump(), - labels=hb.labels, + await reject_if_blocked(session, cred, hb.agent_id, hb.hostname, observed) + + features = hb.features.model_dump() + res = await session.execute(select(Agent).where(Agent.accepted_key_hash == cred.key_hash)) + accepted = res.scalar_one_or_none() + if accepted is not None: + await _reject_hostname_collision(session, accepted.agent_id, hb.hostname) + accepted.hostname = hb.hostname + accepted.status = "alive" + accepted.last_seen_at = observed + accepted.features = features + accepted.labels = hb.labels + metrics.heartbeats_total.labels(result="ok").inc() + return "accepted" + + res = await session.execute(select(Agent).where(Agent.pending_key_hash == cred.key_hash)) + pending = res.scalar_one_or_none() + if pending is not None: + await _reject_hostname_collision(session, pending.agent_id, hb.hostname) + _set_pending(pending, hb, cred, observed, features) + if pending.accepted_key_hash is None: + pending.hostname = hb.hostname + pending.status = "alive" + pending.last_seen_at = observed + pending.features = features + pending.labels = hb.labels + metrics.heartbeats_total.labels(result="pending").inc() + return "pending" + + hostname_row = ( + await session.execute(select(Agent).where(Agent.hostname == hb.hostname)) + ).scalar_one_or_none() + if hostname_row is not None: + if hostname_row.pending_key_hash is not None: + raise HTTPException( + status_code=409, + detail="agent already has a different pending key", + ) + await _ensure_pending_capacity(session) + _set_pending(hostname_row, hb, cred, observed, features) + metrics.heartbeats_total.labels(result="pending").inc() + return "pending" + + agent_id_row = ( + await session.execute(select(Agent).where(Agent.agent_id == hb.agent_id)) + ).scalar_one_or_none() + if agent_id_row is not None: + raise HTTPException(status_code=409, detail="agent_id already exists") + + await _ensure_pending_capacity(session) + session.add( + Agent( + agent_id=hb.agent_id, + hostname=hb.hostname, + status="alive", + last_seen_at=observed, + features=features, + labels=hb.labels, + pending_key_hash=cred.key_hash, + pending_key_fingerprint=cred.key_fingerprint, + pending_hostname=hb.hostname, + pending_seen_at=observed, + pending_features=features, + pending_labels=hb.labels, + ) ) - stmt = stmt.on_conflict_do_update( - index_elements=[Agent.agent_id], - set_={ - "hostname": stmt.excluded.hostname, - "status": "alive", - "last_seen_at": stmt.excluded.last_seen_at, - "features": stmt.excluded.features, - "labels": stmt.excluded.labels, - }, - ) - await session.execute(stmt) - metrics.heartbeats_total.labels(result="ok").inc() + metrics.heartbeats_total.labels(result="pending").inc() + return "pending" + + +async def accepted_agent_for_key(session: AsyncSession, cred: AgentCredential) -> Agent | None: + if await _blacklist_row(session, cred) is not None: + raise HTTPException(status_code=403, detail="agent blocked") + res = await session.execute(select(Agent).where(Agent.accepted_key_hash == cred.key_hash)) + return res.scalar_one_or_none() async def _apply_incident( diff --git a/server/monlet_server/services/notifier_worker.py b/server/monlet_server/services/notifier_worker.py index ac3a5f1..9ba7273 100644 --- a/server/monlet_server/services/notifier_worker.py +++ b/server/monlet_server/services/notifier_worker.py @@ -25,6 +25,43 @@ def _backoff(attempts: int) -> timedelta: return timedelta(seconds=_BACKOFF_SCHEDULE_SEC[idx]) +def _apply_output_policy(payload: dict) -> dict: + """PH-020: enforce a single notifier output policy. + + Per-notifier code keeps its own format-specific shortening (e.g. Telegram + 300 chars), but a hard cap and include-flag apply consistently before any + delivery. Truncation uses a clear marker so receivers can see it happened. + """ + settings = get_settings() + out = dict(payload) + text_value = out.get("output") + if text_value is None: + return out + if not settings.notifier_include_output: + out["output"] = None + out["output_truncated"] = True + return out + max_bytes = max(settings.notifier_max_output_bytes, 0) + encoded = text_value.encode("utf-8") + if len(encoded) <= max_bytes: + return out + # Reserve room for the marker so the final string never exceeds max_bytes. + # If max_bytes is smaller than even the marker, drop the body entirely — + # this keeps the absolute cap a hard guarantee under any setting. + dropped = len(encoded) - max_bytes + marker = f"...[truncated {dropped} bytes]" + marker_bytes = len(marker.encode("utf-8")) + if marker_bytes >= max_bytes: + out["output"] = "" + out["output_truncated"] = True + return out + prefix_budget = max_bytes - marker_bytes + truncated = encoded[:prefix_budget].decode("utf-8", errors="ignore") + out["output"] = f"{truncated}{marker}" + out["output_truncated"] = True + return out + + async def _claim_batch(session, limit: int, lease_sec: int) -> list[dict]: res = await session.execute( text( @@ -107,8 +144,10 @@ async def _process_row( metrics.notifications_total.labels(notifier=row["notifier"], result="discarded").inc() return + # PH-020: apply central output policy before handing to any notifier. + delivery_payload = _apply_output_policy(row["payload"]) try: - result = await notifier.deliver(row["event_type"], row["payload"]) + result = await notifier.deliver(row["event_type"], delivery_payload) except Exception as exc: # defensive: notifier should not raise result_ok = False result_perm = False diff --git a/server/monlet_server/settings.py b/server/monlet_server/settings.py index 74760a2..5d7a393 100644 --- a/server/monlet_server/settings.py +++ b/server/monlet_server/settings.py @@ -13,9 +13,9 @@ class Settings(BaseSettings): body_limit_bytes: int = 1_048_576 output_max_bytes: int = 8192 max_batch_events: int = 200 - stale_after_sec: int = 90 - dead_after_sec: int = 300 - detector_tick_sec: int = 15 + stale_after_sec: int = 20 + dead_after_sec: int = 30 + detector_tick_sec: int = 5 host: str = "0.0.0.0" port: int = 8000 enable_detector: bool = True @@ -24,8 +24,18 @@ class Settings(BaseSettings): events_future_partitions: int = 3 event_dedup_retention_days: int = 30 event_dedup_prune_batch_size: int = 5000 + max_pending_agents: int = 10_000 + pending_agent_ttl_days: int = 7 + pending_agent_prune_batch_size: int = 1000 partition_maintenance_interval_sec: int = 3600 outbox_retention_max_rows: int = 50_000 + # PH-012: bounded prune per maintenance tick to avoid one huge DELETE. + outbox_prune_batch_size: int = 5000 + # PH-016: explicit DB pool sizing for production tuning. + db_pool_size: int = 10 + db_max_overflow: int = 10 + db_pool_timeout_sec: float = 30.0 + db_pool_recycle_sec: int = 1800 notifier_tick_sec: int = 5 notifier_batch_size: int = 20 notifier_max_attempts: int = 8 @@ -33,6 +43,11 @@ class Settings(BaseSettings): notifier_http_timeout_sec: float = 10.0 notification_time_zone: str = "UTC" + # PH-020: notifier output policy. include controls whether output text is + # sent at all; max_bytes truncates it before delivery (after redaction). + notifier_include_output: bool = True + notifier_max_output_bytes: int = 1024 + notifier_debug_enabled: bool = True notifier_telegram_enabled: bool = False notifier_telegram_token: str = "" @@ -43,12 +58,26 @@ class Settings(BaseSettings): notifier_alertmanager_enabled: bool = False notifier_alertmanager_url: str = "" + @staticmethod + def _validate_token_list(value: str, env_name: str) -> str: + if not value or value == "changeme": + raise ValueError(f"{env_name} must be set to a non-default value") + tokens = [t for t in value.replace(",", " ").split() if t] + if not tokens: + raise ValueError(f"{env_name} must contain at least one token") + if any(t == "changeme" for t in tokens): + raise ValueError(f"{env_name} must be set to a non-default value") + return value + @field_validator("auth_token") @classmethod def _validate_auth_token(cls, value: str) -> str: - if not value or value == "changeme": - raise ValueError("MONLET_AUTH_TOKEN must be set to a non-default value") - return value + return cls._validate_token_list(value, "MONLET_AUTH_TOKEN") + + @property + def auth_tokens(self) -> list[str]: + """Active Bearer tokens (>=1). Multiple tokens enable overlap rotation.""" + return [t for t in self.auth_token.replace(",", " ").split() if t] @field_validator("notification_time_zone") @classmethod diff --git a/server/tests/_helpers.py b/server/tests/_helpers.py index b23a1f5..056676f 100644 --- a/server/tests/_helpers.py +++ b/server/tests/_helpers.py @@ -49,7 +49,28 @@ def make_heartbeat(agent_id: str = "agent-1") -> dict: return { "agent_id": agent_id, "observed_at": now_iso(), - "hostname": "host-1", + "hostname": agent_id, "features": {"push": True, "metrics": True}, "labels": {}, } + + +def agent_token(agent_id: str = "agent-1") -> str: + safe = "".join(ch if ch.isalnum() else "0" for ch in agent_id) + return f"test-agent-key-{safe}-00000000000000000000000000000000" + + +def agent_headers(agent_id: str = "agent-1") -> dict[str, str]: + return {"Authorization": f"Bearer {agent_token(agent_id)}"} + + +async def register_agent( + client, ui_headers: dict[str, str], agent_id: str = "agent-1" +) -> dict[str, str]: + headers = agent_headers(agent_id) + r = await client.post("/api/v1/heartbeat", json=make_heartbeat(agent_id), headers=headers) + assert r.status_code == 202 + r = await client.post(f"/api/v1/agents/{agent_id}/accept", headers=ui_headers) + assert r.status_code == 200 + assert r.json()["count"] == 1 + return headers diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 6465c10..5742ab1 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -13,7 +13,7 @@ from testcontainers.postgres import PostgresContainer from alembic import command -os.environ.setdefault("MONLET_AUTH_TOKEN", "test-token") +os.environ.setdefault("MONLET_AUTH_TOKEN", "test-ui-token") os.environ.setdefault("MONLET_ENABLE_DETECTOR", "false") os.environ.setdefault("MONLET_ENABLE_NOTIFIER_WORKER", "false") @@ -97,7 +97,7 @@ async def _truncate_tables(engine) -> AsyncIterator[None]: async with engine.begin() as conn: await conn.execute( text( - "TRUNCATE TABLE notification_outbox, incidents, events, event_ingest_dedup, checks, agents RESTART IDENTITY CASCADE" + "TRUNCATE TABLE notification_outbox, incidents, events, event_ingest_dedup, checks, agent_blacklist, agents RESTART IDENTITY CASCADE" ) ) @@ -116,4 +116,11 @@ async def app_client(database_url: str) -> AsyncIterator[AsyncClient]: @pytest.fixture def auth_headers() -> dict[str, str]: - return {"Authorization": "Bearer test-token"} + from tests._helpers import agent_headers + + return agent_headers() + + +@pytest.fixture +def ui_auth_headers() -> dict[str, str]: + return {"Authorization": "Bearer test-ui-token"} diff --git a/server/tests/test_agent_admission.py b/server/tests/test_agent_admission.py new file mode 100644 index 0000000..63169d4 --- /dev/null +++ b/server/tests/test_agent_admission.py @@ -0,0 +1,354 @@ +import pytest +from sqlalchemy import func, select + +from monlet_server.models import Agent, AgentBlacklist, Check +from monlet_server.services.detector import _prune_history +from monlet_server.settings import reset_settings_cache + +from ._helpers import agent_headers, make_event, make_heartbeat, register_agent + + +@pytest.mark.asyncio +async def test_unknown_heartbeat_creates_pending_agent(app_client, auth_headers, session): + r = await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-pending"), headers=auth_headers + ) + assert r.status_code == 202 + agent = ( + await session.execute(select(Agent).where(Agent.agent_id == "agent-pending")) + ).scalar_one() + assert agent.pending_key_hash + assert agent.accepted_key_hash is None + + +@pytest.mark.asyncio +async def test_pending_events_are_dropped_until_accept(app_client, auth_headers, session): + await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) + ev = make_event(status="critical", exit_code=2) + r = await app_client.post( + "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers + ) + assert r.status_code == 202 + assert r.json() == {"accepted": 0, "deduplicated": 0} + n = (await session.execute(select(func.count()).select_from(Check))).scalar_one() + assert n == 0 + + +@pytest.mark.asyncio +async def test_accept_enables_events(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) + ev = make_event(status="critical", exit_code=2) + r = await app_client.post( + "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers + ) + assert r.json() == {"accepted": 1, "deduplicated": 0} + n = (await session.execute(select(func.count()).select_from(Check))).scalar_one() + assert n == 1 + + +@pytest.mark.asyncio +async def test_block_pending_key(app_client, auth_headers, ui_auth_headers, session): + await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-block"), headers=auth_headers + ) + r = await app_client.post("/api/v1/agents/agent-block/block", headers=ui_auth_headers) + assert r.status_code == 200 + assert r.json()["count"] == 1 + assert (await session.execute(select(func.count()).select_from(Agent))).scalar_one() == 0 + assert ( + await session.execute(select(func.count()).select_from(AgentBlacklist)) + ).scalar_one() == 1 + + r = await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-block"), headers=auth_headers + ) + assert r.status_code == 403 + assert r.json()["error"]["code"] == "agent_blocked" + + +@pytest.mark.asyncio +async def test_blacklist_list_delete_and_clear(app_client, auth_headers, ui_auth_headers): + await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-block-a"), headers=auth_headers + ) + await app_client.post("/api/v1/agents/agent-block-a/block", headers=ui_auth_headers) + + other_headers = agent_headers("agent-block-b") + await app_client.post( + "/api/v1/heartbeat", + json=make_heartbeat("agent-block-b"), + headers=other_headers, + ) + await app_client.post("/api/v1/agents/agent-block-b/block", headers=ui_auth_headers) + + r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers) + assert r.status_code == 200 + items = r.json()["items"] + assert len(items) == 2 + + blocked_id = items[0]["id"] + r = await app_client.delete( + f"/api/v1/agents/admission/blacklist/{blocked_id}", + headers=ui_auth_headers, + ) + assert r.json()["count"] == 1 + + r = await app_client.delete("/api/v1/agents/admission/blacklist", headers=ui_auth_headers) + assert r.json()["count"] == 1 + + +@pytest.mark.asyncio +async def test_remove_accepted_agent_reappears_pending(app_client, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers, "agent-remove") + r = await app_client.delete("/api/v1/agents/agent-remove", headers=ui_auth_headers) + assert r.status_code == 200 + + r = await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-remove"), headers=auth_headers + ) + assert r.status_code == 202 + r = await app_client.get("/api/v1/agents/agent-remove", headers=ui_auth_headers) + assert r.json()["admission_state"] == "pending" + + +@pytest.mark.asyncio +async def test_new_key_same_hostname_is_pending_replacement(app_client, ui_auth_headers, session): + old_headers = await register_agent(app_client, ui_auth_headers, "old-id") + hb = make_heartbeat("new-id") + hb["hostname"] = "old-id" + new_headers = agent_headers("new-id") + + r = await app_client.post("/api/v1/heartbeat", json=hb, headers=new_headers) + assert r.status_code == 202 + r = await app_client.get("/api/v1/agents/old-id", headers=ui_auth_headers) + assert r.json()["admission_state"] == "pending" + assert r.json()["rotation_pending"] is True + + ev = make_event(status="ok") + r = await app_client.post( + "/api/v1/events", json={"agent_id": "old-id", "events": [ev]}, headers=old_headers + ) + assert r.json()["accepted"] == 1 + + r = await app_client.post("/api/v1/agents/old-id/block", headers=ui_auth_headers) + assert r.json()["count"] == 1 + agent = (await session.execute(select(Agent).where(Agent.agent_id == "old-id"))).scalar_one() + assert agent.accepted_key_hash + assert agent.pending_key_hash is None + + +@pytest.mark.asyncio +async def test_accept_all_skips_rotations_without_explicit_flag( + app_client, ui_auth_headers, session +): + await register_agent(app_client, ui_auth_headers, "stable-host") + hb = make_heartbeat("replacement-id") + hb["hostname"] = "stable-host" + await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("replacement")) + await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("new-host"), headers=agent_headers("new-host") + ) + + r = await app_client.post("/api/v1/agents/admission/accept-all", headers=ui_auth_headers) + assert r.status_code == 200 + assert r.json()["count"] == 1 + + stable = ( + await session.execute(select(Agent).where(Agent.agent_id == "stable-host")) + ).scalar_one() + assert stable.pending_key_hash is not None + new_host = ( + await session.execute(select(Agent).where(Agent.agent_id == "new-host")) + ).scalar_one() + assert new_host.pending_key_hash is None + assert new_host.accepted_key_hash is not None + + r = await app_client.post( + "/api/v1/agents/admission/accept-all?include_rotation=true", + headers=ui_auth_headers, + ) + assert r.status_code == 200 + assert r.json()["count"] == 1 + await session.refresh(stable) + assert stable.pending_key_hash is None + + +@pytest.mark.asyncio +async def test_new_key_cannot_hijack_existing_pending_hostname(app_client, auth_headers): + await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("pending-host"), headers=auth_headers + ) + r = await app_client.post( + "/api/v1/heartbeat", + json=make_heartbeat("pending-host"), + headers=agent_headers("other-key"), + ) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_new_key_agent_id_collision_is_rejected(app_client, ui_auth_headers): + await register_agent(app_client, ui_auth_headers, "agent-a") + await register_agent(app_client, ui_auth_headers, "agent-b") + + hb = make_heartbeat("agent-a") + hb["hostname"] = "new-host" + r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("new-key")) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_accepted_key_cannot_take_another_hostname(app_client, ui_auth_headers): + headers = await register_agent(app_client, ui_auth_headers, "agent-a") + await register_agent(app_client, ui_auth_headers, "agent-b") + + hb = make_heartbeat("agent-a") + hb["hostname"] = "agent-b" + r = await app_client.post("/api/v1/heartbeat", json=hb, headers=headers) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_hostname_replacement_still_wins_over_body_agent_id(app_client, ui_auth_headers): + await register_agent(app_client, ui_auth_headers, "agent-a") + await register_agent(app_client, ui_auth_headers, "agent-b") + + hb = make_heartbeat("agent-a") + hb["hostname"] = "agent-b" + r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("new-key")) + assert r.status_code == 202 + + assert (await app_client.get("/api/v1/agents/agent-b", headers=ui_auth_headers)).json()[ + "admission_state" + ] == "pending" + assert (await app_client.get("/api/v1/agents/agent-a", headers=ui_auth_headers)).json()[ + "admission_state" + ] == "accepted" + + +@pytest.mark.asyncio +async def test_remove_pending_replacement_keeps_accepted_agent( + app_client, ui_auth_headers, session +): + old_headers = await register_agent(app_client, ui_auth_headers, "stable-host") + hb = make_heartbeat("replacement-id") + hb["hostname"] = "stable-host" + r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("replacement")) + assert r.status_code == 202 + + r = await app_client.delete("/api/v1/agents/stable-host", headers=ui_auth_headers) + assert r.status_code == 200 + + agent = ( + await session.execute(select(Agent).where(Agent.agent_id == "stable-host")) + ).scalar_one() + assert agent.accepted_key_hash + assert agent.pending_key_hash is None + + ev = make_event(status="ok") + r = await app_client.post( + "/api/v1/events", json={"agent_id": "stable-host", "events": [ev]}, headers=old_headers + ) + assert r.json()["accepted"] == 1 + + +@pytest.mark.asyncio +async def test_pending_agent_cap_rejects_new_pending(app_client, auth_headers, monkeypatch): + monkeypatch.setenv("MONLET_MAX_PENDING_AGENTS", "1") + reset_settings_cache() + r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat("one"), headers=auth_headers) + assert r.status_code == 202 + r = await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("two"), headers=agent_headers("two") + ) + assert r.status_code == 429 + assert r.json()["error"]["code"] == "rate_limited" + monkeypatch.delenv("MONLET_MAX_PENDING_AGENTS", raising=False) + reset_settings_cache() + + +@pytest.mark.asyncio +async def test_pending_prune_deletes_pending_only_and_clears_replacement( + app_client, ui_auth_headers, session, monkeypatch +): + from datetime import UTC, datetime, timedelta + + await app_client.post( + "/api/v1/heartbeat", json=make_heartbeat("pending-only"), headers=agent_headers("p1") + ) + await register_agent(app_client, ui_auth_headers, "accepted") + hb = make_heartbeat("replacement") + hb["hostname"] = "accepted" + await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("p2")) + + old = datetime.now(UTC) - timedelta(days=2) + rows = ( + (await session.execute(select(Agent).where(Agent.pending_key_hash.is_not(None)))) + .scalars() + .all() + ) + for row in rows: + row.pending_seen_at = old + await session.commit() + + monkeypatch.setenv("MONLET_PENDING_AGENT_TTL_DAYS", "1") + reset_settings_cache() + await _prune_history(session) + await session.commit() + + assert ( + await session.execute(select(Agent).where(Agent.agent_id == "pending-only")) + ).scalar_one_or_none() is None + accepted = ( + await session.execute(select(Agent).where(Agent.agent_id == "accepted")) + ).scalar_one() + assert accepted.accepted_key_hash + assert accepted.pending_key_hash is None + monkeypatch.delenv("MONLET_PENDING_AGENT_TTL_DAYS", raising=False) + reset_settings_cache() + + +@pytest.mark.asyncio +async def test_blacklist_delete_uses_unique_row_id(app_client, ui_auth_headers, session): + from datetime import UTC, datetime + + now = datetime.now(UTC) + same_fingerprint = "0123456789abcdef" + first_hash = "a" * 64 + second_hash = "b" * 64 + session.add_all( + [ + AgentBlacklist( + key_hash=first_hash, + key_fingerprint=same_fingerprint, + agent_id="agent-a", + hostname="host-a", + last_seen_at=now, + ), + AgentBlacklist( + key_hash=second_hash, + key_fingerprint=same_fingerprint, + agent_id="agent-b", + hostname="host-b", + last_seen_at=now, + ), + ] + ) + await session.commit() + + r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers) + assert r.status_code == 200 + items = r.json()["items"] + assert [i["id"] for i in items] == [first_hash, second_hash] + + r = await app_client.delete( + f"/api/v1/agents/admission/blacklist/{first_hash}", + headers=ui_auth_headers, + ) + assert r.json()["count"] == 1 + + r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers) + assert [i["id"] for i in r.json()["items"]] == [second_hash] diff --git a/server/tests/test_auth.py b/server/tests/test_auth.py index d2aba3f..29f1342 100644 --- a/server/tests/test_auth.py +++ b/server/tests/test_auth.py @@ -30,6 +30,69 @@ async def test_wrong_token_401(app_client): @pytest.mark.asyncio -async def test_valid_token_200(app_client, auth_headers): - r = await app_client.get("/api/v1/agents", headers=auth_headers) +async def test_valid_token_200(app_client, ui_auth_headers): + r = await app_client.get("/api/v1/agents", headers=ui_auth_headers) assert r.status_code == 200 + + +@pytest.mark.asyncio +async def test_ui_token_is_separate_from_agent_self_key(monkeypatch, database_url): + from httpx import ASGITransport, AsyncClient + + from monlet_server import db as db_module + from monlet_server.main import create_app + from monlet_server.settings import reset_settings_cache + from tests._helpers import agent_headers, make_heartbeat + + monkeypatch.setenv("MONLET_AUTH_TOKEN", "ui-token") + reset_settings_cache() + await db_module.dispose_engine() + app = create_app() + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + ui = {"Authorization": "Bearer ui-token"} + agent_a = agent_headers("agent-a") + agent_b = agent_headers("agent-b") + assert (await client.get("/api/v1/agents", headers=ui)).status_code == 200 + assert (await client.get("/api/v1/agents", headers=agent_a)).status_code == 401 + + r = await client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-a"), headers=agent_a + ) + assert r.status_code == 202 + r = await client.post( + "/api/v1/heartbeat", json=make_heartbeat("agent-b"), headers=agent_b + ) + assert r.status_code == 202 + r = await client.post("/api/v1/heartbeat", json=make_heartbeat("agent-c"), headers=ui) + assert r.status_code == 401 + await db_module.dispose_engine() + monkeypatch.setenv("MONLET_AUTH_TOKEN", "test-ui-token") + reset_settings_cache() + + +@pytest.mark.asyncio +async def test_multi_token_overlap(monkeypatch, database_url): + """Two active tokens both authenticate; rotation can overlap without 401s.""" + from httpx import ASGITransport, AsyncClient + + from monlet_server import db as db_module + from monlet_server.main import create_app + from monlet_server.settings import reset_settings_cache + + monkeypatch.setenv("MONLET_AUTH_TOKEN", "old-token new-token") + reset_settings_cache() + await db_module.dispose_engine() + app = create_app() + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + for tok in ("old-token", "new-token"): + r = await client.get("/api/v1/agents", headers={"Authorization": f"Bearer {tok}"}) + assert r.status_code == 200, tok + r = await client.get("/api/v1/agents", headers={"Authorization": "Bearer revoked"}) + assert r.status_code == 401 + await db_module.dispose_engine() + monkeypatch.setenv("MONLET_AUTH_TOKEN", "test-ui-token") + reset_settings_cache() diff --git a/server/tests/test_detector.py b/server/tests/test_detector.py index 99fc769..fe30c7d 100644 --- a/server/tests/test_detector.py +++ b/server/tests/test_detector.py @@ -9,20 +9,14 @@ from monlet_server.models import Agent, Check, Event, Incident, NotificationOutb from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick from monlet_server.settings import reset_settings_cache -from ._helpers import make_event, make_heartbeat +from ._helpers import make_event, register_agent @pytest.mark.asyncio -async def test_detector_transitions(app_client, auth_headers, engine, session): - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers - ) - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers - ) - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers - ) +async def test_detector_transitions(app_client, ui_auth_headers, engine, session): + await register_agent(app_client, ui_auth_headers, "agent-alive") + await register_agent(app_client, ui_auth_headers, "agent-stale") + await register_agent(app_client, ui_auth_headers, "agent-dead") now = datetime.now(UTC) await session.execute( @@ -82,10 +76,57 @@ async def test_detector_transitions(app_client, auth_headers, engine, session): @pytest.mark.asyncio -async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session): - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers +async def test_detector_noop_tick_does_not_rewrite_liveness( + app_client, ui_auth_headers, engine, session +): + """PH-009: when no agent's liveness status changes, the detector must not + rewrite the liveness Check rows. We verify by tagging last_observed_at to a + sentinel before the tick and checking the row was untouched.""" + await register_agent(app_client, ui_auth_headers, "agent-still") + sm = async_sessionmaker(engine, expire_on_commit=False) + # Force a transition first so a liveness Check row exists. + await session.execute( + update(Agent) + .where(Agent.agent_id == "agent-still") + .values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10)) ) + await session.commit() + await _tick(sm) + session.expire_all() + row_before = ( + await session.execute( + select(Check).where( + Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID + ) + ) + ).scalar_one() + sentinel = datetime(2030, 1, 1, tzinfo=UTC) + await session.execute( + update(Check) + .where(Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID) + .values(last_observed_at=sentinel) + ) + await session.commit() + + # Second tick with no liveness change: row must NOT be rewritten. + await _tick(sm) + session.expire_all() + row_after = ( + await session.execute( + select(Check).where( + Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID + ) + ) + ).scalar_one() + assert row_after.last_observed_at == sentinel + assert row_after.last_event_id == row_before.last_event_id + + +@pytest.mark.asyncio +async def test_detector_owns_liveness_check_and_events( + app_client, ui_auth_headers, engine, session +): + await register_agent(app_client, ui_auth_headers, "agent-flap") sm = async_sessionmaker(engine, expire_on_commit=False) # Tick 1: alive → Check exists with status=ok, one event. @@ -144,13 +185,11 @@ async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, @pytest.mark.asyncio -async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch): +async def test_detector_prunes_outbox(app_client, ui_auth_headers, engine, session, monkeypatch): monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2") reset_settings_cache() try: - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers - ) + auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune") critical = make_event(check_id="critical_prune", status="critical", exit_code=2) await app_client.post( "/api/v1/events", @@ -192,3 +231,73 @@ async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, assert terminal_outbox == 2 finally: reset_settings_cache() + + +@pytest.mark.asyncio +async def test_detector_prune_preserves_active_outbox( + app_client, ui_auth_headers, engine, session, monkeypatch +): + """PH-012: prune must never delete active rows (pending/sending/retry), + even when retention is exceeded. The keep-N-newest list applies only to + terminal states; active rows are filtered out of the eligible set.""" + monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "1") + reset_settings_cache() + try: + auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune-active") + critical = make_event(check_id="active_prune", status="critical", exit_code=2) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-prune-active", "events": [critical]}, + headers=auth_headers, + ) + inc = ( + await session.execute(select(Incident).where(Incident.check_id == "active_prune")) + ).scalar_one() + now = datetime.now(UTC) + # Mix of active and terminal states. + for state, count in (("sent", 5), ("pending", 3), ("retry", 2), ("sending", 1)): + for _ in range(count): + session.add( + NotificationOutbox( + id=uuid4(), + incident_id=inc.id, + notifier="debug", + event_type="firing", + state=state, + attempts=1, + next_attempt_at=None, + payload={}, + created_at=now, + updated_at=now, + ) + ) + await session.commit() + + sm = async_sessionmaker(engine, expire_on_commit=False) + await _tick(sm) + + # Active rows must be untouched. Ingestion may add extra pending rows + # for the debug notifier; we only assert ours are still present and + # that no active rows were deleted by the prune. + for state, injected in (("pending", 3), ("retry", 2), ("sending", 1)): + n = ( + await session.execute( + select(func.count()) + .select_from(NotificationOutbox) + .where(NotificationOutbox.state == state) + ) + ).scalar_one() + assert n >= injected, (state, n, injected) + # Terminal rows must be capped to retention=1 (or 2 after notifier worker + # produces extras from the critical event we ingested; but worker is + # disabled in tests so we just check terminal pruning). + sent = ( + await session.execute( + select(func.count()) + .select_from(NotificationOutbox) + .where(NotificationOutbox.state == "sent") + ) + ).scalar_one() + assert sent <= 2 # 1 retained + at most 1 created by detector enqueue + finally: + reset_settings_cache() diff --git a/server/tests/test_events_ingestion.py b/server/tests/test_events_ingestion.py index 0558f97..4f4bbf7 100644 --- a/server/tests/test_events_ingestion.py +++ b/server/tests/test_events_ingestion.py @@ -1,14 +1,14 @@ import pytest from sqlalchemy import func, select -from monlet_server.models import Check, Event +from monlet_server.models import Agent, Check, Event -from ._helpers import make_event, make_heartbeat +from ._helpers import make_event, register_agent @pytest.mark.asyncio -async def test_event_accepted_and_state(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_event_accepted_and_state(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="ok", output="disk ok") r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers @@ -23,8 +23,20 @@ async def test_event_accepted_and_state(app_client, auth_headers, session): @pytest.mark.asyncio -async def test_duplicate_event_deduplicated(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_event_before_heartbeat_is_dropped(app_client, auth_headers, session): + ev = make_event(status="warning", exit_code=1) + r = await app_client.post( + "/api/v1/events", json={"agent_id": "agent-replay", "events": [ev]}, headers=auth_headers + ) + assert r.status_code == 202 + assert r.json() == {"accepted": 0, "deduplicated": 0} + n = (await session.execute(select(func.count()).select_from(Agent))).scalar_one() + assert n == 0 + + +@pytest.mark.asyncio +async def test_duplicate_event_deduplicated(app_client, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2) r1 = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers @@ -46,8 +58,8 @@ async def test_empty_batch_400(app_client, auth_headers): @pytest.mark.asyncio @pytest.mark.parametrize("observed_at", ["2026-05-27T09:00:00", "2026-05-27T09:00:00+04:00"]) -async def test_event_requires_utc_timestamp(app_client, auth_headers, observed_at): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_event_requires_utc_timestamp(app_client, ui_auth_headers, observed_at): + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(observed_at=observed_at) r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers @@ -66,8 +78,8 @@ async def test_oversize_batch_400(app_client, auth_headers): @pytest.mark.asyncio -async def test_repeated_same_status_not_stored(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_repeated_same_status_not_stored(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) evs = [make_event(status="ok") for _ in range(5)] r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers @@ -79,8 +91,8 @@ async def test_repeated_same_status_not_stored(app_client, auth_headers, session @pytest.mark.asyncio -async def test_status_transitions_stored(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_status_transitions_stored(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) seq = [ make_event(status="ok"), make_event(status="warning", exit_code=1), @@ -98,8 +110,8 @@ async def test_status_transitions_stored(app_client, auth_headers, session): @pytest.mark.asyncio -async def test_same_status_different_exit_code_stored(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_same_status_different_exit_code_stored(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) seq = [ make_event(status="warning", exit_code=1), make_event(status="warning", exit_code=3), @@ -113,10 +125,10 @@ async def test_same_status_different_exit_code_stored(app_client, auth_headers, @pytest.mark.asyncio -async def test_late_event_does_not_move_state(app_client, auth_headers, session): +async def test_late_event_does_not_move_state(app_client, ui_auth_headers, session): from datetime import UTC, datetime, timedelta - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) + auth_headers = await register_agent(app_client, ui_auth_headers) newer = datetime.now(UTC).replace(microsecond=0) older = (newer - timedelta(minutes=5)).isoformat() ev_now = make_event( diff --git a/server/tests/test_heartbeat.py b/server/tests/test_heartbeat.py index 3618366..26025f4 100644 --- a/server/tests/test_heartbeat.py +++ b/server/tests/test_heartbeat.py @@ -12,8 +12,10 @@ async def test_heartbeat_creates_agent(app_client, auth_headers, session): assert r.status_code == 202 res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1")) a = res.scalar_one() - assert a.hostname == "host-1" + assert a.hostname == "agent-1" assert a.status == "alive" + assert a.pending_key_hash + assert a.accepted_key_hash is None @pytest.mark.asyncio @@ -24,7 +26,7 @@ async def test_heartbeat_upsert(app_client, auth_headers, session): await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers) res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1")) a = res.scalar_one() - assert a.features == {"push": True, "metrics": False} + assert a.pending_features == {"push": True, "metrics": False} @pytest.mark.asyncio diff --git a/server/tests/test_incident_key.py b/server/tests/test_incident_key.py index a580db3..3ec1061 100644 --- a/server/tests/test_incident_key.py +++ b/server/tests/test_incident_key.py @@ -1,30 +1,30 @@ import pytest -from ._helpers import make_event, make_heartbeat +from ._helpers import make_event, register_agent @pytest.mark.asyncio -async def test_incident_key_ownership_rejected(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-1"), headers=auth_headers) - await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-2"), headers=auth_headers) +async def test_incident_key_ownership_rejected(app_client, ui_auth_headers): + agent_1_headers = await register_agent(app_client, ui_auth_headers, "agent-1") + agent_2_headers = await register_agent(app_client, ui_auth_headers, "agent-2") ev1 = make_event(status="critical", exit_code=2, incident_key="shared-key") r1 = await app_client.post( - "/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=auth_headers + "/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=agent_1_headers ) assert r1.status_code == 202 ev2 = make_event(status="critical", exit_code=2, incident_key="shared-key") r2 = await app_client.post( - "/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=auth_headers + "/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=agent_2_headers ) assert r2.status_code == 400 assert r2.json()["error"]["code"] == "validation" @pytest.mark.asyncio -async def test_incident_key_too_long(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_incident_key_too_long(app_client, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2, incident_key="x" * 257) r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers @@ -33,20 +33,20 @@ async def test_incident_key_too_long(app_client, auth_headers): @pytest.mark.asyncio -async def test_events_query_cursor_pagination(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_events_query_cursor_pagination(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) evs = [make_event(check_id=f"chk-{i}") for i in range(5)] await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers ) - r = await app_client.get("/api/v1/events/query?limit=2", headers=auth_headers) + r = await app_client.get("/api/v1/events/query?limit=2", headers=ui_auth_headers) assert r.status_code == 200 page1 = r.json() assert len(page1["items"]) == 2 assert page1["next_cursor"] r2 = await app_client.get( - f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=auth_headers + f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=ui_auth_headers ) page2 = r2.json() assert len(page2["items"]) == 2 diff --git a/server/tests/test_incidents.py b/server/tests/test_incidents.py index c11721c..15e6a6c 100644 --- a/server/tests/test_incidents.py +++ b/server/tests/test_incidents.py @@ -3,12 +3,12 @@ from sqlalchemy import select from monlet_server.models import Incident, NotificationOutbox -from ._helpers import make_event, make_heartbeat +from ._helpers import make_event, register_agent @pytest.mark.asyncio -async def test_open_escalate_resolve_reopen(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_open_escalate_resolve_reopen(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) a = "agent-1" # warning -> open warning @@ -52,8 +52,8 @@ async def test_open_escalate_resolve_reopen(app_client, auth_headers, session): @pytest.mark.asyncio -async def test_outbox_skipped_when_notifications_disabled(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_outbox_skipped_when_notifications_disabled(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2, notifications_enabled=False) await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers @@ -63,8 +63,8 @@ async def test_outbox_skipped_when_notifications_disabled(app_client, auth_heade @pytest.mark.asyncio -async def test_outbox_firing_and_resolved(app_client, auth_headers, session): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_outbox_firing_and_resolved(app_client, ui_auth_headers, session): + auth_headers = await register_agent(app_client, ui_auth_headers) a = "agent-1" ev_open = make_event(status="critical", exit_code=2) await app_client.post( @@ -77,3 +77,42 @@ async def test_outbox_firing_and_resolved(app_client, auth_headers, session): rows = (await session.execute(select(NotificationOutbox))).scalars().all() types = sorted(r.event_type for r in rows) assert types == ["firing", "resolved"] + + +def test_status_rank_map_covers_all_pairs(): + """PH-review: the SQL-side rank is computed by a STORED generated column + (`incidents.status_rank`). The Python `_STATUS_RANK` map mirrors that CASE + for cursor encoding only. This test pins the map so the two cannot drift — + if a new (state, severity) pair is added, both must update.""" + from monlet_server.api.incidents import _STATUS_RANK + + assert _STATUS_RANK == { + ("open", "critical"): 5, + ("open", "warning"): 4, + ("open", "unknown"): 3, + ("resolved", "critical"): 2, + ("resolved", "warning"): 1, + ("resolved", "unknown"): 0, + } + + +@pytest.mark.asyncio +async def test_status_rank_generated_column_matches_python_map( + app_client, ui_auth_headers, session +): + """End-to-end check: insert one event per (state, severity) pair and + confirm the DB-side generated column equals the Python map value.""" + from monlet_server.api.incidents import _STATUS_RANK + + # Open critical incident comes from a single critical event. + auth_headers = await register_agent(app_client, ui_auth_headers) + crit = make_event(check_id="crit", status="critical", exit_code=2) + warn = make_event(check_id="warn", status="warning", exit_code=1) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-1", "events": [crit, warn]}, + headers=auth_headers, + ) + rows = (await session.execute(select(Incident))).scalars().all() + for r in rows: + assert r.status_rank == _STATUS_RANK[(r.state, r.severity)] diff --git a/server/tests/test_integration_agent.py b/server/tests/test_integration_agent.py index 7301a42..d98ce99 100644 --- a/server/tests/test_integration_agent.py +++ b/server/tests/test_integration_agent.py @@ -10,7 +10,7 @@ EXAMPLES = os.path.join(os.path.dirname(__file__), "..", "..", "examples") @pytest.mark.asyncio -async def test_examples_payload(app_client, auth_headers, session): +async def test_examples_payload(app_client, auth_headers, ui_auth_headers, session): with open(os.path.join(EXAMPLES, "heartbeat.json")) as f: hb = json.load(f) with open(os.path.join(EXAMPLES, "event-batch.json")) as f: @@ -18,6 +18,8 @@ async def test_examples_payload(app_client, auth_headers, session): r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers) assert r.status_code == 202 + r = await app_client.post(f"/api/v1/agents/{hb['agent_id']}/accept", headers=ui_auth_headers) + assert r.status_code == 200 r = await app_client.post("/api/v1/events", json=batch, headers=auth_headers) assert r.status_code == 202 diff --git a/server/tests/test_notifiers.py b/server/tests/test_notifiers.py index fe7becd..5917dee 100644 --- a/server/tests/test_notifiers.py +++ b/server/tests/test_notifiers.py @@ -200,7 +200,7 @@ async def _seed_incident_and_outbox( event_type="firing", state="pending", attempts=0, - next_attempt_at=datetime.now(UTC), + next_attempt_at=datetime.now(UTC) - timedelta(seconds=1), payload=_payload(), ) session.add(row) @@ -386,7 +386,7 @@ async def test_telegram_redacts_secret_in_incident_key(): @pytest.mark.asyncio async def test_fanout_creates_row_per_enabled_notifier( - app_client, auth_headers, session, monkeypatch + app_client, ui_auth_headers, session, monkeypatch ): # Enable telegram (token+chat_id satisfied) and webhook in addition to debug. os.environ["MONLET_NOTIFIER_TELEGRAM_ENABLED"] = "true" @@ -398,9 +398,9 @@ async def test_fanout_creates_row_per_enabled_notifier( reset_settings_cache() try: - from tests._helpers import make_event, make_heartbeat + from tests._helpers import make_event, register_agent - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2) r = await app_client.post( "/api/v1/events", diff --git a/server/tests/test_pagination.py b/server/tests/test_pagination.py index ce50e30..4f4a501 100644 --- a/server/tests/test_pagination.py +++ b/server/tests/test_pagination.py @@ -1,21 +1,19 @@ import pytest -from ._helpers import make_event, make_heartbeat +from ._helpers import agent_headers, make_event, make_heartbeat, register_agent @pytest.mark.asyncio -async def test_agents_cursor(app_client, auth_headers): +async def test_agents_cursor(app_client, ui_auth_headers): for i in range(5): - await app_client.post( - "/api/v1/heartbeat", json=make_heartbeat(f"a-{i}"), headers=auth_headers - ) - r1 = await app_client.get("/api/v1/agents?limit=2", headers=auth_headers) + await register_agent(app_client, ui_auth_headers, f"a-{i}") + r1 = await app_client.get("/api/v1/agents?limit=2", headers=ui_auth_headers) p1 = r1.json() assert len(p1["items"]) == 2 assert p1["next_cursor"] r2 = await app_client.get( - f"/api/v1/agents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers + f"/api/v1/agents?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers ) p2 = r2.json() assert len(p2["items"]) == 2 @@ -25,76 +23,204 @@ async def test_agents_cursor(app_client, auth_headers): @pytest.mark.asyncio -async def test_checks_cursor(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_agents_cursor_sorts_pending_before_accepted(app_client, ui_auth_headers): + for i in range(3): + await register_agent(app_client, ui_auth_headers, f"accepted-{i}") + for i in range(2): + await app_client.post( + "/api/v1/heartbeat", + json=make_heartbeat(f"pending-{i}"), + headers=agent_headers(f"pending-{i}"), + ) + + r1 = await app_client.get("/api/v1/agents?limit=3", headers=ui_auth_headers) + p1 = r1.json() + assert [i["admission_state"] for i in p1["items"][:2]] == ["pending", "pending"] + assert p1["next_cursor"] + + r2 = await app_client.get( + f"/api/v1/agents?limit=3&cursor={p1['next_cursor']}", headers=ui_auth_headers + ) + assert {i["agent_id"] for i in r2.json()["items"]}.isdisjoint( + {i["agent_id"] for i in p1["items"]} + ) + + +@pytest.mark.asyncio +async def test_checks_cursor(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) for i in range(4): ev = make_event(check_id=f"chk-{i}") await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers ) - r1 = await app_client.get("/api/v1/checks?limit=2", headers=auth_headers) + r1 = await app_client.get("/api/v1/checks?limit=2", headers=ui_auth_headers) p1 = r1.json() assert len(p1["items"]) == 2 assert p1["next_cursor"] r2 = await app_client.get( - f"/api/v1/checks?limit=2&cursor={p1['next_cursor']}", headers=auth_headers + f"/api/v1/checks?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers ) p2 = r2.json() assert len(p2["items"]) == 2 @pytest.mark.asyncio -async def test_checks_rejects_invalid_agent_id_query(app_client, auth_headers): - r = await app_client.get("/api/v1/checks?agent_id=bad%20id", headers=auth_headers) +async def test_checks_rejects_invalid_agent_id_query(app_client, ui_auth_headers): + r = await app_client.get("/api/v1/checks?agent_id=bad%20id", headers=ui_auth_headers) assert r.status_code == 400 assert r.json()["error"]["code"] == "validation" @pytest.mark.asyncio -async def test_events_query_rejects_invalid_id_queries(app_client, auth_headers): - r = await app_client.get("/api/v1/events/query?agent_id=bad%20id", headers=auth_headers) +async def test_events_query_rejects_invalid_id_queries(app_client, ui_auth_headers): + r = await app_client.get("/api/v1/events/query?agent_id=bad%20id", headers=ui_auth_headers) assert r.status_code == 400 assert r.json()["error"]["code"] == "validation" - r = await app_client.get("/api/v1/events/query?check_id=bad%20id", headers=auth_headers) + r = await app_client.get("/api/v1/events/query?check_id=bad%20id", headers=ui_auth_headers) assert r.status_code == 400 assert r.json()["error"]["code"] == "validation" @pytest.mark.asyncio -async def test_incidents_cursor(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_incidents_cursor(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) for i in range(3): ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2) await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers ) - r1 = await app_client.get("/api/v1/incidents?limit=2", headers=auth_headers) + r1 = await app_client.get("/api/v1/incidents?limit=2", headers=ui_auth_headers) p1 = r1.json() assert len(p1["items"]) == 2 assert p1["next_cursor"] r2 = await app_client.get( - f"/api/v1/incidents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers + f"/api/v1/incidents?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers ) p2 = r2.json() assert len(p2["items"]) == 1 @pytest.mark.asyncio -async def test_outbox_cursor(app_client, auth_headers): - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) +async def test_incidents_default_sort_is_status(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) + warning = make_event(check_id="warn-check", status="warning", exit_code=1) + critical = make_event(check_id="crit-check", status="critical", exit_code=2) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-1", "events": [warning, critical]}, + headers=auth_headers, + ) + + r = await app_client.get("/api/v1/incidents?limit=10", headers=ui_auth_headers) + assert r.status_code == 200 + items = r.json()["items"] + assert [i["severity"] for i in items[:2]] == ["critical", "warning"] + + +@pytest.mark.asyncio +async def test_outbox_cursor(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) for i in range(3): ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2) await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers ) - r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=auth_headers) + r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=ui_auth_headers) p1 = r1.json() assert len(p1["items"]) == 2 assert p1["next_cursor"] @pytest.mark.asyncio -async def test_invalid_cursor_400(app_client, auth_headers): - r = await app_client.get("/api/v1/agents?cursor=$$$bad", headers=auth_headers) +async def test_invalid_cursor_400(app_client, ui_auth_headers): + r = await app_client.get("/api/v1/agents?cursor=$$$bad", headers=ui_auth_headers) assert r.status_code == 400 + + +@pytest.mark.asyncio +async def test_events_query_invalid_cursor_uuid_400(app_client, ui_auth_headers): + """PH-010: cursor encodes (received_at, event_id) and event_id must parse + as a UUID before reaching the SQL layer.""" + import base64 + + bad = base64.urlsafe_b64encode(b"2026-01-01T00:00:00+00:00|not-a-uuid").decode().rstrip("=") + r = await app_client.get(f"/api/v1/events/query?cursor={bad}", headers=ui_auth_headers) + assert r.status_code == 400 + + +@pytest.mark.asyncio +async def test_incidents_bidirectional_cursor_round_trip(app_client, auth_headers, ui_auth_headers): + """PH-review: page1 → next → prev returns page1 with the same items and + order. prev_cursor must be null on the first page.""" + auth_headers = await register_agent(app_client, ui_auth_headers) + for i in range(6): + ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-1", "events": [ev]}, + headers=auth_headers, + ) + + r1 = await app_client.get("/api/v1/incidents?limit=3", headers=ui_auth_headers) + p1 = r1.json() + assert len(p1["items"]) == 3 + assert p1["next_cursor"] + assert p1.get("prev_cursor") is None # first page + + r2 = await app_client.get( + f"/api/v1/incidents?limit=3&cursor={p1['next_cursor']}", headers=ui_auth_headers + ) + p2 = r2.json() + assert len(p2["items"]) == 3 + assert p2["prev_cursor"] + ids1 = [i["id"] for i in p1["items"]] + ids2 = [i["id"] for i in p2["items"]] + assert set(ids1).isdisjoint(set(ids2)) + + r_back = await app_client.get( + f"/api/v1/incidents?limit=3&cursor={p2['prev_cursor']}&back=true", + headers=ui_auth_headers, + ) + p_back = r_back.json() + assert [i["id"] for i in p_back["items"]] == ids1 + # back to first page => prev_cursor must be null again + assert p_back.get("prev_cursor") is None + assert p_back["next_cursor"] + + +@pytest.mark.asyncio +async def test_outbox_bidirectional_cursor_round_trip(app_client, auth_headers, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers) + for i in range(5): + ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-1", "events": [ev]}, + headers=auth_headers, + ) + + r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=ui_auth_headers) + p1 = r1.json() + assert len(p1["items"]) == 2 + assert p1["next_cursor"] + assert p1.get("prev_cursor") is None + + r2 = await app_client.get( + f"/api/v1/notifiers/outbox?limit=2&cursor={p1['next_cursor']}", + headers=ui_auth_headers, + ) + p2 = r2.json() + assert p2["prev_cursor"] + ids1 = [i["id"] for i in p1["items"]] + ids2 = [i["id"] for i in p2["items"]] + assert set(ids1).isdisjoint(set(ids2)) + + r_back = await app_client.get( + f"/api/v1/notifiers/outbox?limit=2&cursor={p2['prev_cursor']}&back=true", + headers=ui_auth_headers, + ) + p_back = r_back.json() + assert [i["id"] for i in p_back["items"]] == ids1 + assert p_back.get("prev_cursor") is None diff --git a/server/tests/test_partitioning.py b/server/tests/test_partitioning.py index c2dde02..daffa3d 100644 --- a/server/tests/test_partitioning.py +++ b/server/tests/test_partitioning.py @@ -67,14 +67,14 @@ async def test_drop_old_partitions_uses_retention(engine, monkeypatch): @pytest.mark.asyncio -async def test_dedup_prevents_replay_via_ingest(app_client, auth_headers, session): +async def test_dedup_prevents_replay_via_ingest(app_client, ui_auth_headers, session): from sqlalchemy import func, select from monlet_server.models import Event - from ._helpers import make_event, make_heartbeat + from ._helpers import make_event, register_agent - await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) + auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2) r1 = await app_client.post( diff --git a/server/tests/test_system_summary.py b/server/tests/test_system_summary.py new file mode 100644 index 0000000..d5ecddc --- /dev/null +++ b/server/tests/test_system_summary.py @@ -0,0 +1,35 @@ +import pytest + +from ._helpers import make_event, register_agent + + +@pytest.mark.asyncio +async def test_system_summary_aggregate(app_client, ui_auth_headers): + auth_headers = await register_agent(app_client, ui_auth_headers, "agent-1") + await register_agent(app_client, ui_auth_headers, "agent-2") + ev = make_event(check_id="c1", status="critical", exit_code=2) + await app_client.post( + "/api/v1/events", + json={"agent_id": "agent-1", "events": [ev]}, + headers=auth_headers, + ) + + r = await app_client.get("/api/v1/system-summary", headers=ui_auth_headers) + assert r.status_code == 200 + body = r.json() + assert body["agents_online"] == 2 + assert body["agents_offline"] == 0 + assert body["checks_critical"] >= 1 + assert "generated_at" in body + + +@pytest.mark.asyncio +async def test_system_summary_requires_auth(app_client): + r = await app_client.get("/api/v1/system-summary") + assert r.status_code == 401 + + +@pytest.mark.asyncio +async def test_outbox_invalid_state_400(app_client, ui_auth_headers): + r = await app_client.get("/api/v1/notifiers/outbox?state=bogus", headers=ui_auth_headers) + assert r.status_code == 400 diff --git a/web/Dockerfile b/web/Dockerfile index e9df2aa..cec5a81 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -20,5 +20,9 @@ COPY --from=builder /app/.next ./.next COPY --from=builder /app/public ./public COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/package.json ./package.json +# PH-015: run as the prebuilt non-root `node` user shipped by the base image. +RUN chown -R node:node /app +USER node EXPOSE 3000 +STOPSIGNAL SIGTERM CMD ["npm", "run", "start"] diff --git a/web/README.md b/web/README.md index ad50783..55a152b 100644 --- a/web/README.md +++ b/web/README.md @@ -1,6 +1,6 @@ # Monlet Web -Read-only dashboard for Monlet. +Dashboard for Monlet. Monitoring pages are read-only; agent admission and blacklist controls can mutate server state. ## Stack @@ -36,13 +36,17 @@ Do not install Node packages globally. Project dependencies live in `web/node_mo | `MONLET_API_BASE_URL` | `http://127.0.0.1:8000` | Server base URL (server-side fetch) | | `MONLET_API_TOKEN` | _(none)_ | Bearer token sent to server | | `MONLET_WEB_TIME_ZONE` | `UTC` | IANA timezone used to render timestamps | +| `MONLET_WEB_AUTH_USERNAME` | _(none)_ | Optional Basic Auth username for the web UI | +| `MONLET_WEB_AUTH_PASSWORD` | _(none)_ | Optional Basic Auth password for the web UI | +| `MONLET_WEB_TRUST_PROXY_AUTH` | `false` | Trust `X-Forwarded-User` from an upstream auth proxy | -The token never reaches the browser — all server calls happen in Server Components / route handlers. +The API token never reaches the browser — all server calls happen in Server Components / route handlers. Agent admission actions are blocked unless Basic Auth is configured or trusted proxy auth is enabled. ## Pages -- `/` — overview +- `/` — redirects to `/agents` - `/agents` — agent list +- `/agents/blacklist` — blocked agent keys - `/agents/[id]` — agent detail - `/checks` — current check states - `/incidents` — incident list diff --git a/web/src/app/agents/[agent_id]/page.tsx b/web/src/app/agents/[agent_id]/page.tsx index 18b0b5d..1a1c942 100644 --- a/web/src/app/agents/[agent_id]/page.tsx +++ b/web/src/app/agents/[agent_id]/page.tsx @@ -2,7 +2,7 @@ import { AgentDetailBrowser } from "@/components/AgentDetailBrowser"; import { ErrorPanel } from "@/components/DataPanel"; import { ApiError, api } from "@/lib/api"; import { loadChecks } from "@/lib/loaders"; -import { normalizePageSize } from "@/lib/pagination"; +import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -30,6 +30,8 @@ export default async function AgentDetailPage({ tab?: string; events_limit?: string; cursor?: string; + back?: string; + page?: string; }>; }) { const { agent_id } = await params; @@ -38,10 +40,12 @@ export default async function AgentDetailPage({ const tab = normalizeTab(sp.tab); const eventsLimit = normalizePageSize(sp.events_limit, 50); const eventsCursor = sp.cursor; + const eventsPage = normalizePageNumber(sp.page); + const eventsBack = normalizeBack(sp.back); let agent; let checks; - let events: EventsPage = { items: [], next_cursor: null }; + let events: EventsPage = { items: [], next_cursor: null, prev_cursor: null }; try { [agent, checks] = await Promise.all([api.getAgent(agent_id), loadChecks(agent_id)]); if (tab === "events") { @@ -49,6 +53,7 @@ export default async function AgentDetailPage({ agent_id, limit: eventsLimit, cursor: eventsCursor, + back: eventsBack, }); } } catch (e) { @@ -59,13 +64,15 @@ export default async function AgentDetailPage({ } return ( ); } diff --git a/web/src/app/agents/blacklist/page.tsx b/web/src/app/agents/blacklist/page.tsx new file mode 100644 index 0000000..df0a367 --- /dev/null +++ b/web/src/app/agents/blacklist/page.tsx @@ -0,0 +1,15 @@ +import { AgentBlacklistBrowser } from "@/components/AgentBlacklistBrowser"; +import { ErrorPanel } from "@/components/DataPanel"; +import { loadAgentBlacklist } from "@/lib/loaders"; + +export const dynamic = "force-dynamic"; + +export default async function AgentBlacklistPage() { + let rows; + try { + rows = await loadAgentBlacklist(); + } catch (e) { + return ; + } + return ; +} diff --git a/web/src/app/api/agents/action/route.ts b/web/src/app/api/agents/action/route.ts new file mode 100644 index 0000000..ac76b94 --- /dev/null +++ b/web/src/app/api/agents/action/route.ts @@ -0,0 +1,56 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { api } from "@/lib/api"; +import { jsonError } from "@/lib/poll-response"; + +export const dynamic = "force-dynamic"; + +type Action = + | "accept" + | "block" + | "remove" + | "accept_all" + | "block_all" + | "remove_all" + | "blacklist_delete" + | "blacklist_clear"; + +export async function POST(request: NextRequest) { + try { + const body = (await request.json()) as { + action?: Action; + agent_id?: string; + blacklist_id?: string; + include_rotation?: boolean; + }; + switch (body.action) { + case "accept": + return NextResponse.json(await api.acceptAgent(required(body.agent_id, "agent_id"))); + case "block": + return NextResponse.json(await api.blockAgent(required(body.agent_id, "agent_id"))); + case "remove": + return NextResponse.json(await api.removeAgent(required(body.agent_id, "agent_id"))); + case "accept_all": + return NextResponse.json(await api.acceptAllPendingAgents(body.include_rotation === true)); + case "block_all": + return NextResponse.json(await api.blockAllPendingAgents()); + case "remove_all": + return NextResponse.json(await api.removeAllPendingAgents()); + case "blacklist_delete": + return NextResponse.json( + await api.deleteAgentBlacklistItem(required(body.blacklist_id, "blacklist_id")), + ); + case "blacklist_clear": + return NextResponse.json(await api.clearAgentBlacklist()); + default: + return NextResponse.json({ error: "unknown action" }, { status: 400 }); + } + } catch (e) { + return jsonError(e); + } +} + +function required(value: string | undefined, name: string): string { + if (!value) throw new Error(`${name} is required`); + return value; +} diff --git a/web/src/app/api/health/route.ts b/web/src/app/api/health/route.ts new file mode 100644 index 0000000..8beebb1 --- /dev/null +++ b/web/src/app/api/health/route.ts @@ -0,0 +1,7 @@ +import { NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +export function GET() { + return NextResponse.json({ ok: true }); +} diff --git a/web/src/app/api/poll/events/route.ts b/web/src/app/api/poll/events/route.ts index 6bef416..60a8d1b 100644 --- a/web/src/app/api/poll/events/route.ts +++ b/web/src/app/api/poll/events/route.ts @@ -15,6 +15,7 @@ export async function GET(request: Request) { agentId: url.searchParams.get("agent_id") ?? undefined, checkId: url.searchParams.get("check_id") ?? undefined, cursor: url.searchParams.get("cursor") ?? undefined, + back: url.searchParams.get("back") === "1", limit, }); return NextResponse.json(data); diff --git a/web/src/app/api/poll/incidents/route.ts b/web/src/app/api/poll/incidents/route.ts index 86cbf9e..b713da4 100644 --- a/web/src/app/api/poll/incidents/route.ts +++ b/web/src/app/api/poll/incidents/route.ts @@ -1,18 +1,35 @@ import { NextResponse } from "next/server"; -import { loadIncidents } from "@/lib/loaders"; +import { loadIncidentsPage } from "@/lib/loaders"; import { jsonError } from "@/lib/poll-response"; export const dynamic = "force-dynamic"; +const SORTS = new Set(["status", "opened_at", "resolved_at"]); +const DIRS = new Set(["asc", "desc"]); + +// PH-001: poll a single page, not the entire incident history. Sort and +// direction are forwarded so the server orders consistently across pages. export async function GET(request: Request) { const url = new URL(request.url); const stateParam = url.searchParams.get("state"); const state = stateParam === "open" || stateParam === "resolved" ? stateParam : undefined; + const cursor = url.searchParams.get("cursor") ?? undefined; + const back = url.searchParams.get("back") === "1"; + const limitRaw = url.searchParams.get("limit"); + const parsed = limitRaw ? Number(limitRaw) : undefined; + const limit = parsed && Number.isFinite(parsed) ? Math.min(Math.max(parsed, 1), 500) : 100; + const sortRaw = url.searchParams.get("sort") ?? undefined; + const sort = + sortRaw && SORTS.has(sortRaw) + ? (sortRaw as "status" | "opened_at" | "resolved_at") + : undefined; + const dirRaw = url.searchParams.get("direction") ?? undefined; + const direction = dirRaw && DIRS.has(dirRaw) ? (dirRaw as "asc" | "desc") : undefined; try { - const items = await loadIncidents(state); - return NextResponse.json({ items }); + const page = await loadIncidentsPage({ state, cursor, back, limit, sort, direction }); + return NextResponse.json(page); } catch (e) { return jsonError(e); } diff --git a/web/src/app/api/poll/outbox/route.ts b/web/src/app/api/poll/outbox/route.ts index d61b90b..0dcebcb 100644 --- a/web/src/app/api/poll/outbox/route.ts +++ b/web/src/app/api/poll/outbox/route.ts @@ -1,14 +1,38 @@ import { NextResponse } from "next/server"; -import { loadOutboxAll } from "@/lib/loaders"; +import { loadOutboxPage } from "@/lib/loaders"; import { jsonError } from "@/lib/poll-response"; export const dynamic = "force-dynamic"; -export async function GET() { +const SORTS = new Set(["created_at", "updated_at"]); +const DIRS = new Set(["asc", "desc"]); + +// PH-002: poll a single outbox page; the table can grow under notifier failures. +export async function GET(request: Request) { + const url = new URL(request.url); + const cursor = url.searchParams.get("cursor") ?? undefined; + const back = url.searchParams.get("back") === "1"; + const limitRaw = url.searchParams.get("limit"); + const parsed = limitRaw ? Number(limitRaw) : undefined; + const limit = parsed && Number.isFinite(parsed) ? Math.min(Math.max(parsed, 1), 500) : 100; + const state = url.searchParams.get("state") ?? undefined; + const sortRaw = url.searchParams.get("sort") ?? undefined; + const sort = + sortRaw && SORTS.has(sortRaw) ? (sortRaw as "created_at" | "updated_at") : undefined; + const dirRaw = url.searchParams.get("direction") ?? undefined; + const direction = dirRaw && DIRS.has(dirRaw) ? (dirRaw as "asc" | "desc") : undefined; + try { - const items = await loadOutboxAll(); - return NextResponse.json({ items }); + const page = await loadOutboxPage({ + cursor, + back, + limit, + state: state as never, + sort, + direction, + }); + return NextResponse.json(page); } catch (e) { return jsonError(e); } diff --git a/web/src/app/api/poll/overview/route.ts b/web/src/app/api/poll/system-summary/route.ts similarity index 67% rename from web/src/app/api/poll/overview/route.ts rename to web/src/app/api/poll/system-summary/route.ts index 91c9a7f..835746f 100644 --- a/web/src/app/api/poll/overview/route.ts +++ b/web/src/app/api/poll/system-summary/route.ts @@ -1,13 +1,13 @@ import { NextResponse } from "next/server"; -import { loadOverview } from "@/lib/loaders"; +import { loadSystemSummary } from "@/lib/loaders"; import { jsonError } from "@/lib/poll-response"; export const dynamic = "force-dynamic"; export async function GET() { try { - return NextResponse.json(await loadOverview()); + return NextResponse.json(await loadSystemSummary()); } catch (e) { return jsonError(e); } diff --git a/web/src/app/checks/[check_id]/page.tsx b/web/src/app/checks/[check_id]/page.tsx index 350fa93..bf9f807 100644 --- a/web/src/app/checks/[check_id]/page.tsx +++ b/web/src/app/checks/[check_id]/page.tsx @@ -2,7 +2,7 @@ import { CheckDetailBrowser } from "@/components/CheckDetailBrowser"; import { ErrorPanel } from "@/components/DataPanel"; import { api } from "@/lib/api"; import { loadInventory } from "@/lib/loaders"; -import { normalizePageSize } from "@/lib/pagination"; +import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -18,16 +18,24 @@ export default async function CheckDetailPage({ searchParams, }: { params: Promise<{ check_id: string }>; - searchParams: Promise<{ tab?: string; events_limit?: string; cursor?: string }>; + searchParams: Promise<{ + tab?: string; + events_limit?: string; + cursor?: string; + back?: string; + page?: string; + }>; }) { const { check_id } = await params; const sp = await searchParams; const tab = normalizeTab(sp.tab); const eventsLimit = normalizePageSize(sp.events_limit, 50); const eventsCursor = sp.cursor; + const eventsPage = normalizePageNumber(sp.page); + const eventsBack = normalizeBack(sp.back); let data; - let events: EventsPage = { items: [], next_cursor: null }; + let events: EventsPage = { items: [], next_cursor: null, prev_cursor: null }; try { data = await loadInventory(); if (tab === "events") { @@ -35,6 +43,7 @@ export default async function CheckDetailPage({ check_id, limit: eventsLimit, cursor: eventsCursor, + back: eventsBack, }); } } catch (e) { @@ -43,13 +52,15 @@ export default async function CheckDetailPage({ return ( ); } diff --git a/web/src/app/events/page.tsx b/web/src/app/events/page.tsx index 6e5b42b..d3aa8a6 100644 --- a/web/src/app/events/page.tsx +++ b/web/src/app/events/page.tsx @@ -1,7 +1,7 @@ import { ErrorPanel } from "@/components/DataPanel"; import { EventsBrowser } from "@/components/EventsBrowser"; import { loadEvents } from "@/lib/loaders"; -import { normalizePageSize } from "@/lib/pagination"; +import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -10,19 +10,24 @@ export default async function EventsPage({ }: { searchParams: Promise<{ cursor?: string; + back?: string; agent_id?: string; check_id?: string; limit?: string; + page?: string; }>; }) { const sp = await searchParams; const limit = normalizePageSize(sp.limit); + const page = normalizePageNumber(sp.page); + const back = normalizeBack(sp.back); let data; try { data = await loadEvents({ agentId: sp.agent_id, checkId: sp.check_id, cursor: sp.cursor, + back, limit, }); } catch (e) { @@ -30,12 +35,14 @@ export default async function EventsPage({ } return ( ); } diff --git a/web/src/app/incidents/page.tsx b/web/src/app/incidents/page.tsx index eef1aa6..177f04c 100644 --- a/web/src/app/incidents/page.tsx +++ b/web/src/app/incidents/page.tsx @@ -1,29 +1,55 @@ import { ErrorPanel } from "@/components/DataPanel"; import { IncidentsBrowser } from "@/components/IncidentsBrowser"; -import { loadIncidents, loadInventory } from "@/lib/loaders"; +import { loadIncidentsPage, loadInventory } from "@/lib/loaders"; +import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; export const dynamic = "force-dynamic"; export default async function IncidentsPage({ searchParams, }: { - searchParams: Promise<{ state?: string }>; + searchParams: Promise<{ + state?: string; + cursor?: string; + back?: string; + limit?: string; + sort?: string; + direction?: string; + page?: string; + }>; }) { const sp = await searchParams; const state = sp.state === "open" || sp.state === "resolved" ? sp.state : undefined; - let items; + const sort = + sp.sort === "opened_at" || sp.sort === "resolved_at" || sp.sort === "status" + ? sp.sort + : "status"; + const direction = sp.direction === "asc" || sp.direction === "desc" ? sp.direction : "desc"; + const limit = normalizePageSize(sp.limit); + const pageNumber = normalizePageNumber(sp.page); + const back = normalizeBack(sp.back); + let page; let inventory; try { - [items, inventory] = await Promise.all([loadIncidents(state), loadInventory()]); + [page, inventory] = await Promise.all([ + loadIncidentsPage({ state, cursor: sp.cursor, back, limit, sort, direction }), + loadInventory(), + ]); } catch (e) { return ; } return ( ); } diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx index d4204e7..3298a16 100644 --- a/web/src/app/layout.tsx +++ b/web/src/app/layout.tsx @@ -1,7 +1,10 @@ import type { Metadata } from "next"; -import Link from "next/link"; -import { TimeZoneCarousel, TimeZoneProvider } from "@/components/TimeZoneControls"; +import type { ReactNode } from "react"; +import { TimeZoneProvider } from "@/components/TimeZoneControls"; +import { TopNav } from "@/components/TopNav"; +import { loadSystemSummary } from "@/lib/loaders"; import { getUiTimeZone } from "@/lib/timezone"; +import type { SystemSummaryData } from "@/lib/view-types"; import "./globals.css"; export const metadata: Metadata = { @@ -9,50 +12,25 @@ export const metadata: Metadata = { description: "Read-only monitoring dashboard", }; -const NAV = [ - { href: "/", label: "Overview" }, - { href: "/agents", label: "Agents" }, - { href: "/checks", label: "Checks" }, - { href: "/incidents", label: "Incidents" }, - { href: "/events", label: "Events" }, - { href: "/outbox", label: "Outbox" }, -]; +export const dynamic = "force-dynamic"; -function Brand() { - return ( - - - - - - - - - - monlet - - - ); +async function safeLoadSystemSummary(): Promise { + try { + return await loadSystemSummary(); + } catch { + return null; + } } -export default function RootLayout({ children }: { children: React.ReactNode }) { +export default async function RootLayout({ children }: { children: ReactNode }) { const configuredTimeZone = getUiTimeZone(); + const summary = await safeLoadSystemSummary(); return ( -
- - - -
+
{children}
 diff --git a/web/src/app/outbox/page.tsx b/web/src/app/outbox/page.tsx index c3bf351..9f93d6a 100644 --- a/web/src/app/outbox/page.tsx +++ b/web/src/app/outbox/page.tsx @@ -1,15 +1,44 @@ import { ErrorPanel } from "@/components/DataPanel"; import { OutboxBrowser } from "@/components/OutboxBrowser"; -import { loadOutboxAll } from "@/lib/loaders"; +import { loadOutboxPage } from "@/lib/loaders"; +import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; export const dynamic = "force-dynamic"; -export default async function OutboxPage() { - let items; +export default async function OutboxPage({ + searchParams, +}: { + searchParams: Promise<{ + cursor?: string; + back?: string; + limit?: string; + sort?: string; + direction?: string; + page?: string; + }>; +}) { + const sp = await searchParams; + const limit = normalizePageSize(sp.limit); + const sort = sp.sort === "created_at" || sp.sort === "updated_at" ? sp.sort : "updated_at"; + const direction = sp.direction === "asc" || sp.direction === "desc" ? sp.direction : "desc"; + const pageNumber = normalizePageNumber(sp.page); + const back = normalizeBack(sp.back); + let page; try { - items = await loadOutboxAll(); + page = await loadOutboxPage({ cursor: sp.cursor, back, limit, sort, direction }); } catch (e) { return ; } - return ; + return ( + + ); } diff --git a/web/src/app/page.tsx b/web/src/app/page.tsx index eebc291..7bde426 100644 --- a/web/src/app/page.tsx +++ b/web/src/app/page.tsx @@ -1,15 +1,7 @@ -import { ErrorPanel } from "@/components/DataPanel"; -import { OverviewDashboard } from "@/components/OverviewDashboard"; -import { loadOverview } from "@/lib/loaders"; +import { redirect } from "next/navigation"; export const dynamic = "force-dynamic"; -export default async function OverviewPage() { - let data; - try { - data = await loadOverview(); - } catch (e) { - return ; - } - return ; +export default function HomePage() { + redirect("/agents"); } diff --git a/web/src/components/AgentBlacklistBrowser.tsx b/web/src/components/AgentBlacklistBrowser.tsx new file mode 100644 index 0000000..77532d5 --- /dev/null +++ b/web/src/components/AgentBlacklistBrowser.tsx @@ -0,0 +1,104 @@ +"use client"; + +import Link from "next/link"; +import { useState } from "react"; + +import { EmptyPanel, PageHeader, Td, Th } from "@/components/DataPanel"; +import { DateTime } from "@/components/TimeZoneControls"; +import type { AgentBlacklistItem } from "@/lib/api"; + +export function AgentBlacklistBrowser({ initialRows }: { initialRows: AgentBlacklistItem[] }) { + const [rows, setRows] = useState(initialRows); + const [busy, setBusy] = useState(false); + + const runAction = async (action: string, blacklistId?: string, confirmText?: string) => { + if (confirmText && !window.confirm(confirmText)) return; + setBusy(true); + try { + const res = await fetch("/api/agents/action", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action, blacklist_id: blacklistId }), + }); + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new Error(body?.error?.message ?? res.statusText); + } + if (action === "blacklist_clear") { + setRows([]); + } else if (blacklistId) { + setRows((current) => current.filter((r) => r.id !== blacklistId)); + } + } catch (e) { + window.alert(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(false); + } + }; + + return ( + <> + + ← agents + +
+ + {rows.length > 0 ? ( + + ) : null} +
+ {rows.length === 0 ? ( + + ) : ( + + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + ))} + +
fingerprinthostagent_idlast_seen_atactions
{row.key_fingerprint}{row.hostname}{row.agent_id} + + + +
+ )} + + ); +} diff --git a/web/src/components/AgentDetailBrowser.tsx b/web/src/components/AgentDetailBrowser.tsx index b9a3694..a5e96c1 100644 --- a/web/src/components/AgentDetailBrowser.tsx +++ b/web/src/components/AgentDetailBrowser.tsx @@ -27,7 +27,7 @@ import { usePolling } from "@/lib/usePolling"; type Agent = components["schemas"]["Agent"]; type CheckState = components["schemas"]["CheckState"]; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; type SortKey = "severity" | "check_id" | "last_seen"; type TabKey = "checks" | "events"; type AgentDetailData = { agent: Agent; checks: CheckState[] }; @@ -47,6 +47,8 @@ export function AgentDetailBrowser({ tab, eventsLimit, eventsCursor, + eventsBack, + eventsPage, }: { initialData: AgentDetailData; initialEvents: EventsPageData; @@ -54,6 +56,8 @@ export function AgentDetailBrowser({ tab: TabKey; eventsLimit: PageSize; eventsCursor?: string; + eventsBack: boolean; + eventsPage: number; }) { const agentId = initialData.agent.agent_id; const pollUrl = `/api/poll/agent?agent_id=${encodeURIComponent(agentId)}`; @@ -110,6 +114,8 @@ export function AgentDetailBrowser({ sort={sort} eventsLimit={eventsLimit} eventsCursor={eventsCursor} + eventsBack={eventsBack} + eventsPage={eventsPage} initialEvents={initialEvents} /> )} @@ -188,20 +194,25 @@ function EventsTable({ sort, eventsLimit, eventsCursor, + eventsBack, + eventsPage, initialEvents, }: { agentId: string; sort: SortKey; eventsLimit: PageSize; eventsCursor?: string; + eventsBack: boolean; + eventsPage: number; initialEvents: EventsPageData; }) { const router = useRouter(); const pollUrl = useMemo(() => { const qs = new URLSearchParams({ agent_id: agentId, limit: String(eventsLimit) }); if (eventsCursor) qs.set("cursor", eventsCursor); + if (eventsBack) qs.set("back", "1"); return `/api/poll/events?${qs.toString()}`; - }, [agentId, eventsCursor, eventsLimit]); + }, [agentId, eventsCursor, eventsBack, eventsLimit]); const { data } = usePolling(initialEvents, pollUrl); const onLimitChange = (next: PageSize) => { @@ -224,6 +235,7 @@ function EventsTable({ check_id status duration_ms + SUMMARY @@ -237,6 +249,7 @@ function EventsTable({ {e.status} {e.duration_ms} + ))} @@ -244,8 +257,9 @@ function EventsTable({ )} diff --git a/web/src/components/AgentsBrowser.tsx b/web/src/components/AgentsBrowser.tsx index d615a1c..630f8d6 100644 --- a/web/src/components/AgentsBrowser.tsx +++ b/web/src/components/AgentsBrowser.tsx @@ -1,5 +1,6 @@ "use client"; +import Link from "next/link"; import { useMemo, useState } from "react"; import { AgentFeatures } from "@/components/AgentFeatures"; @@ -35,16 +36,25 @@ const statusRank: Record = { alive: 0, stale: 1, dead: const checkSortStatuses: CheckStatus[] = ["critical", "warning", "unknown", "ok"]; export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { - const { data } = usePolling(initialData, "/api/poll/inventory"); + const [reloadKey, setReloadKey] = useState(0); + const { data } = usePolling(initialData, `/api/poll/inventory?v=${reloadKey}`); const [query, setQuery] = useState(""); const [sort, setSort] = useState("status"); const [dir, setDir] = useState("desc"); + const [busy, setBusy] = useState(null); const initialRows = useMemo(() => rowsFromInventory(data), [data]); - const rows = useMemo( + const filteredRows = useMemo( () => sortAgents(filterAgents(initialRows, query), sort, dir), [dir, initialRows, query, sort], ); + const pendingRows = filteredRows.filter(({ agent }) => agent.admission_state === "pending"); + const acceptedRows = filteredRows.filter(({ agent }) => agent.admission_state === "accepted"); + const rows = [...pendingRows, ...acceptedRows]; + const allPendingRows = initialRows.filter(({ agent }) => agent.admission_state === "pending"); + const allRotationPendingCount = allPendingRows.filter( + ({ agent }) => agent.rotation_pending, + ).length; const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); const [page, setPage] = useState(1); @@ -69,14 +79,45 @@ export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { } setPage(1); }; + const runAction = async ( + action: string, + agentId?: string, + confirmText?: string, + includeRotation = false, + ) => { + if (confirmText && !window.confirm(confirmText)) return; + const key = `${action}:${agentId ?? "*"}`; + setBusy(key); + try { + const res = await fetch("/api/agents/action", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ action, agent_id: agentId, include_rotation: includeRotation }), + }); + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new Error(body?.error?.message ?? res.statusText); + } + setReloadKey((v) => v + 1); + } catch (e) { + window.alert(e instanceof Error ? e.message : String(e)); + } finally { + setBusy(null); + } + }; return ( <> -
-

Agents

- - {rows.length} of {initialRows.length} items - +
+
+

Agents

+ + {rows.length} of {initialRows.length} items + +
+ + Blacklist +
) : ( - - - - - - - - - - - - {visible.map(({ agent: a, checks }) => ( - - - - - - - - ))} - -
- - HOST - - - - STATUS - - - - LAST_SEEN_AT - - featureslabels
- - - - - - - - - -
+ )} ); } +function AgentsTable({ + rows, + sort, + dir, + chooseSort, + busy, + runAction, + pendingCount, + rotationPendingCount, +}: { + rows: AgentRow[]; + sort: SortField; + dir: SortDir; + chooseSort: (field: SortField) => void; + busy: string | null; + runAction: ( + action: string, + agentId?: string, + confirmText?: string, + includeRotation?: boolean, + ) => void; + pendingCount: number; + rotationPendingCount: number; +}) { + const acceptAllText = + rotationPendingCount > 0 + ? `ROTATION: existing accepted keys will be replaced for ${rotationPendingCount} agents. Accept all pending agents including rotations?` + : undefined; + return ( + + + + + + + + + + + + + {pendingCount > 0 ? ( + + + + ) : null} + {rows.map(({ agent: a, checks }) => ( + + + + + + + + + ))} + +
+ + HOST + + + + STATUS + + + + LAST_SEEN_AT + + featureslabelsactions
+
+ {pendingCount} pending + + runAction("accept_all", undefined, acceptAllText, rotationPendingCount > 0) + } + /> + runAction("block_all", undefined, "Block all pending agents?")} + /> + runAction("remove_all", undefined, "Remove all pending agents and their data?")} + /> +
+
+
+ + {a.admission_state === "pending" ? ( + + {a.rotation_pending ? "ROTATION" : "pending"} + + ) : null} +
+ 		+ + + + + + + + + +
+ ); +} + +function AgentActions({ + agent, + busy, + runAction, +}: { + agent: Agent; + busy: string | null; + runAction: ( + action: string, + agentId?: string, + confirmText?: string, + includeRotation?: boolean, + ) => void; +}) { + const disabled = busy !== null; + if (agent.admission_state === "pending") { + const acceptText = agent.rotation_pending + ? `ROTATION: existing accepted key for ${agent.hostname} will be replaced. Accept?` + : undefined; + return ( +
+ runAction("accept", agent.agent_id, acceptText)} + /> + runAction("block", agent.agent_id, `Block ${agent.hostname}?`)} + /> + runAction("remove", agent.agent_id, `Remove ${agent.hostname} and all its data?`)} + /> +
+ ); + } + return ( + runAction("remove", agent.agent_id, `Remove ${agent.hostname} and all its data?`)} + /> + ); +} + +function ActionButton({ + label, + title, + disabled, + onClick, +}: { + label: string; + title: string; + disabled: boolean; + onClick: () => void; +}) { + return ( + + ); +} + function filterAgents(rows: AgentRow[], query: string): AgentRow[] { const needle = query.trim().toLowerCase(); if (!needle) return rows; diff --git a/web/src/components/CheckDetailBrowser.tsx b/web/src/components/CheckDetailBrowser.tsx index 8d19aab..4fd7fbd 100644 --- a/web/src/components/CheckDetailBrowser.tsx +++ b/web/src/components/CheckDetailBrowser.tsx @@ -26,7 +26,7 @@ import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; type TabKey = "agents" | "events"; export function CheckDetailBrowser({ @@ -36,6 +36,8 @@ export function CheckDetailBrowser({ tab, eventsLimit, eventsCursor, + eventsBack, + eventsPage, }: { checkId: string; initialData: InventoryData; @@ -43,6 +45,8 @@ export function CheckDetailBrowser({ tab: TabKey; eventsLimit: PageSize; eventsCursor?: string; + eventsBack: boolean; + eventsPage: number; }) { const { data } = usePolling(initialData, "/api/poll/inventory"); const agentsById = useMemo( @@ -110,6 +114,8 @@ export function CheckDetailBrowser({ agentsById={agentsById} eventsLimit={eventsLimit} eventsCursor={eventsCursor} + eventsBack={eventsBack} + eventsPage={eventsPage} /> )}
@@ -177,12 +183,16 @@ function EventsForCheck({ agentsById, eventsLimit, eventsCursor, + eventsBack, + eventsPage, }: { checkId: string; initialData: EventsPageData; agentsById: Map; eventsLimit: PageSize; eventsCursor?: string; + eventsBack: boolean; + eventsPage: number; }) { const router = useRouter(); const pollUrl = useMemo(() => { @@ -191,8 +201,9 @@ function EventsForCheck({ limit: String(eventsLimit), }); if (eventsCursor) qs.set("cursor", eventsCursor); + if (eventsBack) qs.set("back", "1"); return `/api/poll/events?${qs.toString()}`; - }, [checkId, eventsCursor, eventsLimit]); + }, [checkId, eventsCursor, eventsBack, eventsLimit]); const { data } = usePolling(initialData, pollUrl); const onLimitChange = (next: PageSize) => { @@ -245,8 +256,9 @@ function EventsForCheck({ )} diff --git a/web/src/components/EventsBrowser.tsx b/web/src/components/EventsBrowser.tsx index be5278c..e30a89e 100644 --- a/web/src/components/EventsBrowser.tsx +++ b/web/src/components/EventsBrowser.tsx @@ -15,25 +15,29 @@ import { type PageSize } from "@/lib/pagination"; import { usePolling } from "@/lib/usePolling"; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; export function EventsBrowser({ initialData, cursor, + back, agentId, checkId, limit, + page, }: { initialData: EventsPageData; cursor?: string; + back: boolean; agentId?: string; checkId?: string; limit: PageSize; + page: number; }) { const router = useRouter(); const url = useMemo( - () => eventsPollUrl({ cursor, agentId, checkId, limit }), - [agentId, checkId, cursor, limit], + () => eventsPollUrl({ cursor, back, agentId, checkId, limit }), + [agentId, back, checkId, cursor, limit], ); const { data } = usePolling(initialData, url); @@ -86,8 +90,9 @@ export function EventsBrowser({ )} @@ -160,17 +165,20 @@ function eventsHref(params: { function eventsPollUrl({ cursor, + back, agentId, checkId, limit, }: { cursor?: string; + back: boolean; agentId?: string; checkId?: string; limit: PageSize; }) { const qs = new URLSearchParams(); if (cursor) qs.set("cursor", cursor); + if (back) qs.set("back", "1"); if (agentId) qs.set("agent_id", agentId); if (checkId) qs.set("check_id", checkId); qs.set("limit", String(limit)); diff --git a/web/src/components/IncidentsBrowser.tsx b/web/src/components/IncidentsBrowser.tsx index e91b97f..c1059a7 100644 --- a/web/src/components/IncidentsBrowser.tsx +++ b/web/src/components/IncidentsBrowser.tsx @@ -1,24 +1,19 @@ "use client"; import Link from "next/link"; -import { useMemo, useState } from "react"; +import { useRouter } from "next/navigation"; +import { useMemo } from "react"; import { AgentLink } from "@/components/AgentLink"; import { CheckEventsLink } from "@/components/CheckEventsLink"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, PageHeader, SummaryCell, Td, Th } from "@/components/DataPanel"; import { PageSizeSelect } from "@/components/PageSizeSelect"; +import { Pager } from "@/components/Pager"; import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { type PageSize } from "@/lib/pagination"; import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; @@ -26,70 +21,75 @@ const STATES = ["all", "open", "resolved"] as const; type Incident = components["schemas"]["Incident"]; type Agent = components["schemas"]["Agent"]; -type IncidentsData = { items: Incident[] }; -type SortField = "status" | "host" | "check" | "opened_at" | "resolved_at"; - -const stateRank: Record = { open: 0, resolved: 1 }; -const severityRank: Record = { - critical: 0, - warning: 1, - unknown: 2, -}; +type IncidentsData = { items: Incident[]; next_cursor?: string | null; prev_cursor?: string | null }; +type SortField = "status" | "opened_at" | "resolved_at"; export function IncidentsBrowser({ - initialItems, + initialData, initialInventory, state, + cursor, + back, + sort, + direction, + limit, + page, }: { - initialItems: Incident[]; + initialData: IncidentsData; initialInventory: InventoryData; state?: "open" | "resolved"; + cursor?: string; + back: boolean; + sort: SortField; + direction: SortDir; + limit: PageSize; + page: number; }) { - const url = useMemo( - () => (state ? `/api/poll/incidents?state=${state}` : "/api/poll/incidents"), - [state], - ); - const { data } = usePolling({ items: initialItems }, url); + const router = useRouter(); + const url = useMemo(() => { + const q = new URLSearchParams(); + if (state) q.set("state", state); + q.set("sort", sort); + q.set("direction", direction); + q.set("limit", String(limit)); + if (cursor) q.set("cursor", cursor); + if (back) q.set("back", "1"); + return `/api/poll/incidents?${q.toString()}`; + }, [cursor, back, direction, limit, state, sort]); + const { data } = usePolling(initialData, url); const { data: inventory } = usePolling(initialInventory, "/api/poll/inventory"); const agentsById = useMemo( () => new Map(inventory.agents.map((agent) => [agent.agent_id, agent])), [inventory.agents], ); - const [sort, setSort] = useState("status"); - const [dir, setDir] = useState("asc"); - const sorted = useMemo( - () => sortIncidents(data.items, agentsById, sort, dir), - [agentsById, data.items, dir, sort], - ); - - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); + router.push(incidentsHref({ state, sort, direction, limit: next })); }; const chooseSort = (field: SortField) => { + let nextSort = sort; + let nextDirection = direction; if (field === sort) { - setDir((current) => (current === "asc" ? "desc" : "asc")); + nextDirection = direction === "asc" ? "desc" : "asc"; } else { - setSort(field); - setDir(fieldDefaultDir(field)); + nextSort = field; + nextDirection = fieldDefaultDir(); } - setPage(1); + router.push(incidentsHref({ state, sort: nextSort, direction: nextDirection, limit })); }; - const pageCount = pageCountOf(sorted.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(sorted, currentPage, pageSize); - return ( <> - +
{STATES.map((s) => { - const href = s === "all" ? "/incidents" : `/incidents?state=${s}`; + const href = incidentsHref({ + state: s === "all" ? undefined : s, + sort: sort === "resolved_at" && s !== "resolved" ? "status" : sort, + direction, + limit, + }); const active = (s === "all" && !state) || s === state; return ( - +
- {sorted.length === 0 ? ( + {data.items.length === 0 ? ( ) : ( + + - - - {visible.map((incident) => ( + {data.items.map((incident) => (
- + STATUS HOSTCHECK - - HOST - - - - CHECK - - - + OPENED_AT - - RESOLVED_AT - + {state === "resolved" ? ( + + RESOLVED_AT + + ) : ( + // PH-review: backend rejects sort=resolved_at when state is + // not "resolved" (server/monlet_server/api/incidents.py + // auto-applies state=resolved when none is set, which would + // silently filter the "all" tab). Sort is available only on + // the resolved tab. + "RESOLVED_AT" + )} SUMMARY
)} - + ); } @@ -203,56 +220,26 @@ function incidentCheckId(incident: Incident): string { return incident.check_id ?? ""; } -function fieldDefaultDir(field: SortField): SortDir { - switch (field) { - case "status": - return "asc"; - case "host": - case "check": - return "asc"; - case "opened_at": - case "resolved_at": - return "desc"; - } +function fieldDefaultDir(): SortDir { + return "desc"; } -function sortIncidents( - items: Incident[], - agentsById: Map, - sort: SortField, - dir: SortDir, -): Incident[] { - return [...items].sort((a, b) => { - const primary = compareIncidents(a, b, agentsById, sort); - if (primary !== 0) return dir === "asc" ? primary : -primary; - return (b.opened_at ?? "").localeCompare(a.opened_at ?? ""); - }); -} - -function compareIncidents( - a: Incident, - b: Incident, - agentsById: Map, - sort: SortField, -): number { - switch (sort) { - case "status": { - const s = stateRank[a.state] - stateRank[b.state]; - if (s !== 0) return s; - return severityRank[a.severity] - severityRank[b.severity]; - } - case "host": - return hostFor(a, agentsById).localeCompare(hostFor(b, agentsById)); - case "check": - return incidentCheckId(a).localeCompare(incidentCheckId(b)); - case "opened_at": - return (a.opened_at ?? "").localeCompare(b.opened_at ?? ""); - case "resolved_at": - return (a.resolved_at ?? "").localeCompare(b.resolved_at ?? ""); - } -} - -function hostFor(incident: Incident, agentsById: Map): string { - const id = incidentAgentId(incident); - return agentsById.get(id)?.hostname || id; +function incidentsHref({ + state, + sort, + direction, + limit, +}: { + state?: "open" | "resolved"; + sort: SortField; + direction: SortDir; + limit: PageSize; +}) { + const qs = new URLSearchParams(); + if (state) qs.set("state", state); + qs.set("sort", sort); + qs.set("direction", direction); + qs.set("limit", String(limit)); + const query = qs.toString(); + return query ? `/incidents?${query}` : "/incidents"; } diff --git a/web/src/components/OutboxBrowser.tsx b/web/src/components/OutboxBrowser.tsx index 09cd4fb..7ba50b9 100644 --- a/web/src/components/OutboxBrowser.tsx +++ b/web/src/components/OutboxBrowser.tsx @@ -1,44 +1,64 @@ "use client"; -import { useState } from "react"; +import { useRouter } from "next/navigation"; +import { useMemo } from "react"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, PageHeader, Td, Th } from "@/components/DataPanel"; import { PageSizeSelect } from "@/components/PageSizeSelect"; +import { Pager } from "@/components/Pager"; +import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { type PageSize } from "@/lib/pagination"; import { usePolling } from "@/lib/usePolling"; type OutboxItem = components["schemas"]["NotificationOutboxItem"]; -type OutboxData = { items: OutboxItem[] }; +type OutboxData = { items: OutboxItem[]; next_cursor?: string | null; prev_cursor?: string | null }; +type SortField = "created_at" | "updated_at"; -export function OutboxBrowser({ initialItems }: { initialItems: OutboxItem[] }) { - const { data } = usePolling({ items: initialItems }, "/api/poll/outbox"); +export function OutboxBrowser({ + initialData, + cursor, + back, + sort, + direction, + limit, + page, +}: { + initialData: OutboxData; + cursor?: string; + back: boolean; + sort: SortField; + direction: SortDir; + limit: PageSize; + page: number; +}) { + const router = useRouter(); + const url = useMemo(() => { + const q = new URLSearchParams(); + q.set("sort", sort); + q.set("direction", direction); + q.set("limit", String(limit)); + if (cursor) q.set("cursor", cursor); + if (back) q.set("back", "1"); + return `/api/poll/outbox?${q.toString()}`; + }, [cursor, back, direction, limit, sort]); + const { data } = usePolling(initialData, url); - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); + router.push(outboxHref({ sort, direction, limit: next })); + }; + const chooseSort = (field: SortField) => { + const nextDirection = field === sort && direction === "desc" ? "asc" : "desc"; + router.push(outboxHref({ sort: field, direction: nextDirection, limit })); }; - - const pageCount = pageCountOf(data.items.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(data.items, currentPage, pageSize); return ( <>
- +
{data.items.length === 0 ? ( @@ -46,7 +66,11 @@ export function OutboxBrowser({ initialItems }: { initialItems: OutboxItem[] }) - + @@ -56,7 +80,7 @@ export function OutboxBrowser({ initialItems }: { initialItems: OutboxItem[] }) - {visible.map((o) => ( + {data.items.map((o) => (
updated_at + + updated_at + + notifier event state
@@ -74,7 +98,29 @@ export function OutboxBrowser({ initialItems }: { initialItems: OutboxItem[] })
)} - + ); } + +function outboxHref({ + sort, + direction, + limit, +}: { + sort: SortField; + direction: SortDir; + limit: PageSize; +}) { + const qs = new URLSearchParams(); + qs.set("sort", sort); + qs.set("direction", direction); + qs.set("limit", String(limit)); + return `/outbox?${qs.toString()}`; +} diff --git a/web/src/components/OverviewDashboard.tsx b/web/src/components/OverviewDashboard.tsx deleted file mode 100644 index bbb0b9e..0000000 --- a/web/src/components/OverviewDashboard.tsx +++ /dev/null @@ -1,56 +0,0 @@ -"use client"; - -import Link from "next/link"; - -import { statusBadgeClass } from "@/lib/format"; -import { usePolling } from "@/lib/usePolling"; -import type { OverviewData } from "@/lib/view-types"; - -export function OverviewDashboard({ initialData }: { initialData: OverviewData }) { - const { data } = usePolling(initialData, "/api/poll/overview"); - - return ( -
-

Overview

-
- - - 0 ? "open" : undefined} - /> -
-
- ); -} - -function Card({ - title, - total, - href, - tally, - highlight, -}: { - title: string; - total: number; - href: string; - tally: Record; - highlight?: string; -}) { - return ( - -
{title}
-
{total}
-
- {Object.entries(tally).map(([k, v]) => ( - - {k} {v} - - ))} -
- - ); -} diff --git a/web/src/components/Pager.tsx b/web/src/components/Pager.tsx index a48a3fe..97a592d 100644 --- a/web/src/components/Pager.tsx +++ b/web/src/components/Pager.tsx @@ -2,27 +2,55 @@ import Link from "next/link"; export function Pager({ basePath, - cursor, + prevCursor, nextCursor, + page = 1, extra = {}, }: { basePath: string; - cursor?: string; + prevCursor?: string | null; nextCursor?: string | null; + page?: number; extra?: Record; }) { - const make = (c?: string) => { + const make = ({ + nextPage, + nextPageCursor, + back, + }: { + nextPage: number; + nextPageCursor?: string; + back?: boolean; + }) => { const sp = new URLSearchParams(); for (const [k, v] of Object.entries(extra)) if (v) sp.set(k, v); - if (c) sp.set("cursor", c); + if (nextPageCursor) sp.set("cursor", nextPageCursor); + if (back) sp.set("back", "1"); + if (nextPage > 1) sp.set("page", String(nextPage)); const qs = sp.toString(); return qs ? `${basePath}?${qs}` : basePath; }; + + const previousHref = + page > 1 && prevCursor + ? make({ nextPage: page - 1, nextPageCursor: prevCursor, back: true }) + : null; + const nextHref = nextCursor + ? make({ nextPage: page + 1, nextPageCursor: nextCursor }) + : null; + return (
- {cursor ? `cursor: ${cursor.slice(0, 12)}…` : ""} - {nextCursor ? ( - + {previousHref ? ( + + ← prev + + ) : ( + ← prev + )} + page {page} + {nextHref ? ( + next → ) : ( diff --git a/web/src/components/TopNav.tsx b/web/src/components/TopNav.tsx new file mode 100644 index 0000000..e236a6c --- /dev/null +++ b/web/src/components/TopNav.tsx @@ -0,0 +1,93 @@ +"use client"; + +import Link from "next/link"; +import type { ReactNode } from "react"; + +import { TimeZoneCarousel } from "@/components/TimeZoneControls"; +import { usePolling } from "@/lib/usePolling"; +import type { SystemSummaryData } from "@/lib/view-types"; + +const ZERO_SUMMARY: SystemSummaryData = { + agents_online: 0, + agents_offline: 0, + checks_ok: 0, + checks_warning: 0, + checks_critical: 0, + checks_unknown: 0, + open_incidents: 0, + generated_at: "", +}; + +export function TopNav({ initialSummary }: { initialSummary: SystemSummaryData | null }) { + const { data } = usePolling(initialSummary ?? ZERO_SUMMARY, "/api/poll/system-summary"); + + return ( +
+ + + +
+ ); +} + +function Brand() { + return ( + + + + + + + + + + monlet + + + ); +} + +function NavLink({ + href, + label, + children, +}: { + href: string; + label: string; + children: ReactNode; +}) { + return ( + + {label} + + ( + {children} + ) + + + ); +} diff --git a/web/src/lib/api-types.ts b/web/src/lib/api-types.ts index 37ed602..30aa372 100644 --- a/web/src/lib/api-types.ts +++ b/web/src/lib/api-types.ts @@ -89,6 +89,92 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/agents/admission/blacklist": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List blocked agent keys */ + get: operations["listAgentBlacklist"]; + put?: never; + post?: never; + /** Clear blocked agent keys */ + delete: operations["clearAgentBlacklist"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/admission/blacklist/{key_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post?: never; + /** Remove one blocked agent key */ + delete: operations["deleteAgentBlacklistItem"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/admission/accept-all": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Accept all pending agents */ + post: operations["acceptAllPendingAgents"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/admission/block-all": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Block all pending agents */ + post: operations["blockAllPendingAgents"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/admission/remove-all": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Remove all pending agents */ + post: operations["removeAllPendingAgents"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/agents/{agent_id}": { parameters: { query?: never; @@ -100,6 +186,41 @@ export interface paths { get: operations["getAgent"]; put?: never; post?: never; + /** Remove agent and all owned state */ + delete: operations["deleteAgent"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/{agent_id}/accept": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Accept pending agent key */ + post: operations["acceptAgent"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/v1/agents/{agent_id}/block": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Block pending agent key */ + post: operations["blockAgent"]; delete?: never; options?: never; head?: never; @@ -147,7 +268,7 @@ export interface paths { path?: never; cookie?: never; }; - /** Query stored events */ + /** Query check status-change history */ get: operations["queryEvents"]; put?: never; post?: never; @@ -157,6 +278,27 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/system-summary": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * System navigation summary + * @description Bounded aggregate counts for the top navigation. Safe to poll from many + * open dashboards because response size is independent of row counts. + */ + get: operations["getSystemSummary"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/notifiers/outbox": { parameters: { query?: never; @@ -185,7 +327,7 @@ export interface components { ErrorResponse: { error: { /** @enum {string} */ - code: "unauthorized" | "not_found" | "validation" | "payload_too_large" | "rate_limited" | "internal" | "not_ready"; + code: "unauthorized" | "forbidden" | "conflict" | "agent_blocked" | "not_found" | "validation" | "payload_too_large" | "rate_limited" | "internal" | "not_ready"; message: string; /** @description Server-generated request id, mirrored from X-Request-Id. */ request_id: string; @@ -194,14 +336,21 @@ export interface components { PageEnvelope: { /** @description Pass back as `cursor` to fetch the next page. Null/absent if no more results. */ next_cursor?: string | null; + /** @description Pass back as `cursor` with `back=true` to fetch the previous page. Null on the first page. */ + prev_cursor?: string | null; }; AcceptedResponse: { accepted: boolean; }; EventBatchAcceptedResponse: { - /** @description Number of events stored on this request. */ + /** @description Number of events that produced a new status-change row on this request. */ accepted: number; - /** @description Number of events whose `event_id` was already known. */ + /** + * @description Number of events that were not stored as a change. Includes events whose + * `event_id` was already known, events whose `(status, exit_code)` matched + * the previously observed value for the same check, and events received late + * (older than the current `last_observed_at` of the check). + */ deduplicated: number; }; HeartbeatRequest: { @@ -231,7 +380,7 @@ export interface components { /** * @description Event body used inside `EventBatchRequest.events`. The owning `agent_id` * is supplied once at the batch envelope; per-event `agent_id` is intentionally - * absent so requests cannot mix or spoof other agents under a shared token. + * absent so one HTTP request cannot mix multiple agents. * Server response endpoints (`GET /events/query`) include `agent_id` separately * in their item schema. */ @@ -265,7 +414,12 @@ export interface components { notifications_enabled: boolean; incident_key?: string; }; - /** @description Stored event returned by `GET /events/query`. Includes the resolved `agent_id` and server-side `received_at`. */ + /** + * @description Stored status-change event returned by `GET /events/query`. The server persists + * only events that change `(status, exit_code)` versus the previous observed value + * for the same `(agent_id, check_id)`. Repeated identical results are not stored. + * Includes the resolved `agent_id` and server-side `received_at`. + */ StoredCheckResultEvent: components["schemas"]["CheckResultEvent"] & { agent_id: string; /** Format: date-time */ @@ -276,6 +430,14 @@ export interface components { hostname: string; /** @enum {string} */ status: "alive" | "stale" | "dead"; + /** @enum {string} */ + admission_state: "pending" | "accepted"; + /** + * @description True when a pending key would replace an already accepted key for this agent. + * @default false + */ + rotation_pending: boolean; + key_fingerprint?: string | null; /** Format: date-time */ last_seen_at: string; features: components["schemas"]["AgentFeatures"]; @@ -283,6 +445,20 @@ export interface components { [key: string]: string; }; }; + AgentBlacklistItem: { + /** @description Opaque blacklist row id. Use for delete operations; display `key_fingerprint` to humans. */ + id: string; + key_fingerprint: string; + agent_id: string; + hostname: string; + /** Format: date-time */ + created_at: string; + /** Format: date-time */ + last_seen_at: string; + }; + ActionCountResponse: { + count: number; + }; CheckState: { agent_id: string; check_id: string; @@ -324,6 +500,18 @@ export interface components { /** Format: date-time */ updated_at?: string; }; + /** @description Bounded aggregate counts for the web navigation. */ + SystemSummaryResponse: { + agents_online: number; + agents_offline: number; + checks_ok: number; + checks_warning: number; + checks_critical: number; + checks_unknown: number; + open_incidents: number; + /** Format: date-time */ + generated_at: string; + }; }; responses: { /** @description Authentication failed. */ @@ -344,6 +532,15 @@ export interface components { "application/json": components["schemas"]["ErrorResponse"]; }; }; + /** @description Request conflicts with current server state. */ + Conflict: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; /** @description Payload exceeds configured limit (1 MiB). */ PayloadTooLarge: { headers: { @@ -353,6 +550,15 @@ export interface components { "application/json": components["schemas"]["ErrorResponse"]; }; }; + /** @description Request rejected by configured rate/volume limits. */ + RateLimited: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; /** @description Request body or parameters failed validation. */ ValidationError: { headers: { @@ -374,8 +580,10 @@ export interface components { }; parameters: { AgentId: string; - /** @description Opaque pagination cursor returned in `next_cursor`. */ + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ Cursor: string; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + Back: boolean; Limit: number; }; requestBodies: never; @@ -457,7 +665,18 @@ export interface operations { }; 400: components["responses"]["ValidationError"]; 401: components["responses"]["Unauthorized"]; + /** @description Agent key is blocked. */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + 409: components["responses"]["Conflict"]; 413: components["responses"]["PayloadTooLarge"]; + 429: components["responses"]["RateLimited"]; 500: components["responses"]["InternalError"]; }; }; @@ -485,6 +704,15 @@ export interface operations { }; 400: components["responses"]["ValidationError"]; 401: components["responses"]["Unauthorized"]; + /** @description Agent key is blocked. */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; 413: components["responses"]["PayloadTooLarge"]; 500: components["responses"]["InternalError"]; }; @@ -492,8 +720,10 @@ export interface operations { listAgents: { parameters: { query?: { - /** @description Opaque pagination cursor returned in `next_cursor`. */ + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ cursor?: components["parameters"]["Cursor"]; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + back?: components["parameters"]["Back"]; limit?: components["parameters"]["Limit"]; }; header?: never; @@ -516,6 +746,143 @@ export interface operations { 401: components["responses"]["Unauthorized"]; }; }; + listAgentBlacklist: { + parameters: { + query?: { + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ + cursor?: components["parameters"]["Cursor"]; + limit?: components["parameters"]["Limit"]; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Agent blacklist. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PageEnvelope"] & { + items: components["schemas"]["AgentBlacklistItem"][]; + }; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + clearAgentBlacklist: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of removed blacklist rows. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + deleteAgentBlacklistItem: { + parameters: { + query?: never; + header?: never; + path: { + key_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of removed blacklist rows. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + acceptAllPendingAgents: { + parameters: { + query?: { + /** @description When false, accepted agents with pending replacement keys are skipped. */ + include_rotation?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of accepted agents. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + blockAllPendingAgents: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of blocked agent keys. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + removeAllPendingAgents: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of removed agents. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; getAgent: { parameters: { query?: never; @@ -540,12 +907,86 @@ export interface operations { 404: components["responses"]["NotFound"]; }; }; + deleteAgent: { + parameters: { + query?: never; + header?: never; + path: { + agent_id: components["parameters"]["AgentId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of removed agents. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 404: components["responses"]["NotFound"]; + }; + }; + acceptAgent: { + parameters: { + query?: never; + header?: never; + path: { + agent_id: components["parameters"]["AgentId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of accepted agents. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 404: components["responses"]["NotFound"]; + }; + }; + blockAgent: { + parameters: { + query?: never; + header?: never; + path: { + agent_id: components["parameters"]["AgentId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Number of blocked agent keys. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ActionCountResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 404: components["responses"]["NotFound"]; + }; + }; listChecks: { parameters: { query?: { agent_id?: string; - /** @description Opaque pagination cursor returned in `next_cursor`. */ + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ cursor?: components["parameters"]["Cursor"]; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + back?: components["parameters"]["Back"]; limit?: components["parameters"]["Limit"]; }; header?: never; @@ -572,8 +1013,15 @@ export interface operations { parameters: { query?: { state?: "open" | "resolved"; - /** @description Opaque pagination cursor returned in `next_cursor`. */ + severity?: "warning" | "critical" | "unknown"; + agent_id?: string; + check_id?: string; + sort?: "status" | "opened_at" | "resolved_at"; + direction?: "asc" | "desc"; + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ cursor?: components["parameters"]["Cursor"]; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + back?: components["parameters"]["Back"]; limit?: components["parameters"]["Limit"]; }; header?: never; @@ -601,8 +1049,10 @@ export interface operations { query?: { agent_id?: string; check_id?: string; - /** @description Opaque pagination cursor returned in `next_cursor`. */ + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ cursor?: components["parameters"]["Cursor"]; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + back?: components["parameters"]["Back"]; limit?: components["parameters"]["Limit"]; }; header?: never; @@ -625,11 +1075,39 @@ export interface operations { 401: components["responses"]["Unauthorized"]; }; }; + getSystemSummary: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description System summary snapshot. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SystemSummaryResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; listOutbox: { parameters: { query?: { - /** @description Opaque pagination cursor returned in `next_cursor`. */ + state?: "pending" | "sending" | "sent" | "retry" | "failed" | "discarded"; + notifier?: "debug" | "telegram" | "webhook" | "alertmanager"; + event_type?: "firing" | "resolved"; + sort?: "created_at" | "updated_at"; + direction?: "asc" | "desc"; + /** @description Opaque pagination cursor returned in `next_cursor` (or `prev_cursor` when `back=true`). */ cursor?: components["parameters"]["Cursor"]; + /** @description When true, treat the `cursor` as a `prev_cursor` and return the previous page. */ + back?: components["parameters"]["Back"]; limit?: components["parameters"]["Limit"]; }; header?: never; diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index c681633..a865a0b 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -3,6 +3,7 @@ import "server-only"; import type { components, operations } from "./api-types"; export type Agent = components["schemas"]["Agent"]; +export type AgentBlacklistItem = components["schemas"]["AgentBlacklistItem"]; export type CheckState = components["schemas"]["CheckState"]; export type Incident = components["schemas"]["Incident"]; export type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; @@ -18,6 +19,9 @@ type ChecksBody = OkBody; type IncidentsBody = OkBody; type EventsBody = OkBody; type OutboxBody = OkBody; +type SystemSummaryBody = OkBody; +type AgentBlacklistBody = OkBody; +type ActionCountBody = components["schemas"]["ActionCountResponse"]; const BASE = process.env.MONLET_API_BASE_URL ?? "http://127.0.0.1:8000"; const TOKEN = process.env.MONLET_API_TOKEN ?? ""; @@ -28,11 +32,15 @@ export class ApiError extends Error { } } -async function get(path: string, params?: Record): Promise { +async function get( + path: string, + params?: Record, +): Promise { const url = new URL(path, BASE); if (params) { for (const [k, v] of Object.entries(params)) { - if (v !== undefined && v !== "") url.searchParams.set(k, String(v)); + if (v === undefined || v === "" || v === false) continue; + url.searchParams.set(k, String(v)); } } const headers: Record = { Accept: "application/json" }; @@ -51,10 +59,51 @@ async function get(path: string, params?: Record; } +async function send(method: "POST" | "DELETE", path: string): Promise { + const url = new URL(path, BASE); + const headers: Record = { Accept: "application/json" }; + if (TOKEN) headers.Authorization = `Bearer ${TOKEN}`; + const res = await fetch(url, { method, headers, cache: "no-store" }); + if (!res.ok) { + let code = "http_error"; + let message = res.statusText; + try { + const body = await res.json(); + code = body?.error?.code ?? code; + message = body?.error?.message ?? message; + } catch {} + throw new ApiError(res.status, code, message); + } + return res.json() as Promise; +} + export const api = { listAgents: (q: QueryOf = {}) => get("/api/v1/agents", q), getAgent: (agentId: string) => get(`/api/v1/agents/${encodeURIComponent(agentId)}`), + acceptAgent: (agentId: string) => + send("POST", `/api/v1/agents/${encodeURIComponent(agentId)}/accept`), + blockAgent: (agentId: string) => + send("POST", `/api/v1/agents/${encodeURIComponent(agentId)}/block`), + removeAgent: (agentId: string) => + send("DELETE", `/api/v1/agents/${encodeURIComponent(agentId)}`), + acceptAllPendingAgents: (includeRotation = false) => + send( + "POST", + `/api/v1/agents/admission/accept-all${includeRotation ? "?include_rotation=true" : ""}`, + ), + blockAllPendingAgents: () => + send("POST", "/api/v1/agents/admission/block-all"), + removeAllPendingAgents: () => + send("POST", "/api/v1/agents/admission/remove-all"), + listAgentBlacklist: (q: QueryOf = {}) => + get("/api/v1/agents/admission/blacklist", q), + deleteAgentBlacklistItem: (id: string) => + send( + "DELETE", + `/api/v1/agents/admission/blacklist/${encodeURIComponent(id)}`, + ), + clearAgentBlacklist: () => send("DELETE", "/api/v1/agents/admission/blacklist"), listChecks: (q: QueryOf = {}) => get("/api/v1/checks", q), listIncidents: (q: QueryOf = {}) => @@ -63,5 +112,6 @@ export const api = { get("/api/v1/events/query", q), listOutbox: (q: QueryOf = {}) => get("/api/v1/notifiers/outbox", q), + getSystemSummary: () => get("/api/v1/system-summary"), health: () => get<{ status: string }>("/api/v1/health"), }; diff --git a/web/src/lib/loaders.ts b/web/src/lib/loaders.ts index 8d028db..33eead5 100644 --- a/web/src/lib/loaders.ts +++ b/web/src/lib/loaders.ts @@ -1,18 +1,33 @@ import "server-only"; -import { api, type Agent, type CheckState, type OutboxItem, type StoredEvent } from "@/lib/api"; -import type { components } from "@/lib/api-types"; -import type { InventoryData, OverviewData } from "@/lib/view-types"; +import { + api, + type Agent, + type AgentBlacklistItem, + type CheckState, + type OutboxItem, + type StoredEvent, +} from "@/lib/api"; +import type { components, operations } from "@/lib/api-types"; +import type { InventoryData, SystemSummaryData } from "@/lib/view-types"; type Incident = components["schemas"]["Incident"]; +// PH-004: inventory is bounded current-state. Enforce a hard cap so a runaway +// agent inventory cannot drain the UI process. +const INVENTORY_PAGE = 500; +const INVENTORY_HARD_CAP_PAGES = 20; // 10_000 rows worst case for inventory. + export async function loadAgents(): Promise { const agents: Agent[] = []; let cursor: string | undefined; + let pages = 0; do { - const page = await api.listAgents({ cursor, limit: 500 }); + const page = await api.listAgents({ cursor, limit: INVENTORY_PAGE }); agents.push(...page.items); cursor = page.next_cursor ?? undefined; + pages += 1; + if (pages >= INVENTORY_HARD_CAP_PAGES) break; } while (cursor); return agents; } @@ -20,10 +35,13 @@ export async function loadAgents(): Promise { export async function loadChecks(agentId?: string): Promise { const checks: CheckState[] = []; let cursor: string | undefined; + let pages = 0; do { - const page = await api.listChecks({ agent_id: agentId, cursor, limit: 500 }); + const page = await api.listChecks({ agent_id: agentId, cursor, limit: INVENTORY_PAGE }); checks.push(...page.items); cursor = page.next_cursor ?? undefined; + pages += 1; + if (pages >= INVENTORY_HARD_CAP_PAGES) break; } while (cursor); return checks; } @@ -33,72 +51,66 @@ export async function loadInventory(): Promise { return { agents, checks }; } +export async function loadAgentBlacklist(): Promise { + const rows: AgentBlacklistItem[] = []; + let cursor: string | undefined; + let pages = 0; + do { + const page = await api.listAgentBlacklist({ cursor, limit: INVENTORY_PAGE }); + rows.push(...page.items); + cursor = page.next_cursor ?? undefined; + pages += 1; + if (pages >= INVENTORY_HARD_CAP_PAGES) break; + } while (cursor); + return rows; +} + export async function loadEvents({ agentId, checkId, cursor, + back, limit, }: { agentId?: string; checkId?: string; cursor?: string; + back?: boolean; limit?: number; -} = {}): Promise<{ items: StoredEvent[]; next_cursor?: string | null }> { - return api.queryEvents({ agent_id: agentId, check_id: checkId, cursor, limit }); +} = {}): Promise<{ items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }> { + return api.queryEvents({ agent_id: agentId, check_id: checkId, cursor, back, limit }); } -export async function loadIncidents(state?: "open" | "resolved"): Promise { - const items: Incident[] = []; - let cursor: string | undefined; - do { - const page = await api.listIncidents({ state, cursor, limit: 500 }); - items.push(...page.items); - cursor = page.next_cursor ?? undefined; - } while (cursor); - return items; +// PH-001: incidents are unbounded history. Always page through the server. +export async function loadIncidentsPage( + q: NonNullable = {}, +): Promise<{ items: Incident[]; next_cursor?: string | null; prev_cursor?: string | null }> { + return api.listIncidents(q); +} + +// PH-002: outbox is unbounded. Always page through the server. +export async function loadOutboxPage( + q: NonNullable = {}, +): Promise<{ items: OutboxItem[]; next_cursor?: string | null; prev_cursor?: string | null }> { + return api.listOutbox(q); +} + +// First-page convenience loaders for SSR. Replace the previous drain-all paths +// (PH-001, PH-002): they returned the entire table on every initial render and +// did not scale past a few thousand rows. Pages now show one page and the UI +// owns subsequent fetches via /api/poll/* with cursors. +export async function loadIncidents( + state?: "open" | "resolved", +): Promise { + const page = await api.listIncidents({ state, limit: 100 }); + return page.items; } export async function loadOutboxAll(): Promise { - const items: OutboxItem[] = []; - let cursor: string | undefined; - do { - const page = await api.listOutbox({ cursor, limit: 500 }); - items.push(...page.items); - cursor = page.next_cursor ?? undefined; - } while (cursor); - return items; + const page = await api.listOutbox({ limit: 100 }); + return page.items; } -export async function loadOverview(): Promise { - const [agents, checks, openIncidents] = await Promise.all([ - loadAgents(), - loadChecks(), - countOpenIncidents(), - ]); - return { - agents: tally(agents), - checks: tally(checks), - openIncidents, - totalAgents: agents.length, - totalChecks: checks.length, - }; -} - -async function countOpenIncidents(): Promise { - let total = 0; - let cursor: string | undefined; - do { - const page = await api.listIncidents({ state: "open", cursor, limit: 500 }); - total += page.items.length; - cursor = page.next_cursor ?? undefined; - } while (cursor); - return total; -} - -function tally(xs: { status?: string }[]): Record { - return xs.reduce>((acc, x) => { - const k = x.status ?? "unknown"; - acc[k] = (acc[k] ?? 0) + 1; - return acc; - }, {}); +export async function loadSystemSummary(): Promise { + return api.getSystemSummary(); } diff --git a/web/src/lib/pagination.ts b/web/src/lib/pagination.ts index 2e8d38b..d56c8c8 100644 --- a/web/src/lib/pagination.ts +++ b/web/src/lib/pagination.ts @@ -21,3 +21,12 @@ export function normalizePageSize(value: unknown, fallback: PageSize = DEFAULT_P const n = Number(value); return (PAGE_SIZE_OPTIONS as readonly number[]).includes(n) ? (n as PageSize) : fallback; } + +export function normalizePageNumber(value: unknown): number { + const n = Number(value); + return Number.isInteger(n) && n > 0 ? n : 1; +} + +export function normalizeBack(value: unknown): boolean { + return value === "1" || value === "true"; +} diff --git a/web/src/lib/usePolling.ts b/web/src/lib/usePolling.ts index 16c3c58..a0fb479 100644 --- a/web/src/lib/usePolling.ts +++ b/web/src/lib/usePolling.ts @@ -3,21 +3,51 @@ import { useEffect, useState } from "react"; const DEFAULT_POLL_MS = 10_000; +const BACKOFF_AFTER_ERRORS = 2; +const BACKOFF_CAP_MS = 60_000; + +export type PollingState = { + data: T; + error: string | null; + isStale: boolean; + lastUpdatedAt: number | null; +}; export function usePolling(initialData: T, url: string, intervalMs = pollIntervalMs()) { const [data, setData] = useState(initialData); const [error, setError] = useState(null); + const [lastUpdatedAt, setLastUpdatedAt] = useState(null); useEffect(() => { if (intervalMs <= 0) return; - let timer: ReturnType | undefined; + let timer: ReturnType | undefined; let inFlight = false; let stopped = false; + let errors = 0; let controller: AbortController | undefined; + // PH-005: small random jitter prevents synchronized poll spikes when many + // dashboards refresh together; bounded exponential backoff after repeated + // failures keeps a degraded server from being hot-looped. + const nextDelay = () => { + const base = errors <= BACKOFF_AFTER_ERRORS + ? intervalMs + : Math.min(intervalMs * 2 ** (errors - BACKOFF_AFTER_ERRORS), BACKOFF_CAP_MS); + const jitter = base * (Math.random() * 0.2); + return base + jitter; + }; + + const schedule = () => { + if (stopped) return; + timer = setTimeout(refresh, nextDelay()); + }; + const refresh = () => { - if (stopped || inFlight || document.visibilityState !== "visible") return; + if (stopped || inFlight || document.visibilityState !== "visible") { + schedule(); + return; + } inFlight = true; controller = new AbortController(); @@ -32,51 +62,44 @@ export function usePolling(initialData: T, url: string, intervalMs = pollInte .then((body) => { setData(body); setError(null); + setLastUpdatedAt(Date.now()); + errors = 0; }) .catch((e: unknown) => { if (!controller?.signal.aborted) { setError(e instanceof Error ? e.message : String(e)); + errors += 1; } }) .finally(() => { inFlight = false; + schedule(); }); }; - const start = () => { - stopTimer(); - timer = setInterval(refresh, intervalMs); - }; - - const stopTimer = () => { - if (timer !== undefined) { - clearInterval(timer); - timer = undefined; - } - }; - const handleVisibilityChange = () => { if (document.visibilityState === "visible") { - start(); refresh(); } else { - stopTimer(); + if (timer !== undefined) clearTimeout(timer); controller?.abort(); } }; - if (document.visibilityState === "visible") start(); + if (document.visibilityState === "visible") refresh(); document.addEventListener("visibilitychange", handleVisibilityChange); return () => { stopped = true; - stopTimer(); + if (timer !== undefined) clearTimeout(timer); controller?.abort(); document.removeEventListener("visibilitychange", handleVisibilityChange); }; }, [intervalMs, url]); - return { data, error }; + // PH-005: surface stale data flag so pages can show "data is stale" badges. + const isStale = error !== null; + return { data, error, isStale, lastUpdatedAt } as PollingState; } export function pollIntervalMs() { diff --git a/web/src/lib/view-types.ts b/web/src/lib/view-types.ts index 1096cfa..74cb323 100644 --- a/web/src/lib/view-types.ts +++ b/web/src/lib/view-types.ts @@ -8,10 +8,13 @@ export type InventoryData = { checks: CheckState[]; }; -export type OverviewData = { - agents: Record; - checks: Record; - openIncidents: number; - totalAgents: number; - totalChecks: number; +export type SystemSummaryData = { + agents_online: number; + agents_offline: number; + checks_ok: number; + checks_warning: number; + checks_critical: number; + checks_unknown: number; + open_incidents: number; + generated_at: string; }; diff --git a/web/src/proxy.ts b/web/src/proxy.ts new file mode 100644 index 0000000..7f40dca --- /dev/null +++ b/web/src/proxy.ts @@ -0,0 +1,48 @@ +import type { NextRequest } from "next/server"; + +const ACTION_PATH = "/api/agents/action"; +const HEALTH_PATH = "/api/health"; + +export const config = { + matcher: "/((?!_next/static|_next/image|favicon.ico).*)", +}; + +export function proxy(request: NextRequest) { + const path = request.nextUrl.pathname; + if (path === HEALTH_PATH) return; + + const user = process.env.MONLET_WEB_AUTH_USERNAME; + const password = process.env.MONLET_WEB_AUTH_PASSWORD; + if (user && password) { + if (validBasicAuth(request, user, password)) return; + return new Response("authentication required", { + status: 401, + headers: { "WWW-Authenticate": 'Basic realm="monlet"' }, + }); + } + + if (process.env.MONLET_WEB_TRUST_PROXY_AUTH === "true") { + if (request.headers.get("x-forwarded-user")) return; + return new Response("proxy authentication required", { status: 401 }); + } + + if (path === ACTION_PATH) { + return Response.json( + { error: { code: "forbidden", message: "web auth required" } }, + { status: 403 }, + ); + } +} + +function validBasicAuth(request: NextRequest, user: string, password: string): boolean { + const header = request.headers.get("authorization"); + if (!header?.startsWith("Basic ")) return false; + try { + const raw = atob(header.slice("Basic ".length)); + const split = raw.indexOf(":"); + if (split < 0) return false; + return raw.slice(0, split) === user && raw.slice(split + 1) === password; + } catch { + return false; + } +} diff --git a/web/tests/mock-server.mjs b/web/tests/mock-server.mjs index 074a2aa..b3bea0a 100644 --- a/web/tests/mock-server.mjs +++ b/web/tests/mock-server.mjs @@ -6,12 +6,23 @@ const now = new Date().toISOString(); const fixtures = { "/api/v1/health": { status: "ok" }, + "/api/v1/system-summary": { + agents_online: 1, + agents_offline: 1, + checks_ok: 2, + checks_warning: 1, + checks_critical: 1, + open_incidents: 2, + generated_at: now, + }, "/api/v1/agents": { items: [ { agent_id: "agent-1", hostname: "host-1", status: "alive", + admission_state: "accepted", + key_fingerprint: "1111111111111111", last_seen_at: now, features: { push: true, metrics: false }, labels: { env: "dev", monlet_agent_version: "0.1.0" }, @@ -20,6 +31,8 @@ const fixtures = { agent_id: "agent-2", hostname: "host-2", status: "dead", + admission_state: "accepted", + key_fingerprint: "2222222222222222", last_seen_at: now, features: { push: true, metrics: false }, labels: {}, @@ -31,6 +44,8 @@ const fixtures = { agent_id: "agent-1", hostname: "host-1", status: "alive", + admission_state: "accepted", + key_fingerprint: "1111111111111111", last_seen_at: now, features: { push: true, metrics: false }, labels: { env: "dev", monlet_agent_version: "0.1.0" }, diff --git a/web/tests/smoke.spec.ts b/web/tests/smoke.spec.ts index 6105da1..a87b0fd 100644 --- a/web/tests/smoke.spec.ts +++ b/web/tests/smoke.spec.ts @@ -1,10 +1,12 @@ import { expect, test } from "@playwright/test"; -test("overview renders tallies", async ({ page }) => { +test("root redirects to agents with nav counters", async ({ page }) => { await page.goto("/"); - await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); - await expect(page.locator("body")).toContainText("Agents"); - await expect(page.locator("body")).toContainText("Checks"); + await expect(page).toHaveURL(/\/agents$/); + await expect(page.getByRole("heading", { name: "Agents" })).toBeVisible(); + await expect(page.getByRole("link", { name: /Agents.*1.*1/ })).toBeVisible(); + await expect(page.getByRole("link", { name: /Checks.*2.*1.*1/ })).toBeVisible(); + await expect(page.getByRole("link", { name: /Incidents.*2/ })).toBeVisible(); }); test("agents list links to detail", async ({ page }) => {