diff --git a/CONFIG_REFERENCE.ru.md b/CONFIG_REFERENCE.ru.md new file mode 100644 index 0000000..f03a31d --- /dev/null +++ b/CONFIG_REFERENCE.ru.md @@ -0,0 +1,206 @@ +# Monlet: краткий справочник по настройкам + +Monlet состоит из агента на хостах и центрального сервера. Агент сам генерирует локальный ключ в `state_dir/agent.key`; сервер видит новый ключ как `pending`, пока оператор не примет агента в UI. + +## Agent TOML + +Файл: обычно `/etc/monlet/agent.toml`. + +```toml +# Необязательно. Если не задано, используется hostname ОС. +# agent_id = "host-01" + +state_dir = "/var/lib/monlet-agent" + +[labels] +env = "prod" +role = "api" + +[server] +enabled = true +url = "https://monlet.example.com" +heartbeat_interval = "10s" +batch_interval = "10s" + +[metrics] +enabled = true +listen = "127.0.0.1:9465" + +[[checks]] +id = "disk_root" +name = "Root disk usage" +command = ''' +/usr/local/lib/monlet/check_disk_root.sh --warn 80 --crit 90 +''' +interval = "60s" +timeout = "10s" +notifications_enabled = true +resource_limits = { cpu_time = "5s", memory = "256MiB", open_files = 128 } +``` + +Ключи: + +- `agent_id` — стабильный ID агента; необязателен, по умолчанию hostname. +- `state_dir` — локальное состояние, ключ агента и spool. +- `[labels]` — метки инвентаря; секреты сюда не класть. +- `server.enabled` — отправлять heartbeat/events на сервер. +- `server.url` — базовый URL сервера. +- `metrics.enabled` — поднять Prometheus `/metrics`. +- `metrics.listen` — адрес `/metrics`. +- `checks[].id` — стабильный ID проверки. +- `checks[].command` — shell-команда строкой; однострочно в `"..."`, многострочно в `'''...'''`. +- `checks[].interval` — период запуска. +- `checks[].timeout` — таймаут; при таймауте результат `critical`. +- `checks[].notifications_enabled` — разрешить серверные уведомления по этой проверке. +- `checks[].resource_limits` — опциональные лимиты процесса: CPU-время, virtual memory, open files. + +Exit code: `0=ok`, `1=warning`, `2=critical`, `3+=unknown`. + +Env агента: + +- `MONLET_AGENT_MAX_CHECKS` — максимум проверок в конфиге (по умолчанию 256). Лимит ограничивает кардинальность метрики `{check_id}`. Положительное целое; превышение лимита — ошибка загрузки конфига. + +Reload: + +- `systemctl reload monlet-agent` перечитывает локальный TOML через `SIGHUP`. +- Ошибочный config не применяется, старый продолжает работать. +- Уже запущенный check при reload не убивается, а доживает до своего `timeout`. +- Для изменения `agent_id`, `state_dir`, `server.*`, `metrics.*` нужен restart. + +## Server env + +Файл: `.env` или переменные контейнера. + +```env +MONLET_AUTH_TOKEN=replace-with-random-ui-token +MONLET_DATABASE_URL=postgresql+asyncpg://monlet:monlet@postgres:5432/monlet +MONLET_LOG_LEVEL=INFO + +MONLET_HOST=0.0.0.0 +MONLET_PORT=8000 + +MONLET_BODY_LIMIT_BYTES=1048576 +MONLET_OUTPUT_MAX_BYTES=8192 +MONLET_MAX_BATCH_EVENTS=200 + +MONLET_ENABLE_DETECTOR=true +MONLET_STALE_AFTER_SEC=20 +MONLET_DEAD_AFTER_SEC=30 +MONLET_DETECTOR_TICK_SEC=5 + +MONLET_DB_POOL_SIZE=10 +MONLET_DB_MAX_OVERFLOW=10 +MONLET_DB_POOL_TIMEOUT_SEC=30 +MONLET_DB_POOL_RECYCLE_SEC=1800 + +MONLET_EVENTS_RETENTION_MONTHS=36 +MONLET_EVENTS_FUTURE_PARTITIONS=3 +MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC=3600 +MONLET_OUTBOX_RETENTION_MAX_ROWS=50000 + +MONLET_EVENT_DEDUP_RETENTION_DAYS=30 +MONLET_EVENT_DEDUP_PRUNE_BATCH_SIZE=5000 + +MONLET_MAX_PENDING_AGENTS=10000 +MONLET_PENDING_AGENT_TTL_DAYS=7 + +MONLET_ENABLE_NOTIFIER_WORKER=true +MONLET_NOTIFIER_TICK_SEC=5 +MONLET_NOTIFIER_BATCH_SIZE=20 +MONLET_NOTIFIER_CONCURRENCY=4 +MONLET_NOTIFIER_MAX_ATTEMPTS=8 +MONLET_NOTIFIER_LEASE_SEC=300 +MONLET_NOTIFIER_HTTP_TIMEOUT_SEC=10 +MONLET_NOTIFIER_INCLUDE_OUTPUT=true +MONLET_NOTIFIER_MAX_OUTPUT_BYTES=1024 +MONLET_NOTIFICATION_TIME_ZONE=UTC + +MONLET_NOTIFIER_DEBUG_ENABLED=true +MONLET_NOTIFIER_TELEGRAM_ENABLED=false +MONLET_NOTIFIER_WEBHOOK_ENABLED=false +MONLET_NOTIFIER_ALERTMANAGER_ENABLED=false +``` + +Основное: + +- `MONLET_AUTH_TOKEN` — Bearer-токен(ы) для UI/operator API, не для агентов. Список через запятую/пробел для ротации. +- `MONLET_DATABASE_URL` — PostgreSQL DSN (`postgresql+asyncpg://...`). +- `MONLET_LOG_LEVEL` — уровень логирования. +- `MONLET_HOST` / `MONLET_PORT` — адрес и порт HTTP-сервера (по умолчанию `0.0.0.0:8000`). + +Лимиты приёма: + +- `MONLET_BODY_LIMIT_BYTES` — максимальный размер тела запроса (по умолчанию 1 MiB = 1048576). +- `MONLET_OUTPUT_MAX_BYTES` — лимит хранимого `output` события (по умолчанию 8192). +- `MONLET_MAX_BATCH_EVENTS` — максимум событий в одном batch (по умолчанию 200). + +Liveness-детектор: + +- `MONLET_ENABLE_DETECTOR` — включить фоновый пересчёт stale/dead (по умолчанию `true`). +- `MONLET_STALE_AFTER_SEC` — через сколько секунд без heartbeat агент становится `stale`. +- `MONLET_DEAD_AFTER_SEC` — через сколько секунд без heartbeat агент становится `dead`. +- `MONLET_DETECTOR_TICK_SEC` — как часто сервер пересчитывает liveness. + +Пул БД (тюнинг под нагрузку): + +- `MONLET_DB_POOL_SIZE` — постоянных соединений в пуле (по умолчанию 10). +- `MONLET_DB_MAX_OVERFLOW` — дополнительных соединений сверх пула (по умолчанию 10). +- `MONLET_DB_POOL_TIMEOUT_SEC` — ожидание свободного соединения (по умолчанию 30). +- `MONLET_DB_POOL_RECYCLE_SEC` — пересоздание соединения по возрасту (по умолчанию 1800). + +Хранение и обслуживание: + +- `MONLET_EVENTS_RETENTION_MONTHS` — хранение event-партиций (по умолчанию 36). +- `MONLET_EVENTS_FUTURE_PARTITIONS` — сколько будущих месячных партиций создавать заранее (по умолчанию 3). +- `MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC` — период обслуживания партиций и prune (по умолчанию 3600). +- `MONLET_OUTBOX_RETENTION_MAX_ROWS` — верхняя граница строк в outbox; лишние обрезаются (по умолчанию 50000). +- `MONLET_EVENT_DEDUP_RETENTION_DAYS` — хранение записей идемпотентности `event_id` (по умолчанию 30). +- `MONLET_EVENT_DEDUP_PRUNE_BATCH_SIZE` — размер пакета при очистке dedup-записей (по умолчанию 5000). +- `MONLET_MAX_PENDING_AGENTS` — лимит новых непринятых агентов (по умолчанию 10000). +- `MONLET_PENDING_AGENT_TTL_DAYS` — очистка старых pending-ключей (по умолчанию 7). + +Notifier worker: + +- `MONLET_ENABLE_NOTIFIER_WORKER` — включить worker доставки уведомлений (по умолчанию `true`). +- `MONLET_NOTIFIER_TICK_SEC` — период опроса outbox (по умолчанию 5). +- `MONLET_NOTIFIER_BATCH_SIZE` — сколько outbox-записей worker забирает за тик (по умолчанию 20). +- `MONLET_NOTIFIER_CONCURRENCY` — параллелизм доставки внутри тика (по умолчанию 4). +- `MONLET_NOTIFIER_MAX_ATTEMPTS` — максимум попыток доставки до перевода в permanent failure (по умолчанию 8). +- `MONLET_NOTIFIER_LEASE_SEC` — аренда записи на время доставки, чтобы её не взял другой worker (по умолчанию 300). +- `MONLET_NOTIFIER_HTTP_TIMEOUT_SEC` — таймаут HTTP-доставки (по умолчанию 10). +- `MONLET_NOTIFIER_INCLUDE_OUTPUT` — включать ли текст `output` в уведомления (по умолчанию `true`). +- `MONLET_NOTIFIER_MAX_OUTPUT_BYTES` — обрезка `output` в уведомлении после редакции (по умолчанию 1024). +- `MONLET_NOTIFICATION_TIME_ZONE` — таймзона человекочитаемых дат в уведомлениях. + +Notifier каналы: + +- Telegram: `MONLET_NOTIFIER_TELEGRAM_ENABLED=true`, `MONLET_NOTIFIER_TELEGRAM_TOKEN`, `MONLET_NOTIFIER_TELEGRAM_CHAT_ID`. +- Webhook: `MONLET_NOTIFIER_WEBHOOK_ENABLED=true`, `MONLET_NOTIFIER_WEBHOOK_URL`, опционально `MONLET_NOTIFIER_WEBHOOK_TOKEN`. +- Alertmanager: `MONLET_NOTIFIER_ALERTMANAGER_ENABLED=true`, `MONLET_NOTIFIER_ALERTMANAGER_URL`. +- Debug/log: `MONLET_NOTIFIER_DEBUG_ENABLED=true`. + +## Web env + +Веб-UI читает auth-настройки из env. Без них read-only страницы доступны локально, но admission-мутации возвращают `403`. + +```env +MONLET_WEB_AUTH_USERNAME=admin +MONLET_WEB_AUTH_PASSWORD=replace-with-strong-password +MONLET_WEB_SESSION_SECRET=replace-with-random-32-bytes-min +MONLET_WEB_SESSION_TTL_SEC=604800 +# или вместо локального логина — доверие reverse-proxy: +# MONLET_WEB_TRUST_PROXY_AUTH=true +``` + +- `MONLET_WEB_AUTH_USERNAME` / `MONLET_WEB_AUTH_PASSWORD` — учётные данные локального cookie-логина. +- `MONLET_WEB_SESSION_SECRET` — секрет подписи сессии, минимум 32 байта; короче — UI считает auth неправильно настроенным. +- `MONLET_WEB_SESSION_TTL_SEC` — срок жизни сессии (по умолчанию 604800 = 7 дней). +- `MONLET_WEB_TRUST_PROXY_AUTH` — `true` включает доверие заголовку `X-Forwarded-User` от reverse-proxy. Несовместимо с локальным логином: одновременная настройка обоих режимов — ошибка (UI отвечает `500`). + +## Минимальная схема + +1. Сервер стартует с PostgreSQL и `MONLET_AUTH_TOKEN`. +2. Агент стартует с TOML, создаёт `state_dir/agent.key` и шлёт heartbeat. +3. Новый агент появляется в UI как `pending`. +4. Оператор принимает агента. +5. После accept сервер начинает учитывать events, checks, incidents и notifications. diff --git a/agent/README.md b/agent/README.md index a7a3f83..4230f12 100644 --- a/agent/README.md +++ b/agent/README.md @@ -26,6 +26,7 @@ See `config.example.toml`. Required fields: top-level `state_dir`, at least one | `metrics.listen` | `127.0.0.1:9465` | low-cardinality only | | `checks[].command` | required | Shell string executed as `/bin/sh -c`. Both forms accepted: `command = "check_disk --warn 80 --crit 90"` and triple-quoted `command = '''...'''`. Argv arrays are rejected. See [Command security contract](#command-security-contract). | | `checks[].notifications_enabled` | `true` | Set `false` for checks that must not create server notifications. | +| `checks[].resource_limits` | `{}` | Optional per-check best-effort `ulimit` values: `cpu_time`, `memory`, `open_files`. | Examples — one-line and multi-line forms: @@ -44,6 +45,7 @@ set -eu ''' interval = "60s" timeout = "10s" +resource_limits = { cpu_time = "5s", memory = "256MiB", open_files = 128 } ``` ## Command security contract @@ -67,9 +69,25 @@ executes it as `/bin/sh -c ` on the host where the agent runs. validation to keep secrets out of heartbeats. - Each run uses `exec.CommandContext` with `Setpgid`. On timeout the whole process group is killed and the check result becomes `critical`. +- Optional `resource_limits` are applied inside the child shell before the + configured command runs. `cpu_time` uses `ulimit -t` and maps SIGXCPU to + `critical`; `memory` uses virtual memory/address-space `ulimit -v`; `open_files` + uses `ulimit -n`. These limits are best-effort and are not a cgroup/RSS limit. - Combined stdout+stderr is truncated by UTF-8 byte length to 8 KiB with marker `...[truncated N bytes]` before being persisted, exposed, or notified. +## Config reload + +Send `SIGHUP` or run `systemctl reload monlet-agent` to reload the local TOML file. + +- Reload first validates the full new config. On error, the old config keeps running. +- Hot-reloadable: labels and checks, including `command`, `interval`, `timeout`, + `notifications_enabled`, `dedupe_key`, and `resource_limits`. +- Restart required: `agent_id`, `state_dir`, `server.*`, and `metrics.*`. +- A check already running during reload is allowed to finish under its old config. + Removed or changed checks do not start new old-config runs; changed checks start + with the new config after the in-flight run finishes. + ## Behaviour - Per-check scheduler with no-overlap: a tick is skipped if the previous run is still in flight (`monlet_agent_check_skipped_total`). @@ -79,7 +97,9 @@ executes it as `/bin/sh -c ` on the host where the agent runs. - `output` (stdout+stderr) is truncated by **UTF-8 byte length** to 8 KiB with marker `...[truncated N bytes]`. - `event_id` is UUIDv7 generated on check completion and preserved across spool replay (ADR-0006 idempotency). - Spool: per-event JSON files in `state_dir/spool/`, FIFO, limits 10 000 events / 50 MiB, drop oldest on overflow (ADR-0005). Drops are counted in `monlet_agent_events_dropped_total{reason}` with bounded reasons; one warning is logged per ~30s drop burst, not per event. +- Event flushes are split before POST so each JSON request body stays within the 1 MiB server limit. - Retry backoff for heartbeat and events: 1s → 30s, exp ×2, jitter ±20%. 4xx are dropped (not retried) to avoid hot loop; dropped batches are counted as `reason=send_non_retryable` and logged once per batch. +- On shutdown the agent asks workers to stop cleanly; if they do not drain within 30s, the process exits non-zero so the service manager can restart or mark failure. - Bearer auth header on both endpoints uses the local `state_dir/agent.key` value (ADR-0007). - Heartbeats include configured labels plus reserved `monlet_agent_version=`. - Heartbeats include explicit feature flags: `push` from `server.enabled`, `metrics` from `metrics.enabled`. @@ -97,6 +117,11 @@ Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `statu - `monlet_agent_check_last_success_timestamp_seconds{check_id}` - `monlet_agent_check_interval_seconds{check_id}` - `monlet_agent_checks_configured` +- `monlet_agent_config_load_success` +- `monlet_agent_config_reload_total{result}` +- `monlet_agent_config_last_reload_success_timestamp_seconds` +- `monlet_agent_resource_limit_apply_failures_total{check_id,resource}` +- `monlet_agent_events_channel_full_total` - `monlet_agent_spool_events`, `monlet_agent_spool_bytes` - `monlet_agent_send_attempts_total{kind}`, `monlet_agent_send_failures_total{kind}` (`kind` ∈ `heartbeat`, `events`) - `monlet_agent_events_accepted_total`, `monlet_agent_events_deduplicated_total` diff --git a/agent/cmd/monlet-agent/main.go b/agent/cmd/monlet-agent/main.go index 657acaa..181c8c5 100644 --- a/agent/cmd/monlet-agent/main.go +++ b/agent/cmd/monlet-agent/main.go @@ -8,6 +8,7 @@ import ( "os/signal" "path/filepath" "syscall" + "time" "github.com/monlet/agent/internal/agentkey" "github.com/monlet/agent/internal/app" @@ -20,6 +21,13 @@ import ( var version = "0.1.0" +const shutdownTimeout = 30 * time.Second + +type reloadTarget interface { + RecordReloadFailure() + Reload(*config.Config, string) error +} + func main() { cfgPath := flag.String("config", "config.toml", "path to TOML config") flag.Parse() @@ -65,6 +73,12 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() + startShutdownWatchdog(ctx, shutdownTimeout, log, os.Exit) + + hup := make(chan os.Signal, 1) + signal.Notify(hup, syscall.SIGHUP) + defer signal.Stop(hup) + go runReloadLoop(ctx, *cfgPath, hup, a, log) log.Info("monlet-agent starting", "agent_id", agentID, "version", version, "checks", len(cfg.Checks)) if err := a.Run(ctx); err != nil { @@ -73,3 +87,45 @@ func main() { } log.Info("monlet-agent stopped") } + +func startShutdownWatchdog(ctx context.Context, timeout time.Duration, log *slog.Logger, exit func(int)) { + go func() { + <-ctx.Done() + t := time.NewTimer(timeout) + defer t.Stop() + <-t.C + log.Error("shutdown timeout exceeded", "timeout", timeout.String()) + exit(1) + }() +} + +func runReloadLoop(ctx context.Context, cfgPath string, hup <-chan os.Signal, target reloadTarget, log *slog.Logger) { + for { + select { + case <-ctx.Done(): + return + case <-hup: + reloadOnce(cfgPath, target, log) + } + } +} + +func reloadOnce(cfgPath string, target reloadTarget, log *slog.Logger) { + nextCfg, err := config.Load(cfgPath) + if err != nil { + target.RecordReloadFailure() + log.Warn("reload config failed", "err", err) + return + } + nextAgentID, err := ids.ResolveAgentID(nextCfg.AgentID, nextCfg.Hostname) + if err != nil { + target.RecordReloadFailure() + log.Warn("reload agent_id failed", "err", err) + return + } + if err := target.Reload(nextCfg, nextAgentID); err != nil { + log.Warn("reload rejected", "err", err) + return + } + log.Info("reload complete", "checks", len(nextCfg.Checks)) +} diff --git a/agent/cmd/monlet-agent/main_test.go b/agent/cmd/monlet-agent/main_test.go new file mode 100644 index 0000000..bb70619 --- /dev/null +++ b/agent/cmd/monlet-agent/main_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/monlet/agent/internal/config" +) + +type reloadCall struct { + cfg *config.Config + agentID string +} + +type fakeReloadTarget struct { + reloads chan reloadCall + failures chan struct{} +} + +func (f *fakeReloadTarget) RecordReloadFailure() { + f.failures <- struct{}{} +} + +func (f *fakeReloadTarget) Reload(cfg *config.Config, agentID string) error { + f.reloads <- reloadCall{cfg: cfg, agentID: agentID} + return nil +} + +func TestRunReloadLoopHandlesSIGHUP(t *testing.T) { + cfgPath := writeTestConfig(t) + target := &fakeReloadTarget{ + reloads: make(chan reloadCall, 1), + failures: make(chan struct{}, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hup := make(chan os.Signal, 1) + go runReloadLoop(ctx, cfgPath, hup, target, discardLogger()) + + hup <- syscall.SIGHUP + select { + case call := <-target.reloads: + if call.agentID != "agent-test" || len(call.cfg.Checks) != 1 { + t.Fatalf("unexpected reload call: agent_id=%q cfg=%+v", call.agentID, call.cfg) + } + case <-time.After(time.Second): + t.Fatal("reload was not called") + } +} + +func TestRunReloadLoopRecordsLoadFailure(t *testing.T) { + target := &fakeReloadTarget{ + reloads: make(chan reloadCall, 1), + failures: make(chan struct{}, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hup := make(chan os.Signal, 1) + go runReloadLoop(ctx, filepath.Join(t.TempDir(), "missing.toml"), hup, target, discardLogger()) + + hup <- syscall.SIGHUP + select { + case <-target.failures: + case <-time.After(time.Second): + t.Fatal("reload failure was not recorded") + } +} + +func TestShutdownWatchdogExitsAfterTimeout(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + exits := make(chan int, 1) + startShutdownWatchdog(ctx, time.Millisecond, discardLogger(), func(code int) { + exits <- code + }) + cancel() + select { + case code := <-exits: + if code != 1 { + t.Fatalf("exit code: %d", code) + } + case <-time.After(time.Second): + t.Fatal("watchdog did not exit") + } +} + +func writeTestConfig(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "agent.toml") + body := fmt.Sprintf(` +agent_id = "agent-test" +state_dir = %q + +[server] +enabled = true +url = "http://localhost" + +[[checks]] +id = "c1" +command = "true" +interval = "10s" +timeout = "5s" +`, t.TempDir()) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} diff --git a/agent/config.example.toml b/agent/config.example.toml index 262815d..17342b8 100644 --- a/agent/config.example.toml +++ b/agent/config.example.toml @@ -24,6 +24,7 @@ command = ''' ''' interval = "60s" timeout = "10s" +resource_limits = { cpu_time = "5s", memory = "256MiB", open_files = 128 } [[checks]] id = "uptime" diff --git a/agent/internal/app/app.go b/agent/internal/app/app.go index 6535c72..1cb5893 100644 --- a/agent/internal/app/app.go +++ b/agent/internal/app/app.go @@ -3,6 +3,7 @@ package app import ( "context" "errors" + "fmt" "log/slog" "sync" "time" @@ -24,6 +25,9 @@ type App struct { Log *slog.Logger lastDropLog time.Time + cfgMu sync.RWMutex + schedulerMu sync.RWMutex + scheduler *scheduler.Manager } const ( @@ -36,47 +40,61 @@ const ( func (a *App) Run(ctx context.Context) error { events := make(chan scheduler.Event, 256) + cfg := a.config() // Account drops accumulated during spool Open (e.g. shrunk limits on restart). a.recordSpoolDrops(a.Spool.Drops()) - a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks))) - a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(a.Cfg.PushesToServer()), boolLabel(a.Cfg.ExposesMetrics())).Set(1) - for _, c := range a.Cfg.Checks { + a.Metrics.ConfigLoadSuccess.Set(1) + a.Metrics.ChecksConfigured.Set(float64(len(cfg.Checks))) + a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(cfg.PushesToServer()), boolLabel(cfg.ExposesMetrics())).Set(1) + for _, c := range cfg.Checks { a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds()) } var wg sync.WaitGroup - if a.Cfg.ExposesMetrics() { + if cfg.ExposesMetrics() { wg.Add(1) go func() { defer wg.Done() - if err := a.Metrics.Serve(ctx, a.Cfg.Metrics.Listen); err != nil { + if err := a.Metrics.Serve(ctx, cfg.Metrics.Listen); err != nil { a.Log.Error("metrics server", "err", err) } }() } + mgr := scheduler.NewManager(ctx, a.AgentID, events, scheduler.Hooks{ + OnSkipped: func(id string) { a.Metrics.CheckSkipped.WithLabelValues(id).Inc() }, + OnResult: func(ev scheduler.Event) { + a.Metrics.CheckRuns.WithLabelValues(ev.CheckID, ev.Status).Inc() + a.Metrics.CheckDuration.WithLabelValues(ev.CheckID).Observe(float64(ev.DurationMs) / 1000.0) + a.Metrics.CheckStatus.WithLabelValues(ev.CheckID).Set(metrics.StatusCode(ev.Status)) + a.Metrics.CheckExitCode.WithLabelValues(ev.CheckID).Set(float64(ev.ExitCode)) + ts := float64(ev.ObservedAt.Unix()) + a.Metrics.CheckLastRun.WithLabelValues(ev.CheckID).Set(ts) + if ev.Status == "ok" { + a.Metrics.CheckLastSuccess.WithLabelValues(ev.CheckID).Set(ts) + } + }, + OnEventChannelFull: func() { a.Metrics.EventsChannelFull.Inc() }, + OnResourceLimitFailure: func(id string, resource string) { + a.Metrics.ResourceLimitApplyFailures.WithLabelValues(id, resource).Inc() + }, + OnStopped: func(id string) { a.Metrics.DeleteCheck(id) }, + }) + mgr.Update(cfg.Checks) + a.schedulerMu.Lock() + a.scheduler = mgr + a.schedulerMu.Unlock() wg.Add(1) go func() { defer wg.Done() - scheduler.Run(ctx, a.AgentID, a.Cfg.Checks, events, scheduler.Hooks{ - OnSkipped: func(id string) { a.Metrics.CheckSkipped.WithLabelValues(id).Inc() }, - OnResult: func(ev scheduler.Event) { - a.Metrics.CheckRuns.WithLabelValues(ev.CheckID, ev.Status).Inc() - a.Metrics.CheckDuration.WithLabelValues(ev.CheckID).Observe(float64(ev.DurationMs) / 1000.0) - a.Metrics.CheckStatus.WithLabelValues(ev.CheckID).Set(metrics.StatusCode(ev.Status)) - a.Metrics.CheckExitCode.WithLabelValues(ev.CheckID).Set(float64(ev.ExitCode)) - ts := float64(ev.ObservedAt.Unix()) - a.Metrics.CheckLastRun.WithLabelValues(ev.CheckID).Set(ts) - if ev.Status == "ok" { - a.Metrics.CheckLastSuccess.WithLabelValues(ev.CheckID).Set(ts) - } - }, - }) + <-ctx.Done() + mgr.Stop() + mgr.Wait() }() - if a.Cfg.PushesToServer() { + if cfg.PushesToServer() { wg.Add(1) go func() { defer wg.Done() @@ -128,18 +146,19 @@ func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) { } func (a *App) heartbeatLoop(ctx context.Context) { - interval := a.Cfg.Server.HeartbeatInterval.Duration attempt := 0 for { + cfg := a.config() + interval := cfg.Server.HeartbeatInterval.Duration hb := client.HeartbeatRequest{ AgentID: a.AgentID, ObservedAt: time.Now().UTC(), - Hostname: a.Cfg.Hostname, + Hostname: cfg.Hostname, Features: client.AgentFeatures{ - Push: a.Cfg.PushesToServer(), - Metrics: a.Cfg.ExposesMetrics(), + Push: cfg.PushesToServer(), + Metrics: cfg.ExposesMetrics(), }, - Labels: a.Cfg.HeartbeatLabels(a.Version), + Labels: cfg.HeartbeatLabels(a.Version), } a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc() err := a.Client.SendHeartbeat(ctx, hb) @@ -167,9 +186,14 @@ func (a *App) heartbeatLoop(ctx context.Context) { } func (a *App) sendLoop(ctx context.Context) { - interval := a.Cfg.Server.BatchInterval.Duration attempt := 0 for { + select { + case <-ctx.Done(): + return + default: + } + interval := a.config().Server.BatchInterval.Duration items, err := a.Spool.Peek(client.MaxBatchEvents) if err != nil { a.Log.Error("spool peek", "err", err) @@ -217,6 +241,74 @@ func (a *App) sendLoop(ctx context.Context) { } } +// Reload applies a validated hot-reload config if immutable fields match. +func (a *App) Reload(newCfg *config.Config, newAgentID string) error { + oldCfg := a.config() + if err := validateReload(oldCfg, newCfg, a.AgentID, newAgentID); err != nil { + a.RecordReloadFailure() + return err + } + a.cfgMu.Lock() + a.Cfg = newCfg + a.cfgMu.Unlock() + + a.Metrics.ConfigLoadSuccess.Set(1) + a.Metrics.ConfigReloadTotal.WithLabelValues("success").Inc() + a.Metrics.ConfigLastReloadSuccess.Set(float64(time.Now().Unix())) + a.Metrics.ChecksConfigured.Set(float64(len(newCfg.Checks))) + for _, c := range newCfg.Checks { + a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds()) + } + + a.schedulerMu.RLock() + mgr := a.scheduler + a.schedulerMu.RUnlock() + if mgr != nil { + mgr.Update(newCfg.Checks) + } + return nil +} + +// RecordReloadFailure updates config reload metrics after a failed reload attempt. +func (a *App) RecordReloadFailure() { + a.Metrics.ConfigLoadSuccess.Set(0) + a.Metrics.ConfigReloadTotal.WithLabelValues("failure").Inc() +} + +func (a *App) config() *config.Config { + a.cfgMu.RLock() + defer a.cfgMu.RUnlock() + return a.Cfg +} + +func validateReload(oldCfg, newCfg *config.Config, oldAgentID, newAgentID string) error { + if oldAgentID != newAgentID { + return fmt.Errorf("agent_id change requires restart") + } + if oldCfg.StateDir != newCfg.StateDir { + return fmt.Errorf("state_dir change requires restart") + } + if oldCfg.PushesToServer() != newCfg.PushesToServer() { + return fmt.Errorf("server.enabled change requires restart") + } + if oldCfg.Server.URL != newCfg.Server.URL { + return fmt.Errorf("server.url change requires restart") + } + if oldCfg.Server.HeartbeatInterval.Duration != newCfg.Server.HeartbeatInterval.Duration { + return fmt.Errorf("server.heartbeat_interval change requires restart") + } + if oldCfg.Server.BatchInterval.Duration != newCfg.Server.BatchInterval.Duration { + return fmt.Errorf("server.batch_interval change requires restart") + } + if oldCfg.ExposesMetrics() != newCfg.ExposesMetrics() { + return fmt.Errorf("metrics.enabled change requires restart") + } + if oldCfg.Metrics.Listen != newCfg.Metrics.Listen { + return fmt.Errorf("metrics.listen change requires restart") + } + return nil +} + // sleep returns true if context was cancelled while waiting. func sleep(ctx context.Context, d time.Duration) bool { t := time.NewTimer(d) diff --git a/agent/internal/app/app_test.go b/agent/internal/app/app_test.go index cd70e30..31b3297 100644 --- a/agent/internal/app/app_test.go +++ b/agent/internal/app/app_test.go @@ -182,6 +182,81 @@ func TestEndToEndPushAndReplay(t *testing.T) { } } +func TestReloadRejectsImmutableChangesAndKeepsOldConfig(t *testing.T) { + oldCfg := &config.Config{ + AgentID: "a1", + Hostname: "h", + StateDir: "/tmp/old", + Server: config.ServerConfig{ + Enabled: boolPtr(true), + URL: "http://old", + HeartbeatInterval: config.Duration{Duration: 10 * time.Second}, + BatchInterval: config.Duration{Duration: 10 * time.Second}, + }, + Metrics: config.MetricsConfig{Enabled: false}, + Checks: []config.CheckConfig{{ + ID: "c1", + Command: "true", + Interval: config.Duration{Duration: time.Second}, + Timeout: config.Duration{Duration: time.Second}, + }}, + } + newCfg := *oldCfg + newCfg.Server.URL = "http://new" + a := &App{ + Cfg: oldCfg, + AgentID: "a1", + Metrics: metrics.New(), + } + if err := a.Reload(&newCfg, "a1"); err == nil { + t.Fatal("expected immutable reload rejection") + } + if a.config().Server.URL != "http://old" { + t.Fatalf("old config was replaced: %+v", a.config().Server) + } +} + +func TestReloadAcceptsLabelsAndChecks(t *testing.T) { + oldCfg := &config.Config{ + AgentID: "a1", + Hostname: "h", + StateDir: "/tmp/old", + Labels: map[string]string{"env": "old"}, + Server: config.ServerConfig{ + Enabled: boolPtr(true), + URL: "http://server", + HeartbeatInterval: config.Duration{Duration: 10 * time.Second}, + BatchInterval: config.Duration{Duration: 10 * time.Second}, + }, + Metrics: config.MetricsConfig{Enabled: false}, + Checks: []config.CheckConfig{{ + ID: "c1", + Command: "true", + Interval: config.Duration{Duration: time.Second}, + Timeout: config.Duration{Duration: time.Second}, + }}, + } + newCfg := *oldCfg + newCfg.Labels = map[string]string{"env": "new"} + newCfg.Checks = []config.CheckConfig{{ + ID: "c2", + Command: "true", + Interval: config.Duration{Duration: 2 * time.Second}, + Timeout: config.Duration{Duration: time.Second}, + }} + a := &App{ + Cfg: oldCfg, + AgentID: "a1", + Metrics: metrics.New(), + } + if err := a.Reload(&newCfg, "a1"); err != nil { + t.Fatal(err) + } + if a.config().Labels["env"] != "new" || len(a.config().Checks) != 1 || a.config().Checks[0].ID != "c2" { + t.Fatalf("reload did not apply: %+v", a.config()) + } +} + func boolPtr(v bool) *bool { return &v } diff --git a/agent/internal/client/client.go b/agent/internal/client/client.go index 518ddec..8858d03 100644 --- a/agent/internal/client/client.go +++ b/agent/internal/client/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -15,6 +16,7 @@ import ( const ( MaxBatchEvents = 200 + MaxBatchBytes = 1 << 20 ) type Client struct { @@ -68,18 +70,81 @@ func (c *Client) SendEvents(ctx context.Context, events []scheduler.Event) (*Eve return &EventBatchResponse{}, nil } if len(events) > MaxBatchEvents { - return nil, fmt.Errorf("batch exceeds %d events", MaxBatchEvents) + return nil, nonRetryablef("batch exceeds %d events", MaxBatchEvents) } wireEvents := make([]scheduler.Event, len(events)) for i, ev := range events { wireEvents[i] = ev.WireEvent() } req := EventBatchRequest{AgentID: c.agentID, Events: wireEvents} - var resp EventBatchResponse - if err := c.post(ctx, "/api/v1/events", req, &resp); err != nil { + batches, err := c.splitEventBatches(req.Events) + if err != nil { return nil, err } - return &resp, nil + var total EventBatchResponse + for _, batch := range batches { + var resp EventBatchResponse + if err := c.post(ctx, "/api/v1/events", EventBatchRequest{AgentID: c.agentID, Events: batch}, &resp); err != nil { + return nil, err + } + total.Accepted += resp.Accepted + total.Deduplicated += resp.Deduplicated + } + return &total, nil +} + +func (c *Client) splitEventBatches(events []scheduler.Event) ([][]scheduler.Event, error) { + overhead, err := c.eventBatchOverhead() + if err != nil { + return nil, err + } + batches := make([][]scheduler.Event, 0, 1) + current := make([]scheduler.Event, 0, len(events)) + currentSize := overhead + for _, ev := range events { + eventSize, err := jsonSize(ev) + if err != nil { + return nil, err + } + nextSize := currentSize + eventSize + if len(current) > 0 { + nextSize++ + } + if nextSize <= MaxBatchBytes { + current = append(current, ev) + currentSize = nextSize + continue + } + if len(current) == 0 { + return nil, nonRetryablef("single event batch exceeds %d bytes", MaxBatchBytes) + } + batches = append(batches, current) + current = []scheduler.Event{ev} + currentSize = overhead + eventSize + if currentSize > MaxBatchBytes { + return nil, nonRetryablef("single event batch exceeds %d bytes", MaxBatchBytes) + } + } + if len(current) > 0 { + batches = append(batches, current) + } + return batches, nil +} + +func (c *Client) eventBatchOverhead() (int, error) { + size, err := jsonSize(EventBatchRequest{AgentID: c.agentID, Events: []scheduler.Event{}}) + if err != nil { + return 0, err + } + return size, nil +} + +func jsonSize(v any) (int, error) { + b, err := json.Marshal(v) + if err != nil { + return 0, err + } + return len(b), nil } func (c *Client) post(ctx context.Context, path string, body any, out any) error { @@ -115,30 +180,28 @@ type HTTPError struct { func (e *HTTPError) Error() string { return fmt.Sprintf("http %d: %s", e.Status, e.Body) } +type NonRetryableError struct{ Err error } + +func (e *NonRetryableError) Error() string { return e.Err.Error() } +func (e *NonRetryableError) Unwrap() error { return e.Err } + +func nonRetryablef(format string, args ...any) error { + return &NonRetryableError{Err: fmt.Errorf(format, args...)} +} + // Retryable reports whether the error should trigger a retry (network errors // and 5xx are retryable; 4xx are dropped to avoid hot-loop on bad payload). func Retryable(err error) bool { if err == nil { return false } + var ne *NonRetryableError + if errors.As(err, &ne) { + return false + } var he *HTTPError - if asErr(err, &he) { + if errors.As(err, &he) { return he.Status >= 500 } return true } - -func asErr(err error, target **HTTPError) bool { - for e := err; e != nil; { - if h, ok := e.(*HTTPError); ok { - *target = h - return true - } - u, ok := e.(interface{ Unwrap() error }) - if !ok { - return false - } - e = u.Unwrap() - } - return false -} diff --git a/agent/internal/client/client_test.go b/agent/internal/client/client_test.go index 4a2fd77..9480c6a 100644 --- a/agent/internal/client/client_test.go +++ b/agent/internal/client/client_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -57,6 +58,59 @@ func TestSendEventsBatch(t *testing.T) { } } +func TestSendEventsSplitsByJSONSize(t *testing.T) { + var ( + requests int + total int + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + http.Error(w, "read body", http.StatusInternalServerError) + return + } + if len(body) > MaxBatchBytes { + t.Errorf("body size %d exceeds %d", len(body), MaxBatchBytes) + http.Error(w, "too large", http.StatusRequestEntityTooLarge) + return + } + var got EventBatchRequest + if err := json.Unmarshal(body, &got); err != nil { + t.Error(err) + http.Error(w, "bad json", http.StatusBadRequest) + return + } + requests++ + total += len(got.Events) + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(EventBatchResponse{Accepted: len(got.Events)}) + })) + defer srv.Close() + + events := make([]scheduler.Event, MaxBatchEvents) + for i := range events { + events[i] = scheduler.Event{ + EventID: "event-" + time.Unix(int64(i), 0).UTC().Format("20060102150405"), + CheckID: "check", + ObservedAt: time.Unix(int64(i), 0).UTC(), + Status: "ok", + Output: strings.Repeat("x", 8192), + } + } + c := New(srv.URL, "tok", "a1") + resp, err := c.SendEvents(context.Background(), events) + if err != nil { + t.Fatal(err) + } + if requests < 2 { + t.Fatalf("expected split batch, got %d request", requests) + } + if total != MaxBatchEvents || resp.Accepted != MaxBatchEvents { + t.Fatalf("total=%d resp=%+v", total, resp) + } +} + func TestSendEventsTooBig(t *testing.T) { c := New("http://x", "t", "a1") big := make([]scheduler.Event, MaxBatchEvents+1) @@ -64,6 +118,24 @@ func TestSendEventsTooBig(t *testing.T) { if err == nil { t.Fatal("expected error") } + if Retryable(err) { + t.Fatal("oversized batch should not retry") + } +} + +func TestSendEventsSingleEventTooLargeIsNotRetryable(t *testing.T) { + c := New("http://x", "t", "a1") + _, err := c.SendEvents(context.Background(), []scheduler.Event{{ + EventID: "x", + CheckID: strings.Repeat("c", MaxBatchBytes), + Status: "ok", + }}) + if err == nil { + t.Fatal("expected error") + } + if Retryable(err) { + t.Fatal("oversized single event should not retry") + } } func TestRetryableClassification(t *testing.T) { diff --git a/agent/internal/config/config.go b/agent/internal/config/config.go index 8c884a8..8f47e74 100644 --- a/agent/internal/config/config.go +++ b/agent/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "math" "os" "regexp" "strconv" @@ -53,17 +54,28 @@ type MetricsConfig struct { } type CheckConfig struct { - ID string `toml:"id"` - Name string `toml:"name"` - Command string `toml:"command"` - Interval Duration `toml:"interval"` - Timeout Duration `toml:"timeout"` - NotificationsEnabled *bool `toml:"notifications_enabled"` - DedupeKey string `toml:"dedupe_key"` + ID string `toml:"id"` + Name string `toml:"name"` + Command string `toml:"command"` + Interval Duration `toml:"interval"` + Timeout Duration `toml:"timeout"` + NotificationsEnabled *bool `toml:"notifications_enabled"` + DedupeKey string `toml:"dedupe_key"` + ResourceLimits ResourceLimits `toml:"resource_limits"` } type Duration struct{ time.Duration } +// ByteSize stores a parsed TOML byte size. +type ByteSize struct{ Bytes int64 } + +// ResourceLimits defines optional per-check OS resource limits. +type ResourceLimits struct { + CPUTime Duration `toml:"cpu_time"` + Memory ByteSize `toml:"memory"` + OpenFiles int `toml:"open_files"` +} + func (d *Duration) UnmarshalText(b []byte) error { v, err := time.ParseDuration(string(b)) if err != nil { @@ -73,6 +85,43 @@ func (d *Duration) UnmarshalText(b []byte) error { return nil } +func (s *ByteSize) UnmarshalText(b []byte) error { + raw := strings.TrimSpace(string(b)) + if raw == "" { + return fmt.Errorf("empty byte size") + } + valueEnd := 0 + for valueEnd < len(raw) && ((raw[valueEnd] >= '0' && raw[valueEnd] <= '9') || raw[valueEnd] == '.') { + valueEnd++ + } + if valueEnd == 0 { + return fmt.Errorf("invalid byte size %q", raw) + } + value, err := strconv.ParseFloat(raw[:valueEnd], 64) + if err != nil || value <= 0 { + return fmt.Errorf("invalid byte size %q", raw) + } + unit := strings.ToLower(strings.TrimSpace(raw[valueEnd:])) + mul, ok := map[string]float64{ + "": 1, "b": 1, + "k": 1000, "kb": 1000, + "m": 1000 * 1000, "mb": 1000 * 1000, + "g": 1000 * 1000 * 1000, "gb": 1000 * 1000 * 1000, + "kib": 1024, + "mib": 1024 * 1024, + "gib": 1024 * 1024 * 1024, + }[unit] + if !ok { + return fmt.Errorf("unknown byte size unit %q", unit) + } + bytes := value * mul + if bytes > math.MaxInt64 { + return fmt.Errorf("byte size %q is too large", raw) + } + s.Bytes = int64(bytes) + return nil +} + func Load(path string) (*Config, error) { var c Config md, err := toml.DecodeFile(path, &c) @@ -174,6 +223,9 @@ func (c *Config) Validate() error { if len(ch.DedupeKey) > 256 { return fmt.Errorf("check %q: dedupe_key too long", ch.ID) } + if err := validateResourceLimits(ch.ID, ch.ResourceLimits); err != nil { + return err + } } return nil } @@ -201,8 +253,23 @@ func (c CheckConfig) NotificationsOn() bool { return c.NotificationsEnabled == nil || *c.NotificationsEnabled } -func (c CheckConfig) Argv() []string { - return []string{"/bin/sh", "-c", c.Command} +func validateResourceLimits(checkID string, limits ResourceLimits) error { + if limits.CPUTime.Duration < 0 { + return fmt.Errorf("check %q: resource_limits.cpu_time must be > 0", checkID) + } + if limits.Memory.Bytes < 0 { + return fmt.Errorf("check %q: resource_limits.memory must be > 0", checkID) + } + if limits.Memory.Bytes > 0 && limits.Memory.Bytes < 1024*1024 { + return fmt.Errorf("check %q: resource_limits.memory must be >= 1MiB", checkID) + } + if limits.OpenFiles < 0 { + return fmt.Errorf("check %q: resource_limits.open_files must be > 0", checkID) + } + if limits.OpenFiles > 0 && limits.OpenFiles < 16 { + return fmt.Errorf("check %q: resource_limits.open_files must be >= 16", checkID) + } + return nil } func ValidateID(field, v string) error { diff --git a/agent/internal/config/config_test.go b/agent/internal/config/config_test.go index 1ce3eb0..07dcb94 100644 --- a/agent/internal/config/config_test.go +++ b/agent/internal/config/config_test.go @@ -48,6 +48,59 @@ func TestLoadMinimal(t *testing.T) { } } +func TestLoadResourceLimits(t *testing.T) { + c, err := Load(writeTmp(t, ` +agent_id = "host-1" +state_dir = "/tmp/x" + +[server] +enabled = true +url = "http://localhost" + +[[checks]] +id = "c1" +command = "true" +interval = "10s" +timeout = "5s" +resource_limits = { cpu_time = "2s", memory = "256MiB", open_files = 128 } +`)) + if err != nil { + t.Fatal(err) + } + limits := c.Checks[0].ResourceLimits + if limits.CPUTime.Duration != 2_000_000_000 || limits.Memory.Bytes != 256*1024*1024 || limits.OpenFiles != 128 { + t.Fatalf("unexpected limits: %+v", limits) + } +} + +func TestValidateBadResourceLimits(t *testing.T) { + for name, line := range map[string]string{ + "memory_too_small": `resource_limits = { memory = "512KiB" }`, + "open_files_too_small": `resource_limits = { open_files = 8 }`, + } { + t.Run(name, func(t *testing.T) { + _, err := Load(writeTmp(t, ` +agent_id = "host-1" +state_dir = "/tmp/x" + +[server] +enabled = true +url = "http://localhost" + +[[checks]] +id = "c1" +command = "true" +interval = "10s" +timeout = "5s" +`+line+` +`)) + if err == nil { + t.Fatal("expected resource limit error") + } + }) + } +} + func TestLoadMultilineCommand(t *testing.T) { c, err := Load(writeTmp(t, ` agent_id = "host-1" @@ -72,9 +125,6 @@ timeout = "5s" if !strings.Contains(c.Checks[0].Command, "echo ok") { t.Fatalf("command not loaded: %q", c.Checks[0].Command) } - if got := c.Checks[0].Argv(); len(got) != 3 || got[0] != "/bin/sh" || got[1] != "-c" { - t.Fatalf("argv: %#v", got) - } } func TestCommandArrayRejected(t *testing.T) { diff --git a/agent/internal/metrics/metrics.go b/agent/internal/metrics/metrics.go index c8e80b7..bd9fe62 100644 --- a/agent/internal/metrics/metrics.go +++ b/agent/internal/metrics/metrics.go @@ -13,24 +13,29 @@ import ( type Metrics struct { Registry *prometheus.Registry - CheckRuns *prometheus.CounterVec - CheckSkipped *prometheus.CounterVec - CheckDuration *prometheus.HistogramVec - CheckStatus *prometheus.GaugeVec - CheckExitCode *prometheus.GaugeVec - CheckLastRun *prometheus.GaugeVec - CheckLastSuccess *prometheus.GaugeVec - CheckInterval *prometheus.GaugeVec - ChecksConfigured prometheus.Gauge - SpoolDepth prometheus.Gauge - SpoolBytes prometheus.Gauge - SendAttempts *prometheus.CounterVec - SendFailures *prometheus.CounterVec - EventsAccepted prometheus.Counter - EventsDedup prometheus.Counter - EventsDropped *prometheus.CounterVec - LastHeartbeat prometheus.Gauge - BuildInfo *prometheus.GaugeVec + CheckRuns *prometheus.CounterVec + CheckSkipped *prometheus.CounterVec + CheckDuration *prometheus.HistogramVec + CheckStatus *prometheus.GaugeVec + CheckExitCode *prometheus.GaugeVec + CheckLastRun *prometheus.GaugeVec + CheckLastSuccess *prometheus.GaugeVec + CheckInterval *prometheus.GaugeVec + ChecksConfigured prometheus.Gauge + ConfigLoadSuccess prometheus.Gauge + ConfigReloadTotal *prometheus.CounterVec + ConfigLastReloadSuccess prometheus.Gauge + ResourceLimitApplyFailures *prometheus.CounterVec + EventsChannelFull prometheus.Counter + SpoolDepth prometheus.Gauge + SpoolBytes prometheus.Gauge + SendAttempts *prometheus.CounterVec + SendFailures *prometheus.CounterVec + EventsAccepted prometheus.Counter + EventsDedup prometheus.Counter + EventsDropped *prometheus.CounterVec + LastHeartbeat prometheus.Gauge + BuildInfo *prometheus.GaugeVec } // StatusCode maps a CheckResultEvent status string to a numeric gauge value. @@ -115,6 +120,26 @@ func New() *Metrics { Name: "monlet_agent_checks_configured", Help: "Number of checks configured.", }) + m.ConfigLoadSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "monlet_agent_config_load_success", + Help: "Whether the last agent config load or reload succeeded.", + }) + m.ConfigReloadTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "monlet_agent_config_reload_total", + Help: "Number of config reload attempts by result.", + }, []string{"result"}) + m.ConfigLastReloadSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "monlet_agent_config_last_reload_success_timestamp_seconds", + Help: "Unix timestamp of the last successful config reload.", + }) + m.ResourceLimitApplyFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "monlet_agent_resource_limit_apply_failures_total", + Help: "Number of failed resource limit applications by check and resource.", + }, []string{"check_id", "resource"}) + m.EventsChannelFull = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "monlet_agent_events_channel_full_total", + Help: "Number of times scheduler event delivery would block because the agent event channel was full.", + }) m.LastHeartbeat = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "monlet_agent_last_heartbeat_timestamp_seconds", Help: "Unix timestamp of the last successful heartbeat ack.", @@ -126,6 +151,9 @@ func New() *Metrics { reg.MustRegister(m.CheckRuns, m.CheckSkipped, m.CheckDuration, m.CheckStatus, m.CheckExitCode, m.CheckLastRun, m.CheckLastSuccess, m.CheckInterval, m.ChecksConfigured, + m.ConfigLoadSuccess, m.ConfigReloadTotal, m.ConfigLastReloadSuccess, + m.ResourceLimitApplyFailures, + m.EventsChannelFull, m.SpoolDepth, m.SpoolBytes, m.SendAttempts, m.SendFailures, m.EventsAccepted, m.EventsDedup, m.EventsDropped, @@ -133,6 +161,19 @@ func New() *Metrics { return m } +// DeleteCheck removes metric series for a check that no longer exists. +func (m *Metrics) DeleteCheck(checkID string) { + m.CheckRuns.DeletePartialMatch(prometheus.Labels{"check_id": checkID}) + m.CheckSkipped.DeleteLabelValues(checkID) + m.CheckDuration.DeleteLabelValues(checkID) + m.CheckStatus.DeleteLabelValues(checkID) + m.CheckExitCode.DeleteLabelValues(checkID) + m.CheckLastRun.DeleteLabelValues(checkID) + m.CheckLastSuccess.DeleteLabelValues(checkID) + m.CheckInterval.DeleteLabelValues(checkID) + m.ResourceLimitApplyFailures.DeletePartialMatch(prometheus.Labels{"check_id": checkID}) +} + // Serve blocks until ctx is done; the listener is closed on shutdown. func (m *Metrics) Serve(ctx context.Context, addr string) error { mux := http.NewServeMux() diff --git a/agent/internal/metrics/metrics_test.go b/agent/internal/metrics/metrics_test.go index 0bed268..255bf43 100644 --- a/agent/internal/metrics/metrics_test.go +++ b/agent/internal/metrics/metrics_test.go @@ -14,6 +14,7 @@ func TestMetricsExposeRegistered(t *testing.T) { m := New() m.CheckRuns.WithLabelValues("c1", "ok").Inc() m.SpoolDepth.Set(7) + m.EventsChannelFull.Inc() srv := httptest.NewServer(promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{})) defer srv.Close() resp, err := http.Get(srv.URL) @@ -26,6 +27,7 @@ func TestMetricsExposeRegistered(t *testing.T) { for _, want := range []string{ `monlet_agent_check_runs_total{check_id="c1",status="ok"} 1`, `monlet_agent_spool_events 7`, + `monlet_agent_events_channel_full_total 1`, } { if !strings.Contains(s, want) { t.Errorf("missing %q in output", want) diff --git a/agent/internal/runner/runner.go b/agent/internal/runner/runner.go index 2ec851d..332c47c 100644 --- a/agent/internal/runner/runner.go +++ b/agent/internal/runner/runner.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "os/exec" + "strconv" + "strings" "syscall" "time" "unicode/utf8" @@ -14,8 +16,9 @@ import ( var replacementRune = []byte("�") const ( - MaxOutputBytes = 8192 - truncMarkerFmt = "...[truncated %d bytes]" + MaxOutputBytes = 8192 + truncMarkerFmt = "...[truncated %d bytes]" + limitFailPrefix = "__MONLET_RESOURCE_LIMIT_FAILED__:" ) type Status string @@ -28,16 +31,41 @@ const ( ) type Result struct { - Status Status - ExitCode int - DurationMs int64 - Output string - OutputTruncated bool + Status Status + ExitCode int + DurationMs int64 + Output string + OutputTruncated bool + ResourceLimitFailure string +} + +// ResourceLimits contains best-effort shell ulimit values for a check command. +type ResourceLimits struct { + CPUTime time.Duration + MemoryBytes int64 + OpenFiles int +} + +func (l ResourceLimits) IsZero() bool { + return l.CPUTime == 0 && l.MemoryBytes == 0 && l.OpenFiles == 0 } // Run executes argv with timeout. Stdout+stderr are captured into a single // buffer; both are truncated to 8 KiB UTF-8 bytes if exceeded. func Run(ctx context.Context, argv []string, timeout time.Duration) Result { + return runArgv(ctx, argv, timeout) +} + +// RunCommand executes a shell command with optional resource limits. +func RunCommand(ctx context.Context, command string, timeout time.Duration, limits ResourceLimits) Result { + argv := []string{"/bin/sh", "-c", command} + if !limits.IsZero() { + argv = limitedShellArgv(command, limits) + } + return runArgv(ctx, argv, timeout) +} + +func runArgv(ctx context.Context, argv []string, timeout time.Duration) Result { start := time.Now() cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -67,6 +95,23 @@ func Run(ctx context.Context, argv []string, timeout time.Duration) Result { r.Output = fmt.Sprintf("critical: check timed out after %s", timeout) return r } + if isExitCode(err, 125) { + if resource, ok := detectLimitFailure(out); ok { + r.Status = StatusUnknown + r.ExitCode = -1 + r.Output = "unknown: failed to apply resource limit: " + resource + r.OutputTruncated = false + r.ResourceLimitFailure = resource + return r + } + } + if isSignal(err, syscall.SIGXCPU) { + r.Status = StatusCritical + r.ExitCode = 2 + r.Output = "critical: CPU time limit exceeded" + r.OutputTruncated = false + return r + } if err != nil { var ee *exec.ExitError if errors.As(err, &ee) { @@ -74,6 +119,8 @@ func Run(ctx context.Context, argv []string, timeout time.Duration) Result { } else { r.Status = StatusUnknown r.ExitCode = -1 + // Launch failure (e.g. shell missing); surface exec error, never check output. + r.Output, r.OutputTruncated = TruncateUTF8([]byte("unknown: failed to start check: "+err.Error()), MaxOutputBytes) return r } } @@ -81,6 +128,60 @@ func Run(ctx context.Context, argv []string, timeout time.Duration) Result { return r } +func isExitCode(err error, code int) bool { + var ee *exec.ExitError + return errors.As(err, &ee) && ee.ExitCode() == code +} + +func limitedShellArgv(command string, limits ResourceLimits) []string { + cpu := "" + if limits.CPUTime > 0 { + cpu = strconv.FormatInt(int64((limits.CPUTime+time.Second-1)/time.Second), 10) + } + mem := "" + if limits.MemoryBytes > 0 { + mem = strconv.FormatInt((limits.MemoryBytes+1023)/1024, 10) + } + openFiles := "" + if limits.OpenFiles > 0 { + openFiles = strconv.Itoa(limits.OpenFiles) + } + script := ` +cmd=$1 +cpu=$2 +mem=$3 +open_files=$4 +if [ -n "$cpu" ]; then ulimit -t "$cpu" 2>/dev/null || { printf '%s\n' '` + limitFailPrefix + `cpu_time'; exit 125; }; fi +if [ -n "$mem" ]; then ulimit -v "$mem" 2>/dev/null || { printf '%s\n' '` + limitFailPrefix + `memory'; exit 125; }; fi +if [ -n "$open_files" ]; then ulimit -n "$open_files" 2>/dev/null || { printf '%s\n' '` + limitFailPrefix + `open_files'; exit 125; }; fi +exec /bin/sh -c "$cmd" +` + return []string{"/bin/sh", "-c", script, "monlet-limit-wrapper", command, cpu, mem, openFiles} +} + +func detectLimitFailure(output string) (string, bool) { + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, limitFailPrefix) { + resource := strings.TrimPrefix(line, limitFailPrefix) + if resource != "cpu_time" && resource != "memory" && resource != "open_files" { + resource = "unknown" + } + return resource, true + } + } + return "", false +} + +func isSignal(err error, sig syscall.Signal) bool { + var ee *exec.ExitError + if !errors.As(err, &ee) { + return false + } + ws, ok := ee.Sys().(syscall.WaitStatus) + return ok && ws.Signaled() && ws.Signal() == sig +} + func MapExit(code int) Status { switch code { case 0: diff --git a/agent/internal/runner/runner_test.go b/agent/internal/runner/runner_test.go index 50bdde5..4a39120 100644 --- a/agent/internal/runner/runner_test.go +++ b/agent/internal/runner/runner_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "runtime" "strings" "testing" "time" @@ -114,3 +115,41 @@ func TestRunNoExec(t *testing.T) { t.Fatalf("got %+v", r) } } + +func TestRunCommandWithLimitsOK(t *testing.T) { + r := RunCommand(context.Background(), "echo ok", 2*time.Second, ResourceLimits{ + CPUTime: time.Second, + OpenFiles: 64, + }) + if r.Status != StatusOK || r.ExitCode != 0 { + t.Fatalf("got %+v", r) + } + if !strings.Contains(r.Output, "ok") { + t.Fatalf("output: %q", r.Output) + } +} + +func TestRunCommandCPULimitExceeded(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("ulimit is Unix-only") + } + r := RunCommand(context.Background(), "while :; do :; done", 5*time.Second, ResourceLimits{ + CPUTime: time.Second, + }) + if r.Status != StatusCritical || r.ExitCode != 2 { + t.Fatalf("got %+v", r) + } + if !strings.Contains(r.Output, "CPU time limit exceeded") { + t.Fatalf("output: %q", r.Output) + } +} + +func TestDetectLimitFailureBoundsResourceLabel(t *testing.T) { + resource, ok := detectLimitFailure(limitFailPrefix + "dynamic-user-output") + if !ok { + t.Fatal("expected limit failure") + } + if resource != "unknown" { + t.Fatalf("resource label must be bounded, got %q", resource) + } +} diff --git a/agent/internal/scheduler/scheduler.go b/agent/internal/scheduler/scheduler.go index 495fb34..c4360b0 100644 --- a/agent/internal/scheduler/scheduler.go +++ b/agent/internal/scheduler/scheduler.go @@ -2,6 +2,7 @@ package scheduler import ( "context" + "reflect" "sync" "time" @@ -33,82 +34,316 @@ func (e Event) WireEvent() Event { } type Hooks struct { - OnSkipped func(checkID string) - OnResult func(ev Event) + OnSkipped func(checkID string) + OnResult func(ev Event) + OnEventChannelFull func() + OnResourceLimitFailure func(checkID string, resource string) + OnStopped func(checkID string) } -// Run starts one goroutine per check. Emits events to out. Blocks until ctx is done. -func Run(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) { - var wg sync.WaitGroup - for i := range checks { - c := checks[i] - wg.Add(1) - go func() { - defer wg.Done() - runCheck(ctx, agentID, c, out, h) - }() +// Manager owns per-check workers and applies hot config updates. +type Manager struct { + ctx context.Context + agentID string + out chan<- Event + hooks Hooks + + mu sync.Mutex + workers map[string]*checkWorker + draining map[string]struct{} + pending map[string]config.CheckConfig + all []*checkWorker + stopped bool +} + +// NewManager creates a scheduler manager bound to the parent context. +func NewManager(ctx context.Context, agentID string, out chan<- Event, h Hooks) *Manager { + return &Manager{ + ctx: ctx, + agentID: agentID, + out: out, + hooks: h, + workers: make(map[string]*checkWorker), + draining: make(map[string]struct{}), + pending: make(map[string]config.CheckConfig), } - wg.Wait() } -func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out chan<- Event, h Hooks) { - t := time.NewTicker(c.Interval.Duration) - defer t.Stop() - var running bool - var mu sync.Mutex +// Update applies the current check set without killing in-flight runs. +func (m *Manager) Update(checks []config.CheckConfig) { + m.mu.Lock() + defer m.mu.Unlock() + if m.stopped { + return + } + next := make(map[string]config.CheckConfig, len(checks)) + for _, c := range checks { + next[c.ID] = c + if w, ok := m.workers[c.ID]; ok { + if _, draining := m.draining[c.ID]; draining { + m.pending[c.ID] = c + } else { + w.Update(c) + } + continue + } + id := c.ID + w := newCheckWorker(m.ctx, m.agentID, c, m.out, m.hooks, func(stopped *checkWorker) { + m.workerStopped(id, stopped) + }) + m.workers[c.ID] = w + m.all = append(m.all, w) + } + for id, w := range m.workers { + if _, ok := next[id]; ok { + continue + } + m.draining[id] = struct{}{} + delete(m.pending, id) + w.Stop() + } +} - exec := func() { - mu.Lock() +// Stop prevents new runs and lets in-flight checks finish or observe ctx cancel. +func (m *Manager) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + if m.stopped { + return + } + m.stopped = true + for _, w := range m.workers { + w.Stop() + } +} + +// Wait blocks until all workers have drained. +func (m *Manager) Wait() { + m.mu.Lock() + workers := append([]*checkWorker(nil), m.all...) + m.mu.Unlock() + for _, w := range workers { + w.Wait() + } +} + +func (m *Manager) workerStopped(checkID string, w *checkWorker) { + m.mu.Lock() + defer m.mu.Unlock() + if current, ok := m.workers[checkID]; !ok || current != w { + return + } + delete(m.workers, checkID) + delete(m.draining, checkID) + if m.stopped { + delete(m.pending, checkID) + return + } + if cfg, ok := m.pending[checkID]; ok { + delete(m.pending, checkID) + next := newCheckWorker(m.ctx, m.agentID, cfg, m.out, m.hooks, func(stopped *checkWorker) { + m.workerStopped(checkID, stopped) + }) + m.workers[checkID] = next + m.all = append(m.all, next) + } +} + +type checkWorker struct { + ctx context.Context + agentID string + out chan<- Event + hooks Hooks + onDone func(*checkWorker) + + updateCh chan config.CheckConfig + stopCh chan struct{} + doneCh chan struct{} + stopOnce sync.Once + wg sync.WaitGroup +} + +func newCheckWorker(ctx context.Context, agentID string, c config.CheckConfig, out chan<- Event, h Hooks, onDone func(*checkWorker)) *checkWorker { + w := &checkWorker{ + ctx: ctx, + agentID: agentID, + out: out, + hooks: h, + onDone: onDone, + updateCh: make(chan config.CheckConfig, 1), + stopCh: make(chan struct{}), + doneCh: make(chan struct{}, 1), + } + w.wg.Add(1) + go w.loop(c) + return w +} + +func (w *checkWorker) Update(c config.CheckConfig) { + select { + case w.updateCh <- c: + default: + select { + case <-w.updateCh: + default: + } + w.updateCh <- c + } +} + +func (w *checkWorker) Stop() { + w.stopOnce.Do(func() { close(w.stopCh) }) +} + +func (w *checkWorker) Wait() { + w.wg.Wait() +} + +func (w *checkWorker) loop(initial config.CheckConfig) { + defer func() { + if w.onDone != nil { + w.onDone(w) + } + w.wg.Done() + }() + cfg := initial + t := time.NewTicker(cfg.Interval.Duration) + defer t.Stop() + + running := false + stopping := false + runAfterCurrent := false + ctxDone := w.ctx.Done() + stopCh := w.stopCh + + start := func(c config.CheckConfig) { if running { - mu.Unlock() - if h.OnSkipped != nil { - h.OnSkipped(c.ID) + if w.hooks.OnSkipped != nil { + w.hooks.OnSkipped(c.ID) } return } running = true - mu.Unlock() - defer func() { - mu.Lock() - running = false - mu.Unlock() + go func() { + w.run(c) + w.doneCh <- struct{}{} }() - - res := runner.Run(ctx, c.Argv(), c.Timeout.Duration) - id, err := eventid.New() - if err != nil { - return - } - ev := Event{ - EventID: id, - CheckID: c.ID, - ObservedAt: time.Now().UTC(), - Status: string(res.Status), - ExitCode: res.ExitCode, - DurationMs: res.DurationMs, - Output: res.Output, - OutputTruncated: res.OutputTruncated, - NotificationsEnabled: boolPtr(c.NotificationsOn()), - IncidentKey: incidentKey(agentID, c), - } - if h.OnResult != nil { - h.OnResult(ev) - } - select { - case out <- ev: - case <-ctx.Done(): - } } - // Fire first run immediately so smoke tests don't wait a full interval. - go exec() + start(cfg) for { select { - case <-ctx.Done(): - return + case <-ctxDone: + stopping = true + ctxDone = nil + if !running { + if w.hooks.OnStopped != nil { + w.hooks.OnStopped(cfg.ID) + } + return + } + case <-stopCh: + stopping = true + stopCh = nil + if !running { + if w.hooks.OnStopped != nil { + w.hooks.OnStopped(cfg.ID) + } + return + } + case next := <-w.updateCh: + if reflect.DeepEqual(cfg, next) { + continue + } + cfg = next + t.Reset(cfg.Interval.Duration) + if running { + runAfterCurrent = true + } else { + start(cfg) + } + case <-w.doneCh: + running = false + if !stopping { + select { + case <-stopCh: + stopping = true + stopCh = nil + default: + } + } + if stopping { + if w.hooks.OnStopped != nil { + w.hooks.OnStopped(cfg.ID) + } + return + } + if runAfterCurrent { + runAfterCurrent = false + for { + select { + case next := <-w.updateCh: + if !reflect.DeepEqual(cfg, next) { + cfg = next + t.Reset(cfg.Interval.Duration) + } + default: + start(cfg) + goto nextLoop + } + } + nextLoop: + continue + } case <-t.C: - go exec() + if stopping { + continue + } + start(cfg) + } + } +} + +func (w *checkWorker) run(c config.CheckConfig) { + res := runner.RunCommand(w.ctx, c.Command, c.Timeout.Duration, runner.ResourceLimits{ + CPUTime: c.ResourceLimits.CPUTime.Duration, + MemoryBytes: c.ResourceLimits.Memory.Bytes, + OpenFiles: c.ResourceLimits.OpenFiles, + }) + if res.ResourceLimitFailure != "" && w.hooks.OnResourceLimitFailure != nil { + w.hooks.OnResourceLimitFailure(c.ID, res.ResourceLimitFailure) + } + id, err := eventid.New() + if err != nil { + return + } + ev := Event{ + EventID: id, + CheckID: c.ID, + ObservedAt: time.Now().UTC(), + Status: string(res.Status), + ExitCode: res.ExitCode, + DurationMs: res.DurationMs, + Output: res.Output, + OutputTruncated: res.OutputTruncated, + NotificationsEnabled: boolPtr(c.NotificationsOn()), + IncidentKey: incidentKey(w.agentID, c), + } + if w.hooks.OnResult != nil { + w.hooks.OnResult(ev) + } + select { + case w.out <- ev: + case <-w.ctx.Done(): + default: + if w.hooks.OnEventChannelFull != nil { + w.hooks.OnEventChannelFull() + } + select { + case w.out <- ev: + case <-w.ctx.Done(): + case <-w.stopCh: } } } diff --git a/agent/internal/scheduler/scheduler_test.go b/agent/internal/scheduler/scheduler_test.go index 7495368..d47fcf9 100644 --- a/agent/internal/scheduler/scheduler_test.go +++ b/agent/internal/scheduler/scheduler_test.go @@ -2,6 +2,7 @@ package scheduler import ( "context" + "strings" "sync/atomic" "testing" "time" @@ -9,6 +10,14 @@ import ( "github.com/monlet/agent/internal/config" ) +func runScheduler(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) { + m := NewManager(ctx, agentID, out, h) + m.Update(checks) + <-ctx.Done() + m.Stop() + m.Wait() +} + func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig { return config.CheckConfig{ ID: id, @@ -22,7 +31,7 @@ func TestEmitsEvents(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() out := make(chan Event, 10) - go Run(ctx, "a1", []config.CheckConfig{mkCheck("c1", "echo hi", 200*time.Millisecond, 100*time.Millisecond)}, out, Hooks{}) + go runScheduler(ctx, "a1", []config.CheckConfig{mkCheck("c1", "echo hi", 200*time.Millisecond, 100*time.Millisecond)}, out, Hooks{}) select { case ev := <-out: if ev.CheckID != "c1" || ev.Status != "ok" { @@ -43,9 +52,168 @@ func TestNoOverlapSkips(t *testing.T) { var skipped int32 hooks := Hooks{OnSkipped: func(string) { atomic.AddInt32(&skipped, 1) }} c := mkCheck("slow", "sleep 1", 100*time.Millisecond, 900*time.Millisecond) - go Run(ctx, "a1", []config.CheckConfig{c}, out, hooks) + go runScheduler(ctx, "a1", []config.CheckConfig{c}, out, hooks) <-ctx.Done() if atomic.LoadInt32(&skipped) == 0 { t.Fatal("expected at least one skipped tick") } } + +func TestReportsFullEventChannel(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + out := make(chan Event, 1) + out <- Event{EventID: "occupied"} + var full int32 + m := NewManager(ctx, "a1", out, Hooks{ + OnEventChannelFull: func() { atomic.AddInt32(&full, 1) }, + }) + m.Update([]config.CheckConfig{mkCheck("c1", "echo hi", time.Hour, time.Second)}) + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if atomic.LoadInt32(&full) > 0 { + break + } + time.Sleep(10 * time.Millisecond) + } + if atomic.LoadInt32(&full) == 0 { + t.Fatal("expected full channel hook") + } + <-out + select { + case ev := <-out: + if ev.CheckID != "c1" { + t.Fatalf("unexpected event: %+v", ev) + } + case <-ctx.Done(): + t.Fatal("blocked event was not delivered") + } + m.Stop() + m.Wait() +} + +func TestManagerReloadChangedCheckWaitsForRunning(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + out := make(chan Event, 10) + m := NewManager(ctx, "a1", out, Hooks{}) + m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.3; echo old", time.Second, time.Second)}) + time.Sleep(50 * time.Millisecond) + m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)}) + + var first, second Event + select { + case first = <-out: + case <-ctx.Done(): + t.Fatal("no first event") + } + select { + case second = <-out: + case <-ctx.Done(): + t.Fatal("no second event") + } + m.Stop() + m.Wait() + + if !strings.Contains(first.Output, "old") { + t.Fatalf("first output: %q", first.Output) + } + if !strings.Contains(second.Output, "new") { + t.Fatalf("second output: %q", second.Output) + } +} + +func TestManagerReloadRemovedCheckLetsRunningFinish(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + out := make(chan Event, 10) + stopped := make(chan string, 1) + m := NewManager(ctx, "a1", out, Hooks{OnStopped: func(id string) { stopped <- id }}) + m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.2; echo old", time.Second, time.Second)}) + time.Sleep(50 * time.Millisecond) + m.Update(nil) + + select { + case ev := <-out: + if !strings.Contains(ev.Output, "old") { + t.Fatalf("output: %q", ev.Output) + } + case <-ctx.Done(): + t.Fatal("removed running check did not finish") + } + select { + case id := <-stopped: + if id != "c1" { + t.Fatalf("stopped id: %q", id) + } + case <-ctx.Done(): + t.Fatal("removed check did not stop") + } + m.Stop() + m.Wait() +} + +func TestManagerRemoveThenReaddSameCheckDoesNotOverlap(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out := make(chan Event, 10) + m := NewManager(ctx, "a1", out, Hooks{}) + m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)}) + time.Sleep(50 * time.Millisecond) + m.Update(nil) + m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)}) + + var first, second Event + select { + case first = <-out: + case <-ctx.Done(): + t.Fatal("no first event") + } + select { + case second = <-out: + case <-ctx.Done(): + t.Fatal("no second event") + } + m.Stop() + m.Wait() + + if !strings.Contains(first.Output, "old") { + t.Fatalf("first output: %q", first.Output) + } + if !strings.Contains(second.Output, "new") { + t.Fatalf("second output: %q", second.Output) + } +} + +func TestManagerUsesLatestPendingReloadAfterRunningFinishes(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out := make(chan Event, 10) + m := NewManager(ctx, "a1", out, Hooks{}) + m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)}) + time.Sleep(50 * time.Millisecond) + m.Update([]config.CheckConfig{mkCheck("c1", "echo intermediate", time.Second, time.Second)}) + m.Update([]config.CheckConfig{mkCheck("c1", "echo latest", time.Second, time.Second)}) + + var first, second Event + select { + case first = <-out: + case <-ctx.Done(): + t.Fatal("no first event") + } + select { + case second = <-out: + case <-ctx.Done(): + t.Fatal("no second event") + } + m.Stop() + m.Wait() + + if !strings.Contains(first.Output, "old") { + t.Fatalf("first output: %q", first.Output) + } + if strings.Contains(second.Output, "intermediate") || !strings.Contains(second.Output, "latest") { + t.Fatalf("second output: %q", second.Output) + } +} diff --git a/agent/systemd/monlet-agent.service.example b/agent/systemd/monlet-agent.service.example index 892e9c0..2fc3a9c 100644 --- a/agent/systemd/monlet-agent.service.example +++ b/agent/systemd/monlet-agent.service.example @@ -6,8 +6,10 @@ Wants=network-online.target [Service] Type=simple ExecStart=/usr/local/bin/monlet-agent -config /etc/monlet/agent.toml +ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=5s +TimeoutStopSec=35s User=monlet Group=monlet StateDirectory=monlet-agent diff --git a/api/openapi.yaml b/api/openapi.yaml index 6d8ee2e..8b67665 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -107,7 +107,9 @@ paths: "401": $ref: "#/components/responses/Unauthorized" "403": - description: Agent key is blocked. + description: >- + Agent key is blocked (`agent_blocked`) or not yet accepted + (`agent_not_accepted`). content: application/json: schema: @@ -627,6 +629,7 @@ components: - forbidden - conflict - agent_blocked + - agent_not_accepted - not_found - validation - payload_too_large diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index fd02c2f..9edf627 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -65,6 +65,8 @@ services: MONLET_WEB_TIME_ZONE: ${MONLET_WEB_TIME_ZONE:-UTC} MONLET_WEB_AUTH_USERNAME: ${MONLET_WEB_AUTH_USERNAME:-} MONLET_WEB_AUTH_PASSWORD: ${MONLET_WEB_AUTH_PASSWORD:-} + MONLET_WEB_SESSION_SECRET: ${MONLET_WEB_SESSION_SECRET:-} + MONLET_WEB_SESSION_TTL_SEC: ${MONLET_WEB_SESSION_TTL_SEC:-604800} MONLET_WEB_TRUST_PROXY_AUTH: ${MONLET_WEB_TRUST_PROXY_AUTH:-false} TZ: ${MONLET_WEB_TIME_ZONE:-UTC} ports: diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 898464f..b60443e 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -50,7 +50,7 @@ Web is read-only for monitoring state in v1. Agent admission and blacklist contr Auth plan: -- reverse proxy basic auth or equivalent gateway auth; +- local web login or equivalent gateway auth through a trusted reverse proxy; - no full RBAC in v1. ## Observability diff --git a/docs/ops/deployment-checklist.md b/docs/ops/deployment-checklist.md index 3a5959c..c157f7f 100644 --- a/docs/ops/deployment-checklist.md +++ b/docs/ops/deployment-checklist.md @@ -5,6 +5,7 @@ A pragmatic list for the first production-ish Monlet stand-up. Adjust per enviro ## 1. Secrets - [ ] Generate `MONLET_AUTH_TOKEN` (`openssl rand -hex 32`) and store it in your secret manager. +- [ ] Generate `MONLET_WEB_SESSION_SECRET` (`openssl rand -hex 32`) if local web login is enabled. - [ ] Generate a PostgreSQL password and store it in your secret manager. - [ ] Decide which notifier channels you want and gather their credentials (Telegram bot token + chat id, webhook URL, Alertmanager URL). @@ -41,7 +42,8 @@ A pragmatic list for the first production-ish Monlet stand-up. Adjust per enviro ## 6. Web UI - [ ] Set `MONLET_API_BASE_URL`, `MONLET_API_TOKEN`, and optional `MONLET_WEB_TIME_ZONE` for the web container. -- [ ] Protect the web UI with either `MONLET_WEB_AUTH_USERNAME`/`MONLET_WEB_AUTH_PASSWORD` or a trusted reverse proxy that sets `X-Forwarded-User` plus `MONLET_WEB_TRUST_PROXY_AUTH=true`. +- [ ] Protect the web UI with either local login (`MONLET_WEB_AUTH_USERNAME`, `MONLET_WEB_AUTH_PASSWORD`, `MONLET_WEB_SESSION_SECRET`) or a trusted reverse proxy that sets `X-Forwarded-User` plus `MONLET_WEB_TRUST_PROXY_AUTH=true`. +- [ ] Rate-limit `/login` at the reverse proxy when local web login is enabled. - [ ] Set optional `MONLET_NOTIFICATION_TIME_ZONE` for human-readable notifier timestamps. - [ ] Browse `/`, `/agents`, `/checks`, `/incidents`, `/events`, `/outbox`. - [ ] Verify the web image does not log the token (`docker logs` greps clean). diff --git a/server/monlet_server/api/events.py b/server/monlet_server/api/events.py index 2b91503..35c24fb 100644 --- a/server/monlet_server/api/events.py +++ b/server/monlet_server/api/events.py @@ -7,8 +7,10 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy import and_, or_, select from sqlalchemy.ext.asyncio import AsyncSession +from .. import metrics from ..agent_auth import AgentCredential, get_agent_credential from ..db import get_session +from ..logging_config import get_logger from ..models import Event from ..schemas import ( ID_PATTERN, @@ -22,6 +24,8 @@ from ..timeutil import to_utc router = APIRouter() +log = get_logger("monlet.events") + def _encode_cursor(received_at: datetime, event_id) -> str: raw = f"{to_utc(received_at).isoformat()}|{event_id}".encode() @@ -52,12 +56,24 @@ async def events( ) -> EventBatchAcceptedResponse: agent = await accepted_agent_for_key(session, credential) if agent is None: - await session.commit() - return EventBatchAcceptedResponse(accepted=0, deduplicated=0) + raise HTTPException(status_code=403, detail="agent not accepted") accepted = 0 deduplicated = 0 for ev in body.events: - ok = await ingest_event(session, agent.agent_id, ev) + try: + async with session.begin_nested(): + ok = await ingest_event(session, agent.agent_id, ev) + except HTTPException as exc: + if exc.status_code != 400: + raise + log.warning( + "event_rejected", + check_id=ev.check_id, + agent_id=agent.agent_id, + code=exc.detail, + ) + metrics.events_total.labels(result="rejected").inc() + continue if ok: accepted += 1 else: diff --git a/server/monlet_server/errors.py b/server/monlet_server/errors.py index 6ab9995..4847374 100644 --- a/server/monlet_server/errors.py +++ b/server/monlet_server/errors.py @@ -46,11 +46,12 @@ def install_error_handlers(app: FastAPI) -> None: @app.exception_handler(StarletteHTTPException) async def _http(request: Request, exc: StarletteHTTPException): - code = ( - "agent_blocked" - if exc.status_code == 403 and exc.detail == "agent blocked" - else _STATUS_TO_CODE.get(exc.status_code, "internal") - ) + code: ErrorCode = _STATUS_TO_CODE.get(exc.status_code, "internal") + if exc.status_code == 403: + if exc.detail == "agent blocked": + code = "agent_blocked" + elif exc.detail == "agent not accepted": + code = "agent_not_accepted" msg = exc.detail if isinstance(exc.detail, str) else code return error_response(request, exc.status_code, code, msg) diff --git a/server/monlet_server/metrics.py b/server/monlet_server/metrics.py index 009f7c6..39889e4 100644 --- a/server/monlet_server/metrics.py +++ b/server/monlet_server/metrics.py @@ -1,4 +1,4 @@ -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, generate_latest +from prometheus_client import CollectorRegistry, Counter, Gauge, generate_latest from prometheus_client.exposition import CONTENT_TYPE_LATEST REGISTRY = CollectorRegistry() @@ -42,12 +42,6 @@ notifications_total = Counter( ["notifier", "result"], registry=REGISTRY, ) -request_duration = Histogram( - "monlet_server_request_duration_seconds", - "Request duration seconds.", - ["path"], - registry=REGISTRY, -) def render() -> tuple[bytes, str]: diff --git a/server/monlet_server/schemas.py b/server/monlet_server/schemas.py index 1930128..f362644 100644 --- a/server/monlet_server/schemas.py +++ b/server/monlet_server/schemas.py @@ -1,7 +1,6 @@ import re from datetime import datetime from typing import Literal -from uuid import UUID from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -39,6 +38,7 @@ ErrorCode = Literal[ "internal", "not_ready", "agent_blocked", + "agent_not_accepted", ] @@ -233,12 +233,6 @@ class OutboxPage(PageEnvelope): items: list[NotificationOutboxItem] -class UUIDStr(BaseModel): - """Helper to validate generated UUIDs.""" - - value: UUID - - class SystemSummaryResponse(BaseModel): agents_online: int agents_offline: int diff --git a/server/monlet_server/services/ingestion.py b/server/monlet_server/services/ingestion.py index 2e859e7..5754ead 100644 --- a/server/monlet_server/services/ingestion.py +++ b/server/monlet_server/services/ingestion.py @@ -53,13 +53,7 @@ async def _blacklist_row(session: AsyncSession, cred: AgentCredential) -> AgentB return res.scalar_one_or_none() -async def reject_if_blocked( - session: AsyncSession, - cred: AgentCredential, - agent_id: str, - hostname: str, - observed: datetime | None = None, -) -> None: +async def reject_if_blocked(session: AsyncSession, cred: AgentCredential) -> None: row = await _blacklist_row(session, cred) if row is None: return @@ -108,7 +102,7 @@ async def upsert_agent_heartbeat( session: AsyncSession, hb: HeartbeatRequest, cred: AgentCredential ) -> str: observed = to_utc(hb.observed_at) - await reject_if_blocked(session, cred, hb.agent_id, hb.hostname, observed) + await reject_if_blocked(session, cred) features = hb.features.model_dump() res = await session.execute(select(Agent).where(Agent.accepted_key_hash == cred.key_hash)) @@ -330,7 +324,6 @@ async def ingest_event( incident_key=incident_key, ) await session.execute(stmt) - metrics.events_total.labels(result="accepted").inc() else: metrics.events_total.labels(result="unchanged").inc() @@ -366,6 +359,9 @@ async def ingest_event( return False transition, inc = await _apply_incident(session, agent_id, ev, incident_key) + # Count accepted only after _apply_incident: an incident_key collision raises + # 400 and the savepoint rollback discards the Event insert above. + metrics.events_total.labels(result="accepted").inc() if transition and inc is not None and ev.notifications_enabled: event_type = "resolved" if ev.status == "ok" else "firing" await _enqueue_outbox( diff --git a/server/monlet_server/services/notifier_worker.py b/server/monlet_server/services/notifier_worker.py index 9ba7273..bbcd435 100644 --- a/server/monlet_server/services/notifier_worker.py +++ b/server/monlet_server/services/notifier_worker.py @@ -208,9 +208,16 @@ async def tick( await session.commit() if not rows: return 0 - await asyncio.gather( - *(_process_row(sm, registry, r, settings.notifier_max_attempts) for r in rows) - ) + + # Bound concurrent DB sessions: each _process_row opens its own session, so an + # unbounded gather over a large batch can exhaust the connection pool. + sem = asyncio.Semaphore(settings.notifier_concurrency) + + async def _guarded(row: dict) -> None: + async with sem: + await _process_row(sm, registry, row, settings.notifier_max_attempts) + + await asyncio.gather(*(_guarded(r) for r in rows)) return len(rows) diff --git a/server/monlet_server/services/notifiers/http.py b/server/monlet_server/services/notifiers/http.py index 0d710b8..16d0a6b 100644 --- a/server/monlet_server/services/notifiers/http.py +++ b/server/monlet_server/services/notifiers/http.py @@ -1,7 +1,5 @@ from __future__ import annotations -import httpx - def classify_response(status: int) -> tuple[bool, bool]: """Return (ok, permanent_failure).""" @@ -10,9 +8,3 @@ def classify_response(status: int) -> tuple[bool, bool]: if 400 <= status < 500 and status not in (408, 425, 429): return False, True return False, False - - -def classify_exception(exc: Exception) -> tuple[bool, bool]: - if isinstance(exc, httpx.TimeoutException | httpx.NetworkError): - return False, False - return False, False diff --git a/server/monlet_server/settings.py b/server/monlet_server/settings.py index 5d7a393..439a62b 100644 --- a/server/monlet_server/settings.py +++ b/server/monlet_server/settings.py @@ -1,4 +1,4 @@ -from pydantic import field_validator +from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict from .timeutil import validate_time_zone @@ -40,6 +40,7 @@ class Settings(BaseSettings): notifier_batch_size: int = 20 notifier_max_attempts: int = 8 notifier_lease_sec: int = 300 + notifier_concurrency: int = Field(default=4, ge=1) notifier_http_timeout_sec: float = 10.0 notification_time_zone: str = "UTC" diff --git a/server/tests/test_agent_admission.py b/server/tests/test_agent_admission.py index 63169d4..fa634bd 100644 --- a/server/tests/test_agent_admission.py +++ b/server/tests/test_agent_admission.py @@ -22,14 +22,14 @@ async def test_unknown_heartbeat_creates_pending_agent(app_client, auth_headers, @pytest.mark.asyncio -async def test_pending_events_are_dropped_until_accept(app_client, auth_headers, session): +async def test_pending_events_are_rejected_until_accept(app_client, auth_headers, session): await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers) ev = make_event(status="critical", exit_code=2) r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers ) - assert r.status_code == 202 - assert r.json() == {"accepted": 0, "deduplicated": 0} + assert r.status_code == 403 + assert r.json()["error"]["code"] == "agent_not_accepted" n = (await session.execute(select(func.count()).select_from(Check))).scalar_one() assert n == 0 diff --git a/server/tests/test_events_ingestion.py b/server/tests/test_events_ingestion.py index 4f4bbf7..4ef9ac4 100644 --- a/server/tests/test_events_ingestion.py +++ b/server/tests/test_events_ingestion.py @@ -23,13 +23,13 @@ async def test_event_accepted_and_state(app_client, ui_auth_headers, session): @pytest.mark.asyncio -async def test_event_before_heartbeat_is_dropped(app_client, auth_headers, session): +async def test_event_before_heartbeat_is_rejected(app_client, auth_headers, session): ev = make_event(status="warning", exit_code=1) r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-replay", "events": [ev]}, headers=auth_headers ) - assert r.status_code == 202 - assert r.json() == {"accepted": 0, "deduplicated": 0} + assert r.status_code == 403 + assert r.json()["error"]["code"] == "agent_not_accepted" n = (await session.execute(select(func.count()).select_from(Agent))).scalar_one() assert n == 0 diff --git a/server/tests/test_incident_key.py b/server/tests/test_incident_key.py index 3ec1061..1de74f2 100644 --- a/server/tests/test_incident_key.py +++ b/server/tests/test_incident_key.py @@ -18,8 +18,8 @@ async def test_incident_key_ownership_rejected(app_client, ui_auth_headers): r2 = await app_client.post( "/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=agent_2_headers ) - assert r2.status_code == 400 - assert r2.json()["error"]["code"] == "validation" + assert r2.status_code == 202 + assert r2.json() == {"accepted": 0, "deduplicated": 0} @pytest.mark.asyncio diff --git a/web/README.md b/web/README.md index 55a152b..570a6b2 100644 --- a/web/README.md +++ b/web/README.md @@ -36,11 +36,17 @@ Do not install Node packages globally. Project dependencies live in `web/node_mo | `MONLET_API_BASE_URL` | `http://127.0.0.1:8000` | Server base URL (server-side fetch) | | `MONLET_API_TOKEN` | _(none)_ | Bearer token sent to server | | `MONLET_WEB_TIME_ZONE` | `UTC` | IANA timezone used to render timestamps | -| `MONLET_WEB_AUTH_USERNAME` | _(none)_ | Optional Basic Auth username for the web UI | -| `MONLET_WEB_AUTH_PASSWORD` | _(none)_ | Optional Basic Auth password for the web UI | +| `MONLET_WEB_AUTH_USERNAME` | _(none)_ | Optional local web login username | +| `MONLET_WEB_AUTH_PASSWORD` | _(none)_ | Optional local web login password | +| `MONLET_WEB_SESSION_SECRET` | _(none)_ | Secret used to sign the web session cookie; required with local web login; at least 32 bytes | +| `MONLET_WEB_SESSION_TTL_SEC` | `604800` | Local web session lifetime | | `MONLET_WEB_TRUST_PROXY_AUTH` | `false` | Trust `X-Forwarded-User` from an upstream auth proxy | -The API token never reaches the browser — all server calls happen in Server Components / route handlers. Agent admission actions are blocked unless Basic Auth is configured or trusted proxy auth is enabled. +The API token never reaches the browser — all server calls happen in Server Components / route handlers. Agent admission actions are blocked unless local web login is configured or trusted proxy auth is enabled. + +Production deployments should rate-limit `/login` at the reverse proxy. The app applies constant-time credential checks and a small fixed delay on failed login attempts, but it is not a full auth gateway. + +The timezone switcher stores the selected mode in browser cookies, so each operator keeps their own default/browser/UTC preference. ## Pages diff --git a/web/src/app/agents/[agent_id]/page.tsx b/web/src/app/agents/[agent_id]/page.tsx index 1a1c942..7972be2 100644 --- a/web/src/app/agents/[agent_id]/page.tsx +++ b/web/src/app/agents/[agent_id]/page.tsx @@ -1,8 +1,10 @@ +import { redirect } from "next/navigation"; + import { AgentDetailBrowser } from "@/components/AgentDetailBrowser"; import { ErrorPanel } from "@/components/DataPanel"; import { ApiError, api } from "@/lib/api"; import { loadChecks } from "@/lib/loaders"; -import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -38,22 +40,19 @@ export default async function AgentDetailPage({ const sp = await searchParams; const sort = normalizeSort(sp.sort); const tab = normalizeTab(sp.tab); - const eventsLimit = normalizePageSize(sp.events_limit, 50); - const eventsCursor = sp.cursor; - const eventsPage = normalizePageNumber(sp.page); - const eventsBack = normalizeBack(sp.back); + if (sp.cursor || sp.back || sp.page || sp.events_limit) { + redirect(agentDetailHref({ agentId: agent_id, tab, sort })); + } let agent; let checks; - let events: EventsPage = { items: [], next_cursor: null, prev_cursor: null }; + let events: EventsPage = { items: [], next_cursor: null }; try { [agent, checks] = await Promise.all([api.getAgent(agent_id), loadChecks(agent_id)]); if (tab === "events") { events = await api.queryEvents({ agent_id, - limit: eventsLimit, - cursor: eventsCursor, - back: eventsBack, + limit: INFINITE_BATCH_SIZE, }); } } catch (e) { @@ -64,15 +63,24 @@ export default async function AgentDetailPage({ } return ( ); } + +function agentDetailHref({ + agentId, + tab, + sort, +}: { + agentId: string; + tab: TabKey; + sort: SortKey; +}) { + const qs = new URLSearchParams({ tab, sort }); + return `/agents/${encodeURIComponent(agentId)}?${qs.toString()}`; +} diff --git a/web/src/app/api/agents/action/route.ts b/web/src/app/api/agents/action/route.ts index ac76b94..a364ac1 100644 --- a/web/src/app/api/agents/action/route.ts +++ b/web/src/app/api/agents/action/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { api } from "@/lib/api"; import { jsonError } from "@/lib/poll-response"; +import { assertSameOrigin } from "@/lib/web-auth-server"; export const dynamic = "force-dynamic"; @@ -16,6 +17,12 @@ type Action = | "blacklist_clear"; export async function POST(request: NextRequest) { + if (!assertSameOrigin(request)) { + return NextResponse.json( + { error: { code: "forbidden", message: "cross-origin request rejected" } }, + { status: 403 }, + ); + } try { const body = (await request.json()) as { action?: Action; diff --git a/web/src/app/api/auth/login/route.ts b/web/src/app/api/auth/login/route.ts new file mode 100644 index 0000000..da4207e --- /dev/null +++ b/web/src/app/api/auth/login/route.ts @@ -0,0 +1,58 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { + cookieSecure, + constantTimeEqualString, + createSessionToken, + delayLoginFailure, + localAuthEnabled, + proxyAuthEnabled, + sanitizeNext, + SESSION_COOKIE, + sessionTtlSec, +} from "@/lib/web-auth"; +import { assertSameOrigin } from "@/lib/web-auth-server"; + +export const dynamic = "force-dynamic"; + +export async function POST(request: NextRequest) { + if (!assertSameOrigin(request)) return new NextResponse("forbidden", { status: 403 }); + const form = await request.formData(); + const next = sanitizeNext(stringField(form, "next")); + if (proxyAuthEnabled() || !localAuthEnabled()) return redirectLogin(request, next, "config"); + + const username = stringField(form, "username"); + const password = stringField(form, "password"); + const [usernameOk, passwordOk] = await Promise.all([ + constantTimeEqualString(username, process.env.MONLET_WEB_AUTH_USERNAME ?? ""), + constantTimeEqualString(password, process.env.MONLET_WEB_AUTH_PASSWORD ?? ""), + ]); + if (!usernameOk || !passwordOk) { + await delayLoginFailure(); + return redirectLogin(request, next, "invalid"); + } + + const ttl = sessionTtlSec(); + const token = await createSessionToken(username, process.env.MONLET_WEB_SESSION_SECRET ?? "", ttl); + const response = NextResponse.redirect(new URL(next, request.url), { status: 303 }); + response.cookies.set(SESSION_COOKIE, token, { + httpOnly: true, + sameSite: "lax", + secure: cookieSecure(request), + path: "/", + maxAge: ttl, + }); + return response; +} + +function redirectLogin(request: NextRequest, next: string, error: string) { + const url = new URL("/login", request.url); + url.searchParams.set("next", next); + url.searchParams.set("error", error); + return NextResponse.redirect(url, { status: 303 }); +} + +function stringField(form: FormData, name: string): string { + const value = form.get(name); + return typeof value === "string" ? value : ""; +} diff --git a/web/src/app/api/auth/logout/route.ts b/web/src/app/api/auth/logout/route.ts new file mode 100644 index 0000000..86cd397 --- /dev/null +++ b/web/src/app/api/auth/logout/route.ts @@ -0,0 +1,19 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { cookieSecure, SESSION_COOKIE } from "@/lib/web-auth"; +import { assertSameOrigin } from "@/lib/web-auth-server"; + +export const dynamic = "force-dynamic"; + +export async function POST(request: NextRequest) { + if (!assertSameOrigin(request)) return new NextResponse("forbidden", { status: 403 }); + const response = NextResponse.redirect(new URL("/login", request.url), { status: 303 }); + response.cookies.set(SESSION_COOKIE, "", { + httpOnly: true, + sameSite: "lax", + secure: cookieSecure(request), + path: "/", + maxAge: 0, + }); + return response; +} diff --git a/web/src/app/api/poll/events/route.ts b/web/src/app/api/poll/events/route.ts index 60a8d1b..5152a67 100644 --- a/web/src/app/api/poll/events/route.ts +++ b/web/src/app/api/poll/events/route.ts @@ -1,21 +1,20 @@ import { NextResponse } from "next/server"; import { loadEvents } from "@/lib/loaders"; -import { normalizePageSize } from "@/lib/pagination"; +import { normalizeFetchLimit } from "@/lib/pagination"; import { jsonError } from "@/lib/poll-response"; export const dynamic = "force-dynamic"; export async function GET(request: Request) { const url = new URL(request.url); - const limit = normalizePageSize(url.searchParams.get("limit"), 50); + const limit = normalizeFetchLimit(url.searchParams.get("limit")); try { const data = await loadEvents({ agentId: url.searchParams.get("agent_id") ?? undefined, checkId: url.searchParams.get("check_id") ?? undefined, cursor: url.searchParams.get("cursor") ?? undefined, - back: url.searchParams.get("back") === "1", limit, }); return NextResponse.json(data); diff --git a/web/src/app/api/poll/incidents/route.ts b/web/src/app/api/poll/incidents/route.ts index b713da4..4319df4 100644 --- a/web/src/app/api/poll/incidents/route.ts +++ b/web/src/app/api/poll/incidents/route.ts @@ -15,7 +15,6 @@ export async function GET(request: Request) { const stateParam = url.searchParams.get("state"); const state = stateParam === "open" || stateParam === "resolved" ? stateParam : undefined; const cursor = url.searchParams.get("cursor") ?? undefined; - const back = url.searchParams.get("back") === "1"; const limitRaw = url.searchParams.get("limit"); const parsed = limitRaw ? Number(limitRaw) : undefined; const limit = parsed && Number.isFinite(parsed) ? Math.min(Math.max(parsed, 1), 500) : 100; @@ -28,7 +27,7 @@ export async function GET(request: Request) { const direction = dirRaw && DIRS.has(dirRaw) ? (dirRaw as "asc" | "desc") : undefined; try { - const page = await loadIncidentsPage({ state, cursor, back, limit, sort, direction }); + const page = await loadIncidentsPage({ state, cursor, limit, sort, direction }); return NextResponse.json(page); } catch (e) { return jsonError(e); diff --git a/web/src/app/api/poll/outbox/route.ts b/web/src/app/api/poll/outbox/route.ts index 0dcebcb..ebaa4da 100644 --- a/web/src/app/api/poll/outbox/route.ts +++ b/web/src/app/api/poll/outbox/route.ts @@ -12,7 +12,6 @@ const DIRS = new Set(["asc", "desc"]); export async function GET(request: Request) { const url = new URL(request.url); const cursor = url.searchParams.get("cursor") ?? undefined; - const back = url.searchParams.get("back") === "1"; const limitRaw = url.searchParams.get("limit"); const parsed = limitRaw ? Number(limitRaw) : undefined; const limit = parsed && Number.isFinite(parsed) ? Math.min(Math.max(parsed, 1), 500) : 100; @@ -26,7 +25,6 @@ export async function GET(request: Request) { try { const page = await loadOutboxPage({ cursor, - back, limit, state: state as never, sort, diff --git a/web/src/app/checks/[check_id]/page.tsx b/web/src/app/checks/[check_id]/page.tsx index bf9f807..6e1a54e 100644 --- a/web/src/app/checks/[check_id]/page.tsx +++ b/web/src/app/checks/[check_id]/page.tsx @@ -1,8 +1,10 @@ +import { redirect } from "next/navigation"; + import { CheckDetailBrowser } from "@/components/CheckDetailBrowser"; import { ErrorPanel } from "@/components/DataPanel"; import { api } from "@/lib/api"; import { loadInventory } from "@/lib/loaders"; -import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -29,21 +31,18 @@ export default async function CheckDetailPage({ const { check_id } = await params; const sp = await searchParams; const tab = normalizeTab(sp.tab); - const eventsLimit = normalizePageSize(sp.events_limit, 50); - const eventsCursor = sp.cursor; - const eventsPage = normalizePageNumber(sp.page); - const eventsBack = normalizeBack(sp.back); + if (sp.cursor || sp.back || sp.page || sp.events_limit) { + redirect(checkDetailHref({ checkId: check_id, tab })); + } let data; - let events: EventsPage = { items: [], next_cursor: null, prev_cursor: null }; + let events: EventsPage = { items: [], next_cursor: null }; try { data = await loadInventory(); if (tab === "events") { events = await api.queryEvents({ check_id, - limit: eventsLimit, - cursor: eventsCursor, - back: eventsBack, + limit: INFINITE_BATCH_SIZE, }); } } catch (e) { @@ -52,15 +51,22 @@ export default async function CheckDetailPage({ return ( ); } + +function checkDetailHref({ + checkId, + tab, +}: { + checkId: string; + tab: TabKey; +}) { + const qs = new URLSearchParams({ tab }); + return `/checks/${encodeURIComponent(checkId)}?${qs.toString()}`; +} diff --git a/web/src/app/events/page.tsx b/web/src/app/events/page.tsx index d3aa8a6..cfb1600 100644 --- a/web/src/app/events/page.tsx +++ b/web/src/app/events/page.tsx @@ -1,7 +1,9 @@ +import { redirect } from "next/navigation"; + import { ErrorPanel } from "@/components/DataPanel"; import { EventsBrowser } from "@/components/EventsBrowser"; import { loadEvents } from "@/lib/loaders"; -import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -18,31 +20,39 @@ export default async function EventsPage({ }>; }) { const sp = await searchParams; - const limit = normalizePageSize(sp.limit); - const page = normalizePageNumber(sp.page); - const back = normalizeBack(sp.back); + if (sp.cursor || sp.back || sp.page || sp.limit) { + redirect(eventsHref({ agentId: sp.agent_id, checkId: sp.check_id })); + } let data; try { data = await loadEvents({ agentId: sp.agent_id, checkId: sp.check_id, - cursor: sp.cursor, - back, - limit, + limit: INFINITE_BATCH_SIZE, }); } catch (e) { return ; } return ( ); } + +function eventsHref({ + agentId, + checkId, +}: { + agentId?: string; + checkId?: string; +}) { + const qs = new URLSearchParams(); + if (agentId) qs.set("agent_id", agentId); + if (checkId) qs.set("check_id", checkId); + const query = qs.toString(); + return query ? `/events?${query}` : "/events"; +} diff --git a/web/src/app/incidents/page.tsx b/web/src/app/incidents/page.tsx index 177f04c..66cb214 100644 --- a/web/src/app/incidents/page.tsx +++ b/web/src/app/incidents/page.tsx @@ -1,7 +1,9 @@ +import { redirect } from "next/navigation"; + import { ErrorPanel } from "@/components/DataPanel"; import { IncidentsBrowser } from "@/components/IncidentsBrowser"; import { loadIncidentsPage, loadInventory } from "@/lib/loaders"; -import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -25,14 +27,14 @@ export default async function IncidentsPage({ ? sp.sort : "status"; const direction = sp.direction === "asc" || sp.direction === "desc" ? sp.direction : "desc"; - const limit = normalizePageSize(sp.limit); - const pageNumber = normalizePageNumber(sp.page); - const back = normalizeBack(sp.back); + if (sp.cursor || sp.back || sp.page || sp.limit) { + redirect(incidentsHref({ state, sort, direction })); + } let page; let inventory; try { [page, inventory] = await Promise.all([ - loadIncidentsPage({ state, cursor: sp.cursor, back, limit, sort, direction }), + loadIncidentsPage({ state, limit: INFINITE_BATCH_SIZE, sort, direction }), loadInventory(), ]); } catch (e) { @@ -40,16 +42,28 @@ export default async function IncidentsPage({ } return ( ); } + +function incidentsHref({ + state, + sort, + direction, +}: { + state?: "open" | "resolved"; + sort: "status" | "opened_at" | "resolved_at"; + direction: "asc" | "desc"; +}) { + const qs = new URLSearchParams(); + if (state) qs.set("state", state); + qs.set("sort", sort); + qs.set("direction", direction); + return `/incidents?${qs.toString()}`; +} diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx index 3298a16..8e5932f 100644 --- a/web/src/app/layout.tsx +++ b/web/src/app/layout.tsx @@ -3,8 +3,9 @@ import type { ReactNode } from "react"; import { TimeZoneProvider } from "@/components/TimeZoneControls"; import { TopNav } from "@/components/TopNav"; import { loadSystemSummary } from "@/lib/loaders"; -import { getUiTimeZone } from "@/lib/timezone"; +import { getInitialTimeZonePreference, getUiTimeZone } from "@/lib/timezone"; import type { SystemSummaryData } from "@/lib/view-types"; +import { requestHasAppAccess, showLocalLogout } from "@/lib/web-auth-server"; import "./globals.css"; export const metadata: Metadata = { @@ -24,13 +25,19 @@ async function safeLoadSystemSummary(): Promise { export default async function RootLayout({ children }: { children: ReactNode }) { const configuredTimeZone = getUiTimeZone(); - const summary = await safeLoadSystemSummary(); + const timezonePreference = await getInitialTimeZonePreference(); + const hasAppAccess = await requestHasAppAccess(); + const summary = hasAppAccess ? await safeLoadSystemSummary() : null; return ( - - + + {hasAppAccess ? : null}
{children}
 diff --git a/web/src/app/login/page.tsx b/web/src/app/login/page.tsx new file mode 100644 index 0000000..1c06dd8 --- /dev/null +++ b/web/src/app/login/page.tsx @@ -0,0 +1,96 @@ +import { redirect } from "next/navigation"; + +import { + localAuthEnabled, + localAuthMisconfigured, + proxyAuthEnabled, + sanitizeNext, +} from "@/lib/web-auth"; +import { requestHasAppAccess } from "@/lib/web-auth-server"; + +export const dynamic = "force-dynamic"; + +export default async function LoginPage({ + searchParams, +}: { + searchParams: Promise<{ next?: string; error?: string }>; +}) { + const sp = await searchParams; + const next = sanitizeNext(sp.next); + + if (await requestHasAppAccess()) redirect(next); + + const proxyMode = proxyAuthEnabled(); + const enabled = localAuthEnabled(); + const misconfigured = localAuthMisconfigured(); + const error = + sp.error === "invalid" + ? "Invalid username or password." + : sp.error === "config" || misconfigured + ? "Web authentication is not configured correctly." + : null; + + return ( +
+
+ + + + + + +

+ monlet +

+
+ +

Sign in

+ {error ? ( +
+ {error} +
+ ) : null} + {misconfigured ? ( +
+ Web authentication is misconfigured. +
+ ) : proxyMode ? ( +
+ Proxy authentication is enabled. This page is only shown when the proxy did not pass + an authenticated user. +
+ ) : enabled ? ( +
+ + + + +
+ ) : ( +
+ Local web authentication is disabled. +
+ )} +
+ ); +} diff --git a/web/src/app/outbox/page.tsx b/web/src/app/outbox/page.tsx index 9f93d6a..7f11e24 100644 --- a/web/src/app/outbox/page.tsx +++ b/web/src/app/outbox/page.tsx @@ -1,7 +1,9 @@ +import { redirect } from "next/navigation"; + import { ErrorPanel } from "@/components/DataPanel"; import { OutboxBrowser } from "@/components/OutboxBrowser"; import { loadOutboxPage } from "@/lib/loaders"; -import { normalizeBack, normalizePageNumber, normalizePageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; export const dynamic = "force-dynamic"; @@ -18,27 +20,34 @@ export default async function OutboxPage({ }>; }) { const sp = await searchParams; - const limit = normalizePageSize(sp.limit); const sort = sp.sort === "created_at" || sp.sort === "updated_at" ? sp.sort : "updated_at"; const direction = sp.direction === "asc" || sp.direction === "desc" ? sp.direction : "desc"; - const pageNumber = normalizePageNumber(sp.page); - const back = normalizeBack(sp.back); + if (sp.cursor || sp.back || sp.page || sp.limit) redirect(outboxHref({ sort, direction })); let page; try { - page = await loadOutboxPage({ cursor: sp.cursor, back, limit, sort, direction }); + page = await loadOutboxPage({ limit: INFINITE_BATCH_SIZE, sort, direction }); } catch (e) { return ; } return ( ); } + +function outboxHref({ + sort, + direction, +}: { + sort: "created_at" | "updated_at"; + direction: "asc" | "desc"; +}) { + const qs = new URLSearchParams(); + qs.set("sort", sort); + qs.set("direction", direction); + return `/outbox?${qs.toString()}`; +} diff --git a/web/src/components/AgentDetailBrowser.tsx b/web/src/components/AgentDetailBrowser.tsx index a5e96c1..68e6512 100644 --- a/web/src/components/AgentDetailBrowser.tsx +++ b/web/src/components/AgentDetailBrowser.tsx @@ -1,33 +1,25 @@ "use client"; import Link from "next/link"; -import { useRouter } from "next/navigation"; -import { useMemo, useState } from "react"; +import { useCallback, useMemo } from "react"; import { AgentFeatures } from "@/components/AgentFeatures"; import { CheckEventsLink } from "@/components/CheckEventsLink"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, SummaryCell, Td, Th } from "@/components/DataPanel"; import { DetailTabs } from "@/components/DetailTabs"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { LabelChips } from "@/components/Labels"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { Pager } from "@/components/Pager"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteClientList, useInfiniteCursorList } from "@/lib/useInfiniteList"; import { usePolling } from "@/lib/usePolling"; type Agent = components["schemas"]["Agent"]; type CheckState = components["schemas"]["CheckState"]; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; type SortKey = "severity" | "check_id" | "last_seen"; type TabKey = "checks" | "events"; type AgentDetailData = { agent: Agent; checks: CheckState[] }; @@ -45,19 +37,11 @@ export function AgentDetailBrowser({ initialEvents, sort, tab, - eventsLimit, - eventsCursor, - eventsBack, - eventsPage, }: { initialData: AgentDetailData; initialEvents: EventsPageData; sort: SortKey; tab: TabKey; - eventsLimit: PageSize; - eventsCursor?: string; - eventsBack: boolean; - eventsPage: number; }) { const agentId = initialData.agent.agent_id; const pollUrl = `/api/poll/agent?agent_id=${encodeURIComponent(agentId)}`; @@ -102,22 +86,14 @@ export function AgentDetailBrowser({ { key: "events", label: "Events", - href: `/agents/${encodeURIComponent(agentId)}?tab=events&sort=${sort}&events_limit=${eventsLimit}`, + href: `/agents/${encodeURIComponent(agentId)}?tab=events&sort=${sort}`, }, ]} /> {tab === "checks" ? ( ) : ( - + )} ); @@ -132,22 +108,19 @@ function ChecksTable({ checks: CheckState[]; sort: SortKey; }) { - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); - const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); - }; - const pageCount = pageCountOf(checks.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(checks, currentPage, pageSize); + const { + visibleItems: visible, + hasMore, + sentinelRef, + } = useInfiniteClientList({ + items: checks, + batchSize: INFINITE_BATCH_SIZE, + resetKey: `${agentId}:${sort}`, + }); if (checks.length === 0) return ; return (
-
- -
@@ -184,48 +157,45 @@ function ChecksTable({ ))}
- +
); } function EventsTable({ agentId, - sort, - eventsLimit, - eventsCursor, - eventsBack, - eventsPage, initialEvents, }: { agentId: string; - sort: SortKey; - eventsLimit: PageSize; - eventsCursor?: string; - eventsBack: boolean; - eventsPage: number; initialEvents: EventsPageData; }) { - const router = useRouter(); const pollUrl = useMemo(() => { - const qs = new URLSearchParams({ agent_id: agentId, limit: String(eventsLimit) }); - if (eventsCursor) qs.set("cursor", eventsCursor); - if (eventsBack) qs.set("back", "1"); + const qs = new URLSearchParams({ agent_id: agentId, limit: String(INFINITE_BATCH_SIZE) }); return `/api/poll/events?${qs.toString()}`; - }, [agentId, eventsCursor, eventsBack, eventsLimit]); - const { data } = usePolling(initialEvents, pollUrl); - - const onLimitChange = (next: PageSize) => { - const qs = new URLSearchParams({ tab: "events", sort, events_limit: String(next) }); - router.push(`/agents/${encodeURIComponent(agentId)}?${qs.toString()}`); - }; + }, [agentId]); + const getPageUrl = useCallback( + (cursor: string) => `${pollUrl}&cursor=${encodeURIComponent(cursor)}`, + [pollUrl], + ); + const { + items, + hasMore, + loadingMore, + error, + sentinelRef, + hasNewItems, + showLatest, + } = useInfiniteCursorList({ + initialData: initialEvents, + firstPageUrl: pollUrl, + getPageUrl, + getKey: eventKey, + resetKey: agentId, + }); return (
-
- -
- {data.items.length === 0 ? ( + {items.length === 0 ? ( ) : ( @@ -239,7 +209,7 @@ function EventsTable({ - {data.items.map((e) => ( + {items.map((e) => (
@@ -255,12 +225,13 @@ function EventsTable({
)} -
); @@ -294,3 +265,7 @@ function sortChecks(checks: CheckState[], sort: SortKey): CheckState[] { return severityRank[a.status] - severityRank[b.status] || a.check_id.localeCompare(b.check_id); }); } + +function eventKey(event: StoredEvent): string { + return event.event_id; +} diff --git a/web/src/components/AgentsBrowser.tsx b/web/src/components/AgentsBrowser.tsx index 630f8d6..69e13cd 100644 --- a/web/src/components/AgentsBrowser.tsx +++ b/web/src/components/AgentsBrowser.tsx @@ -5,21 +5,16 @@ import { useMemo, useState } from "react"; import { AgentFeatures } from "@/components/AgentFeatures"; import { AgentLink } from "@/components/AgentLink"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, Td, Th } from "@/components/DataPanel"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { LabelChips } from "@/components/Labels"; import { ListToolbar } from "@/components/ListToolbar"; import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteClientList } from "@/lib/useInfiniteList"; import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; @@ -56,19 +51,18 @@ export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { ({ agent }) => agent.rotation_pending, ).length; - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); - const pageCount = pageCountOf(rows.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(rows, currentPage, pageSize); + const { + visibleItems: visible, + hasMore, + sentinelRef, + } = useInfiniteClientList({ + items: rows, + batchSize: INFINITE_BATCH_SIZE, + resetKey: `${query}:${sort}:${dir}`, + }); const onQueryChange = (value: string) => { setQuery(value); - setPage(1); - }; - const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); }; const chooseSort = (field: SortField) => { if (field === sort) { @@ -77,7 +71,6 @@ export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { setSort(field); setDir(fieldDefaultDir(field)); } - setPage(1); }; const runAction = async ( action: string, @@ -123,8 +116,6 @@ export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { query={query} onQueryChange={onQueryChange} placeholder="search agent" - pageSize={pageSize} - onPageSizeChange={onPageSizeChange} /> {rows.length === 0 ? ( @@ -140,7 +131,7 @@ export function AgentsBrowser({ initialData }: { initialData: InventoryData }) { rotationPendingCount={allRotationPendingCount} /> )} - + ); } @@ -173,6 +164,8 @@ function AgentsTable({ rotationPendingCount > 0 ? `ROTATION: existing accepted keys will be replaced for ${rotationPendingCount} agents. Accept all pending agents including rotations?` : undefined; + const visiblePendingRows = rows.filter(({ agent }) => agent.admission_state === "pending"); + const visibleAcceptedRows = rows.filter(({ agent }) => agent.admission_state === "accepted"); return ( @@ -198,69 +191,105 @@ function AgentsTable({ - {pendingCount > 0 ? ( + {visiblePendingRows.length > 0 ? ( + + ) : null} + {visiblePendingRows.map((row) => ( + + ))} + {visiblePendingRows.length > 0 ? ( + + ) : null} - {rows.map(({ agent: a, checks }) => ( - - - - - - - - + {visibleAcceptedRows.map((row) => ( + ))}
-
+
{pendingCount} pending - 0 ? ( + {rotationPendingCount} replacements + ) : null} +
+
+
+ runAction("accept_all", undefined, acceptAllText, rotationPendingCount > 0) } /> - runAction("block_all", undefined, "Block all pending agents?")} /> - runAction("remove_all", undefined, "Remove all pending agents and their data?")} + onClick={() => + runAction("remove_all", undefined, "Remove all pending agents and their data?") + } />
-
- - {a.admission_state === "pending" ? ( - - {a.rotation_pending ? "ROTATION" : "pending"} - - ) : null} -
- 		- - - - - - - - - -
); } +function AgentTableRow({ + row, + busy, + runAction, +}: { + row: AgentRow; + busy: string | null; + runAction: ( + action: string, + agentId?: string, + confirmText?: string, + includeRotation?: boolean, + ) => void; +}) { + const { agent: a, checks } = row; + return ( + + +
+ + {a.admission_state === "pending" ? ( + + {a.rotation_pending ? "ROTATION" : "pending"} + + ) : null} +
+ + + + + + + + + + + + + + + + + + ); +} + function AgentActions({ agent, busy, @@ -313,6 +342,31 @@ function AgentActions({ ); } +function BulkActionButton({ + label, + title, + disabled, + onClick, +}: { + label: string; + title: string; + disabled: boolean; + onClick: () => void; +}) { + return ( + + ); +} + function ActionButton({ label, title, diff --git a/web/src/components/CheckDetailBrowser.tsx b/web/src/components/CheckDetailBrowser.tsx index 4fd7fbd..b45ae09 100644 --- a/web/src/components/CheckDetailBrowser.tsx +++ b/web/src/components/CheckDetailBrowser.tsx @@ -1,32 +1,24 @@ "use client"; import Link from "next/link"; -import { useRouter } from "next/navigation"; -import { useMemo, useState } from "react"; +import { useCallback, useMemo } from "react"; import { AgentLink } from "@/components/AgentLink"; import { CheckStatusSummary } from "@/components/CheckStatusSummary"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, SummaryCell, Td, Th } from "@/components/DataPanel"; import { DetailTabs } from "@/components/DetailTabs"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { Pager } from "@/components/Pager"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { groupChecks, statusSeverity, type Agent, type CheckGroup } from "@/lib/check-groups"; import { statusBadgeClass } from "@/lib/format"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteClientList, useInfiniteCursorList } from "@/lib/useInfiniteList"; import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; type TabKey = "agents" | "events"; export function CheckDetailBrowser({ @@ -34,19 +26,11 @@ export function CheckDetailBrowser({ initialData, initialEvents, tab, - eventsLimit, - eventsCursor, - eventsBack, - eventsPage, }: { checkId: string; initialData: InventoryData; initialEvents: EventsPageData; tab: TabKey; - eventsLimit: PageSize; - eventsCursor?: string; - eventsBack: boolean; - eventsPage: number; }) { const { data } = usePolling(initialData, "/api/poll/inventory"); const agentsById = useMemo( @@ -101,7 +85,7 @@ export function CheckDetailBrowser({ { key: "events", label: "Events", - href: `/checks/${encodeURIComponent(checkId)}?tab=events&events_limit=${eventsLimit}`, + href: `/checks/${encodeURIComponent(checkId)}?tab=events`, }, ]} /> @@ -112,10 +96,6 @@ export function CheckDetailBrowser({ checkId={checkId} initialData={initialEvents} agentsById={agentsById} - eventsLimit={eventsLimit} - eventsCursor={eventsCursor} - eventsBack={eventsBack} - eventsPage={eventsPage} /> )} @@ -133,21 +113,18 @@ function AgentsForCheck({ group }: { group: CheckGroup }) { [group.checks], ); - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); - const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); - }; - const pageCount = pageCountOf(rows.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(rows, currentPage, pageSize); + const { + visibleItems: visible, + hasMore, + sentinelRef, + } = useInfiniteClientList({ + items: rows, + batchSize: INFINITE_BATCH_SIZE, + resetKey: group.checkId, + }); return (
-
- -
@@ -172,7 +149,7 @@ function AgentsForCheck({ group }: { group: CheckGroup }) { ))}
- +
); } @@ -181,43 +158,41 @@ function EventsForCheck({ checkId, initialData, agentsById, - eventsLimit, - eventsCursor, - eventsBack, - eventsPage, }: { checkId: string; initialData: EventsPageData; agentsById: Map; - eventsLimit: PageSize; - eventsCursor?: string; - eventsBack: boolean; - eventsPage: number; }) { - const router = useRouter(); const pollUrl = useMemo(() => { const qs = new URLSearchParams({ check_id: checkId, - limit: String(eventsLimit), + limit: String(INFINITE_BATCH_SIZE), }); - if (eventsCursor) qs.set("cursor", eventsCursor); - if (eventsBack) qs.set("back", "1"); return `/api/poll/events?${qs.toString()}`; - }, [checkId, eventsCursor, eventsBack, eventsLimit]); - const { data } = usePolling(initialData, pollUrl); - - const onLimitChange = (next: PageSize) => { - router.push( - `/checks/${encodeURIComponent(checkId)}?tab=events&events_limit=${next}`, - ); - }; + }, [checkId]); + const getPageUrl = useCallback( + (cursor: string) => `${pollUrl}&cursor=${encodeURIComponent(cursor)}`, + [pollUrl], + ); + const { + items, + hasMore, + loadingMore, + error, + sentinelRef, + hasNewItems, + showLatest, + } = useInfiniteCursorList({ + initialData, + firstPageUrl: pollUrl, + getPageUrl, + getKey: eventKey, + resetKey: checkId, + }); return (
-
- -
- {data.items.length === 0 ? ( + {items.length === 0 ? ( ) : ( @@ -232,7 +207,7 @@ function EventsForCheck({ - {data.items.map((event) => { + {items.map((event) => { const agent = agentsById.get(event.agent_id); return ( @@ -254,12 +229,13 @@ function EventsForCheck({
)} -
); @@ -268,3 +244,7 @@ function EventsForCheck({ function hostName(row: { agent?: Agent; check: { agent_id: string } }): string { return row.agent?.hostname || row.check.agent_id; } + +function eventKey(event: StoredEvent): string { + return event.event_id; +} diff --git a/web/src/components/ChecksBrowser.tsx b/web/src/components/ChecksBrowser.tsx index ae3a6ce..ddc420b 100644 --- a/web/src/components/ChecksBrowser.tsx +++ b/web/src/components/ChecksBrowser.tsx @@ -4,8 +4,8 @@ import Link from "next/link"; import { useMemo, useState } from "react"; import { CheckStatusSummary } from "@/components/CheckStatusSummary"; -import { ClientPager } from "@/components/ClientPager"; import { EmptyPanel, Td, Th } from "@/components/DataPanel"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { ListToolbar } from "@/components/ListToolbar"; import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; @@ -14,13 +14,8 @@ import { groupChecks, type CheckGroup, } from "@/lib/check-groups"; -import { - DEFAULT_PAGE_SIZE, - clampPage, - pageCountOf, - paginate, - type PageSize, -} from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteClientList } from "@/lib/useInfiniteList"; import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; @@ -39,19 +34,18 @@ export function ChecksBrowser({ initialData }: { initialData: InventoryData }) { [dir, groups, query, sort], ); - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [page, setPage] = useState(1); - const pageCount = pageCountOf(rows.length, pageSize); - const currentPage = clampPage(page, pageCount); - const visible = paginate(rows, currentPage, pageSize); + const { + visibleItems: visible, + hasMore, + sentinelRef, + } = useInfiniteClientList({ + items: rows, + batchSize: INFINITE_BATCH_SIZE, + resetKey: `${query}:${sort}:${dir}`, + }); const onQueryChange = (value: string) => { setQuery(value); - setPage(1); - }; - const onPageSizeChange = (next: PageSize) => { - setPageSize(next); - setPage(1); }; const chooseSort = (field: SortField) => { if (field === sort) { @@ -60,7 +54,6 @@ export function ChecksBrowser({ initialData }: { initialData: InventoryData }) { setSort(field); setDir(fieldDefaultDir(field)); } - setPage(1); }; return ( @@ -75,8 +68,6 @@ export function ChecksBrowser({ initialData }: { initialData: InventoryData }) { query={query} onQueryChange={onQueryChange} placeholder="search check" - pageSize={pageSize} - onPageSizeChange={onPageSizeChange} /> {rows.length === 0 ? ( @@ -125,7 +116,7 @@ export function ChecksBrowser({ initialData }: { initialData: InventoryData }) { )} - + ); } diff --git a/web/src/components/ClientPager.tsx b/web/src/components/ClientPager.tsx deleted file mode 100644 index 4529841..0000000 --- a/web/src/components/ClientPager.tsx +++ /dev/null @@ -1,39 +0,0 @@ -"use client"; - -export function ClientPager({ - page, - pageCount, - onPageChange, -}: { - page: number; - pageCount: number; - onPageChange: (page: number) => void; -}) { - const prev = () => onPageChange(Math.max(1, page - 1)); - const next = () => onPageChange(Math.min(pageCount, page + 1)); - const disabledPrev = page <= 1; - const disabledNext = page >= pageCount; - return ( -
- - - page {page} of {pageCount} - - -
- ); -} diff --git a/web/src/components/EventsBrowser.tsx b/web/src/components/EventsBrowser.tsx index e30a89e..30812e6 100644 --- a/web/src/components/EventsBrowser.tsx +++ b/web/src/components/EventsBrowser.tsx @@ -1,58 +1,60 @@ "use client"; import Link from "next/link"; -import { useRouter } from "next/navigation"; -import { useMemo } from "react"; +import { useCallback, useMemo } from "react"; import { CheckEventsLink } from "@/components/CheckEventsLink"; import { EmptyPanel, PageHeader, SummaryCell, Td, Th } from "@/components/DataPanel"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { Pager } from "@/components/Pager"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { type PageSize } from "@/lib/pagination"; -import { usePolling } from "@/lib/usePolling"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteCursorList } from "@/lib/useInfiniteList"; type StoredEvent = components["schemas"]["StoredCheckResultEvent"]; -type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }; +type EventsPageData = { items: StoredEvent[]; next_cursor?: string | null }; export function EventsBrowser({ initialData, - cursor, - back, agentId, checkId, - limit, - page, }: { initialData: EventsPageData; - cursor?: string; - back: boolean; agentId?: string; checkId?: string; - limit: PageSize; - page: number; }) { - const router = useRouter(); - const url = useMemo( - () => eventsPollUrl({ cursor, back, agentId, checkId, limit }), - [agentId, back, checkId, cursor, limit], + const firstPageUrl = useMemo( + () => eventsPollUrl({ agentId, checkId }), + [agentId, checkId], ); - const { data } = usePolling(initialData, url); - - const onLimitChange = (next: PageSize) => { - router.push(eventsHref({ agent_id: agentId, check_id: checkId, limit: next })); - }; + const getPageUrl = useCallback( + (cursor: string) => `${firstPageUrl}&cursor=${encodeURIComponent(cursor)}`, + [firstPageUrl], + ); + const { + items, + hasMore, + loadingMore, + error, + sentinelRef, + hasNewItems, + showLatest, + } = useInfiniteCursorList({ + initialData, + firstPageUrl, + getPageUrl, + getKey: eventKey, + resetKey: `${agentId ?? ""}:${checkId ?? ""}`, + }); return ( <> - -
- - + +
+
- {data.items.length === 0 ? ( + {items.length === 0 ? ( ) : ( @@ -68,7 +70,7 @@ export function EventsBrowser({ - {data.items.map((e) => ( + {items.map((e) => (
@@ -88,12 +90,13 @@ export function EventsBrowser({
)} - ); @@ -102,11 +105,9 @@ export function EventsBrowser({ function ActiveFilters({ agentId, checkId, - limit, }: { agentId?: string; checkId?: string; - limit: PageSize; }) { if (!agentId && !checkId) return
; @@ -116,18 +117,18 @@ function ActiveFilters({ ) : null} {checkId ? ( ) : null} clear @@ -153,34 +154,28 @@ function FilterChip({ label, value, href }: { label: string; value: string; href function eventsHref(params: { agent_id?: string; check_id?: string; - limit?: PageSize; }): string { const qs = new URLSearchParams(); if (params.agent_id) qs.set("agent_id", params.agent_id); if (params.check_id) qs.set("check_id", params.check_id); - if (params.limit) qs.set("limit", String(params.limit)); const query = qs.toString(); return query ? `/events?${query}` : "/events"; } function eventsPollUrl({ - cursor, - back, agentId, checkId, - limit, }: { - cursor?: string; - back: boolean; agentId?: string; checkId?: string; - limit: PageSize; }) { const qs = new URLSearchParams(); - if (cursor) qs.set("cursor", cursor); - if (back) qs.set("back", "1"); if (agentId) qs.set("agent_id", agentId); if (checkId) qs.set("check_id", checkId); - qs.set("limit", String(limit)); + qs.set("limit", String(INFINITE_BATCH_SIZE)); return `/api/poll/events?${qs.toString()}`; } + +function eventKey(event: StoredEvent): string { + return event.event_id; +} diff --git a/web/src/components/IncidentsBrowser.tsx b/web/src/components/IncidentsBrowser.tsx index c1059a7..22ed970 100644 --- a/web/src/components/IncidentsBrowser.tsx +++ b/web/src/components/IncidentsBrowser.tsx @@ -2,18 +2,18 @@ import Link from "next/link"; import { useRouter } from "next/navigation"; -import { useMemo } from "react"; +import { useCallback, useMemo } from "react"; import { AgentLink } from "@/components/AgentLink"; import { CheckEventsLink } from "@/components/CheckEventsLink"; import { EmptyPanel, PageHeader, SummaryCell, Td, Th } from "@/components/DataPanel"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { Pager } from "@/components/Pager"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { type PageSize } from "@/lib/pagination"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteCursorList } from "@/lib/useInfiniteList"; import { usePolling } from "@/lib/usePolling"; import type { InventoryData } from "@/lib/view-types"; @@ -21,51 +21,56 @@ const STATES = ["all", "open", "resolved"] as const; type Incident = components["schemas"]["Incident"]; type Agent = components["schemas"]["Agent"]; -type IncidentsData = { items: Incident[]; next_cursor?: string | null; prev_cursor?: string | null }; +type IncidentsData = { items: Incident[]; next_cursor?: string | null }; type SortField = "status" | "opened_at" | "resolved_at"; export function IncidentsBrowser({ initialData, initialInventory, state, - cursor, - back, sort, direction, - limit, - page, }: { initialData: IncidentsData; initialInventory: InventoryData; state?: "open" | "resolved"; - cursor?: string; - back: boolean; sort: SortField; direction: SortDir; - limit: PageSize; - page: number; }) { const router = useRouter(); - const url = useMemo(() => { + const firstPageUrl = useMemo(() => { const q = new URLSearchParams(); if (state) q.set("state", state); q.set("sort", sort); q.set("direction", direction); - q.set("limit", String(limit)); - if (cursor) q.set("cursor", cursor); - if (back) q.set("back", "1"); + q.set("limit", String(INFINITE_BATCH_SIZE)); return `/api/poll/incidents?${q.toString()}`; - }, [cursor, back, direction, limit, state, sort]); - const { data } = usePolling(initialData, url); + }, [direction, state, sort]); + const getPageUrl = useCallback( + (cursor: string) => `${firstPageUrl}&cursor=${encodeURIComponent(cursor)}`, + [firstPageUrl], + ); + const { + items, + hasMore, + loadingMore, + error, + sentinelRef, + hasNewItems, + showLatest, + } = useInfiniteCursorList({ + initialData, + firstPageUrl, + getPageUrl, + getKey: incidentKey, + resetKey: `${state ?? "all"}:${sort}:${direction}`, + }); const { data: inventory } = usePolling(initialInventory, "/api/poll/inventory"); const agentsById = useMemo( () => new Map(inventory.agents.map((agent) => [agent.agent_id, agent])), [inventory.agents], ); - const onPageSizeChange = (next: PageSize) => { - router.push(incidentsHref({ state, sort, direction, limit: next })); - }; const chooseSort = (field: SortField) => { let nextSort = sort; let nextDirection = direction; @@ -75,21 +80,20 @@ export function IncidentsBrowser({ nextSort = field; nextDirection = fieldDefaultDir(); } - router.push(incidentsHref({ state, sort: nextSort, direction: nextDirection, limit })); + router.push(incidentsHref({ state, sort: nextSort, direction: nextDirection })); }; return ( <> - +
{STATES.map((s) => { - const href = incidentsHref({ - state: s === "all" ? undefined : s, - sort: sort === "resolved_at" && s !== "resolved" ? "status" : sort, - direction, - limit, - }); + const href = incidentsHref({ + state: s === "all" ? undefined : s, + sort: sort === "resolved_at" && s !== "resolved" ? "status" : sort, + direction, + }); const active = (s === "all" && !state) || s === state; return ( -
- {data.items.length === 0 ? ( + {items.length === 0 ? ( ) : ( @@ -150,7 +153,7 @@ export function IncidentsBrowser({ - {data.items.map((incident) => ( + {items.map((incident) => (
)} - ); @@ -220,6 +224,10 @@ function incidentCheckId(incident: Incident): string { return incident.check_id ?? ""; } +function incidentKey(incident: Incident): string { + return String(incident.id); +} + function fieldDefaultDir(): SortDir { return "desc"; } @@ -228,18 +236,15 @@ function incidentsHref({ state, sort, direction, - limit, }: { state?: "open" | "resolved"; sort: SortField; direction: SortDir; - limit: PageSize; }) { const qs = new URLSearchParams(); if (state) qs.set("state", state); qs.set("sort", sort); qs.set("direction", direction); - qs.set("limit", String(limit)); const query = qs.toString(); return query ? `/incidents?${query}` : "/incidents"; } diff --git a/web/src/components/InfiniteListFooter.tsx b/web/src/components/InfiniteListFooter.tsx new file mode 100644 index 0000000..eda65b7 --- /dev/null +++ b/web/src/components/InfiniteListFooter.tsx @@ -0,0 +1,40 @@ +"use client"; + +import type { RefObject } from "react"; + +export function InfiniteListFooter({ + sentinelRef, + hasMore, + loading, + error, + hasNewItems, + onShowLatest, +}: { + sentinelRef: RefObject; + hasMore: boolean; + loading?: boolean; + error?: string | null; + hasNewItems?: boolean; + onShowLatest?: () => void; +}) { + if (!hasMore && !loading && !error && !hasNewItems) return null; + + return ( + <> + {hasNewItems && onShowLatest ? ( + + ) : null} +
+ {error ? {error} : null} + {!error && loading ? "loading..." : null} + {!error && !loading && hasMore ? "scroll for more" : null} +
+ + ); +} diff --git a/web/src/components/ListToolbar.tsx b/web/src/components/ListToolbar.tsx index d7816f0..65ca079 100644 --- a/web/src/components/ListToolbar.tsx +++ b/web/src/components/ListToolbar.tsx @@ -1,20 +1,13 @@ "use client"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { type PageSize } from "@/lib/pagination"; - export function ListToolbar({ query, onQueryChange, placeholder, - pageSize, - onPageSizeChange, }: { query: string; onQueryChange: (value: string) => void; placeholder: string; - pageSize: PageSize; - onPageSizeChange: (next: PageSize) => void; }) { return (
@@ -36,7 +29,6 @@ export function ListToolbar({ ) : null}
-
); } diff --git a/web/src/components/OutboxBrowser.tsx b/web/src/components/OutboxBrowser.tsx index 7ba50b9..ba6ad59 100644 --- a/web/src/components/OutboxBrowser.tsx +++ b/web/src/components/OutboxBrowser.tsx @@ -1,66 +1,67 @@ "use client"; import { useRouter } from "next/navigation"; -import { useMemo } from "react"; +import { useCallback, useMemo } from "react"; import { EmptyPanel, PageHeader, Td, Th } from "@/components/DataPanel"; -import { PageSizeSelect } from "@/components/PageSizeSelect"; -import { Pager } from "@/components/Pager"; +import { InfiniteListFooter } from "@/components/InfiniteListFooter"; import { SortHeaderButton, type SortDir } from "@/components/SortHeaderButton"; import { DateTime } from "@/components/TimeZoneControls"; import type { components } from "@/lib/api-types"; import { statusBadgeClass } from "@/lib/format"; -import { type PageSize } from "@/lib/pagination"; -import { usePolling } from "@/lib/usePolling"; +import { INFINITE_BATCH_SIZE } from "@/lib/pagination"; +import { useInfiniteCursorList } from "@/lib/useInfiniteList"; type OutboxItem = components["schemas"]["NotificationOutboxItem"]; -type OutboxData = { items: OutboxItem[]; next_cursor?: string | null; prev_cursor?: string | null }; +type OutboxData = { items: OutboxItem[]; next_cursor?: string | null }; type SortField = "created_at" | "updated_at"; export function OutboxBrowser({ initialData, - cursor, - back, sort, direction, - limit, - page, }: { initialData: OutboxData; - cursor?: string; - back: boolean; sort: SortField; direction: SortDir; - limit: PageSize; - page: number; }) { const router = useRouter(); - const url = useMemo(() => { + const firstPageUrl = useMemo(() => { const q = new URLSearchParams(); q.set("sort", sort); q.set("direction", direction); - q.set("limit", String(limit)); - if (cursor) q.set("cursor", cursor); - if (back) q.set("back", "1"); + q.set("limit", String(INFINITE_BATCH_SIZE)); return `/api/poll/outbox?${q.toString()}`; - }, [cursor, back, direction, limit, sort]); - const { data } = usePolling(initialData, url); + }, [direction, sort]); + const getPageUrl = useCallback( + (cursor: string) => `${firstPageUrl}&cursor=${encodeURIComponent(cursor)}`, + [firstPageUrl], + ); + const { + items, + hasMore, + loadingMore, + error, + sentinelRef, + hasNewItems, + showLatest, + } = useInfiniteCursorList({ + initialData, + firstPageUrl, + getPageUrl, + getKey: outboxKey, + resetKey: `${sort}:${direction}`, + }); - const onPageSizeChange = (next: PageSize) => { - router.push(outboxHref({ sort, direction, limit: next })); - }; const chooseSort = (field: SortField) => { const nextDirection = field === sort && direction === "desc" ? "asc" : "desc"; - router.push(outboxHref({ sort: field, direction: nextDirection, limit })); + router.push(outboxHref({ sort: field, direction: nextDirection })); }; return ( <> - -
- -
- {data.items.length === 0 ? ( + + {items.length === 0 ? ( ) : ( @@ -80,7 +81,7 @@ export function OutboxBrowser({ - {data.items.map((o) => ( + {items.map((o) => (
@@ -98,12 +99,13 @@ export function OutboxBrowser({
)} - ); @@ -112,15 +114,16 @@ export function OutboxBrowser({ function outboxHref({ sort, direction, - limit, }: { sort: SortField; direction: SortDir; - limit: PageSize; }) { const qs = new URLSearchParams(); qs.set("sort", sort); qs.set("direction", direction); - qs.set("limit", String(limit)); return `/outbox?${qs.toString()}`; } + +function outboxKey(item: OutboxItem): string { + return String(item.id); +} diff --git a/web/src/components/PageSizeSelect.tsx b/web/src/components/PageSizeSelect.tsx deleted file mode 100644 index b691be7..0000000 --- a/web/src/components/PageSizeSelect.tsx +++ /dev/null @@ -1,30 +0,0 @@ -"use client"; - -import { PAGE_SIZE_OPTIONS, type PageSize } from "@/lib/pagination"; - -export function PageSizeSelect({ - value, - onChange, - options = PAGE_SIZE_OPTIONS, -}: { - value: PageSize; - onChange: (size: PageSize) => void; - options?: readonly number[]; -}) { - return ( - - ); -} diff --git a/web/src/components/Pager.tsx b/web/src/components/Pager.tsx deleted file mode 100644 index 97a592d..0000000 --- a/web/src/components/Pager.tsx +++ /dev/null @@ -1,61 +0,0 @@ -import Link from "next/link"; - -export function Pager({ - basePath, - prevCursor, - nextCursor, - page = 1, - extra = {}, -}: { - basePath: string; - prevCursor?: string | null; - nextCursor?: string | null; - page?: number; - extra?: Record; -}) { - const make = ({ - nextPage, - nextPageCursor, - back, - }: { - nextPage: number; - nextPageCursor?: string; - back?: boolean; - }) => { - const sp = new URLSearchParams(); - for (const [k, v] of Object.entries(extra)) if (v) sp.set(k, v); - if (nextPageCursor) sp.set("cursor", nextPageCursor); - if (back) sp.set("back", "1"); - if (nextPage > 1) sp.set("page", String(nextPage)); - const qs = sp.toString(); - return qs ? `${basePath}?${qs}` : basePath; - }; - - const previousHref = - page > 1 && prevCursor - ? make({ nextPage: page - 1, nextPageCursor: prevCursor, back: true }) - : null; - const nextHref = nextCursor - ? make({ nextPage: page + 1, nextPageCursor: nextCursor }) - : null; - - return ( -
- {previousHref ? ( - - ← prev - - ) : ( - ← prev - )} - page {page} - {nextHref ? ( - - next → - - ) : ( - end - )} -
- ); -} diff --git a/web/src/components/TimeZoneControls.tsx b/web/src/components/TimeZoneControls.tsx index 2edb8b0..fcb49cb 100644 --- a/web/src/components/TimeZoneControls.tsx +++ b/web/src/components/TimeZoneControls.tsx @@ -1,10 +1,15 @@ "use client"; -import { createContext, useContext, useMemo, useState } from "react"; +import { createContext, useContext, useEffect, useMemo, useState } from "react"; import { DEFAULT_TIME_ZONE, fmtDate, normalizeTimeZone } from "@/lib/format"; - -type TimeZoneMode = "configured" | "browser" | "utc"; +import { + normalizeTimeZoneMode, + TZ_BROWSER_COOKIE, + TZ_COOKIE_MAX_AGE_SEC, + TZ_MODE_COOKIE, + type TimeZoneMode, +} from "@/lib/timezone-preference"; const modes: TimeZoneMode[] = ["configured", "browser", "utc"]; @@ -20,14 +25,29 @@ const TimeZoneContext = createContext(null); export function TimeZoneProvider({ configuredTimeZone, + initialMode, + initialBrowserTimeZone, children, }: { configuredTimeZone: string; + initialMode: TimeZoneMode; + initialBrowserTimeZone: string | null; children: React.ReactNode; }) { const configured = normalizeTimeZone(configuredTimeZone); - const [mode, setMode] = useState("configured"); - const [browserTimeZone, setBrowserTimeZone] = useState(null); + const [mode, setMode] = useState(normalizeTimeZoneMode(initialMode)); + const [browserTimeZone, setBrowserTimeZone] = useState( + initialBrowserTimeZone ? normalizeTimeZone(initialBrowserTimeZone) : null, + ); + + useEffect(() => { + if (mode !== "browser") return; + const detected = normalizeTimeZone(Intl.DateTimeFormat().resolvedOptions().timeZone); + writeCookie(TZ_BROWSER_COOKIE, detected); + if (detected === browserTimeZone) return; + const id = window.setTimeout(() => setBrowserTimeZone(detected), 0); + return () => window.clearTimeout(id); + }, [browserTimeZone, mode]); const value = useMemo(() => { const timeZone = @@ -43,8 +63,13 @@ export function TimeZoneProvider({ timeZone, cycleMode: () => { const next = modes[(modes.indexOf(mode) + 1) % modes.length]; + writeCookie(TZ_MODE_COOKIE, next); if (next === "browser" && browserTimeZone === null) { - setBrowserTimeZone(normalizeTimeZone(Intl.DateTimeFormat().resolvedOptions().timeZone)); + const detected = normalizeTimeZone(Intl.DateTimeFormat().resolvedOptions().timeZone); + setBrowserTimeZone(detected); + writeCookie(TZ_BROWSER_COOKIE, detected); + } else if (next !== "browser") { + deleteCookie(TZ_BROWSER_COOKIE); } setMode(next); }, @@ -54,6 +79,15 @@ export function TimeZoneProvider({ return {children}; } +function writeCookie(name: string, value: string) { + const secure = window.location.protocol === "https:" ? "; Secure" : ""; + document.cookie = `${name}=${encodeURIComponent(value)}; Path=/; SameSite=Lax; Max-Age=${TZ_COOKIE_MAX_AGE_SEC}${secure}`; +} + +function deleteCookie(name: string) { + document.cookie = `${name}=; Path=/; SameSite=Lax; Max-Age=0`; +} + export function DateTime({ iso }: { iso?: string | null }) { const { timeZone } = useDisplayTimeZone(); return ; diff --git a/web/src/components/TopNav.tsx b/web/src/components/TopNav.tsx index e236a6c..b87e61a 100644 --- a/web/src/components/TopNav.tsx +++ b/web/src/components/TopNav.tsx @@ -18,7 +18,13 @@ const ZERO_SUMMARY: SystemSummaryData = { generated_at: "", }; -export function TopNav({ initialSummary }: { initialSummary: SystemSummaryData | null }) { +export function TopNav({ + initialSummary, + showLogout, +}: { + initialSummary: SystemSummaryData | null; + showLogout: boolean; +}) { const { data } = usePolling(initialSummary ?? ZERO_SUMMARY, "/api/poll/system-summary"); return ( @@ -50,6 +56,16 @@ export function TopNav({ initialSummary }: { initialSummary: SystemSummaryData | + {showLogout ? ( +
+ +
+ ) : null} ); } diff --git a/web/src/lib/api-types.ts b/web/src/lib/api-types.ts index 30aa372..d05f9a4 100644 --- a/web/src/lib/api-types.ts +++ b/web/src/lib/api-types.ts @@ -327,7 +327,7 @@ export interface components { ErrorResponse: { error: { /** @enum {string} */ - code: "unauthorized" | "forbidden" | "conflict" | "agent_blocked" | "not_found" | "validation" | "payload_too_large" | "rate_limited" | "internal" | "not_ready"; + code: "unauthorized" | "forbidden" | "conflict" | "agent_blocked" | "agent_not_accepted" | "not_found" | "validation" | "payload_too_large" | "rate_limited" | "internal" | "not_ready"; message: string; /** @description Server-generated request id, mirrored from X-Request-Id. */ request_id: string; @@ -704,7 +704,7 @@ export interface operations { }; 400: components["responses"]["ValidationError"]; 401: components["responses"]["Unauthorized"]; - /** @description Agent key is blocked. */ + /** @description Agent key is blocked (`agent_blocked`) or not yet accepted (`agent_not_accepted`). */ 403: { headers: { [name: string]: unknown; diff --git a/web/src/lib/loaders.ts b/web/src/lib/loaders.ts index 33eead5..a1f8fd6 100644 --- a/web/src/lib/loaders.ts +++ b/web/src/lib/loaders.ts @@ -69,48 +69,30 @@ export async function loadEvents({ agentId, checkId, cursor, - back, limit, }: { agentId?: string; checkId?: string; cursor?: string; - back?: boolean; limit?: number; -} = {}): Promise<{ items: StoredEvent[]; next_cursor?: string | null; prev_cursor?: string | null }> { - return api.queryEvents({ agent_id: agentId, check_id: checkId, cursor, back, limit }); +} = {}): Promise<{ items: StoredEvent[]; next_cursor?: string | null }> { + return api.queryEvents({ agent_id: agentId, check_id: checkId, cursor, limit }); } // PH-001: incidents are unbounded history. Always page through the server. export async function loadIncidentsPage( q: NonNullable = {}, -): Promise<{ items: Incident[]; next_cursor?: string | null; prev_cursor?: string | null }> { +): Promise<{ items: Incident[]; next_cursor?: string | null }> { return api.listIncidents(q); } // PH-002: outbox is unbounded. Always page through the server. export async function loadOutboxPage( q: NonNullable = {}, -): Promise<{ items: OutboxItem[]; next_cursor?: string | null; prev_cursor?: string | null }> { +): Promise<{ items: OutboxItem[]; next_cursor?: string | null }> { return api.listOutbox(q); } -// First-page convenience loaders for SSR. Replace the previous drain-all paths -// (PH-001, PH-002): they returned the entire table on every initial render and -// did not scale past a few thousand rows. Pages now show one page and the UI -// owns subsequent fetches via /api/poll/* with cursors. -export async function loadIncidents( - state?: "open" | "resolved", -): Promise { - const page = await api.listIncidents({ state, limit: 100 }); - return page.items; -} - -export async function loadOutboxAll(): Promise { - const page = await api.listOutbox({ limit: 100 }); - return page.items; -} - export async function loadSystemSummary(): Promise { return api.getSystemSummary(); } diff --git a/web/src/lib/pagination.ts b/web/src/lib/pagination.ts index d56c8c8..b3cb37f 100644 --- a/web/src/lib/pagination.ts +++ b/web/src/lib/pagination.ts @@ -1,32 +1,6 @@ -export const PAGE_SIZE_OPTIONS = [10, 50, 100] as const; -export type PageSize = (typeof PAGE_SIZE_OPTIONS)[number]; -export const DEFAULT_PAGE_SIZE: PageSize = 10; +export const INFINITE_BATCH_SIZE = 50; -export function pageCountOf(total: number, pageSize: number): number { - return Math.max(1, Math.ceil(total / pageSize)); -} - -export function clampPage(page: number, pageCount: number): number { - if (page < 1) return 1; - if (page > pageCount) return pageCount; - return page; -} - -export function paginate(items: T[], page: number, pageSize: number): T[] { - const start = (page - 1) * pageSize; - return items.slice(start, start + pageSize); -} - -export function normalizePageSize(value: unknown, fallback: PageSize = DEFAULT_PAGE_SIZE): PageSize { +export function normalizeFetchLimit(value: unknown, fallback = INFINITE_BATCH_SIZE): number { const n = Number(value); - return (PAGE_SIZE_OPTIONS as readonly number[]).includes(n) ? (n as PageSize) : fallback; -} - -export function normalizePageNumber(value: unknown): number { - const n = Number(value); - return Number.isInteger(n) && n > 0 ? n : 1; -} - -export function normalizeBack(value: unknown): boolean { - return value === "1" || value === "true"; + return Number.isFinite(n) ? Math.min(Math.max(Math.floor(n), 1), 500) : fallback; } diff --git a/web/src/lib/redirect-to-login.ts b/web/src/lib/redirect-to-login.ts new file mode 100644 index 0000000..d16dd75 --- /dev/null +++ b/web/src/lib/redirect-to-login.ts @@ -0,0 +1,5 @@ +export function redirectToLogin() { + const next = `${window.location.pathname}${window.location.search}`; + const params = new URLSearchParams({ next }); + window.location.assign(`/login?${params.toString()}`); +} diff --git a/web/src/lib/timezone-preference.ts b/web/src/lib/timezone-preference.ts new file mode 100644 index 0000000..e4a4166 --- /dev/null +++ b/web/src/lib/timezone-preference.ts @@ -0,0 +1,9 @@ +export type TimeZoneMode = "configured" | "browser" | "utc"; + +export const TZ_MODE_COOKIE = "monlet_tz_mode"; +export const TZ_BROWSER_COOKIE = "monlet_tz_browser"; +export const TZ_COOKIE_MAX_AGE_SEC = 31_536_000; + +export function normalizeTimeZoneMode(value?: string | null): TimeZoneMode { + return value === "browser" || value === "utc" ? value : "configured"; +} diff --git a/web/src/lib/timezone.ts b/web/src/lib/timezone.ts index cc58acc..2de9fda 100644 --- a/web/src/lib/timezone.ts +++ b/web/src/lib/timezone.ts @@ -1,7 +1,32 @@ import "server-only"; +import { cookies } from "next/headers"; + import { DEFAULT_TIME_ZONE, normalizeTimeZone } from "@/lib/format"; +import { + normalizeTimeZoneMode, + TZ_BROWSER_COOKIE, + TZ_MODE_COOKIE, + type TimeZoneMode, +} from "@/lib/timezone-preference"; export function getUiTimeZone(): string { return normalizeTimeZone(process.env.MONLET_WEB_TIME_ZONE ?? process.env.TZ ?? DEFAULT_TIME_ZONE); } + +export async function getInitialTimeZonePreference(): Promise<{ + mode: TimeZoneMode; + browserTimeZone: string | null; +}> { + const store = await cookies(); + const mode = normalizeTimeZoneMode(store.get(TZ_MODE_COOKIE)?.value); + const browserRaw = store.get(TZ_BROWSER_COOKIE)?.value; + const browserTimeZone = browserRaw ? normalizeTimeZone(browserRaw) : null; + return { + mode, + browserTimeZone: + browserTimeZone === DEFAULT_TIME_ZONE && browserRaw !== DEFAULT_TIME_ZONE + ? null + : browserTimeZone, + }; +} diff --git a/web/src/lib/useInfiniteList.ts b/web/src/lib/useInfiniteList.ts new file mode 100644 index 0000000..a5eb9a3 --- /dev/null +++ b/web/src/lib/useInfiniteList.ts @@ -0,0 +1,235 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; + +import { redirectToLogin } from "@/lib/redirect-to-login"; +import { usePolling } from "@/lib/usePolling"; + +export type CursorPage = { + items: T[]; + next_cursor?: string | null; +}; + +export function useInfiniteClientList({ + items, + batchSize, + resetKey, +}: { + items: T[]; + batchSize: number; + resetKey: string; +}) { + const stateKey = `${resetKey}:${batchSize}`; + const [state, setState] = useState({ key: stateKey, visibleCount: batchSize }); + + const visibleCount = state.key === stateKey ? state.visibleCount : batchSize; + + const hasMore = visibleCount < items.length; + const loadMore = useCallback(() => { + setState((current) => { + const currentCount = current.key === stateKey ? current.visibleCount : batchSize; + return { + key: stateKey, + visibleCount: Math.min(currentCount + batchSize, items.length), + }; + }); + }, [batchSize, items.length, stateKey]); + const sentinelRef = useInfiniteSentinel(hasMore, loadMore); + + return { + visibleItems: items.slice(0, visibleCount), + hasMore, + sentinelRef, + shown: Math.min(visibleCount, items.length), + total: items.length, + }; +} + +export function useInfiniteCursorList({ + initialData, + firstPageUrl, + getPageUrl, + getKey, + resetKey, +}: { + initialData: CursorPage; + firstPageUrl: string; + getPageUrl: (cursor: string) => string; + getKey: (item: T) => string; + resetKey: string; +}) { + const { data: firstPage, error, isStale, lastUpdatedAt } = usePolling(initialData, firstPageUrl); + const [windowState, setWindowState] = useState(() => + initialWindowState(resetKey, initialData, getKey), + ); + const [loadingMore, setLoadingMore] = useState(false); + const [loadError, setLoadError] = useState(null); + const loadingMoreRef = useRef(false); + const abortRef = useRef(null); + const generationRef = useRef(0); + const nearTop = useNearTop(); + const firstPageSignature = pageSignature(firstPage, getKey); + const activeWindow = + windowState.key === resetKey ? windowState : initialWindowState(resetKey, initialData, getKey); + + // Persist the head rebase into state (single source of truth) so an in-flight + // loadMore cannot glue a stale tail onto an outdated base and revert a fresh + // head. Skip while a tail load is in flight: that request extends the current + // base and must resolve against it. + useEffect(() => { + if (loadingMoreRef.current) return; + setWindowState((current) => { + const w = + current.key === resetKey ? current : initialWindowState(resetKey, initialData, getKey); + if (w.tailItems.length === 0 && w.baseSignature !== firstPageSignature && nearTop) { + return initialWindowState(resetKey, firstPage, getKey); + } + return w; + }); + }, [firstPageSignature, nearTop, resetKey, firstPage, initialData, getKey]); + + const hasNewHead = activeWindow.baseSignature !== firstPageSignature; + const baseItems = activeWindow.baseItems; + const tailItems = activeWindow.tailItems; + const items = appendUnique(baseItems, tailItems, getKey); + const nextCursor = activeWindow.nextCursor; + + const showLatest = useCallback(() => { + generationRef.current += 1; + abortRef.current?.abort(); + abortRef.current = null; + loadingMoreRef.current = false; + setLoadingMore(false); + setLoadError(null); + setWindowState(initialWindowState(resetKey, firstPage, getKey)); + if (typeof window !== "undefined") window.scrollTo({ top: 0, behavior: "smooth" }); + }, [firstPage, getKey, resetKey]); + + const loadMore = useCallback(() => { + if (!nextCursor || loadingMoreRef.current) return; + loadingMoreRef.current = true; + setLoadingMore(true); + setLoadError(null); + const controller = new AbortController(); + abortRef.current = controller; + const generation = generationRef.current; + const requestBaseSignature = activeWindow.baseSignature; + + fetch(getPageUrl(nextCursor), { cache: "no-store", signal: controller.signal }) + .then(async (response) => { + if (response.status === 401) { + redirectToLogin(); + throw new Error("authentication required"); + } + if (!response.ok) { + const body = await response.json().catch(() => null); + throw new Error(body?.error?.message ?? response.statusText); + } + return response.json() as Promise>; + }) + .then((page) => { + if (generation !== generationRef.current) return; + setWindowState((current) => { + if (current.key !== resetKey) return current; + // The cursor was computed against requestBaseSignature. If a poll + // rebased the base in state meanwhile, gluing this tail onto a + // different base would skip the rows between the new head and the old + // cursor. Drop the response instead; the user reloads via showLatest. + if (current.baseSignature !== requestBaseSignature) return current; + return { + key: resetKey, + baseItems: current.baseItems, + baseSignature: current.baseSignature, + tailItems: appendUnique(current.tailItems, page.items, getKey), + nextCursor: page.next_cursor ?? null, + }; + }); + }) + .catch((e: unknown) => { + if (generation === generationRef.current && !controller.signal.aborted) { + setLoadError(e instanceof Error ? e.message : String(e)); + } + }) + .finally(() => { + if (generation !== generationRef.current) return; + if (abortRef.current === controller) abortRef.current = null; + loadingMoreRef.current = false; + setLoadingMore(false); + }); + }, [activeWindow, getKey, getPageUrl, nextCursor, resetKey]); + + useEffect(() => () => abortRef.current?.abort(), []); + + const sentinelRef = useInfiniteSentinel(Boolean(nextCursor), loadMore); + + return { + items, + hasMore: Boolean(nextCursor), + loadingMore, + error: loadError ?? error, + isStale, + lastUpdatedAt, + sentinelRef, + hasNewItems: hasNewHead, + showLatest, + }; +} + +function useInfiniteSentinel(enabled: boolean, onIntersect: () => void) { + const ref = useRef(null); + const callbackRef = useRef(onIntersect); + + useEffect(() => { + callbackRef.current = onIntersect; + }, [onIntersect]); + + useEffect(() => { + const node = ref.current; + if (!enabled || !node) return; + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) callbackRef.current(); + }, + { rootMargin: "360px 0px" }, + ); + observer.observe(node); + return () => observer.disconnect(); + }, [enabled]); + + return ref; +} + +function appendUnique(current: T[], next: T[], getKey: (item: T) => string): T[] { + const keys = new Set(current.map(getKey)); + return [...current, ...next.filter((item) => !keys.has(getKey(item)))]; +} + +function initialWindowState(key: string, page: CursorPage, getKey: (item: T) => string) { + return { + key, + baseItems: page.items, + baseSignature: pageSignature(page, getKey), + tailItems: [] as T[], + nextCursor: page.next_cursor ?? null, + }; +} + +function pageSignature(page: CursorPage, getKey: (item: T) => string): string { + return `${page.next_cursor ?? ""}:${page.items.map(getKey).join("\u0000")}`; +} + +function useNearTop() { + const [nearTop, setNearTop] = useState(true); + + useEffect(() => { + const update = () => setNearTop(window.scrollY < 120); + const frame = window.requestAnimationFrame(update); + window.addEventListener("scroll", update, { passive: true }); + return () => { + window.cancelAnimationFrame(frame); + window.removeEventListener("scroll", update); + }; + }, []); + + return nearTop; +} diff --git a/web/src/lib/usePolling.ts b/web/src/lib/usePolling.ts index a0fb479..ecefc15 100644 --- a/web/src/lib/usePolling.ts +++ b/web/src/lib/usePolling.ts @@ -2,6 +2,8 @@ import { useEffect, useState } from "react"; +import { redirectToLogin } from "@/lib/redirect-to-login"; + const DEFAULT_POLL_MS = 10_000; const BACKOFF_AFTER_ERRORS = 2; const BACKOFF_CAP_MS = 60_000; @@ -53,6 +55,10 @@ export function usePolling(initialData: T, url: string, intervalMs = pollInte fetch(url, { cache: "no-store", signal: controller.signal }) .then(async (response) => { + if (response.status === 401) { + redirectToLogin(); + throw new Error("authentication required"); + } if (!response.ok) { const body = await response.json().catch(() => null); throw new Error(body?.error?.message ?? response.statusText); diff --git a/web/src/lib/view-types.ts b/web/src/lib/view-types.ts index 74cb323..b771a6d 100644 --- a/web/src/lib/view-types.ts +++ b/web/src/lib/view-types.ts @@ -8,13 +8,4 @@ export type InventoryData = { checks: CheckState[]; }; -export type SystemSummaryData = { - agents_online: number; - agents_offline: number; - checks_ok: number; - checks_warning: number; - checks_critical: number; - checks_unknown: number; - open_incidents: number; - generated_at: string; -}; +export type SystemSummaryData = components["schemas"]["SystemSummaryResponse"]; diff --git a/web/src/lib/web-auth-server.ts b/web/src/lib/web-auth-server.ts new file mode 100644 index 0000000..02b1881 --- /dev/null +++ b/web/src/lib/web-auth-server.ts @@ -0,0 +1,52 @@ +import "server-only"; + +import { cookies, headers } from "next/headers"; + +import { + localAuthEnabled, + localAuthMisconfigured, + proxyAuthEnabled, + SESSION_COOKIE, + verifySessionToken, +} from "@/lib/web-auth"; + +export async function requestHasAppAccess(): Promise { + if (localAuthMisconfigured()) return false; + if (proxyAuthEnabled()) return Boolean((await headers()).get("x-forwarded-user")); + if (!localAuthEnabled()) return true; + return verifySessionToken( + (await cookies()).get(SESSION_COOKIE)?.value, + process.env.MONLET_WEB_SESSION_SECRET ?? "", + process.env.MONLET_WEB_AUTH_USERNAME, + ); +} + +export function showLocalLogout(): boolean { + return localAuthEnabled() && !proxyAuthEnabled() && !localAuthMisconfigured(); +} + +// CSRF defense for state-changing routes. Prefer the Fetch Metadata signal; +// fall back to Origin/Host comparison for clients that omit Sec-Fetch-Site. +export function assertSameOrigin(req: Request): boolean { + const fetchSite = req.headers.get("sec-fetch-site"); + // Strict same-origin: same-site would let a sibling subdomain pass the guard. + if (fetchSite) return fetchSite === "same-origin"; + + const origin = req.headers.get("origin"); + if (!origin) return false; + let originHost: string; + try { + originHost = new URL(origin).host; + } catch { + return false; + } + return originHost === requestHost(req); +} + +// Trust X-Forwarded-Host like cookieSecure() trusts X-Forwarded-Proto: the app +// is deployed behind a reverse proxy that terminates the public origin. +function requestHost(req: Request): string { + const forwarded = req.headers.get("x-forwarded-host"); + if (forwarded) return forwarded.split(",")[0]!.trim(); + return req.headers.get("host") ?? new URL(req.url).host; +} diff --git a/web/src/lib/web-auth.ts b/web/src/lib/web-auth.ts new file mode 100644 index 0000000..cdfa06c --- /dev/null +++ b/web/src/lib/web-auth.ts @@ -0,0 +1,170 @@ +export const SESSION_COOKIE = "monlet_session"; +export const DEFAULT_SESSION_TTL_SEC = 604_800; +export const MIN_SESSION_SECRET_BYTES = 32; +const LOGIN_FAILURE_DELAY_MS = 350; + +type SessionPayload = { + u: string; + exp: number; +}; + +export function localAuthEnabled(): boolean { + return Boolean( + process.env.MONLET_WEB_AUTH_USERNAME && + process.env.MONLET_WEB_AUTH_PASSWORD && + sessionSecretStrong(process.env.MONLET_WEB_SESSION_SECRET), + ); +} + +export function localAuthMisconfigured(): boolean { + const hasAny = Boolean( + process.env.MONLET_WEB_AUTH_USERNAME || + process.env.MONLET_WEB_AUTH_PASSWORD || + process.env.MONLET_WEB_SESSION_SECRET, + ); + const enabled = localAuthEnabled(); + return (hasAny && !enabled) || (proxyAuthEnabled() && enabled); +} + +export function proxyAuthEnabled(): boolean { + return process.env.MONLET_WEB_TRUST_PROXY_AUTH === "true"; +} + +export function sessionSecretStrong(secret?: string): boolean { + return new TextEncoder().encode(secret ?? "").byteLength >= MIN_SESSION_SECRET_BYTES; +} + +export function sessionTtlSec(): number { + const raw = Number(process.env.MONLET_WEB_SESSION_TTL_SEC); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_SESSION_TTL_SEC; +} + +export function sanitizeNext(value?: string | null): string { + const fallback = "/agents"; + const raw = value?.trim(); + if (!raw || !raw.startsWith("/") || raw.startsWith("//") || raw.includes("\\")) return fallback; + + let decoded: string; + try { + decoded = decodeURIComponent(raw); + } catch { + return fallback; + } + if ( + !decoded.startsWith("/") || + decoded.startsWith("//") || + decoded.includes("\\") || + hasControlChar(decoded) + ) { + return fallback; + } + + let url: URL; + try { + url = new URL(raw, "https://monlet.local"); + } catch { + return fallback; + } + if (url.origin !== "https://monlet.local") return fallback; + if (url.pathname === "/login" || url.pathname.startsWith("/api/auth/")) return fallback; + return `${url.pathname}${url.search}${url.hash}`; +} + +export function cookieSecure(request: Request): boolean { + const forwarded = request.headers.get("x-forwarded-proto"); + return forwarded === "https" || new URL(request.url).protocol === "https:"; +} + +export async function createSessionToken(username: string, secret: string, ttlSec: number) { + const payload: SessionPayload = { + u: username, + exp: Math.floor(Date.now() / 1000) + ttlSec, + }; + const payloadBytes = new TextEncoder().encode(JSON.stringify(payload)); + const signature = await sign(payloadBytes, secret); + return `${base64UrlEncode(payloadBytes)}.${base64UrlEncode(signature)}`; +} + +export async function constantTimeEqualString(actual: string, expected: string): Promise { + const [actualHash, expectedHash] = await Promise.all([sha256(actual), sha256(expected)]); + return equalBytes(actualHash, expectedHash); +} + +export async function delayLoginFailure(): Promise { + await new Promise((resolve) => setTimeout(resolve, LOGIN_FAILURE_DELAY_MS)); +} + +export async function verifySessionToken( + token: string | undefined, + secret: string, + expectedUsername?: string, +): Promise { + if (!token) return false; + const [payloadPart, signaturePart] = token.split("."); + if (!payloadPart || !signaturePart) return false; + const payloadBytes = base64UrlDecode(payloadPart); + const signature = base64UrlDecode(signaturePart); + if (!payloadBytes || !signature) return false; + const expected = await sign(payloadBytes, secret); + if (!equalBytes(signature, expected)) return false; + try { + const payload = JSON.parse(new TextDecoder().decode(payloadBytes)) as SessionPayload; + return ( + typeof payload.u === "string" && + (!expectedUsername || payload.u === expectedUsername) && + payload.exp > Math.floor(Date.now() / 1000) + ); + } catch { + return false; + } +} + +async function sign(payload: Uint8Array, secret: string): Promise { + const key = await crypto.subtle.importKey( + "raw", + new TextEncoder().encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"], + ); + return new Uint8Array(await crypto.subtle.sign("HMAC", key, arrayBufferOf(payload))); +} + +async function sha256(value: string): Promise { + const bytes = new TextEncoder().encode(value); + return new Uint8Array(await crypto.subtle.digest("SHA-256", arrayBufferOf(bytes))); +} + +function arrayBufferOf(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; +} + +function equalBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff |= a[i] ^ b[i]; + return diff === 0; +} + +function hasControlChar(value: string): boolean { + return /[\u0000-\u001F\u007F]/.test(value); +} + +function base64UrlEncode(bytes: Uint8Array): string { + return btoa(String.fromCharCode(...bytes)) + .replaceAll("+", "-") + .replaceAll("/", "_") + .replaceAll("=", ""); +} + +function base64UrlDecode(value: string): Uint8Array | null { + try { + const padded = value.replaceAll("-", "+").replaceAll("_", "/").padEnd( + Math.ceil(value.length / 4) * 4, + "=", + ); + return Uint8Array.from(atob(padded), (c) => c.charCodeAt(0)); + } catch { + return null; + } +} diff --git a/web/src/proxy.ts b/web/src/proxy.ts index 7f40dca..fe9a139 100644 --- a/web/src/proxy.ts +++ b/web/src/proxy.ts @@ -1,29 +1,44 @@ -import type { NextRequest } from "next/server"; +import { NextRequest, NextResponse } from "next/server"; + +import { + localAuthEnabled, + localAuthMisconfigured, + proxyAuthEnabled, + sanitizeNext, + SESSION_COOKIE, + verifySessionToken, +} from "@/lib/web-auth"; const ACTION_PATH = "/api/agents/action"; const HEALTH_PATH = "/api/health"; +const LOGIN_PATH = "/login"; +const AUTH_PREFIX = "/api/auth/"; export const config = { matcher: "/((?!_next/static|_next/image|favicon.ico).*)", }; -export function proxy(request: NextRequest) { +export async function proxy(request: NextRequest) { const path = request.nextUrl.pathname; - if (path === HEALTH_PATH) return; + if (path === HEALTH_PATH || path === LOGIN_PATH || path.startsWith(AUTH_PREFIX)) return; - const user = process.env.MONLET_WEB_AUTH_USERNAME; - const password = process.env.MONLET_WEB_AUTH_PASSWORD; - if (user && password) { - if (validBasicAuth(request, user, password)) return; - return new Response("authentication required", { - status: 401, - headers: { "WWW-Authenticate": 'Basic realm="monlet"' }, - }); + if (localAuthMisconfigured()) { + return new Response("web auth is misconfigured", { status: 500 }); } - if (process.env.MONLET_WEB_TRUST_PROXY_AUTH === "true") { + if (proxyAuthEnabled()) { if (request.headers.get("x-forwarded-user")) return; - return new Response("proxy authentication required", { status: 401 }); + return unauthorized(request, "proxy authentication required"); + } + + if (localAuthEnabled()) { + const ok = await verifySessionToken( + request.cookies.get(SESSION_COOKIE)?.value, + process.env.MONLET_WEB_SESSION_SECRET ?? "", + process.env.MONLET_WEB_AUTH_USERNAME, + ); + if (ok) return; + return unauthorized(request, "authentication required"); } if (path === ACTION_PATH) { @@ -34,15 +49,11 @@ export function proxy(request: NextRequest) { } } -function validBasicAuth(request: NextRequest, user: string, password: string): boolean { - const header = request.headers.get("authorization"); - if (!header?.startsWith("Basic ")) return false; - try { - const raw = atob(header.slice("Basic ".length)); - const split = raw.indexOf(":"); - if (split < 0) return false; - return raw.slice(0, split) === user && raw.slice(split + 1) === password; - } catch { - return false; +function unauthorized(request: NextRequest, message: string) { + if (request.nextUrl.pathname.startsWith("/api/")) { + return Response.json({ error: { code: "unauthorized", message } }, { status: 401 }); } + const login = new URL(LOGIN_PATH, request.url); + login.searchParams.set("next", sanitizeNext(`${request.nextUrl.pathname}${request.nextUrl.search}`)); + return NextResponse.redirect(login, { status: 303 }); } diff --git a/web/tests/mock-server.mjs b/web/tests/mock-server.mjs index b3bea0a..4916e19 100644 --- a/web/tests/mock-server.mjs +++ b/web/tests/mock-server.mjs @@ -3,6 +3,60 @@ import { createServer } from "node:http"; const PORT = Number(process.env.MOCK_PORT ?? 8765); const now = new Date().toISOString(); +const eventItems = [ + { + event_id: "00000000-0000-7000-8000-000000000001", + agent_id: "agent-1", + check_id: "disk", + observed_at: now, + received_at: now, + status: "ok", + exit_code: 0, + duration_ms: 12, + output: "disk ok", + output_truncated: false, + notifications_enabled: true, + }, + { + event_id: "00000000-0000-7000-8000-000000000002", + agent_id: "agent-1", + check_id: "load", + observed_at: now, + received_at: now, + status: "warning", + exit_code: 1, + duration_ms: 34, + output: "load high", + output_truncated: false, + notifications_enabled: true, + }, + { + event_id: "00000000-0000-7000-8000-000000000003", + agent_id: "agent-2", + check_id: "agent_liveness", + observed_at: now, + received_at: now, + status: "critical", + exit_code: 2, + duration_ms: 0, + output: "agent is dead", + output_truncated: false, + notifications_enabled: true, + }, + ...Array.from({ length: 12 }, (_, i) => ({ + event_id: `00000000-0000-7000-8000-0000000001${String(i).padStart(2, "0")}`, + agent_id: "agent-1", + check_id: "disk", + observed_at: now, + received_at: now, + status: "ok", + exit_code: 0, + duration_ms: 10 + i, + output: `disk extra ${i + 1}`, + output_truncated: false, + notifications_enabled: true, + })), +]; const fixtures = { "/api/v1/health": { status: "ok" }, @@ -12,6 +66,7 @@ const fixtures = { checks_ok: 2, checks_warning: 1, checks_critical: 1, + checks_unknown: 0, open_incidents: 2, generated_at: now, }, @@ -128,48 +183,8 @@ const fixtures = { next_cursor: null, }, "/api/v1/events/query": { - items: [ - { - event_id: "00000000-0000-7000-8000-000000000001", - agent_id: "agent-1", - check_id: "disk", - observed_at: now, - received_at: now, - status: "ok", - exit_code: 0, - duration_ms: 12, - output: "disk ok", - output_truncated: false, - notifications_enabled: true, - }, - { - event_id: "00000000-0000-7000-8000-000000000002", - agent_id: "agent-1", - check_id: "load", - observed_at: now, - received_at: now, - status: "warning", - exit_code: 1, - duration_ms: 34, - output: "load high", - output_truncated: false, - notifications_enabled: true, - }, - { - event_id: "00000000-0000-7000-8000-000000000003", - agent_id: "agent-2", - check_id: "agent_liveness", - observed_at: now, - received_at: now, - status: "critical", - exit_code: 2, - duration_ms: 0, - output: "agent is dead", - output_truncated: false, - notifications_enabled: true, - }, - ], - next_cursor: null, + items: eventItems, + next_cursor: "10", }, "/api/v1/notifiers/outbox": { items: [ @@ -205,15 +220,12 @@ const server = createServer((req, res) => { if (url.pathname === "/api/v1/events/query" && body) { const agentId = url.searchParams.get("agent_id"); const checkId = url.searchParams.get("check_id"); - body = { - ...body, - items: body.items.filter((item) => { - if (agentId && item.agent_id !== agentId) return false; - if (checkId && item.check_id !== checkId) return false; - return true; - }), - next_cursor: null, - }; + const items = body.items.filter((item) => { + if (agentId && item.agent_id !== agentId) return false; + if (checkId && item.check_id !== checkId) return false; + return true; + }); + body = paginate(body, items, url); } if (url.pathname === "/api/v1/incidents" && body) { const state = url.searchParams.get("state"); @@ -239,3 +251,16 @@ const server = createServer((req, res) => { server.listen(PORT, "127.0.0.1", () => { console.log(`mock server on ${PORT}`); }); + +function paginate(body, items, url) { + const limit = Number(url.searchParams.get("limit") ?? 100); + const start = Number(url.searchParams.get("cursor") ?? 0); + const safeLimit = Number.isFinite(limit) && limit > 0 ? limit : 100; + const safeStart = Number.isFinite(start) && start > 0 ? start : 0; + const next = safeStart + safeLimit; + return { + ...body, + items: items.slice(safeStart, next), + next_cursor: next < items.length ? String(next) : null, + }; +} diff --git a/web/tests/smoke.spec.ts b/web/tests/smoke.spec.ts index a87b0fd..271a221 100644 --- a/web/tests/smoke.spec.ts +++ b/web/tests/smoke.spec.ts @@ -107,6 +107,14 @@ test("events page", async ({ page }) => { await expect(page.getByRole("link", { name: /check_id=.*disk/ })).toBeVisible(); }); +test("events page loads the next cursor page on scroll", async ({ page }) => { + await page.goto("/events"); + await expect(page.locator("tbody")).toContainText("disk extra 7"); + await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); + await expect(page.locator("tbody")).toContainText("disk extra 11"); + await expect(page).toHaveURL(/\/events$/); +}); + test("outbox page", async ({ page }) => { await page.goto("/outbox"); await expect(page.locator("body")).toContainText("debug");