Fix scheduler cleanup edge cases

CONFIG_REFERENCE.md

@@ -0,0 +1,231 @@
+English | [Русский](CONFIG_REFERENCE.ru.md)
+
+# Monlet: Configuration Reference
+
+Monlet consists of agents running on hosts and a central server. The agent generates its local key in `state_dir/agent.key`; the server sees a new key as `pending` until an operator accepts the agent in the UI.
+
+## Agent TOML
+
+File: usually `/etc/monlet/agent.toml`.
+
+```toml
+# Optional. If omitted, the OS hostname is used.
+# agent_id = "host-01"
+
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+env = "prod"
+role = "api"
+
+[server]
+enabled = true
+url = "https://monlet.example.com"
+heartbeat_interval = "10s"
+batch_interval = "10s"
+
+[metrics]
+enabled = true
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "disk_root"
+name = "Root disk usage"
+command = '''
+/usr/local/lib/monlet/check_disk_root.sh --warn 80 --crit 90
+'''
+interval = "60s"
+timeout = "10s"
+notifications_enabled = true
+resource_limits = { cpu_time = "5s", memory = "256MiB", open_files = 128 }
+
+[[checks]]
+id = "daily_backup"
+name = "Daily backup freshness"
+command = "/usr/local/lib/monlet/check_backup_freshness.sh"
+cron = "CRON_TZ=UTC 15 6 * * *"
+timeout = "2m"
+```
+
+Keys:
+
+- `agent_id` - stable agent ID; optional, defaults to hostname.
+- `state_dir` - local state, agent key, and spool.
+- `[labels]` - inventory labels; do not put secrets here.
+- `server.enabled` - send heartbeats/events to the server.
+- `server.url` - server base URL.
+- `metrics.enabled` - expose Prometheus `/metrics`.
+- `metrics.listen` - `/metrics` listen address.
+- `checks[].id` - stable check ID.
+- `checks[].command` - shell command string; one-line `"..."` and multi-line `'''...'''` forms are supported.
+- `checks[].interval` - run period (`60s`, `5m`, `1h`); required when `checks[].cron` is not set.
+- `checks[].cron` - cron-style schedule; required when `checks[].interval` is not set. Standard 5-field specs, `@hourly`/`@daily`, and `CRON_TZ=...` are supported; seconds-format specs, `@every`, and `@reboot` are rejected.
+- `checks[].timeout` - timeout; timed-out checks return `critical`.
+- `checks[].notifications_enabled` - allow server-side notifications for this check.
+- `checks[].resource_limits` - optional process limits: CPU time, virtual memory, open files.
+
+Each check must define exactly one schedule mode: `interval` or `cron`. Interval checks run immediately after start/reload and then on the configured period. Cron checks wait for the next cron slot and do not run immediately on start.
+
+Exit code: `0=ok`, `1=warning`, `2=critical`, `3+=unknown`.
+
+Agent env:
+
+- `MONLET_AGENT_MAX_CHECKS` - maximum checks in config (default 256). This bounds `{check_id}` metric cardinality. Must be a positive integer; exceeding the limit is a config load error.
+
+Reload:
+
+- `systemctl reload monlet-agent` reloads the local TOML via `SIGHUP`.
+- Invalid config is not applied; the old config keeps running.
+- A running check is not killed on reload; it runs until completion or its `timeout`.
+- When a schedule changes, an interval check runs after the current in-flight run; a cron check waits for the next cron slot.
+- Changing `agent_id`, `state_dir`, `server.*`, or `metrics.*` requires restart.
+
+## Server env
+
+File: `.env` or container environment.
+
+```env
+MONLET_AUTH_TOKEN=replace-with-random-ui-token
+MONLET_DATABASE_URL=postgresql+asyncpg://monlet:monlet@postgres:5432/monlet
+MONLET_LOG_LEVEL=INFO
+
+MONLET_HOST=0.0.0.0
+MONLET_PORT=8000
+
+MONLET_BODY_LIMIT_BYTES=1048576
+MONLET_OUTPUT_MAX_BYTES=8192
+MONLET_MAX_BATCH_EVENTS=200
+
+MONLET_ENABLE_DETECTOR=true
+MONLET_STALE_AFTER_SEC=20
+MONLET_DEAD_AFTER_SEC=30
+MONLET_DETECTOR_TICK_SEC=5
+
+MONLET_DB_POOL_SIZE=10
+MONLET_DB_MAX_OVERFLOW=10
+MONLET_DB_POOL_TIMEOUT_SEC=30
+MONLET_DB_POOL_RECYCLE_SEC=1800
+
+MONLET_EVENTS_RETENTION_MONTHS=36
+MONLET_EVENTS_FUTURE_PARTITIONS=3
+MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC=3600
+MONLET_OUTBOX_RETENTION_MAX_ROWS=50000
+MONLET_OUTBOX_PRUNE_BATCH_SIZE=5000
+
+MONLET_EVENT_DEDUP_RETENTION_DAYS=30
+MONLET_EVENT_DEDUP_PRUNE_BATCH_SIZE=5000
+
+MONLET_MAX_PENDING_AGENTS=10000
+MONLET_PENDING_AGENT_TTL_DAYS=7
+MONLET_PENDING_AGENT_PRUNE_BATCH_SIZE=1000
+
+MONLET_ENABLE_NOTIFIER_WORKER=true
+MONLET_NOTIFIER_TICK_SEC=5
+MONLET_NOTIFIER_BATCH_SIZE=20
+MONLET_NOTIFIER_CONCURRENCY=4
+MONLET_NOTIFIER_MAX_ATTEMPTS=8
+MONLET_NOTIFIER_LEASE_SEC=300
+MONLET_NOTIFIER_HTTP_TIMEOUT_SEC=10
+MONLET_NOTIFIER_INCLUDE_OUTPUT=true
+MONLET_NOTIFIER_MAX_OUTPUT_BYTES=1024
+MONLET_NOTIFICATION_TIME_ZONE=UTC
+
+MONLET_NOTIFIER_DEBUG_ENABLED=true
+MONLET_NOTIFIER_TELEGRAM_ENABLED=false
+MONLET_NOTIFIER_WEBHOOK_ENABLED=false
+MONLET_NOTIFIER_ALERTMANAGER_ENABLED=false
+```
+
+Basics:
+
+- `MONLET_AUTH_TOKEN` - Bearer token(s) for UI/operator API, not for agents. Space/comma-separated lists support overlap rotation.
+- `MONLET_DATABASE_URL` - PostgreSQL DSN (`postgresql+asyncpg://...`).
+- `MONLET_LOG_LEVEL` - log level.
+- `MONLET_HOST` / `MONLET_PORT` - HTTP server host and port (default `0.0.0.0:8000`).
+
+Ingestion limits:
+
+- `MONLET_BODY_LIMIT_BYTES` - maximum request body size (default 1 MiB = 1048576).
+- `MONLET_OUTPUT_MAX_BYTES` - stored event `output` limit (default 8192).
+- `MONLET_MAX_BATCH_EVENTS` - maximum events in one batch (default 200).
+
+Liveness detector:
+
+- `MONLET_ENABLE_DETECTOR` - enable stale/dead background recalculation (default `true`).
+- `MONLET_STALE_AFTER_SEC` - seconds without heartbeat before an agent becomes `stale`.
+- `MONLET_DEAD_AFTER_SEC` - seconds without heartbeat before an agent becomes `dead`.
+- `MONLET_DETECTOR_TICK_SEC` - liveness recalculation period.
+
+DB pool:
+
+- `MONLET_DB_POOL_SIZE` - persistent pool connections (default 10).
+- `MONLET_DB_MAX_OVERFLOW` - extra connections above the pool (default 10).
+- `MONLET_DB_POOL_TIMEOUT_SEC` - wait time for a free connection (default 30).
+- `MONLET_DB_POOL_RECYCLE_SEC` - recycle connections by age (default 1800).
+
+Storage and maintenance:
+
+- `MONLET_EVENTS_RETENTION_MONTHS` - event partition retention (default 36).
+- `MONLET_EVENTS_FUTURE_PARTITIONS` - future monthly partitions to pre-create (default 3).
+- `MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC` - partition maintenance and prune period (default 3600).
+- `MONLET_OUTBOX_RETENTION_MAX_ROWS` - upper bound for outbox rows; extra terminal rows are pruned (default 50000).
+- `MONLET_OUTBOX_PRUNE_BATCH_SIZE` - terminal outbox prune batch size (default 5000).
+- `MONLET_EVENT_DEDUP_RETENTION_DAYS` - `event_id` idempotency record retention (default 30).
+- `MONLET_EVENT_DEDUP_PRUNE_BATCH_SIZE` - dedup prune batch size (default 5000).
+- `MONLET_MAX_PENDING_AGENTS` - maximum unaccepted new agents (default 10000).
+- `MONLET_PENDING_AGENT_TTL_DAYS` - old pending-key cleanup TTL (default 7).
+- `MONLET_PENDING_AGENT_PRUNE_BATCH_SIZE` - old pending-key prune batch size (default 1000).
+
+Notifier worker:
+
+- `MONLET_ENABLE_NOTIFIER_WORKER` - enable notification delivery worker (default `true`).
+- `MONLET_NOTIFIER_TICK_SEC` - outbox poll period (default 5).
+- `MONLET_NOTIFIER_BATCH_SIZE` - outbox rows claimed per tick (default 20).
+- `MONLET_NOTIFIER_CONCURRENCY` - parallel deliveries inside one tick (default 4).
+- `MONLET_NOTIFIER_MAX_ATTEMPTS` - maximum delivery attempts before permanent failure (default 8).
+- `MONLET_NOTIFIER_LEASE_SEC` - row lease duration during delivery, to prevent another worker from claiming it (default 300).
+- `MONLET_NOTIFIER_HTTP_TIMEOUT_SEC` - HTTP delivery timeout (default 10).
+- `MONLET_NOTIFIER_INCLUDE_OUTPUT` - include `output` text in notifications (default `true`).
+- `MONLET_NOTIFIER_MAX_OUTPUT_BYTES` - notification output truncation after redaction (default 1024).
+- `MONLET_NOTIFICATION_TIME_ZONE` - timezone for human-readable notification timestamps.
+
+Notifier channels:
+
+- Telegram: `MONLET_NOTIFIER_TELEGRAM_ENABLED=true`, `MONLET_NOTIFIER_TELEGRAM_TOKEN`, `MONLET_NOTIFIER_TELEGRAM_CHAT_ID`.
+- Webhook: `MONLET_NOTIFIER_WEBHOOK_ENABLED=true`, `MONLET_NOTIFIER_WEBHOOK_URL`, optional `MONLET_NOTIFIER_WEBHOOK_TOKEN`.
+- Alertmanager: `MONLET_NOTIFIER_ALERTMANAGER_ENABLED=true`, `MONLET_NOTIFIER_ALERTMANAGER_URL`.
+- Debug/log: `MONLET_NOTIFIER_DEBUG_ENABLED=true`.
+
+## Web env
+
+The web UI reads API/auth/timezone settings from env. Without auth settings, read-only pages are available locally, but admission mutations return `403`.
+
+```env
+MONLET_API_BASE_URL=http://127.0.0.1:8000
+MONLET_API_TOKEN=replace-with-server-token
+MONLET_WEB_TIME_ZONE=UTC
+MONLET_WEB_AUTH_USERNAME=admin
+MONLET_WEB_AUTH_PASSWORD=replace-with-strong-password
+MONLET_WEB_SESSION_SECRET=replace-with-random-32-bytes-min
+MONLET_WEB_SESSION_TTL_SEC=604800
+NEXT_PUBLIC_MONLET_POLL_MS=10000
+# Or, instead of local login, trust reverse-proxy auth:
+# MONLET_WEB_TRUST_PROXY_AUTH=true
+```
+
+- `MONLET_API_BASE_URL` - server base URL for server-side fetch.
+- `MONLET_API_TOKEN` - Bearer token for UI requests to the server; it is not sent to the browser.
+- `MONLET_WEB_TIME_ZONE` - IANA timezone used to render timestamps (default `UTC`).
+- `MONLET_WEB_AUTH_USERNAME` / `MONLET_WEB_AUTH_PASSWORD` - local cookie-login credentials.
+- `MONLET_WEB_SESSION_SECRET` - session signing secret, at least 32 bytes; shorter values make UI auth misconfigured.
+- `MONLET_WEB_SESSION_TTL_SEC` - session lifetime (default 604800 = 7 days).
+- `MONLET_WEB_TRUST_PROXY_AUTH` - `true` trusts the `X-Forwarded-User` header from a reverse proxy. Mutually exclusive with local login; configuring both modes is an error (UI returns `500`).
+- `NEXT_PUBLIC_MONLET_POLL_MS` - browser polling interval in milliseconds (default 10000); for production Next.js builds, this is a build-time variable.
+
+## Minimal Flow
+
+1. Server starts with PostgreSQL and `MONLET_AUTH_TOKEN`.
+2. Agent starts from TOML, creates `state_dir/agent.key`, and sends heartbeat.
+3. The new agent appears in UI as `pending`.
+4. Operator accepts the agent.
+5. After accept, the server starts accounting for events, checks, incidents, and notifications.

CONFIG_REFERENCE.ru.md

@@ -1,3 +1,5 @@
+[English](CONFIG_REFERENCE.md) | Русский
+
 # Monlet: краткий справочник по настройкам
 
 Monlet состоит из агента на хостах и центрального сервера. Агент сам генерирует локальный ключ в `state_dir/agent.key`; сервер видит новый ключ как `pending`, пока оператор не примет агента в UI.

README.md

@@ -134,7 +134,8 @@ Start with:
 3. [server/README.md](server/README.md)
 4. [web/README.md](web/README.md)
 5. [api/README.md](api/README.md)
-6. [docs/index.md](docs/index.md)
+6. [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md) ([Русский](CONFIG_REFERENCE.ru.md))
+7. [docs/index.md](docs/index.md)
 
 Dependency caches and virtual environments are kept inside the repository. See
 the component READMEs for current local development commands.

README.ru.md

@@ -135,7 +135,8 @@ Monlet 0.1.0 намеренно не является:
 3. [server/README.md](server/README.md)
 4. [web/README.md](web/README.md)
 5. [api/README.md](api/README.md)
-6. [docs/index.md](docs/index.md)
+6. [CONFIG_REFERENCE.ru.md](CONFIG_REFERENCE.ru.md) ([English](CONFIG_REFERENCE.md))
+7. [docs/index.md](docs/index.md)
 
 Dependency caches и virtual environments хранятся внутри репозитория. Актуальные
 команды local development описаны в component README.

agent/internal/config/config.go

@@ -217,7 +217,7 @@ func (c *Config) Validate() error {
 			return fmt.Errorf("check %q: timeout must be > 0", ch.ID)
 		}
 		if ch.Interval.Duration < 0 {
-			return fmt.Errorf("check %q: interval must be > 0", ch.ID)
+			return fmt.Errorf("check %q: interval must not be negative", ch.ID)
 		}
 		ch.Cron = strings.TrimSpace(ch.Cron)
 		hasInterval := ch.Interval.Duration > 0

agent/internal/config/config_test.go

@@ -123,6 +123,29 @@ func TestValidateCheckScheduleChoice(t *testing.T) {
 	}
 }
 
+func TestValidateNegativeIntervalMessage(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+enabled = true
+url = "http://localhost"
+
+[[checks]]
+id = "c1"
+command = "true"
+interval = "-1s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected negative interval error")
+	}
+	if !strings.Contains(err.Error(), "interval must not be negative") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func TestLoadResourceLimits(t *testing.T) {
 	c, err := Load(writeTmp(t, `
 	agent_id = "host-1"

agent/internal/scheduler/scheduler.go

@@ -202,13 +202,24 @@ func (w *checkWorker) Wait() {
 }
 
 func (w *checkWorker) loop(initial config.CheckConfig) {
+	cfg := initial
+	stoppedNotified := false
+	notifyStopped := func() {
+		if stoppedNotified {
+			return
+		}
+		stoppedNotified = true
+		if w.hooks.OnStopped != nil {
+			w.hooks.OnStopped(cfg.ID)
+		}
+	}
 	defer func() {
+		notifyStopped()
 		if w.onDone != nil {
 			w.onDone(w)
 		}
 		w.wg.Done()
 	}()
-	cfg := initial
 	t := time.NewTimer(time.Hour)
 	stopTimer(t)
 	defer t.Stop()
@@ -249,9 +260,6 @@ func (w *checkWorker) loop(initial config.CheckConfig) {
 	}
 
 	if !arm(cfg, true) {
-		if w.hooks.OnStopped != nil {
-			w.hooks.OnStopped(cfg.ID)
-		}
 		return
 	}
 
@@ -261,18 +269,12 @@ func (w *checkWorker) loop(initial config.CheckConfig) {
 			stopping = true
 			ctxDone = nil
 			if !running {
-				if w.hooks.OnStopped != nil {
-					w.hooks.OnStopped(cfg.ID)
-				}
 				return
 			}
 		case <-stopCh:
 			stopping = true
 			stopCh = nil
 			if !running {
-				if w.hooks.OnStopped != nil {
-					w.hooks.OnStopped(cfg.ID)
-				}
 				return
 			}
 		case next := <-w.updateCh:
@@ -307,9 +309,6 @@ func (w *checkWorker) loop(initial config.CheckConfig) {
 				}
 			}
 			if stopping {
-				if w.hooks.OnStopped != nil {
-					w.hooks.OnStopped(cfg.ID)
-				}
 				return
 			}
 			if runAfterCurrent {

agent/internal/scheduler/scheduler_test.go

@@ -2,6 +2,7 @@ package scheduler
 
 import (
 	"context"
+	"errors"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -107,6 +108,38 @@ func TestCronEmitsOnDueSlot(t *testing.T) {
 	m.Wait()
 }
 
+func TestCronArmErrorCallsStoppedHook(t *testing.T) {
+	old := parseCronSchedule
+	var calls int32
+	parseCronSchedule = func(string) (cron.Schedule, error) {
+		if atomic.AddInt32(&calls, 1) == 1 {
+			return fakeCronSchedule{delay: 10 * time.Millisecond}, nil
+		}
+		return nil, errors.New("boom")
+	}
+	t.Cleanup(func() { parseCronSchedule = old })
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	out := make(chan Event, 10)
+	stopped := make(chan string, 1)
+	m := NewManager(ctx, "a1", out, Hooks{
+		OnStopped: func(id string) { stopped <- id },
+	})
+	m.Update([]config.CheckConfig{mkCronCheck("cron", "true", 100*time.Millisecond)})
+
+	select {
+	case id := <-stopped:
+		if id != "cron" {
+			t.Fatalf("unexpected stopped id: %q", id)
+		}
+	case <-ctx.Done():
+		t.Fatal("stopped hook was not called")
+	}
+	m.Stop()
+	m.Wait()
+}
+
 func TestNoOverlapSkips(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 600*time.Millisecond)
 	defer cancel()

docs/index.md

@@ -8,6 +8,7 @@ This directory contains operator-facing docs that do not fit in the root or comp
 - [Backup and restore](ops/backup-restore.md) - PostgreSQL backup, retention, and disaster recovery notes.
 - [Token rotation](ops/token-rotation.md) - UI token overlap and agent-key rotation.
 - [Deployment notes](ops/deployment.md) - deployment shape and production boundaries.
+- [Configuration reference](../CONFIG_REFERENCE.md) - agent TOML plus server and web environment variables.
 
 ## Conventions
 

web/src/app/globals.css

@@ -13,12 +13,6 @@
   --font-mono: var(--font-geist-mono);
 }
 
-:root[data-theme="dark"] {
-  --background: #0a0a0a;
-  --foreground: #ededed;
-  color-scheme: dark;
-}
-
 body {
   background: var(--background);
   color: var(--foreground);