Implement Monlet MVP stack and UI updates

.github/workflows/ci.yml

@@ -0,0 +1,93 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  openapi:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - name: Lint OpenAPI
+        run: npx --yes @redocly/cli@1.34.6 lint api/openapi.yaml
+
+  agent:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: agent
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.25.7"
+          cache: true
+          cache-dependency-path: agent/go.sum
+      - name: gofmt
+        run: |
+          out=$(gofmt -l .)
+          if [ -n "$out" ]; then echo "$out"; exit 1; fi
+      - name: vet
+        run: go vet ./...
+      - name: test
+        run: go test ./...
+      - name: race
+        run: go test -race ./...
+
+  server:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: server
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: install
+        run: uv sync --extra dev
+      - name: lint
+        run: uv run ruff check . && uv run ruff format --check .
+      - name: test
+        run: uv run pytest -q
+
+  web:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+      - run: npm ci --no-audit --no-fund
+      - run: npm run gen:api
+      - run: npm run lint
+      - run: npm run typecheck
+      - run: npm run build
+      - name: install playwright browser
+        run: npx playwright install chromium
+      - run: npm run smoke
+
+  stack-smoke:
+    runs-on: ubuntu-latest
+    needs: [server, agent]
+    steps:
+      - uses: actions/checkout@v4
+      - name: stack smoke
+        run: bash deploy/smoke.sh

agent/README.md

@@ -14,19 +14,36 @@ Do not use `go install` or `go env -w`. Module cache, build cache, and tool bina
 
 ## Configuration
 
-See `config.example.toml`. Required sections: top-level `state_dir`, `[server]` (`url`, `token`), at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`). `agent_id` is optional — if empty, the agent generates and persists a UUID under `state_dir/agent_id`.
+See `config.example.toml`. Required fields: top-level `state_dir`, at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`), and at least one output enabled: `server.enabled = true` or `metrics.enabled = true`. `agent_id` is optional; if omitted, it defaults to `hostname`.
 
 | Key | Default | Notes |
 |---|---|---|
-| `mode` | `hybrid` | `prometheus_only` (only `/metrics`), `push_only` (only heartbeat+events), `hybrid` (both). `prometheus_only` does not require `server.url`/`server.token`. |
+| `server.enabled` | `false` | Enables heartbeat/event push. Requires `server.url` and `server.token` when true. |
+| `metrics.enabled` | `false` | Enables `/metrics`. |
+| `labels` | `{}` | Optional heartbeat labels. Max 31 custom labels because `monlet_agent_version` is generated by the agent. Secret-like keys are rejected. |
 | `server.heartbeat_interval` | `30s` | ADR-0005 |
 | `server.batch_interval` | `10s` | events flush cadence |
 | `metrics.listen` | `127.0.0.1:9465` | low-cardinality only |
-| `checks[].notification_owner` | `server` | `server`, `prometheus`, `none` |
+| `checks[].command` | required | Shell string executed as `/bin/sh -c`; TOML multi-line strings are supported. |
+| `checks[].notifications_enabled` | `true` | Set `false` for checks that must not create server notifications. |
+
+Example:
+
+```toml
+[[checks]]
+id = "disk_root"
+command = '''
+set -eu
+/usr/local/lib/monlet/check_disk_root.sh
+'''
+interval = "60s"
+timeout = "10s"
+```
 
 ## Behaviour
 
 - Per-check scheduler with no-overlap: a tick is skipped if the previous run is still in flight (`monlet_agent_check_skipped_total`).
+- Check commands run through `/bin/sh -c`; the old argv-array form is intentionally unsupported.
 - Timeout via `exec.CommandContext` with `Setpgid` so the process group is killed on expiry; timeout → `unknown`.
 - Exit code mapping: 0=ok, 1=warning, 2=critical, 3+=unknown (ADR-0005).
 - `output` (stdout+stderr) is truncated by **UTF-8 byte length** to 8 KiB with marker `...[truncated N bytes]`.
@@ -34,6 +51,8 @@ See `config.example.toml`. Required sections: top-level `state_dir`, `[server]`
 - Spool: per-event JSON files in `state_dir/spool/`, FIFO, limits 10 000 events / 50 MiB, drop oldest on overflow (ADR-0005).
 - Retry backoff for heartbeat and events: 1s → 30s, exp ×2, jitter ±20%. 4xx are dropped (not retried) to avoid hot loop.
 - Bearer auth header on both endpoints (ADR-0007).
+- Heartbeats include configured labels plus reserved `monlet_agent_version=<binary version>`.
+- Heartbeats include explicit feature flags: `push` from `server.enabled`, `metrics` from `metrics.enabled`.
 
 ## Prometheus metrics
 
@@ -52,7 +71,7 @@ Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `statu
 - `monlet_agent_send_attempts_total{kind}`, `monlet_agent_send_failures_total{kind}` (`kind` ∈ `heartbeat`, `events`)
 - `monlet_agent_events_accepted_total`, `monlet_agent_events_deduplicated_total`
 - `monlet_agent_last_heartbeat_timestamp_seconds`
-- `monlet_agent_build_info{version,mode}` (constant 1)
+- `monlet_agent_build_info{version,push,metrics}` (constant 1)
 
 ## systemd
 

agent/cmd/monlet-agent/main.go

@@ -17,7 +17,7 @@ import (
 	"github.com/monlet/agent/internal/spool"
 )
 
-var version = "dev"
+var version = "0.1.0"
 
 func main() {
 	cfgPath := flag.String("config", "config.toml", "path to TOML config")
@@ -31,7 +31,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	agentID, err := ids.LoadOrCreateAgentID(cfg.StateDir, cfg.AgentID)
+	agentID, err := ids.ResolveAgentID(cfg.AgentID, cfg.Hostname)
 	if err != nil {
 		log.Error("agent_id", "err", err)
 		os.Exit(1)
@@ -47,7 +47,7 @@ func main() {
 		Cfg:     cfg,
 		AgentID: agentID,
 		Version: version,
-		Client:  client.New(cfg.Server.URL, cfg.Server.Token, agentID, version),
+		Client:  client.New(cfg.Server.URL, cfg.Server.Token, agentID),
 		Spool:   sp,
 		Metrics: metrics.New(),
 		Log:     log,

agent/config.example.toml

@@ -1,9 +1,12 @@
-agent_id = "example-host-01"
 hostname = "example-host-01"
-mode = "hybrid"
 state_dir = "/var/lib/monlet-agent"
 
+[labels]
+env = "prod"
+role = "api"
+
 [server]
+enabled = true
 url = "http://127.0.0.1:8000"
 token = "replace-me"
 heartbeat_interval = "30s"
@@ -16,7 +19,8 @@ listen = "127.0.0.1:9465"
 [[checks]]
 id = "disk_root"
 name = "Root disk usage"
-command = ["/usr/local/lib/monlet/check_disk_root.sh"]
+command = '''
+/usr/local/lib/monlet/check_disk_root.sh
+'''
 interval = "60s"
 timeout = "10s"
-notification_owner = "server"

agent/internal/app/app.go

@@ -28,7 +28,7 @@ func (a *App) Run(ctx context.Context) error {
 	events := make(chan scheduler.Event, 256)
 
 	a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
-	a.Metrics.BuildInfo.WithLabelValues(a.Version, a.Cfg.Mode).Set(1)
+	a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(a.Cfg.PushesToServer()), boolLabel(a.Cfg.ExposesMetrics())).Set(1)
 	for _, c := range a.Cfg.Checks {
 		a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds())
 	}
@@ -122,8 +122,11 @@ func (a *App) heartbeatLoop(ctx context.Context) {
 			AgentID:    a.AgentID,
 			ObservedAt: time.Now().UTC(),
 			Hostname:   a.Cfg.Hostname,
-			Version:    a.Version,
+			Features: client.AgentFeatures{
-			Mode:       a.Cfg.Mode,
+				Push:    a.Cfg.PushesToServer(),
+				Metrics: a.Cfg.ExposesMetrics(),
+			},
+			Labels: a.Cfg.HeartbeatLabels(a.Version),
 		}
 		a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc()
 		err := a.Client.SendHeartbeat(ctx, hb)
@@ -210,3 +213,10 @@ func sleep(ctx context.Context, d time.Duration) bool {
 		return false
 	}
 }
+
+func boolLabel(v bool) string {
+	if v {
+		return "true"
+	}
+	return "false"
+}

agent/internal/app/app_test.go

@@ -30,22 +30,20 @@ func TestPrometheusOnlyDoesNotPush(t *testing.T) {
 	dir := t.TempDir()
 	cfg := &config.Config{
 		AgentID:  "a1",
-		Mode:     "prometheus_only",
 		StateDir: dir,
-		Server:   config.ServerConfig{URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
+		Server:   config.ServerConfig{Enabled: boolPtr(false), URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
-		Metrics:  config.MetricsConfig{Enabled: false},
+		Metrics:  config.MetricsConfig{Enabled: true, Listen: "127.0.0.1:0"},
 		Checks: []config.CheckConfig{{
-			ID:                "c1",
+			ID:       "c1",
-			Command:           []string{"sh", "-c", "echo hi"},
+			Command:  "echo hi",
-			Interval:          config.Duration{Duration: 100 * time.Millisecond},
+			Interval: config.Duration{Duration: 100 * time.Millisecond},
-			Timeout:           config.Duration{Duration: 50 * time.Millisecond},
+			Timeout:  config.Duration{Duration: 50 * time.Millisecond},
-			NotificationOwner: "server",
 		}},
 	}
 	sp, _ := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
 	a := &App{
 		Cfg: cfg, AgentID: "a1", Version: "t",
-		Client: client.New(srv.URL, "t", "a1", "t"),
+		Client: client.New(srv.URL, "t", "a1"),
 		Spool:  sp, Metrics: metrics.New(),
 		Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
 	}
@@ -62,16 +60,22 @@ func TestPrometheusOnlyDoesNotPush(t *testing.T) {
 
 func TestEndToEndPushAndReplay(t *testing.T) {
 	var (
-		mu           sync.Mutex
+		mu            sync.Mutex
-		seenEventIDs = make(map[string]int)
+		seenEventIDs  = make(map[string]int)
-		hbCount      int32
+		lastHeartbeat client.HeartbeatRequest
-		fail         atomic.Bool
+		hbCount       int32
+		fail          atomic.Bool
 	)
 	fail.Store(true)
 
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
 		case "/api/v1/heartbeat":
+			var req client.HeartbeatRequest
+			_ = json.NewDecoder(r.Body).Decode(&req)
+			mu.Lock()
+			lastHeartbeat = req
+			mu.Unlock()
 			atomic.AddInt32(&hbCount, 1)
 			w.WriteHeader(http.StatusAccepted)
 			_, _ = io.WriteString(w, `{"accepted":true}`)
@@ -97,9 +101,10 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 	cfg := &config.Config{
 		AgentID:  "a1",
 		Hostname: "h",
-		Mode:     "hybrid",
 		StateDir: dir,
+		Labels:   map[string]string{"env": "test"},
 		Server: config.ServerConfig{
+			Enabled:           boolPtr(true),
 			URL:               srv.URL,
 			Token:             "t",
 			HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond},
@@ -107,11 +112,10 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 		},
 		Metrics: config.MetricsConfig{Enabled: false},
 		Checks: []config.CheckConfig{{
-			ID:                "c1",
+			ID:       "c1",
-			Command:           []string{"sh", "-c", "echo hi"},
+			Command:  "echo hi",
-			Interval:          config.Duration{Duration: 150 * time.Millisecond},
+			Interval: config.Duration{Duration: 150 * time.Millisecond},
-			Timeout:           config.Duration{Duration: 100 * time.Millisecond},
+			Timeout:  config.Duration{Duration: 100 * time.Millisecond},
-			NotificationOwner: "server",
 		}},
 	}
 	sp, err := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
@@ -122,7 +126,7 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 		Cfg:     cfg,
 		AgentID: "a1",
 		Version: "test",
-		Client:  client.New(cfg.Server.URL, "t", "a1", "test"),
+		Client:  client.New(cfg.Server.URL, "t", "a1"),
 		Spool:   sp,
 		Metrics: metrics.New(),
 		Log:     slog.New(slog.NewTextHandler(io.Discard, nil)),
@@ -168,4 +172,17 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 	if atomic.LoadInt32(&hbCount) == 0 {
 		t.Fatal("no heartbeats observed")
 	}
+	if lastHeartbeat.Labels["env"] != "test" {
+		t.Fatalf("configured heartbeat label missing: %#v", lastHeartbeat.Labels)
+	}
+	if lastHeartbeat.Labels[config.AgentVersionLabel] != "test" {
+		t.Fatalf("version heartbeat label missing: %#v", lastHeartbeat.Labels)
+	}
+	if !lastHeartbeat.Features.Push || lastHeartbeat.Features.Metrics {
+		t.Fatalf("unexpected heartbeat features: %+v", lastHeartbeat.Features)
+	}
+}
+
+func boolPtr(v bool) *bool {
+	return &v
 }

agent/internal/client/client.go

@@ -21,26 +21,28 @@ type Client struct {
 	baseURL string
 	token   string
 	agentID string
-	version string
 	http    *http.Client
 }
 
-func New(baseURL, token, agentID, version string) *Client {
+func New(baseURL, token, agentID string) *Client {
 	return &Client{
 		baseURL: strings.TrimRight(baseURL, "/"),
 		token:   token,
 		agentID: agentID,
-		version: version,
 		http:    &http.Client{Timeout: 15 * time.Second},
 	}
 }
 
+type AgentFeatures struct {
+	Push    bool `json:"push"`
+	Metrics bool `json:"metrics"`
+}
+
 type HeartbeatRequest struct {
 	AgentID    string            `json:"agent_id"`
 	ObservedAt time.Time         `json:"observed_at"`
 	Hostname   string            `json:"hostname"`
-	Version    string            `json:"version"`
+	Features   AgentFeatures     `json:"features"`
-	Mode       string            `json:"mode,omitempty"`
 	Labels     map[string]string `json:"labels,omitempty"`
 }
 
@@ -65,7 +67,11 @@ func (c *Client) SendEvents(ctx context.Context, events []scheduler.Event) (*Eve
 	if len(events) > MaxBatchEvents {
 		return nil, fmt.Errorf("batch exceeds %d events", MaxBatchEvents)
 	}
-	req := EventBatchRequest{AgentID: c.agentID, Events: events}
+	wireEvents := make([]scheduler.Event, len(events))
+	for i, ev := range events {
+		wireEvents[i] = ev.WireEvent()
+	}
+	req := EventBatchRequest{AgentID: c.agentID, Events: wireEvents}
 	var resp EventBatchResponse
 	if err := c.post(ctx, "/api/v1/events", req, &resp); err != nil {
 		return nil, err

agent/internal/client/client_test.go

@@ -24,8 +24,13 @@ func TestSendHeartbeatSuccess(t *testing.T) {
 		_, _ = io.WriteString(w, `{"accepted":true}`)
 	}))
 	defer srv.Close()
-	c := New(srv.URL, "tok", "a1", "v0")
+	c := New(srv.URL, "tok", "a1")
-	err := c.SendHeartbeat(context.Background(), HeartbeatRequest{AgentID: "a1", ObservedAt: time.Now().UTC(), Hostname: "h", Version: "v0"})
+	err := c.SendHeartbeat(context.Background(), HeartbeatRequest{
+		AgentID:    "a1",
+		ObservedAt: time.Now().UTC(),
+		Hostname:   "h",
+		Features:   AgentFeatures{Push: true, Metrics: false},
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -42,7 +47,7 @@ func TestSendEventsBatch(t *testing.T) {
 		_, _ = io.WriteString(w, `{"accepted":2,"deduplicated":0}`)
 	}))
 	defer srv.Close()
-	c := New(srv.URL, "tok", "a1", "v0")
+	c := New(srv.URL, "tok", "a1")
 	resp, err := c.SendEvents(context.Background(), []scheduler.Event{{EventID: "x"}, {EventID: "y"}})
 	if err != nil {
 		t.Fatal(err)
@@ -53,7 +58,7 @@ func TestSendEventsBatch(t *testing.T) {
 }
 
 func TestSendEventsTooBig(t *testing.T) {
-	c := New("http://x", "t", "a1", "v0")
+	c := New("http://x", "t", "a1")
 	big := make([]scheduler.Event, MaxBatchEvents+1)
 	_, err := c.SendEvents(context.Background(), big)
 	if err == nil {

agent/internal/config/config.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strings"
 	"time"
+	"unicode/utf8"
 
 	"github.com/BurntSushi/toml"
 )
@@ -13,22 +15,27 @@ var idPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]+$`)
 
 const (
 	maxIDLen           = 128
+	maxLabelKeyLen     = 64
+	maxLabelValueLen   = 256
+	maxLabels          = 32
+	AgentVersionLabel  = "monlet_agent_version"
 	defaultHeartbeat   = 30 * time.Second
 	defaultBatch       = 10 * time.Second
 	defaultMetricsAddr = "127.0.0.1:9465"
 )
 
 type Config struct {
-	AgentID  string        `toml:"agent_id"`
+	AgentID  string            `toml:"agent_id"`
-	Hostname string        `toml:"hostname"`
+	Hostname string            `toml:"hostname"`
-	Mode     string        `toml:"mode"`
+	StateDir string            `toml:"state_dir"`
-	StateDir string        `toml:"state_dir"`
+	Labels   map[string]string `toml:"labels"`
-	Server   ServerConfig  `toml:"server"`
+	Server   ServerConfig      `toml:"server"`
-	Metrics  MetricsConfig `toml:"metrics"`
+	Metrics  MetricsConfig     `toml:"metrics"`
-	Checks   []CheckConfig `toml:"checks"`
+	Checks   []CheckConfig     `toml:"checks"`
 }
 
 type ServerConfig struct {
+	Enabled           *bool    `toml:"enabled"`
 	URL               string   `toml:"url"`
 	Token             string   `toml:"token"`
 	HeartbeatInterval Duration `toml:"heartbeat_interval"`
@@ -41,13 +48,13 @@ type MetricsConfig struct {
 }
 
 type CheckConfig struct {
-	ID                string   `toml:"id"`
+	ID                   string   `toml:"id"`
-	Name              string   `toml:"name"`
+	Name                 string   `toml:"name"`
-	Command           []string `toml:"command"`
+	Command              string   `toml:"command"`
-	Interval          Duration `toml:"interval"`
+	Interval             Duration `toml:"interval"`
-	Timeout           Duration `toml:"timeout"`
+	Timeout              Duration `toml:"timeout"`
-	NotificationOwner string   `toml:"notification_owner"`
+	NotificationsEnabled *bool    `toml:"notifications_enabled"`
-	DedupeKey         string   `toml:"dedupe_key"`
+	DedupeKey            string   `toml:"dedupe_key"`
 }
 
 type Duration struct{ time.Duration }
@@ -63,9 +70,17 @@ func (d *Duration) UnmarshalText(b []byte) error {
 
 func Load(path string) (*Config, error) {
 	var c Config
-	if _, err := toml.DecodeFile(path, &c); err != nil {
+	md, err := toml.DecodeFile(path, &c)
+	if err != nil {
 		return nil, fmt.Errorf("read config: %w", err)
 	}
+	if undecoded := md.Undecoded(); len(undecoded) > 0 {
+		keys := make([]string, len(undecoded))
+		for i, key := range undecoded {
+			keys[i] = key.String()
+		}
+		return nil, fmt.Errorf("unknown config keys: %s", strings.Join(keys, ", "))
+	}
 	c.applyDefaults()
 	if err := c.Validate(); err != nil {
 		return nil, err
@@ -74,9 +89,6 @@ func Load(path string) (*Config, error) {
 }
 
 func (c *Config) applyDefaults() {
-	if c.Mode == "" {
-		c.Mode = "hybrid"
-	}
 	if c.Hostname == "" {
 		if h, err := os.Hostname(); err == nil {
 			c.Hostname = h
@@ -99,21 +111,27 @@ func (c *Config) Validate() error {
 			return err
 		}
 	}
-	switch c.Mode {
+	if c.Hostname == "" {
-	case "prometheus_only", "push_only", "hybrid":
+		return fmt.Errorf("hostname is required")
-	default:
-		return fmt.Errorf("invalid mode %q", c.Mode)
 	}
 	if c.StateDir == "" {
 		return fmt.Errorf("state_dir is required")
 	}
+	if err := validateLabels(c.Labels); err != nil {
+		return err
+	}
 	if c.PushesToServer() {
 		if c.Server.URL == "" {
-			return fmt.Errorf("server.url is required for mode %q", c.Mode)
+			return fmt.Errorf("server.url is required when server.enabled = true")
 		}
 		if c.Server.Token == "" {
-			return fmt.Errorf("server.token is required for mode %q", c.Mode)
+			return fmt.Errorf("server.token is required when server.enabled = true")
 		}
+	} else if c.Server.URL != "" || c.Server.Token != "" {
+		return fmt.Errorf("server.url/token require server.enabled = true")
+	}
+	if !c.PushesToServer() && !c.ExposesMetrics() {
+		return fmt.Errorf("server.enabled or metrics.enabled must be true")
 	}
 	if len(c.Checks) == 0 {
 		return fmt.Errorf("at least one check is required")
@@ -128,7 +146,7 @@ func (c *Config) Validate() error {
 			return fmt.Errorf("duplicate check id %q", ch.ID)
 		}
 		seen[ch.ID] = struct{}{}
-		if len(ch.Command) == 0 {
+		if strings.TrimSpace(ch.Command) == "" {
 			return fmt.Errorf("check %q: command is required", ch.ID)
 		}
 		if ch.Interval.Duration <= 0 {
@@ -140,14 +158,6 @@ func (c *Config) Validate() error {
 		if ch.Timeout.Duration > ch.Interval.Duration {
 			return fmt.Errorf("check %q: timeout must be <= interval", ch.ID)
 		}
-		switch ch.NotificationOwner {
-		case "", "server", "prometheus", "none":
-		default:
-			return fmt.Errorf("check %q: invalid notification_owner %q", ch.ID, ch.NotificationOwner)
-		}
-		if ch.NotificationOwner == "" {
-			ch.NotificationOwner = "server"
-		}
 		if len(ch.DedupeKey) > 256 {
 			return fmt.Errorf("check %q: dedupe_key too long", ch.ID)
 		}
@@ -155,15 +165,31 @@ func (c *Config) Validate() error {
 	return nil
 }
 
-// PushesToServer is true when the mode requires heartbeat/events push.
+// PushesToServer is true when heartbeat/events push is enabled.
 func (c *Config) PushesToServer() bool {
-	return c.Mode == "push_only" || c.Mode == "hybrid"
+	return c.Server.Enabled != nil && *c.Server.Enabled
 }
 
 // ExposesMetrics is true when the agent should serve /metrics.
-// `metrics.enabled` is an independent gate; both must agree.
 func (c *Config) ExposesMetrics() bool {
-	return c.Metrics.Enabled && (c.Mode == "prometheus_only" || c.Mode == "hybrid")
+	return c.Metrics.Enabled
+}
+
+func (c *Config) HeartbeatLabels(version string) map[string]string {
+	labels := make(map[string]string, len(c.Labels)+1)
+	for k, v := range c.Labels {
+		labels[k] = v
+	}
+	labels[AgentVersionLabel] = version
+	return labels
+}
+
+func (c CheckConfig) NotificationsOn() bool {
+	return c.NotificationsEnabled == nil || *c.NotificationsEnabled
+}
+
+func (c CheckConfig) Argv() []string {
+	return []string{"/bin/sh", "-c", c.Command}
 }
 
 func ValidateID(field, v string) error {
@@ -178,3 +204,40 @@ func ValidateID(field, v string) error {
 	}
 	return nil
 }
+
+func validateLabels(labels map[string]string) error {
+	if len(labels) > maxLabels {
+		return fmt.Errorf("labels exceed %d entries", maxLabels)
+	}
+	if _, hasVersion := labels[AgentVersionLabel]; !hasVersion && len(labels) >= maxLabels {
+		return fmt.Errorf("labels leave no room for reserved %q label", AgentVersionLabel)
+	}
+	for k, v := range labels {
+		if k == "" {
+			return fmt.Errorf("label key is empty")
+		}
+		if len(k) > maxLabelKeyLen {
+			return fmt.Errorf("label key %q exceeds %d chars", k, maxLabelKeyLen)
+		}
+		if !idPattern.MatchString(k) {
+			return fmt.Errorf("label key %q has invalid characters", k)
+		}
+		if isSensitiveLabelKey(k) {
+			return fmt.Errorf("label key %q is not allowed", k)
+		}
+		if utf8.RuneCountInString(v) > maxLabelValueLen {
+			return fmt.Errorf("label %q value exceeds %d chars", k, maxLabelValueLen)
+		}
+	}
+	return nil
+}
+
+func isSensitiveLabelKey(k string) bool {
+	normalized := strings.NewReplacer("-", "_", ".", "_", ":", "_").Replace(strings.ToLower(k))
+	for _, word := range []string{"token", "secret", "password", "credential", "authorization", "cookie", "api_key", "apikey"} {
+		if strings.Contains(normalized, word) {
+			return true
+		}
+	}
+	return false
+}

agent/internal/config/config_test.go

@@ -1,8 +1,10 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -20,12 +22,13 @@ agent_id = "host-1"
 state_dir = "/tmp/x"
 
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `
@@ -35,14 +38,105 @@ func TestLoadMinimal(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if c.Mode != "hybrid" {
+	if !c.PushesToServer() || c.ExposesMetrics() {
-		t.Errorf("default mode: %q", c.Mode)
+		t.Errorf("features: push=%v metrics=%v", c.PushesToServer(), c.ExposesMetrics())
 	}
 	if c.Server.HeartbeatInterval.Duration == 0 {
 		t.Error("default heartbeat not applied")
 	}
-	if c.Checks[0].NotificationOwner != "server" {
+	if !c.Checks[0].NotificationsOn() {
-		t.Errorf("default notification_owner: %q", c.Checks[0].NotificationOwner)
+		t.Error("notifications should default on")
+	}
+}
+
+func TestLoadMultilineCommand(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = '''
+set -eu
+echo ok
+'''
+interval = "10s"
+timeout = "5s"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(c.Checks[0].Command, "echo ok") {
+		t.Fatalf("command not loaded: %q", c.Checks[0].Command)
+	}
+	if got := c.Checks[0].Argv(); len(got) != 3 || got[0] != "/bin/sh" || got[1] != "-c" {
+		t.Fatalf("argv: %#v", got)
+	}
+}
+
+func TestCommandArrayRejected(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected command array to be rejected")
+	}
+}
+
+func TestLoadLabels(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[labels]
+env = "prod"
+role = "api"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	labels := c.HeartbeatLabels("1.2.3")
+	if labels["env"] != "prod" || labels["role"] != "api" {
+		t.Fatalf("configured labels missing: %#v", labels)
+	}
+	if labels[AgentVersionLabel] != "1.2.3" {
+		t.Fatalf("version label missing: %#v", labels)
+	}
+}
+
+func TestHeartbeatLabelsVersionLabelWins(t *testing.T) {
+	c := &Config{Labels: map[string]string{AgentVersionLabel: "manual"}}
+	labels := c.HeartbeatLabels("real")
+	if labels[AgentVersionLabel] != "real" {
+		t.Fatalf("version label must be generated from binary version, got %q", labels[AgentVersionLabel])
 	}
 }
 
@@ -52,11 +146,12 @@ func TestValidateBadID(t *testing.T) {
 agent_id = "bad id!"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -70,11 +165,12 @@ func TestValidateTimeoutExceedsInterval(t *testing.T) {
 agent_id = "x"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "5s"
 timeout = "10s"
 `))
@@ -83,13 +179,69 @@ timeout = "10s"
 	}
 }
 
-func TestPrometheusOnlyAllowsMissingServer(t *testing.T) {
+func TestValidateBadLabelKey(t *testing.T) {
-	c, err := Load(writeTmp(t, `
+	_, err := Load(writeTmp(t, `
-mode = "prometheus_only"
+agent_id = "x"
 state_dir = "/tmp/x"
+[labels]
+"bad key" = "x"
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected bad label key error")
+	}
+}
+
+func TestValidateSensitiveLabelKey(t *testing.T) {
+	for _, key := range []string{"token", "api_key", "api-key", "password", "credential", "authorization", "cookie"} {
+		if err := validateLabels(map[string]string{key: "x"}); err == nil {
+			t.Fatalf("expected sensitive label key error for %q", key)
+		}
+	}
+}
+
+func TestValidateLabelsReservedSlot(t *testing.T) {
+	labels := make(map[string]string, maxLabels)
+	for i := 0; i < maxLabels; i++ {
+		labels[fmt.Sprintf("k%d", i)] = "v"
+	}
+	if err := validateLabels(labels); err == nil {
+		t.Fatal("expected reserved label slot error")
+	}
+	delete(labels, "k0")
+	labels[AgentVersionLabel] = "manual"
+	if err := validateLabels(labels); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestValidateLabelValueMaxLength(t *testing.T) {
+	if err := validateLabels(map[string]string{"note": strings.Repeat("ю", maxLabelValueLen)}); err != nil {
+		t.Fatal(err)
+	}
+	if err := validateLabels(map[string]string{"note": strings.Repeat("x", maxLabelValueLen+1)}); err == nil {
+		t.Fatal("expected label value length error")
+	}
+}
+
+func TestPrometheusOnlyAllowsMissingServer(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+state_dir = "/tmp/x"
+[server]
+enabled = false
+[metrics]
+enabled = true
+[[checks]]
+id = "c1"
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -97,17 +249,21 @@ timeout = "5s"
 		t.Fatal(err)
 	}
 	if c.PushesToServer() {
-		t.Fatal("prometheus_only must not push")
+		t.Fatal("metrics-only config must not push")
+	}
+	if !c.ExposesMetrics() {
+		t.Fatal("metrics-only config must expose metrics")
 	}
 }
 
 func TestPushOnlyRequiresServer(t *testing.T) {
 	_, err := Load(writeTmp(t, `
-mode = "push_only"
 state_dir = "/tmp/x"
+[server]
+enabled = true
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -116,21 +272,52 @@ timeout = "5s"
 	}
 }
 
+func TestUnknownConfigKeyRejected(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+mode = "hybrid"
+state_dir = "/tmp/x"
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+[[checks]]
+id = "c1"
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected unknown key error")
+	}
+}
+
+func TestNotificationsEnabledCanDisable(t *testing.T) {
+	c, err := Load(writeTmp(t, minimal+`
+notifications_enabled = false
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.Checks[0].NotificationsOn() {
+		t.Fatal("notifications_enabled=false was not applied")
+	}
+}
+
 func TestExposesMetricsGate(t *testing.T) {
 	cases := []struct {
-		mode    string
+		server  bool
-		enabled bool
+		metrics bool
 		want    bool
 	}{
-		{"hybrid", true, true},
+		{true, true, true},
-		{"hybrid", false, false},
+		{true, false, false},
-		{"prometheus_only", true, true},
+		{false, true, true},
-		{"push_only", true, false},
+		{false, false, false},
 	}
 	for _, tc := range cases {
-		c := &Config{Mode: tc.mode, Metrics: MetricsConfig{Enabled: tc.enabled}}
+		c := &Config{Server: ServerConfig{Enabled: &tc.server}, Metrics: MetricsConfig{Enabled: tc.metrics}}
 		if got := c.ExposesMetrics(); got != tc.want {
-			t.Errorf("mode=%s enabled=%v want=%v got=%v", tc.mode, tc.enabled, tc.want, got)
+			t.Errorf("server=%v metrics=%v want=%v got=%v", tc.server, tc.metrics, tc.want, got)
 		}
 	}
 }
@@ -140,16 +327,17 @@ func TestValidateDuplicateCheckID(t *testing.T) {
 agent_id = "x"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))

agent/internal/ids/ids.go

@@ -2,38 +2,24 @@ package ids
 
 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
 
-	"github.com/google/uuid"
-
 	"github.com/monlet/agent/internal/config"
 )
 
-// LoadOrCreateAgentID returns configured ID if non-empty, otherwise loads
+func ResolveAgentID(configured, hostname string) (string, error) {
-// state_dir/agent_id, generating and persisting it on first run.
-func LoadOrCreateAgentID(stateDir, configured string) (string, error) {
 	if configured != "" {
-		return configured, nil
+		id := strings.TrimSpace(configured)
-	}
-	if err := os.MkdirAll(stateDir, 0o755); err != nil {
-		return "", fmt.Errorf("create state dir: %w", err)
-	}
-	path := filepath.Join(stateDir, "agent_id")
-	b, err := os.ReadFile(path)
-	if err == nil {
-		id := strings.TrimSpace(string(b))
 		if err := config.ValidateID("agent_id", id); err != nil {
-			return "", fmt.Errorf("persisted agent_id invalid: %w", err)
+			return "", err
 		}
 		return id, nil
 	}
-	if !os.IsNotExist(err) {
+	id := strings.TrimSpace(hostname)
-		return "", err
+	if id == "" {
+		return "", fmt.Errorf("hostname is empty")
 	}
-	id := uuid.NewString()
+	if err := config.ValidateID("hostname as agent_id", id); err != nil {
-	if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil {
 		return "", err
 	}
 	return id, nil

agent/internal/ids/ids_test.go

@@ -1,33 +1,17 @@
 package ids
 
-import (
+import "testing"
-	"os"
-	"path/filepath"
-	"testing"
-)
 
 func TestConfiguredWins(t *testing.T) {
-	id, err := LoadOrCreateAgentID(t.TempDir(), "fixed")
+	id, err := ResolveAgentID("fixed", "host")
 	if err != nil || id != "fixed" {
 		t.Fatalf("got %q %v", id, err)
 	}
 }
 
-func TestGenerateAndReuse(t *testing.T) {
+func TestHostnameFallback(t *testing.T) {
-	dir := t.TempDir()
+	id, err := ResolveAgentID("", "host-1")
-	id1, err := LoadOrCreateAgentID(dir, "")
+	if err != nil || id != "host-1" {
-	if err != nil {
+		t.Fatalf("got %q %v", id, err)
-		t.Fatal(err)
-	}
-	id2, err := LoadOrCreateAgentID(dir, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if id1 != id2 {
-		t.Fatalf("not stable: %q vs %q", id1, id2)
-	}
-	b, _ := os.ReadFile(filepath.Join(dir, "agent_id"))
-	if len(b) == 0 {
-		t.Fatal("agent_id not persisted")
 	}
 }

agent/internal/metrics/metrics.go

@@ -117,7 +117,7 @@ func New() *Metrics {
 	m.BuildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "monlet_agent_build_info",
 		Help: "Constant 1 with build labels.",
-	}, []string{"version", "mode"})
+	}, []string{"version", "push", "metrics"})
 	reg.MustRegister(m.CheckRuns, m.CheckSkipped, m.CheckDuration,
 		m.CheckStatus, m.CheckExitCode, m.CheckLastRun, m.CheckLastSuccess,
 		m.CheckInterval, m.ChecksConfigured,

agent/internal/scheduler/scheduler.go

@@ -12,16 +12,24 @@ import (
 
 // Event is a contract-shaped check result emitted by the scheduler.
 type Event struct {
-	EventID           string    `json:"event_id"`
+	EventID              string    `json:"event_id"`
-	CheckID           string    `json:"check_id"`
+	CheckID              string    `json:"check_id"`
-	ObservedAt        time.Time `json:"observed_at"`
+	ObservedAt           time.Time `json:"observed_at"`
-	Status            string    `json:"status"`
+	Status               string    `json:"status"`
-	ExitCode          int       `json:"exit_code"`
+	ExitCode             int       `json:"exit_code"`
-	DurationMs        int64     `json:"duration_ms"`
+	DurationMs           int64     `json:"duration_ms"`
-	Output            string    `json:"output,omitempty"`
+	Output               string    `json:"output,omitempty"`
-	OutputTruncated   bool      `json:"output_truncated,omitempty"`
+	OutputTruncated      bool      `json:"output_truncated,omitempty"`
-	NotificationOwner string    `json:"notification_owner,omitempty"`
+	NotificationsEnabled *bool     `json:"notifications_enabled,omitempty"`
-	IncidentKey       string    `json:"incident_key,omitempty"`
+	IncidentKey          string    `json:"incident_key,omitempty"`
+}
+
+func (e Event) WireEvent() Event {
+	if e.NotificationsEnabled == nil {
+		enabled := true
+		e.NotificationsEnabled = &enabled
+	}
+	return e
 }
 
 type Hooks struct {
@@ -66,22 +74,22 @@ func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out cha
 			mu.Unlock()
 		}()
 
-		res := runner.Run(ctx, c.Command, c.Timeout.Duration)
+		res := runner.Run(ctx, c.Argv(), c.Timeout.Duration)
 		id, err := eventid.New()
 		if err != nil {
 			return
 		}
 		ev := Event{
-			EventID:           id,
+			EventID:              id,
-			CheckID:           c.ID,
+			CheckID:              c.ID,
-			ObservedAt:        time.Now().UTC(),
+			ObservedAt:           time.Now().UTC(),
-			Status:            string(res.Status),
+			Status:               string(res.Status),
-			ExitCode:          res.ExitCode,
+			ExitCode:             res.ExitCode,
-			DurationMs:        res.DurationMs,
+			DurationMs:           res.DurationMs,
-			Output:            res.Output,
+			Output:               res.Output,
-			OutputTruncated:   res.OutputTruncated,
+			OutputTruncated:      res.OutputTruncated,
-			NotificationOwner: c.NotificationOwner,
+			NotificationsEnabled: boolPtr(c.NotificationsOn()),
-			IncidentKey:       incidentKey(agentID, c),
+			IncidentKey:          incidentKey(agentID, c),
 		}
 		if h.OnResult != nil {
 			h.OnResult(ev)
@@ -105,6 +113,10 @@ func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out cha
 	}
 }
 
+func boolPtr(v bool) *bool {
+	return &v
+}
+
 func incidentKey(agentID string, c config.CheckConfig) string {
 	if c.DedupeKey != "" {
 		return c.DedupeKey

agent/internal/scheduler/scheduler_test.go

@@ -11,11 +11,10 @@ import (
 
 func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig {
 	return config.CheckConfig{
-		ID:                id,
+		ID:       id,
-		Command:           []string{"sh", "-c", cmd},
+		Command:  cmd,
-		Interval:          config.Duration{Duration: interval},
+		Interval: config.Duration{Duration: interval},
-		Timeout:           config.Duration{Duration: timeout},
+		Timeout:  config.Duration{Duration: timeout},
-		NotificationOwner: "server",
 	}
 }
 

api/openapi.yaml

@@ -14,6 +14,7 @@ paths:
   /api/v1/health:
     get:
       summary: Health check
+      operationId: getHealth
       security: []
       responses:
         "200":
@@ -25,6 +26,7 @@ paths:
   /api/v1/ready:
     get:
       summary: Readiness check
+      operationId: getReady
       security: []
       responses:
         "200":
@@ -42,6 +44,7 @@ paths:
   /api/v1/heartbeat:
     post:
       summary: Ingest agent heartbeat
+      operationId: ingestHeartbeat
       requestBody:
         required: true
         content:
@@ -66,6 +69,7 @@ paths:
   /api/v1/events:
     post:
       summary: Ingest agent check result events
+      operationId: ingestEvents
       requestBody:
         required: true
         content:
@@ -90,6 +94,7 @@ paths:
   /api/v1/agents:
     get:
       summary: List agents
+      operationId: listAgents
       parameters:
         - $ref: "#/components/parameters/Cursor"
         - $ref: "#/components/parameters/Limit"
@@ -113,6 +118,7 @@ paths:
   /api/v1/agents/{agent_id}:
     get:
       summary: Get agent detail
+      operationId: getAgent
       parameters:
         - $ref: "#/components/parameters/AgentId"
       responses:
@@ -129,7 +135,15 @@ paths:
   /api/v1/checks:
     get:
       summary: List current check states
+      operationId: listChecks
       parameters:
+        - name: agent_id
+          in: query
+          required: false
+          schema:
+            type: string
+            maxLength: 128
+            pattern: "^[A-Za-z0-9._:-]+$"
         - $ref: "#/components/parameters/Cursor"
         - $ref: "#/components/parameters/Limit"
       responses:
@@ -152,6 +166,7 @@ paths:
   /api/v1/incidents:
     get:
       summary: List incidents
+      operationId: listIncidents
       parameters:
         - name: state
           in: query
@@ -181,6 +196,7 @@ paths:
   /api/v1/events/query:
     get:
       summary: Query stored events
+      operationId: queryEvents
       parameters:
         - name: agent_id
           in: query
@@ -218,6 +234,7 @@ paths:
   /api/v1/notifiers/outbox:
     get:
       summary: List notification outbox items
+      operationId: listOutbox
       parameters:
         - $ref: "#/components/parameters/Cursor"
         - $ref: "#/components/parameters/Limit"
@@ -358,7 +375,8 @@ components:
           description: Number of events whose `event_id` was already known.
     HeartbeatRequest:
       type: object
-      required: [agent_id, observed_at, hostname, version]
+      additionalProperties: false
+      required: [agent_id, observed_at, hostname, features]
       properties:
         agent_id:
           type: string
@@ -370,12 +388,8 @@ components:
         hostname:
           type: string
           maxLength: 253
-        version:
+        features:
-          type: string
+          $ref: "#/components/schemas/AgentFeatures"
-          maxLength: 64
-        mode:
-          type: string
-          enum: [prometheus_only, push_only, hybrid]
         labels:
           type: object
           maxProperties: 32
@@ -385,9 +399,21 @@ components:
           additionalProperties:
             type: string
             maxLength: 256
-          description: Up to 32 labels. Key max 64, value max 256.
+          description: Up to 32 labels. Key max 64, value max 256. Monlet agents reserve `monlet_agent_version`. Secret-like keys are rejected.
+    AgentFeatures:
+      type: object
+      additionalProperties: false
+      required: [push, metrics]
+      properties:
+        push:
+          type: boolean
+          description: Agent pushes heartbeats/events to the server.
+        metrics:
+          type: boolean
+          description: Agent exposes a Prometheus-compatible metrics endpoint.
     EventBatchRequest:
       type: object
+      additionalProperties: false
       required: [agent_id, events]
       properties:
         agent_id:
@@ -402,6 +428,7 @@ components:
             $ref: "#/components/schemas/CheckResultEvent"
     CheckResultEvent:
       type: object
+      additionalProperties: false
       description: |
         Event body used inside `EventBatchRequest.events`. The owning `agent_id`
         is supplied once at the batch envelope; per-event `agent_id` is intentionally
@@ -449,9 +476,10 @@ components:
         output_truncated:
           type: boolean
           default: false
-        notification_owner:
+        notifications_enabled:
-          type: string
+          type: boolean
-          enum: [server, prometheus, none]
+          default: true
+          description: If false, the server stores state/incidents but does not enqueue notifications for this event.
         incident_key:
           type: string
           maxLength: 256
@@ -471,7 +499,7 @@ components:
               format: date-time
     Agent:
       type: object
-      required: [agent_id, hostname, status, last_seen_at]
+      required: [agent_id, hostname, status, last_seen_at, features]
       properties:
         agent_id:
           type: string
@@ -483,11 +511,8 @@ components:
         last_seen_at:
           type: string
           format: date-time
-        version:
+        features:
-          type: string
+          $ref: "#/components/schemas/AgentFeatures"
-        mode:
-          type: string
-          enum: [prometheus_only, push_only, hybrid]
         labels:
           type: object
           additionalProperties:

deploy/README.md

@@ -1,15 +1,75 @@
 # Deploy
 
-Deployment examples live here.
+Local stack and example operator configs.
 
-Planned files:
+## Quick start
 
-- Docker Compose for local development;
+```sh
-- PostgreSQL service;
+cd deploy
-- optional Prometheus scrape config;
+MONLET_AUTH_TOKEN=$(openssl rand -hex 16) docker compose up --build
-- optional Alertmanager config;
+```
-- optional Grafana dashboard stub.
 
-No production-ready deployment files are implemented in Stage 0.
+Brings up: PostgreSQL 16, Monlet server (with Alembic upgrade on start), Web UI.
 
-Docker examples must not install Python, Node, or Go project dependencies on the host. Dependency installation belongs inside images/containers or repo-local development directories.
+- Server: http://127.0.0.1:8000 (`/api/v1/health`, `/api/v1/ready`, `/metrics`)
+- Web UI: http://127.0.0.1:3000
+- PostgreSQL: 127.0.0.1:5432, db `monlet`, user `monlet`
+
+Add Prometheus + Alertmanager + Grafana:
+
+```sh
+MONLET_AUTH_TOKEN=$(openssl rand -hex 16) docker compose --profile observability up
+```
+
+- Prometheus: http://127.0.0.1:9090
+- Alertmanager: http://127.0.0.1:9093
+- Grafana: http://127.0.0.1:3001 (admin / `$GF_ADMIN_PASSWORD`, default `admin`)
+
+## Smoke test
+
+```sh
+bash deploy/smoke.sh
+```
+
+Boots the stack, hits `/health`, `/ready`, `/metrics`, ingests example payloads from `examples/`, and verifies the API exposes resulting agent/check/incident/outbox state.
+
+`SMOKE_KEEP=1 bash deploy/smoke.sh` leaves the stack up after the run.
+
+## Showcase / E2E stand (Stage 6.1)
+
+Full-fat demo stack with several dockerized agents driving the system through
+every witness state: agent alive/stale/dead, check ok/warning/critical/unknown,
+incident open/resolved, outbox sent/retry/failed/discarded,
+anti-double-alerting with `notifications_enabled=false`, and agent spool
+replay against a server that becomes available after the agent starts.
+
+```sh
+bash deploy/e2e-showcase.sh
+```
+
+Internally:
+
+```sh
+MONLET_AUTH_TOKEN=monlet-dev-token docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
+```
+
+`SHOWCASE_KEEP=1 bash deploy/e2e-showcase.sh` leaves the stack up so you can
+inspect the web UI at http://127.0.0.1:3000.
+
+Dockerized agents (`deploy/docker/agent.Dockerfile`, `deploy/showcase/`) exist
+for demo/e2e only — production deploys the Go binary under systemd.
+
+## Files
+
+- `docker-compose.yml` — full local stack (server / postgres / web / optional observability profile).
+- `prometheus/prometheus.yml` — example scrape config (Monlet server + agent target).
+- `alertmanager/alertmanager.yml` — example route + receiver.
+- `grafana/provisioning/` — datasource + dashboard providers.
+- `grafana/dashboards/monlet-overview.json` — starter dashboard.
+- `smoke.sh` — local stack smoke entry point.
+- `docker-compose.showcase.yml` — override that adds demo agents + mock webhook.
+- `docker/agent.Dockerfile` — demo/e2e agent image (not for production).
+- `showcase/` — demo agent configs, check scripts, mock webhook, seed SQL.
+- `e2e-showcase.sh` — full showcase runner (Stage 6.1).
+
+Docker images install Python/Node project dependencies inside the image; host venvs are never mounted (see `AGENTS.md`).

deploy/alertmanager/alertmanager.yml

@@ -0,0 +1,15 @@
+# Example Alertmanager config. REPLACE the receiver before using in any real
+# environment — the default 'null' receiver silently drops alerts.
+
+route:
+  group_by: ["alertname", "agent_id", "check_id"]
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+  receiver: "null"
+
+receivers:
+  - name: "null"
+    # Intentionally empty: drops everything routed here. Add your real
+    # receiver (email_configs, telegram_configs, webhook_configs, ...) and
+    # repoint `route.receiver` at it.

deploy/docker-compose.showcase.yml

@@ -0,0 +1,113 @@
+# Showcase / e2e override on top of deploy/docker-compose.yml.
+# Usage:
+#   docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.showcase.yml up -d --build
+# Demo agents only — production uses systemd-managed Go binary.
+
+services:
+  server:
+    environment:
+      MONLET_STALE_AFTER_SEC: "20"
+      MONLET_DEAD_AFTER_SEC: "40"
+      MONLET_DETECTOR_TICK_SEC: "5"
+      MONLET_EVENTS_RETENTION_MAX_ROWS: "5000"
+      MONLET_OUTBOX_RETENTION_MAX_ROWS: "1000"
+      MONLET_NOTIFIER_DEBUG_ENABLED: "true"
+      MONLET_NOTIFIER_WEBHOOK_ENABLED: "true"
+      MONLET_NOTIFIER_WEBHOOK_URL: "http://mock-webhook:8080/"
+
+  mock-webhook:
+    image: python:3.14-alpine
+    command: ["python3", "-u", "/opt/mock_webhook.py"]
+    volumes:
+      - ./showcase/mock_webhook.py:/opt/mock_webhook.py:ro
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- --post-data='{}' http://localhost:8080/ >/dev/null 2>&1 || true"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  agent-mixed:
+    image: monlet-showcase-agent
+    build:
+      context: ..
+      dockerfile: deploy/docker/agent.Dockerfile
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/mixed.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+    ports:
+      - "9465:9465"
+
+  agent-resolve:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/resolve.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+      - resolve_state:/state
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-prometheus-owned:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/prometheus_owned.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-stale:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/stale.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  agent-dead:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/dead.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+    depends_on:
+      server:
+        condition: service_healthy
+      agent-mixed:
+        condition: service_started
+
+  # No depends_on:server — starts in parallel so it must spool events until server is ready.
+  agent-spool-replay:
+    image: monlet-showcase-agent
+    environment:
+      MONLET_CONFIG: /etc/monlet/agent.toml
+    volumes:
+      - ./showcase/agents/spool_replay.toml:/etc/monlet/agent.toml:ro
+      - ./showcase/checks:/opt/monlet/checks:ro
+      - spool_replay_state:/var/lib/monlet-agent
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+volumes:
+  resolve_state:
+  spool_replay_state:

deploy/docker-compose.yml

@@ -0,0 +1,85 @@
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: monlet
+      POSTGRES_PASSWORD: monlet
+      POSTGRES_DB: monlet
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U monlet -d monlet"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    volumes:
+      - monlet_pg:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  server:
+    build:
+      context: ../server
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      MONLET_AUTH_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required}
+      MONLET_DATABASE_URL: postgresql+asyncpg://monlet:monlet@postgres:5432/monlet
+      MONLET_LOG_LEVEL: ${MONLET_LOG_LEVEL:-INFO}
+      MONLET_ENABLE_DETECTOR: "true"
+      MONLET_ENABLE_NOTIFIER_WORKER: "true"
+      MONLET_NOTIFIER_DEBUG_ENABLED: "true"
+    command:
+      - sh
+      - -c
+      - "alembic upgrade head && uvicorn monlet_server.main:app --host 0.0.0.0 --port 8000"
+    ports:
+      - "8000:8000"
+    healthcheck:
+      test: ["CMD-SHELL", "python -c 'import urllib.request,sys;sys.exit(0) if urllib.request.urlopen(\"http://localhost:8000/api/v1/ready\").status==200 else sys.exit(1)'"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+
+  web:
+    build:
+      context: ../web
+    depends_on:
+      server:
+        condition: service_healthy
+    environment:
+      MONLET_API_BASE_URL: http://server:8000
+      MONLET_API_TOKEN: ${MONLET_AUTH_TOKEN:?MONLET_AUTH_TOKEN is required}
+    ports:
+      - "3000:3000"
+
+  prometheus:
+    image: prom/prometheus:v2.55.0
+    profiles: ["observability"]
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    ports:
+      - "9090:9090"
+    depends_on:
+      - server
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    profiles: ["observability"]
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+    ports:
+      - "9093:9093"
+
+  grafana:
+    image: grafana/grafana:11.3.0
+    profiles: ["observability"]
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASSWORD:-admin}
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    ports:
+      - "3001:3000"
+
+volumes:
+  monlet_pg:

deploy/docker/agent.Dockerfile

@@ -0,0 +1,21 @@
+# Dockerized monlet-agent for the showcase / e2e stand only.
+# Production deployment uses a systemd-managed Go binary; do not use this image in prod.
+# Build context: repo root (we need ../agent).
+
+FROM golang:1.25-alpine AS builder
+WORKDIR /src
+ENV CGO_ENABLED=0 GOTOOLCHAIN=auto
+ARG MONLET_AGENT_VERSION=0.1.0
+COPY agent/go.mod agent/go.sum ./
+RUN go mod download
+COPY agent/ ./
+RUN go build -trimpath -ldflags="-s -w -X main.version=${MONLET_AGENT_VERSION}" -o /out/monlet-agent ./cmd/monlet-agent
+
+FROM ubuntu:24.04
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /out/monlet-agent /usr/local/bin/monlet-agent
+RUN mkdir -p /var/lib/monlet-agent /etc/monlet /opt/monlet/checks
+ENV MONLET_CONFIG=/etc/monlet/agent.toml
+ENTRYPOINT ["/bin/sh","-c","exec /usr/local/bin/monlet-agent -config \"$MONLET_CONFIG\""]

deploy/e2e-showcase.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# Stage 6.1 — Docker showcase / e2e stand for Monlet.
+# Brings up the full stack with demo agents, drives scenarios, asserts all
+# witness states (agent alive/stale/dead, check ok/warning/critical/unknown,
+# incident open/resolved, outbox sent/retry/failed/discarded, disabled server notifications,
+# agent spool replay), and verifies metrics + web pages.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+AGENT_METRICS="${MONLET_AGENT_METRICS_URL:-http://127.0.0.1:9465/metrics}"
+
+COMPOSE=(docker compose
+  -f "$HERE/docker-compose.yml"
+  -f "$HERE/docker-compose.showcase.yml")
+
+log()  { printf "\033[1;36m[showcase]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[showcase]\033[0m %s\n" "$*" >&2; exit 1; }
+
+AUTH=(-H "Authorization: Bearer $TOKEN")
+
+cleanup() {
+  if [[ "${SHOWCASE_KEEP:-0}" != "1" ]]; then
+    log "tearing down stack"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  else
+    log "SHOWCASE_KEEP=1 → leaving stack running (web: $WEB_BASE)"
+  fi
+}
+trap cleanup EXIT
+
+wait_url() {
+  local url="$1" label="$2" tries="${3:-60}"
+  for i in $(seq 1 "$tries"); do
+    if curl -sf -o /dev/null "$url"; then return 0; fi
+    sleep 2
+    [[ "$i" == "$tries" ]] && fail "$label not ready: $url"
+  done
+}
+
+api() { curl -sf "${AUTH[@]}" "$BASE$1"; }
+
+log "starting full showcase stack"
+MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build
+
+log "waiting for server /api/v1/ready"
+wait_url "$BASE/api/v1/ready" "server" 90
+
+log "waiting for web /"
+wait_url "$WEB_BASE/" "web" 90
+
+# --- Phase A: let agents run a few cycles ---
+log "letting demo agents push events for ~25s"
+sleep 25
+
+log "asserting agent labels include scenario and monlet_agent_version"
+api "/api/v1/agents/agent-mixed" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+labels=d.get('labels') or {}
+ver=labels.get('monlet_agent_version')
+ok=labels.get('scenario')=='mixed-status' and ver and ver!='dev'
+print('agent-mixed labels:', labels)
+sys.exit(0 if ok else 1)" \
+  || fail "agent-mixed labels missing scenario or numeric monlet_agent_version"
+
+log "asserting agent-mixed has 4 distinct check statuses (ok/warning/critical/unknown)"
+api "/api/v1/checks?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+want={'ok','warning','critical','unknown'}
+have={c['status'] for c in d['items'] if c['agent_id']=='agent-mixed'}
+missing=want-have
+print('agent-mixed statuses:', sorted(have))
+sys.exit(0 if not missing else 1)" \
+  || fail "agent-mixed missing one of ok/warning/critical/unknown"
+
+# --- Phase B: trigger resolve scenario ---
+log "triggering resolve: touch /state/resolve.flag in agent-resolve"
+"${COMPOSE[@]}" exec -T agent-resolve sh -c 'touch /state/resolve.flag' \
+  || fail "could not touch resolve flag"
+
+log "waiting for resolve event to be processed (~15s)"
+sleep 15
+
+log "asserting resolved incident exists for agent-resolve/flapping"
+api "/api/v1/incidents?state=resolved&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+ok=any(i['agent_id']=='agent-resolve' and i['check_id']=='flapping' for i in d['items'])
+sys.exit(0 if ok else 1)" \
+  || fail "no resolved incident for agent-resolve/flapping"
+
+# --- Phase C: disabled server notifications ---
+log "asserting notifications-disabled incident exists but no outbox row for it"
+PROM_INC="$(api "/api/v1/incidents?limit=500" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+for i in d['items']:
+    if i['agent_id']=='agent-prometheus-owned' and i['check_id']=='promcheck':
+        print(i['id']); break")"
+[[ -n "$PROM_INC" ]] || fail "no incident for prometheus-owned agent"
+api "/api/v1/notifiers/outbox?limit=500" | grep -q "\"incident_id\":\"$PROM_INC\"" \
+  && fail "outbox unexpectedly contains row for prometheus-owned incident $PROM_INC"
+log "no outbox row for notifications-disabled incident"
+
+# --- Phase D: stop stale/dead agents to drive lifecycle transitions ---
+log "stopping agent-stale and agent-dead to age their heartbeats"
+"${COMPOSE[@]}" stop agent-stale agent-dead >/dev/null
+
+log "waiting ~25s for stale transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"stale"' \
+  || api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to stale/dead"
+
+log "waiting another ~25s for dead transition"
+sleep 25
+api "/api/v1/agents/agent-stale" | grep -q '"status":"dead"' \
+  || fail "agent-stale did not transition to dead"
+api "/api/v1/agents/agent-dead"  | grep -q '"status":"dead"' \
+  || fail "agent-dead did not transition to dead"
+
+# At least one agent must still be alive (agent-mixed keeps pushing).
+api "/api/v1/agents/agent-mixed" | grep -q '"status":"alive"' \
+  || fail "agent-mixed not alive"
+
+# --- Phase E: outbox state coverage (live via webhook + debug) ---
+log "waiting for notifier worker cycles (~10s)"
+sleep 10
+
+OUTBOX="$(api "/api/v1/notifiers/outbox?limit=500")"
+echo "$OUTBOX" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+states={i['state'] for i in d['items']}
+need={'sent','retry','failed'}
+missing=need-states
+print('outbox states:', sorted(states))
+sys.exit(0 if not missing else 2)" \
+  || fail "outbox missing one of sent/retry/failed"
+
+# --- Phase F: seed discarded row, wait for worker ---
+log "seeding 'discarded' outbox row via SQL"
+"${COMPOSE[@]}" exec -T postgres \
+  psql -U monlet -d monlet -v ON_ERROR_STOP=1 -q \
+  < "$HERE/showcase/seed_discarded.sql" >/dev/null \
+  || fail "seed_discarded.sql failed"
+
+log "waiting for worker to mark unknown-notifier row as discarded (~10s)"
+sleep 10
+api "/api/v1/notifiers/outbox?state=discarded&limit=10" \
+  | grep -q '"state":"discarded"' \
+  || fail "no outbox row in state=discarded"
+
+# --- Phase G: spool replay assertion ---
+log "asserting agent-spool-replay events arrived without duplicates"
+api "/api/v1/events/query?agent_id=agent-spool-replay&check_id=replay_check&limit=500" \
+  | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+items=d['items']
+ids=[e['event_id'] for e in items]
+if len(items) < 2:
+    print(f'too few events: {len(items)}'); sys.exit(1)
+if len(set(ids)) != len(ids):
+    print('duplicate event_ids'); sys.exit(2)
+print(f'spool replay events: {len(items)} unique')" \
+  || fail "spool replay assertion failed"
+
+# --- Phase H: metrics ---
+log "server /metrics contains monlet_server_*"
+curl -sf "$BASE/metrics" | grep -q '^monlet_server_' \
+  || fail "server metrics missing monlet_server_*"
+
+log "agent-mixed /metrics contains monlet_agent_check_status"
+curl -sf "$AGENT_METRICS" | grep -q 'monlet_agent_check_status' \
+  || fail "agent metrics missing monlet_agent_check_status"
+
+# --- Phase I: web UI ---
+log "web pages render with expected content"
+curl -sf "$WEB_BASE/"          | grep -q "Overview"  || fail "web / missing Overview"
+curl -sf "$WEB_BASE/agents"    | grep -q "agent-mixed" || fail "web /agents missing agent-mixed"
+curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
+curl -sf "$WEB_BASE/incidents" | grep -q "agent-resolve" || fail "web /incidents missing agent-resolve"
+curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
+curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
+
+log "ALL SHOWCASE ASSERTIONS PASSED"

deploy/grafana/dashboards/monlet-overview.json

@@ -0,0 +1,42 @@
+{
+  "title": "Monlet Overview",
+  "uid": "monlet-overview",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-1h", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Agents by status",
+      "targets": [{ "expr": "monlet_server_agents", "legendFormat": "{{status}}" }],
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Open incidents",
+      "targets": [{ "expr": "monlet_server_open_incidents" }],
+      "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Events/sec by result",
+      "targets": [
+        { "expr": "rate(monlet_server_events_total[1m])", "legendFormat": "{{result}}" }
+      ],
+      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 6 }
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Notifications/sec by notifier+result",
+      "targets": [
+        { "expr": "rate(monlet_server_notifications_total[1m])", "legendFormat": "{{notifier}} {{result}}" }
+      ],
+      "gridPos": { "h": 8, "w": 16, "x": 0, "y": 14 }
+    }
+  ]
+}

deploy/grafana/provisioning/dashboards/monlet.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+providers:
+  - name: monlet
+    folder: ""
+    type: file
+    options:
+      path: /var/lib/grafana/dashboards

deploy/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true

deploy/prometheus/README.md

@@ -6,4 +6,4 @@ Planned examples:
 
 - scrape Monlet agent `/metrics`;
 - scrape Monlet server `/metrics`;
-- sample rules that avoid duplicate alerting when `notification_owner = server`.
+- sample rules for checks that use `notifications_enabled = false`.

deploy/prometheus/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: monlet-server
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["server:8000"]
+
+  - job_name: monlet-agent
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["host.docker.internal:9100"]

deploy/showcase/agents/dead.toml

@@ -0,0 +1,24 @@
+hostname = "agent-dead"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "dead"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Dead agent placeholder check"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "10s"
+timeout = "3s"

deploy/showcase/agents/mixed.toml

@@ -0,0 +1,46 @@
+hostname = "agent-mixed"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "mixed-status"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = true
+listen = "0.0.0.0:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Showcase ok"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "5s"
+timeout = "3s"
+
+[[checks]]
+id = "warning_check"
+name = "Showcase warning"
+command = "/opt/monlet/checks/exit_1.sh"
+interval = "5s"
+timeout = "3s"
+
+[[checks]]
+id = "critical_check"
+name = "Showcase critical (permanent webhook fail)"
+command = "/opt/monlet/checks/exit_2.sh"
+interval = "5s"
+timeout = "3s"
+dedupe_key = "agent-mixed:critical:fail"
+
+[[checks]]
+id = "unknown_check"
+name = "Showcase unknown"
+command = "/opt/monlet/checks/exit_3.sh"
+interval = "5s"
+timeout = "3s"

deploy/showcase/agents/prometheus_owned.toml

@@ -0,0 +1,25 @@
+hostname = "agent-prometheus-owned"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "prometheus-owned"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "promcheck"
+name = "Prometheus-owned critical (no outbox expected)"
+command = "/opt/monlet/checks/exit_2.sh"
+interval = "5s"
+timeout = "3s"
+notifications_enabled = false

deploy/showcase/agents/resolve.toml

@@ -0,0 +1,25 @@
+hostname = "agent-resolve"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "resolve"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "flapping"
+name = "Flapping check (critical then ok)"
+command = "/opt/monlet/checks/flapping.sh"
+interval = "5s"
+timeout = "3s"
+dedupe_key = "agent-resolve:flapping:retry"

deploy/showcase/agents/spool_replay.toml

@@ -0,0 +1,24 @@
+hostname = "agent-spool-replay"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "spool-replay"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "replay_check"
+name = "Spool replay check"
+command = "/opt/monlet/checks/replay_check.sh"
+interval = "3s"
+timeout = "2s"

deploy/showcase/agents/stale.toml

@@ -0,0 +1,24 @@
+hostname = "agent-stale"
+state_dir = "/var/lib/monlet-agent"
+
+[labels]
+scenario = "stale"
+role = "showcase"
+
+[server]
+enabled = true
+url = "http://server:8000"
+token = "monlet-dev-token"
+heartbeat_interval = "5s"
+batch_interval = "3s"
+
+[metrics]
+enabled = false
+listen = "127.0.0.1:9465"
+
+[[checks]]
+id = "ok_check"
+name = "Stale agent placeholder check"
+command = "/opt/monlet/checks/exit_0.sh"
+interval = "10s"
+timeout = "3s"

deploy/showcase/checks/exit_0.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "ok: $(date -u +%FT%TZ)"
+exit 0

deploy/showcase/checks/exit_1.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "warning: degraded"
+exit 1

deploy/showcase/checks/exit_2.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "critical: probe failed"
+exit 2

deploy/showcase/checks/exit_3.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "unknown: probe returned unexpected exit code"
+exit 3

deploy/showcase/checks/flapping.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+# critical until /state/resolve.flag exists, then ok.
+if [ -f /state/resolve.flag ]; then
+  echo "ok: resolve flag present"
+  exit 0
+fi
+echo "critical: waiting for resolve flag"
+exit 2

deploy/showcase/checks/replay_check.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "ok: replay tick"
+exit 0

deploy/showcase/mock_webhook.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Mock webhook for showcase/e2e.
+
+Decides response status by substring in incident.incident_key from request body:
+- contains 'retry' -> 503 (retryable failure)
+- contains 'fail'  -> 400 (permanent failure)
+- otherwise        -> 200 (success)
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+def _classify(body: bytes) -> int:
+    try:
+        data = json.loads(body or b"{}")
+    except Exception:
+        return 200
+    incident = data.get("incident") or {}
+    key = str(incident.get("incident_key") or "")
+    if "retry" in key:
+        return 503
+    if "fail" in key:
+        return 400
+    return 200
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_POST(self) -> None:  # noqa: N802
+        length = int(self.headers.get("Content-Length") or 0)
+        body = self.rfile.read(length) if length else b""
+        code = _classify(body)
+        self.send_response(code)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+        try:
+            data = json.loads(body or b"{}")
+            inc = (data.get("incident") or {})
+            print(
+                f"webhook {code} event_type={data.get('event_type')!r} "
+                f"incident_key={inc.get('incident_key')!r}",
+                flush=True,
+            )
+        except Exception:
+            print(f"webhook {code} (unparsable body, {length}B)", flush=True)
+
+    def log_message(self, *_args: object) -> None:
+        return
+
+
+def main() -> int:
+    addr = ("0.0.0.0", 8080)
+    srv = ThreadingHTTPServer(addr, Handler)
+    print(f"mock_webhook listening on {addr[0]}:{addr[1]}", flush=True)
+    try:
+        srv.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        srv.server_close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

deploy/showcase/seed_discarded.sql

@@ -0,0 +1,16 @@
+-- Seed one outbox row with an unknown notifier name so the worker marks it 'discarded'.
+-- Attached to any existing incident; if none exists yet the script is a no-op.
+INSERT INTO notification_outbox
+    (id, incident_id, notifier, event_type, state, attempts, next_attempt_at, payload)
+SELECT
+    gen_random_uuid(),
+    i.id,
+    'showcase-unknown',
+    'firing',
+    'pending',
+    0,
+    now(),
+    '{"showcase":"discarded"}'::jsonb
+FROM incidents i
+ORDER BY i.opened_at DESC
+LIMIT 1;

deploy/smoke.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Local stack smoke test for Stage 6.
+# Brings the stack up, hits /health /ready /metrics, ingests sample heartbeat + events,
+# verifies the API exposes agent / check / incident state.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/.." && pwd)"
+TOKEN="${MONLET_AUTH_TOKEN:-monlet-dev-token}"
+BASE="${MONLET_API_BASE_URL:-http://127.0.0.1:8000}"
+COMPOSE=(docker compose -f "$HERE/docker-compose.yml")
+
+log() { printf "\033[1;36m[smoke]\033[0m %s\n" "$*"; }
+fail() { printf "\033[1;31m[smoke]\033[0m %s\n" "$*" >&2; exit 1; }
+
+cleanup() {
+  if [[ "${SMOKE_KEEP:-0}" != "1" ]]; then
+    log "tearing down stack"
+    "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+log "starting stack"
+MONLET_AUTH_TOKEN="$TOKEN" "${COMPOSE[@]}" up -d --build postgres server web
+
+log "waiting for /api/v1/ready"
+for i in $(seq 1 60); do
+  if curl -sf "$BASE/api/v1/ready" >/dev/null; then break; fi
+  sleep 2
+  [[ "$i" == "60" ]] && fail "server did not become ready"
+done
+
+log "GET /api/v1/health"
+curl -sf "$BASE/api/v1/health" | grep -q '"status":"ok"' || fail "health wrong body"
+
+log "GET /metrics"
+curl -sf "$BASE/metrics" | grep -q '^monlet_server_' || fail "metrics missing"
+
+AUTH=(-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json")
+
+log "POST heartbeat (examples/heartbeat.json)"
+curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/heartbeat" --data @"$ROOT/examples/heartbeat.json" \
+  | grep -q '"accepted":true' || fail "heartbeat not accepted"
+
+log "POST events (examples/event-batch.json)"
+curl -sf "${AUTH[@]}" -X POST "$BASE/api/v1/events" --data @"$ROOT/examples/event-batch.json" \
+  | grep -q '"accepted":' || fail "event batch not accepted"
+
+log "GET /agents has host-01.prod"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/agents" | grep -q '"agent_id":"host-01.prod"' \
+  || fail "host-01.prod missing"
+
+log "GET /checks not empty"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/checks" | grep -q '"items":\[{' \
+  || fail "checks empty"
+
+log "GET /incidents not empty (event-batch.json contains a critical event)"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/incidents?state=open" | grep -q '"items":\[{' \
+  || fail "no open incident"
+
+log "GET /notifiers/outbox not empty"
+curl -sf "${AUTH[@]}" "$BASE/api/v1/notifiers/outbox" | grep -q '"items":\[{' \
+  || fail "outbox empty"
+
+WEB_BASE="${MONLET_WEB_BASE_URL:-http://127.0.0.1:3000}"
+log "waiting for web $WEB_BASE"
+for i in $(seq 1 60); do
+  if curl -sf -o /dev/null "$WEB_BASE/"; then break; fi
+  sleep 2
+  [[ "$i" == "60" ]] && fail "web did not become ready"
+done
+
+log "GET / (web overview)"
+curl -sf "$WEB_BASE/" | grep -q "Overview" || fail "web overview missing 'Overview'"
+
+log "GET /agents (web list)"
+curl -sf "$WEB_BASE/agents" | grep -q "host-01.prod" || fail "web /agents missing host-01.prod"
+
+log "ALL SMOKE TESTS PASSED"

docs/ops/backup-restore.md

@@ -0,0 +1,49 @@
+# Backup and restore
+
+All persistent state lives in PostgreSQL (`agents`, `checks`, `events`, `incidents`, `notification_outbox`). The server is stateless; the agent local spool is best-effort and not part of the canonical state (ADR-0006).
+
+## What to back up
+
+- The Monlet PostgreSQL database (logical or physical).
+- The server `MONLET_AUTH_TOKEN` and `.env` (kept in your secret store, not in backups of the DB).
+- The agent `config.toml` per host (kept in your config management).
+
+## Logical backup (recommended for small deployments)
+
+```sh
+PGPASSWORD=monlet pg_dump \
+  -h $PGHOST -U monlet -d monlet \
+  --format=custom --no-owner --no-privileges \
+  --file=monlet-$(date -u +%Y%m%dT%H%M%SZ).dump
+```
+
+Restore into an empty database:
+
+```sh
+createdb -h $PGHOST -U postgres monlet
+PGPASSWORD=monlet pg_restore \
+  -h $PGHOST -U monlet -d monlet \
+  --no-owner --no-privileges \
+  monlet-YYYYMMDDTHHMMSSZ.dump
+```
+
+The schema is owned by Alembic. Restoring a logical dump from the same Monlet version is safe. After restore, run `alembic upgrade head` only if you restored from an older version.
+
+## Retention
+
+- `events` grows unbounded in v1. Decide a retention policy that matches your storage budget (typical: 7–30 days). Stage 6+ may add server-side trimming; until then, run a manual cron:
+
+  ```sql
+  DELETE FROM events WHERE observed_at < now() - interval '14 days';
+  ```
+
+  `checks.last_event_id` is not a FK, so trimming `events` does not break current state.
+
+- `incidents` and `notification_outbox` are small. Trim outbox `state IN ('sent','failed','discarded')` rows older than 7 days if storage is tight.
+
+## Disaster recovery
+
+- Restore the database.
+- Restart the server. Agents will resume sending; spooled events on each agent are replayed and de-duplicated by `event_id` (ADR-0006).
+- The agent spool is bounded (10 000 events / 50 MiB FIFO, ADR-0005). During a long outage the oldest events get dropped to make room for new ones, so observations from the start of the outage are lost first. Plan recovery time accordingly — for an agent producing ~1 event/s per check, ~10 000 events is roughly 2–3 hours of buffering for a handful of checks.
+- If agents were also lost, only events that were not yet persisted to the spool file are lost in addition.

docs/ops/deployment-checklist.md

@@ -0,0 +1,57 @@
+# First deployment checklist
+
+A pragmatic list for the first production-ish Monlet stand-up. Adjust per environment.
+
+## 1. Secrets
+
+- [ ] Generate `MONLET_AUTH_TOKEN` (`openssl rand -hex 32`) and store it in your secret manager.
+- [ ] Generate a PostgreSQL password and store it in your secret manager.
+- [ ] Decide which notifier channels you want and gather their credentials (Telegram bot token + chat id, webhook URL, Alertmanager URL).
+
+## 2. Database
+
+- [ ] Provision PostgreSQL 16+.
+- [ ] Create `monlet` database and role with `CREATE`/`USAGE` on the schema.
+- [ ] Set `MONLET_DATABASE_URL=postgresql+asyncpg://...`.
+- [ ] Run `alembic upgrade head` from a one-shot container or the server image.
+- [ ] Verify `\dt` shows `agents, checks, events, incidents, notification_outbox`.
+
+## 3. Server
+
+- [ ] Set env vars per `server/.env.example` (see also `server/README.md`).
+- [ ] Start container; verify `GET /api/v1/health` and `/api/v1/ready` return 200.
+- [ ] Verify `/metrics` exposes `monlet_server_*`.
+- [ ] Confirm an authenticated `GET /api/v1/agents` returns `{ items: [], next_cursor: null }`.
+- [ ] Confirm logs show no unredacted token/cookie material.
+
+## 4. Agent (per host)
+
+- [ ] Install binary (see `agent/systemd/`).
+- [ ] Render `config.toml` from `agent/config.example.toml` with hostname or explicit `agent_id`, and `[server].token`.
+- [ ] Start under systemd; verify a heartbeat reaches the server (`GET /api/v1/agents`).
+- [ ] Run one configured check and verify a row appears in `/api/v1/checks` and an event in `/api/v1/events/query`.
+
+## 5. Notifications
+
+- [ ] Enable at least one notifier in server env (`MONLET_NOTIFIER_*`).
+- [ ] Force a `critical` event from one agent and confirm an outbox row reaches `sent` (`GET /api/v1/notifiers/outbox`).
+- [ ] Resolve the check and confirm a `resolved` outbox row is emitted.
+
+## 6. Web UI
+
+- [ ] Set `MONLET_API_BASE_URL` and `MONLET_API_TOKEN` for the web container.
+- [ ] Browse `/`, `/agents`, `/checks`, `/incidents`, `/events`, `/outbox`.
+- [ ] Verify the web image does not log the token (`docker logs` greps clean).
+
+## 7. Observability (optional)
+
+- [ ] Add Monlet server to your Prometheus scrape config (`deploy/prometheus/prometheus.yml`).
+- [ ] Import `deploy/grafana/dashboards/monlet-overview.json` for the starter dashboard.
+- [ ] If using Alertmanager as a notifier, point `MONLET_NOTIFIER_ALERTMANAGER_URL` at it.
+
+## 8. Operational
+
+- [ ] Schedule database backups (`docs/ops/backup-restore.md`).
+- [ ] Document token rotation owner and cadence (`docs/ops/token-rotation.md`).
+- [ ] Add `monlet_server_open_incidents` and `monlet_server_agents{status="dead"}` to your dashboards.
+- [ ] Decide an `events` retention policy.

docs/ops/token-rotation.md

@@ -0,0 +1,45 @@
+# Token rotation
+
+Monlet v1 uses a single shared Bearer token (`MONLET_AUTH_TOKEN`) for all agent and UI endpoints (ADR-0007). Per-agent tokens and dual-token acceptance are out of scope for v1.
+
+## Why rotation is disruptive in v1
+
+The server validates exactly one token at a time. Agents that present the old token after the server has switched receive `401`, which the agent treats as a **non-retryable** failure and drops the affected batch from its local spool (`agent/internal/client/client.go`, `agent/internal/app/app.go`). Events flushed during the mismatch window are lost — they are not re-spooled and not re-sent.
+
+Plan rotations during a low-traffic window and either stop the agents first (so spooled events persist on disk until they restart with the new token) or accept the event-loss window.
+
+## When to rotate
+
+- A suspected leak (token committed, log exposure, lost laptop with `config.toml`).
+- Operator turnover.
+- Routine schedule (e.g., quarterly).
+
+## Procedure
+
+1. **Generate a new token**:
+
+   ```sh
+   openssl rand -hex 32
+   ```
+
+2. **Stop or pause agents** (recommended). Either:
+   - `systemctl stop monlet-agent` on each host; spooled events remain on disk and replay after restart, or
+   - skip this step and accept that any events sent during steps 3–4 are lost on the `401`.
+
+3. **Restart server** with the new `MONLET_AUTH_TOKEN`. Verify:
+
+   ```sh
+   curl -sf -H "Authorization: Bearer $NEW" http://server:8000/api/v1/agents | jq .
+   ```
+
+4. **Update agent `config.toml`** (`[server].token`) on each host and start agents.
+
+5. **Update Web UI** (`MONLET_API_TOKEN`) and restart.
+
+6. **Revoke the old token**: remove the old value from secret stores, CI variables, and operator notes.
+
+## Notes
+
+- The token is the only auth in v1; protect it like a database password.
+- The token never appears in metrics labels, incident keys, or notification labels (ADR-0007, redaction policy).
+- The server redacts `Authorization`/`Cookie`-style headers before logging (`monlet_server/redaction.py`).

examples/event-batch.json

@@ -10,7 +10,6 @@
       "duration_ms": 42,
       "output": "/ 18% used (180G/1.0T)",
       "output_truncated": false,
-      "notification_owner": "server",
       "incident_key": "host-01.prod:disk_root"
     },
     {
@@ -22,7 +21,6 @@
       "duration_ms": 1503,
       "output": "could not connect to server: Connection refused",
       "output_truncated": false,
-      "notification_owner": "server",
       "incident_key": "host-01.prod:postgres_up"
     }
   ]

examples/heartbeat.json

@@ -2,9 +2,12 @@
   "agent_id": "host-01.prod",
   "observed_at": "2026-05-26T12:00:00Z",
   "hostname": "host-01.prod.example.com",
-  "version": "0.1.0",
+  "features": {
-  "mode": "push_only",
+    "push": true,
+    "metrics": false
+  },
   "labels": {
+    "monlet_agent_version": "0.1.0",
     "env": "prod",
     "region": "eu-central-1",
     "role": "api"

server/.env.example

@@ -1,6 +1,27 @@
-MONLET_ENV=local
+MONLET_AUTH_TOKEN=replace-me-with-random-token
-MONLET_LOG_LEVEL=info
+MONLET_DATABASE_URL=postgresql+asyncpg://monlet:monlet@localhost:5432/monlet
-MONLET_DATABASE_URL=postgresql+psycopg://monlet:monlet@localhost:5432/monlet
+MONLET_LOG_LEVEL=INFO
-MONLET_AGENT_TOKEN_BOOTSTRAP=replace-me
+MONLET_BODY_LIMIT_BYTES=1048576
-MONLET_MAX_REQUEST_BYTES=1048576
+MONLET_OUTPUT_MAX_BYTES=8192
-MONLET_MAX_CHECK_OUTPUT_BYTES=8192
+MONLET_STALE_AFTER_SEC=90
+MONLET_DEAD_AFTER_SEC=300
+MONLET_DETECTOR_TICK_SEC=15
+MONLET_ENABLE_DETECTOR=true
+MONLET_EVENTS_RETENTION_MAX_ROWS=100000
+MONLET_OUTBOX_RETENTION_MAX_ROWS=50000
+
+MONLET_ENABLE_NOTIFIER_WORKER=true
+MONLET_NOTIFIER_TICK_SEC=5
+MONLET_NOTIFIER_BATCH_SIZE=20
+MONLET_NOTIFIER_MAX_ATTEMPTS=8
+MONLET_NOTIFIER_LEASE_SEC=300
+MONLET_NOTIFIER_HTTP_TIMEOUT_SEC=10
+MONLET_NOTIFIER_DEBUG_ENABLED=true
+MONLET_NOTIFIER_TELEGRAM_ENABLED=false
+MONLET_NOTIFIER_TELEGRAM_TOKEN=
+MONLET_NOTIFIER_TELEGRAM_CHAT_ID=
+MONLET_NOTIFIER_WEBHOOK_ENABLED=false
+MONLET_NOTIFIER_WEBHOOK_URL=
+MONLET_NOTIFIER_WEBHOOK_TOKEN=
+MONLET_NOTIFIER_ALERTMANAGER_ENABLED=false
+MONLET_NOTIFIER_ALERTMANAGER_URL=

server/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.14-slim AS builder
+
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PROJECT_ENVIRONMENT=/app/.venv
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /usr/local/bin/
+
+WORKDIR /app
+COPY pyproject.toml uv.lock ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev --no-install-project
+COPY monlet_server ./monlet_server
+COPY alembic.ini ./
+COPY alembic ./alembic
+RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev
+
+
+FROM python:3.14-slim
+ENV PATH="/app/.venv/bin:$PATH" \
+    PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY --from=builder /app /app
+
+EXPOSE 8000
+CMD ["uvicorn", "monlet_server.main:app", "--host", "0.0.0.0", "--port", "8000"]

server/README.md

@@ -1,28 +1,78 @@
 # Monlet Server
 
-Placeholder for the Python/FastAPI server.
+FastAPI server: heartbeat/event ingestion, current check state, incident lifecycle, stale/dead detector, notification outbox, Prometheus metrics.
 
-Planned stack:
+## Stack
 
-- Python 3.14;
+- Python 3.14+
-- uv;
+- `uv`
-- FastAPI;
+- FastAPI, Pydantic v2
-- Pydantic;
+- SQLAlchemy 2.x async + `asyncpg`
-- SQLAlchemy 2.x style;
+- Alembic (sync `psycopg`)
-- Alembic;
+- `structlog`, `prometheus-client`
-- PostgreSQL;
+- PostgreSQL 16+
-- httpx for outbound notifier calls;
-- structured logging;
-- Prometheus metrics;
-- Docker image.
 
-No server core logic is implemented in Stage 0.
+## Local development
-
-## Planned Commands
 
 ```sh
-UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv sync
+cd server
+UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv sync --extra dev
+```
+
+Run a Postgres instance (Docker or local), set `MONLET_DATABASE_URL`, then:
+
+```sh
+UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run alembic upgrade head
 UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run fastapi dev monlet_server/main.py
 ```
 
-Do not install Python packages globally on the host. Docker builds install server dependencies inside the image; local development uses `server/.venv` and repo-local `.cache/uv`.
+## Environment
+
+| Var | Default | Purpose |
+|---|---|---|
+| `MONLET_AUTH_TOKEN` | required | Shared Bearer token (ADR-0007); `changeme` is rejected |
+| `MONLET_DATABASE_URL` | `postgresql+asyncpg://monlet:monlet@127.0.0.1:5432/monlet` | Async DSN |
+| `MONLET_LOG_LEVEL` | `INFO` | Log level |
+| `MONLET_BODY_LIMIT_BYTES` | `1048576` | Request body limit (ADR-0005) |
+| `MONLET_OUTPUT_MAX_BYTES` | `8192` | Per-event output limit |
+| `MONLET_STALE_AFTER_SEC` | `90` | Stale threshold |
+| `MONLET_DEAD_AFTER_SEC` | `300` | Dead threshold |
+| `MONLET_DETECTOR_TICK_SEC` | `15` | Detector tick |
+| `MONLET_ENABLE_DETECTOR` | `true` | Toggle detector task |
+| `MONLET_EVENTS_RETENTION_MAX_ROWS` | `100000` | Maximum retained check-result events; `0` disables pruning |
+| `MONLET_OUTBOX_RETENTION_MAX_ROWS` | `50000` | Maximum retained terminal outbox rows; `0` disables pruning |
+| `MONLET_ENABLE_NOTIFIER_WORKER` | `true` | Toggle outbox worker |
+| `MONLET_NOTIFIER_TICK_SEC` | `5` | Worker poll interval |
+| `MONLET_NOTIFIER_BATCH_SIZE` | `20` | Rows claimed per tick |
+| `MONLET_NOTIFIER_MAX_ATTEMPTS` | `8` | Terminal-failure threshold (ADR-0005) |
+| `MONLET_NOTIFIER_LEASE_SEC` | `300` | Recovery lease for stuck `sending` rows |
+| `MONLET_NOTIFIER_HTTP_TIMEOUT_SEC` | `10` | Outbound HTTP timeout |
+| `MONLET_NOTIFIER_DEBUG_ENABLED` | `true` | Log-only notifier |
+| `MONLET_NOTIFIER_TELEGRAM_ENABLED` | `false` | Telegram notifier on/off |
+| `MONLET_NOTIFIER_TELEGRAM_TOKEN` | `` | Telegram bot token |
+| `MONLET_NOTIFIER_TELEGRAM_CHAT_ID` | `` | Telegram chat id |
+| `MONLET_NOTIFIER_WEBHOOK_ENABLED` | `false` | Generic webhook notifier on/off |
+| `MONLET_NOTIFIER_WEBHOOK_URL` | `` | Webhook URL |
+| `MONLET_NOTIFIER_WEBHOOK_TOKEN` | `` | Optional Bearer token for webhook |
+| `MONLET_NOTIFIER_ALERTMANAGER_ENABLED` | `false` | Alertmanager notifier on/off |
+| `MONLET_NOTIFIER_ALERTMANAGER_URL` | `` | Alertmanager base URL (e.g. `http://am:9093`) |
+
+## Tests
+
+Tests use `testcontainers-python` to spin up a real PostgreSQL container; Docker must be available.
+
+```sh
+UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run pytest -q
+```
+
+## Docker
+
+Image installs dependencies inside the container only; host venv is never mounted.
+
+```sh
+docker build -t monlet-server .
+```
+
+## Endpoints
+
+All routes under `/api/v1` (per `api/openapi.yaml`). `/metrics` and `/api/v1/health`, `/api/v1/ready` are open; everything else requires `Authorization: Bearer <token>`.

server/alembic.ini

@@ -0,0 +1,38 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+sqlalchemy.url =
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

server/alembic/env.py

@@ -0,0 +1,48 @@
+import os
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+
+from alembic import context
+from monlet_server.models import Base
+
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+database_url = os.environ.get(
+    "MONLET_DATABASE_URL", "postgresql+asyncpg://monlet:monlet@127.0.0.1:5432/monlet"
+)
+config.set_main_option("sqlalchemy.url", database_url.replace("+asyncpg", "+psycopg"))
+
+
+def run_migrations_offline() -> None:
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

server/alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

server/alembic/versions/0001_baseline.py

@@ -0,0 +1,123 @@
+"""baseline schema (ADR-0004)
+
+Revision ID: 0001_baseline
+Revises:
+Create Date: 2026-05-26
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0001_baseline"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE agents (
+            agent_id      TEXT PRIMARY KEY,
+            hostname      TEXT NOT NULL,
+            status        TEXT NOT NULL CHECK (status IN ('alive','stale','dead')) DEFAULT 'alive',
+            last_seen_at  TIMESTAMPTZ NOT NULL,
+            features      JSONB NOT NULL DEFAULT jsonb_build_object('push', false, 'metrics', false),
+            labels        JSONB NOT NULL DEFAULT '{}'::jsonb,
+            created_at    TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+        """
+    )
+    op.execute("CREATE INDEX agents_status_idx ON agents (status);")
+    op.execute("CREATE INDEX agents_last_seen_idx ON agents (last_seen_at);")
+
+    op.execute(
+        """
+        CREATE TABLE checks (
+            agent_id          TEXT NOT NULL REFERENCES agents(agent_id) ON DELETE CASCADE,
+            check_id          TEXT NOT NULL,
+            status            TEXT NOT NULL CHECK (status IN ('ok','warning','critical','unknown')),
+            exit_code         INTEGER NOT NULL,
+            last_observed_at  TIMESTAMPTZ NOT NULL,
+            last_event_id     UUID NOT NULL,
+            incident_key      TEXT NOT NULL,
+            PRIMARY KEY (agent_id, check_id)
+        );
+        """
+    )
+    op.execute("CREATE INDEX checks_status_idx ON checks (status);")
+
+    op.execute(
+        """
+        CREATE TABLE events (
+            event_id            UUID PRIMARY KEY,
+            agent_id            TEXT NOT NULL,
+            check_id            TEXT NOT NULL,
+            observed_at         TIMESTAMPTZ NOT NULL,
+            received_at         TIMESTAMPTZ NOT NULL DEFAULT now(),
+            status              TEXT NOT NULL CHECK (status IN ('ok','warning','critical','unknown')),
+            exit_code           INTEGER NOT NULL,
+            duration_ms         INTEGER NOT NULL CHECK (duration_ms >= 0),
+            output              TEXT,
+            output_truncated    BOOLEAN NOT NULL DEFAULT FALSE,
+            notifications_enabled BOOLEAN NOT NULL DEFAULT TRUE,
+            incident_key        TEXT NOT NULL
+        );
+        """
+    )
+    op.execute(
+        "CREATE INDEX events_agent_check_observed_idx ON events (agent_id, check_id, observed_at DESC);"
+    )
+    op.execute("CREATE INDEX events_observed_at_idx ON events (observed_at DESC);")
+
+    op.execute(
+        """
+        CREATE TABLE incidents (
+            id            UUID PRIMARY KEY,
+            incident_key  TEXT NOT NULL,
+            agent_id      TEXT NOT NULL,
+            check_id      TEXT NOT NULL,
+            state         TEXT NOT NULL CHECK (state IN ('open','resolved')),
+            severity      TEXT NOT NULL CHECK (severity IN ('warning','critical','unknown')),
+            opened_at     TIMESTAMPTZ NOT NULL,
+            resolved_at   TIMESTAMPTZ,
+            summary       TEXT,
+            last_event_id UUID NOT NULL
+        );
+        """
+    )
+    op.execute(
+        "CREATE UNIQUE INDEX incidents_open_key_idx ON incidents (incident_key) WHERE state = 'open';"
+    )
+    op.execute("CREATE INDEX incidents_state_opened_idx ON incidents (state, opened_at DESC);")
+
+    op.execute(
+        """
+        CREATE TABLE notification_outbox (
+            id              UUID PRIMARY KEY,
+            incident_id     UUID NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+            notifier        TEXT NOT NULL,
+            event_type      TEXT NOT NULL CHECK (event_type IN ('firing','resolved')),
+            state           TEXT NOT NULL CHECK (state IN ('pending','sending','sent','retry','failed','discarded')),
+            attempts        INTEGER NOT NULL DEFAULT 0,
+            next_attempt_at TIMESTAMPTZ,
+            last_error      TEXT,
+            payload         JSONB NOT NULL,
+            created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+            updated_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+        """
+    )
+    op.execute(
+        "CREATE INDEX outbox_pending_idx ON notification_outbox (state, next_attempt_at) WHERE state IN ('pending','retry');"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS notification_outbox CASCADE;")
+    op.execute("DROP TABLE IF EXISTS incidents CASCADE;")
+    op.execute("DROP TABLE IF EXISTS events CASCADE;")
+    op.execute("DROP TABLE IF EXISTS checks CASCADE;")
+    op.execute("DROP TABLE IF EXISTS agents CASCADE;")

server/monlet_server/__init__.py

@@ -0,0 +1 @@
+"""Monlet server package."""

server/monlet_server/api/agents.py

@@ -0,0 +1,48 @@
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, encode
+from ..db import get_session
+from ..models import Agent as AgentModel
+from ..schemas import Agent, AgentsPage
+
+router = APIRouter()
+
+
+def _to_schema(a: AgentModel) -> Agent:
+    return Agent(
+        agent_id=a.agent_id,
+        hostname=a.hostname,
+        status=a.status,
+        last_seen_at=a.last_seen_at,
+        features=a.features or {"push": False, "metrics": False},
+        labels=a.labels or {},
+    )
+
+
+@router.get("/agents", response_model=AgentsPage)
+async def list_agents(
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> AgentsPage:
+    stmt = select(AgentModel).order_by(AgentModel.agent_id)
+    if cursor is not None:
+        (last_id,) = decode(cursor, 1)
+        stmt = stmt.where(AgentModel.agent_id > last_id)
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].agent_id)
+    return AgentsPage(items=[_to_schema(a) for a in rows], next_cursor=next_cursor)
+
+
+@router.get("/agents/{agent_id}", response_model=Agent)
+async def get_agent(agent_id: str, session: AsyncSession = Depends(get_session)) -> Agent:
+    res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id))
+    a = res.scalar_one_or_none()
+    if a is None:
+        raise HTTPException(status_code=404, detail="agent not found")
+    return _to_schema(a)

server/monlet_server/api/checks.py

@@ -0,0 +1,47 @@
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, encode
+from ..db import get_session
+from ..models import Check
+from ..schemas import ID_PATTERN, ChecksPage, CheckState
+
+router = APIRouter()
+
+
+@router.get("/checks", response_model=ChecksPage)
+async def list_checks(
+    agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> ChecksPage:
+    stmt = select(Check).order_by(Check.agent_id, Check.check_id)
+    if agent_id is not None:
+        stmt = stmt.where(Check.agent_id == agent_id)
+    if cursor is not None:
+        last_agent, last_check = decode(cursor, 2)
+        stmt = stmt.where(
+            or_(
+                Check.agent_id > last_agent,
+                and_(Check.agent_id == last_agent, Check.check_id > last_check),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].agent_id, rows[-1].check_id)
+    items = [
+        CheckState(
+            agent_id=r.agent_id,
+            check_id=r.check_id,
+            status=r.status,
+            last_observed_at=r.last_observed_at,
+            exit_code=r.exit_code,
+            incident_key=r.incident_key,
+        )
+        for r in rows
+    ]
+    return ChecksPage(items=items, next_cursor=next_cursor)

server/monlet_server/api/events.py

@@ -0,0 +1,102 @@
+import base64
+import binascii
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..models import Event
+from ..schemas import (
+    ID_PATTERN,
+    EventBatchAcceptedResponse,
+    EventBatchRequest,
+    EventsPage,
+    StoredCheckResultEvent,
+)
+from ..services.ingestion import ingest_event
+
+router = APIRouter()
+
+
+def _encode_cursor(received_at: datetime, event_id) -> str:
+    raw = f"{received_at.isoformat()}|{event_id}".encode()
+    return base64.urlsafe_b64encode(raw).decode().rstrip("=")
+
+
+def _decode_cursor(cursor: str) -> tuple[datetime, str]:
+    pad = "=" * (-len(cursor) % 4)
+    try:
+        raw = base64.urlsafe_b64decode(cursor + pad).decode()
+        ts_str, eid = raw.split("|", 1)
+        return datetime.fromisoformat(ts_str), eid
+    except (binascii.Error, ValueError) as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc
+
+
+@router.post(
+    "/events",
+    response_model=EventBatchAcceptedResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+async def events(
+    body: EventBatchRequest, session: AsyncSession = Depends(get_session)
+) -> EventBatchAcceptedResponse:
+    accepted = 0
+    deduplicated = 0
+    for ev in body.events:
+        ok = await ingest_event(session, body.agent_id, ev)
+        if ok:
+            accepted += 1
+        else:
+            deduplicated += 1
+    await session.commit()
+    return EventBatchAcceptedResponse(accepted=accepted, deduplicated=deduplicated)
+
+
+@router.get("/events/query", response_model=EventsPage)
+async def query_events(
+    agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> EventsPage:
+    stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc())
+    if agent_id is not None:
+        stmt = stmt.where(Event.agent_id == agent_id)
+    if check_id is not None:
+        stmt = stmt.where(Event.check_id == check_id)
+    if cursor is not None:
+        ts, eid = _decode_cursor(cursor)
+        stmt = stmt.where(
+            or_(
+                Event.received_at < ts,
+                and_(Event.received_at == ts, Event.event_id < eid),
+            )
+        )
+    stmt = stmt.limit(limit + 1)
+    rows = (await session.execute(stmt)).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id)
+    items = [
+        StoredCheckResultEvent(
+            event_id=str(r.event_id),
+            agent_id=r.agent_id,
+            check_id=r.check_id,
+            observed_at=r.observed_at,
+            received_at=r.received_at,
+            status=r.status,
+            exit_code=r.exit_code,
+            duration_ms=r.duration_ms,
+            output=r.output,
+            output_truncated=r.output_truncated,
+            notifications_enabled=r.notifications_enabled,
+            incident_key=r.incident_key,
+        )
+        for r in rows
+    ]
+    return EventsPage(items=items, next_cursor=next_cursor)

server/monlet_server/api/health.py

@@ -0,0 +1,22 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..schemas import HealthResponse
+
+router = APIRouter()
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health() -> HealthResponse:
+    return HealthResponse()
+
+
+@router.get("/ready", response_model=HealthResponse)
+async def ready(session: AsyncSession = Depends(get_session)) -> HealthResponse:
+    try:
+        await session.execute(text("SELECT 1"))
+    except Exception as exc:
+        raise HTTPException(status_code=503, detail="database not reachable") from exc
+    return HealthResponse()

server/monlet_server/api/heartbeat.py

@@ -0,0 +1,21 @@
+from fastapi import APIRouter, Depends, status
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..schemas import AcceptedResponse, HeartbeatRequest
+from ..services.ingestion import upsert_agent_heartbeat
+
+router = APIRouter()
+
+
+@router.post(
+    "/heartbeat",
+    response_model=AcceptedResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+async def heartbeat(
+    body: HeartbeatRequest, session: AsyncSession = Depends(get_session)
+) -> AcceptedResponse:
+    await upsert_agent_heartbeat(session, body)
+    await session.commit()
+    return AcceptedResponse(accepted=True)

server/monlet_server/api/incidents.py

@@ -0,0 +1,53 @@
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, decode_datetime, encode
+from ..db import get_session
+from ..models import Incident as IncidentModel
+from ..schemas import Incident, IncidentsPage
+
+router = APIRouter()
+
+
+def _to_schema(i: IncidentModel) -> Incident:
+    return Incident(
+        id=str(i.id),
+        incident_key=i.incident_key,
+        state=i.state,
+        severity=i.severity,
+        agent_id=i.agent_id,
+        check_id=i.check_id,
+        opened_at=i.opened_at,
+        resolved_at=i.resolved_at,
+        summary=i.summary,
+    )
+
+
+@router.get("/incidents", response_model=IncidentsPage)
+async def list_incidents(
+    state: str | None = Query(default=None),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> IncidentsPage:
+    if state is not None and state not in ("open", "resolved"):
+        raise HTTPException(status_code=400, detail="invalid state")
+    stmt = select(IncidentModel).order_by(IncidentModel.opened_at.desc(), IncidentModel.id.desc())
+    if state is not None:
+        stmt = stmt.where(IncidentModel.state == state)
+    if cursor is not None:
+        ts_str, last_id = decode(cursor, 2)
+        ts = decode_datetime(ts_str)
+        stmt = stmt.where(
+            or_(
+                IncidentModel.opened_at < ts,
+                and_(IncidentModel.opened_at == ts, IncidentModel.id < last_id),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].opened_at.isoformat(), rows[-1].id)
+    return IncidentsPage(items=[_to_schema(r) for r in rows], next_cursor=next_cursor)

server/monlet_server/api/outbox.py

@@ -0,0 +1,58 @@
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, decode_datetime, encode
+from ..db import get_session
+from ..models import NotificationOutbox
+from ..schemas import NotificationOutboxItem, OutboxPage
+
+router = APIRouter()
+
+
+@router.get("/notifiers/outbox", response_model=OutboxPage)
+async def list_outbox(
+    state: str | None = Query(default=None),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> OutboxPage:
+    stmt = select(NotificationOutbox).order_by(
+        NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()
+    )
+    if state is not None:
+        stmt = stmt.where(NotificationOutbox.state == state)
+    if cursor is not None:
+        ts_str, last_id = decode(cursor, 2)
+        ts = decode_datetime(ts_str)
+        stmt = stmt.where(
+            or_(
+                NotificationOutbox.created_at < ts,
+                and_(
+                    NotificationOutbox.created_at == ts,
+                    NotificationOutbox.id < last_id,
+                ),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].created_at.isoformat(), rows[-1].id)
+    return OutboxPage(
+        items=[
+            NotificationOutboxItem(
+                id=str(r.id),
+                notifier=r.notifier,
+                state=r.state,
+                incident_id=str(r.incident_id),
+                event_type=r.event_type,
+                attempts=r.attempts,
+                next_attempt_at=r.next_attempt_at,
+                last_error=r.last_error,
+                updated_at=r.updated_at,
+            )
+            for r in rows
+        ],
+        next_cursor=next_cursor,
+    )

server/monlet_server/cursors.py

@@ -0,0 +1,29 @@
+import base64
+import binascii
+from datetime import datetime
+
+from fastapi import HTTPException
+
+
+def encode(*parts: object) -> str:
+    raw = "|".join(str(p) for p in parts).encode()
+    return base64.urlsafe_b64encode(raw).decode().rstrip("=")
+
+
+def decode(cursor: str, count: int) -> list[str]:
+    pad = "=" * (-len(cursor) % 4)
+    try:
+        raw = base64.urlsafe_b64decode(cursor + pad).decode()
+    except (binascii.Error, ValueError) as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc
+    parts = raw.split("|", count - 1)
+    if len(parts) != count:
+        raise HTTPException(status_code=400, detail="invalid cursor")
+    return parts
+
+
+def decode_datetime(value: str) -> datetime:
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc

server/monlet_server/db.py

@@ -0,0 +1,43 @@
+from collections.abc import AsyncIterator
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from .settings import get_settings
+
+_engine: AsyncEngine | None = None
+_sessionmaker: async_sessionmaker[AsyncSession] | None = None
+
+
+def get_engine() -> AsyncEngine:
+    global _engine, _sessionmaker
+    if _engine is None:
+        s = get_settings()
+        _engine = create_async_engine(s.database_url, pool_pre_ping=True, future=True)
+        _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False)
+    return _engine
+
+
+def get_sessionmaker() -> async_sessionmaker[AsyncSession]:
+    if _sessionmaker is None:
+        get_engine()
+    assert _sessionmaker is not None
+    return _sessionmaker
+
+
+async def dispose_engine() -> None:
+    global _engine, _sessionmaker
+    if _engine is not None:
+        await _engine.dispose()
+    _engine = None
+    _sessionmaker = None
+
+
+async def get_session() -> AsyncIterator[AsyncSession]:
+    sm = get_sessionmaker()
+    async with sm() as session:
+        yield session

server/monlet_server/errors.py

@@ -0,0 +1,54 @@
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from .logging_config import get_logger, request_id_var
+from .schemas import ErrorBody, ErrorCode, ErrorResponse
+
+log = get_logger("monlet.errors")
+
+
+def _request_id(request: Request) -> str:
+    rid = getattr(request.state, "request_id", None)
+    if rid is None:
+        rid = request_id_var.get() or ""
+    return rid
+
+
+def error_response(
+    request: Request, status_code: int, code: ErrorCode, message: str
+) -> JSONResponse:
+    body = ErrorResponse(
+        error=ErrorBody(code=code, message=message, request_id=_request_id(request))
+    )
+    return JSONResponse(status_code=status_code, content=body.model_dump())
+
+
+_STATUS_TO_CODE: dict[int, ErrorCode] = {
+    400: "validation",
+    401: "unauthorized",
+    404: "not_found",
+    413: "payload_too_large",
+    429: "rate_limited",
+    500: "internal",
+    503: "not_ready",
+}
+
+
+def install_error_handlers(app: FastAPI) -> None:
+    @app.exception_handler(RequestValidationError)
+    async def _validation(request: Request, exc: RequestValidationError):
+        message = "request validation failed"
+        return error_response(request, 400, "validation", message)
+
+    @app.exception_handler(StarletteHTTPException)
+    async def _http(request: Request, exc: StarletteHTTPException):
+        code = _STATUS_TO_CODE.get(exc.status_code, "internal")
+        msg = exc.detail if isinstance(exc.detail, str) else code
+        return error_response(request, exc.status_code, code, msg)
+
+    @app.exception_handler(Exception)
+    async def _unhandled(request: Request, exc: Exception):
+        log.exception("unhandled_exception", error=type(exc).__name__)
+        return error_response(request, 500, "internal", "internal server error")

server/monlet_server/logging_config.py

@@ -0,0 +1,36 @@
+import logging
+from contextvars import ContextVar
+
+import structlog
+
+request_id_var: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def _add_request_id(_logger, _name, event_dict):
+    rid = request_id_var.get()
+    if rid is not None:
+        event_dict["request_id"] = rid
+    return event_dict
+
+
+def configure_logging(level: str = "INFO") -> None:
+    logging.basicConfig(format="%(message)s", level=getattr(logging, level.upper(), logging.INFO))
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            _add_request_id,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso", utc=True),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.JSONRenderer(),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(
+            getattr(logging, level.upper(), logging.INFO)
+        ),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str = "monlet"):
+    return structlog.get_logger(name)

server/monlet_server/main.py

@@ -0,0 +1,74 @@
+import asyncio
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.responses import Response
+
+from . import metrics
+from .api import agents, checks, events, health, heartbeat, incidents, outbox
+from .db import dispose_engine, get_engine, get_sessionmaker
+from .errors import install_error_handlers
+from .logging_config import configure_logging, get_logger
+from .middleware.auth import BearerAuthMiddleware
+from .middleware.body_limit import BodyLimitMiddleware
+from .middleware.request_id import RequestIdMiddleware
+from .services.detector import run_detector
+from .services.notifier_worker import run_notifier_worker
+from .settings import get_settings
+
+log = get_logger("monlet.main")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    s = get_settings()
+    configure_logging(s.log_level)
+    get_engine()
+    sm = get_sessionmaker()
+    stop_event = asyncio.Event()
+    tasks: list[asyncio.Task] = []
+    if s.enable_detector:
+        tasks.append(asyncio.create_task(run_detector(sm, stop_event), name="detector"))
+    if s.enable_notifier_worker:
+        tasks.append(
+            asyncio.create_task(run_notifier_worker(sm, stop_event), name="notifier_worker")
+        )
+    log.info("server_started", host=s.host, port=s.port)
+    try:
+        yield
+    finally:
+        stop_event.set()
+        for t in tasks:
+            try:
+                await asyncio.wait_for(t, timeout=5.0)
+            except TimeoutError:
+                t.cancel()
+        await dispose_engine()
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="Monlet Server", version="1.0.0", lifespan=lifespan)
+
+    app.add_middleware(BearerAuthMiddleware)
+    app.add_middleware(BodyLimitMiddleware)
+    app.add_middleware(RequestIdMiddleware)
+
+    install_error_handlers(app)
+
+    app.include_router(health.router, prefix="/api/v1")
+    app.include_router(heartbeat.router, prefix="/api/v1")
+    app.include_router(events.router, prefix="/api/v1")
+    app.include_router(agents.router, prefix="/api/v1")
+    app.include_router(checks.router, prefix="/api/v1")
+    app.include_router(incidents.router, prefix="/api/v1")
+    app.include_router(outbox.router, prefix="/api/v1")
+
+    @app.get("/metrics")
+    async def metrics_endpoint() -> Response:
+        body, ct = metrics.render()
+        return Response(content=body, media_type=ct)
+
+    return app
+
+
+app = create_app()

server/monlet_server/metrics.py

@@ -0,0 +1,54 @@
+from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, generate_latest
+from prometheus_client.exposition import CONTENT_TYPE_LATEST
+
+REGISTRY = CollectorRegistry()
+
+heartbeats_total = Counter(
+    "monlet_server_heartbeats_total",
+    "Total heartbeats received.",
+    ["result"],
+    registry=REGISTRY,
+)
+events_total = Counter(
+    "monlet_server_events_total",
+    "Total events ingested.",
+    ["result"],
+    registry=REGISTRY,
+)
+incidents_opened_total = Counter(
+    "monlet_server_incidents_opened_total",
+    "Total incidents opened.",
+    registry=REGISTRY,
+)
+incidents_resolved_total = Counter(
+    "monlet_server_incidents_resolved_total",
+    "Total incidents resolved.",
+    registry=REGISTRY,
+)
+agents_gauge = Gauge(
+    "monlet_server_agents",
+    "Agents by status.",
+    ["status"],
+    registry=REGISTRY,
+)
+open_incidents_gauge = Gauge(
+    "monlet_server_open_incidents",
+    "Number of open incidents.",
+    registry=REGISTRY,
+)
+notifications_total = Counter(
+    "monlet_server_notifications_total",
+    "Notification delivery attempts by notifier and result.",
+    ["notifier", "result"],
+    registry=REGISTRY,
+)
+request_duration = Histogram(
+    "monlet_server_request_duration_seconds",
+    "Request duration seconds.",
+    ["path"],
+    registry=REGISTRY,
+)
+
+
+def render() -> tuple[bytes, str]:
+    return generate_latest(REGISTRY), CONTENT_TYPE_LATEST

server/monlet_server/middleware/auth.py

@@ -0,0 +1,22 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..errors import error_response
+from ..settings import get_settings
+
+NO_AUTH_PATHS = {"/api/v1/health", "/api/v1/ready", "/metrics"}
+
+
+class BearerAuthMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        if request.url.path in NO_AUTH_PATHS:
+            return await call_next(request)
+
+        expected = get_settings().auth_token
+        header = request.headers.get("authorization")
+        if not header or not header.lower().startswith("bearer "):
+            return error_response(request, 401, "unauthorized", "authentication required")
+        token = header.split(" ", 1)[1].strip()
+        if token != expected:
+            return error_response(request, 401, "unauthorized", "authentication required")
+        return await call_next(request)

server/monlet_server/middleware/body_limit.py

@@ -0,0 +1,32 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..errors import error_response
+from ..settings import get_settings
+
+
+class BodyLimitMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        limit = get_settings().body_limit_bytes
+        cl = request.headers.get("content-length")
+        if cl is not None:
+            try:
+                if int(cl) > limit:
+                    return error_response(request, 413, "payload_too_large", "body too large")
+            except ValueError:
+                return error_response(request, 400, "validation", "invalid content-length")
+            return await call_next(request)
+
+        total = 0
+        chunks: list[bytes] = []
+        async for chunk in request.stream():
+            total += len(chunk)
+            if total > limit:
+                return error_response(request, 413, "payload_too_large", "body too large")
+            chunks.append(chunk)
+
+        async def receive():
+            return {"type": "http.request", "body": b"".join(chunks), "more_body": False}
+
+        request._receive = receive  # type: ignore[attr-defined]
+        return await call_next(request)

server/monlet_server/middleware/request_id.py

@@ -0,0 +1,23 @@
+import re
+import uuid
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..logging_config import request_id_var
+
+_UUID_RE = re.compile(r"^[0-9a-fA-F-]{8,64}$")
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        incoming = request.headers.get("X-Request-Id")
+        rid = incoming if incoming and _UUID_RE.match(incoming) else str(uuid.uuid4())
+        request.state.request_id = rid
+        token = request_id_var.set(rid)
+        try:
+            response = await call_next(request)
+        finally:
+            request_id_var.reset(token)
+        response.headers["X-Request-Id"] = rid
+        return response

server/monlet_server/models.py

@@ -0,0 +1,181 @@
+from datetime import datetime
+from uuid import UUID
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    CheckConstraint,
+    DateTime,
+    ForeignKey,
+    Index,
+    Integer,
+    PrimaryKeyConstraint,
+    String,
+    Text,
+    func,
+    text,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import UUID as PG_UUID
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+def _jsonb():
+    return JSONB().with_variant(JSON(), "sqlite")
+
+
+def _uuid():
+    return PG_UUID(as_uuid=True).with_variant(String(36), "sqlite")
+
+
+class Agent(Base):
+    __tablename__ = "agents"
+
+    agent_id: Mapped[str] = mapped_column(Text, primary_key=True)
+    hostname: Mapped[str] = mapped_column(Text, nullable=False)
+    status: Mapped[str] = mapped_column(Text, nullable=False, default="alive")
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    features: Mapped[dict] = mapped_column(
+        _jsonb(),
+        nullable=False,
+        default=lambda: {"push": False, "metrics": False},
+        server_default=text("jsonb_build_object('push', false, 'metrics', false)"),
+    )
+    labels: Mapped[dict] = mapped_column(
+        _jsonb(), nullable=False, default=dict, server_default=text("'{}'::jsonb")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+    __table_args__ = (
+        CheckConstraint("status IN ('alive','stale','dead')", name="agents_status_chk"),
+        Index("agents_status_idx", "status"),
+        Index("agents_last_seen_idx", "last_seen_at"),
+    )
+
+
+class Check(Base):
+    __tablename__ = "checks"
+
+    agent_id: Mapped[str] = mapped_column(
+        Text, ForeignKey("agents.agent_id", ondelete="CASCADE"), nullable=False
+    )
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    exit_code: Mapped[int] = mapped_column(Integer, nullable=False)
+    last_observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+
+    __table_args__ = (
+        PrimaryKeyConstraint("agent_id", "check_id", name="checks_pkey"),
+        CheckConstraint(
+            "status IN ('ok','warning','critical','unknown')", name="checks_status_chk"
+        ),
+        Index("checks_status_idx", "status"),
+    )
+
+
+class Event(Base):
+    __tablename__ = "events"
+
+    event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    agent_id: Mapped[str] = mapped_column(Text, nullable=False)
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    received_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    exit_code: Mapped[int] = mapped_column(Integer, nullable=False)
+    duration_ms: Mapped[int] = mapped_column(Integer, nullable=False)
+    output: Mapped[str | None] = mapped_column(Text, nullable=True)
+    output_truncated: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    notifications_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default=text("TRUE")
+    )
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+
+    __table_args__ = (
+        CheckConstraint(
+            "status IN ('ok','warning','critical','unknown')", name="events_status_chk"
+        ),
+        CheckConstraint("duration_ms >= 0", name="events_duration_chk"),
+        Index(
+            "events_agent_check_observed_idx",
+            "agent_id",
+            "check_id",
+            text("observed_at DESC"),
+        ),
+        Index("events_observed_at_idx", text("observed_at DESC")),
+    )
+
+
+class Incident(Base):
+    __tablename__ = "incidents"
+
+    id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+    agent_id: Mapped[str] = mapped_column(Text, nullable=False)
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    state: Mapped[str] = mapped_column(Text, nullable=False)
+    severity: Mapped[str] = mapped_column(Text, nullable=False)
+    opened_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+    last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
+
+    __table_args__ = (
+        CheckConstraint("state IN ('open','resolved')", name="incidents_state_chk"),
+        CheckConstraint(
+            "severity IN ('warning','critical','unknown')", name="incidents_severity_chk"
+        ),
+        Index(
+            "incidents_open_key_idx",
+            "incident_key",
+            unique=True,
+            postgresql_where=text("state = 'open'"),
+        ),
+        Index("incidents_state_opened_idx", "state", text("opened_at DESC")),
+    )
+
+
+class NotificationOutbox(Base):
+    __tablename__ = "notification_outbox"
+
+    id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    incident_id: Mapped[UUID] = mapped_column(
+        _uuid(), ForeignKey("incidents.id", ondelete="CASCADE"), nullable=False
+    )
+    notifier: Mapped[str] = mapped_column(Text, nullable=False)
+    event_type: Mapped[str] = mapped_column(Text, nullable=False)
+    state: Mapped[str] = mapped_column(Text, nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    payload: Mapped[dict] = mapped_column(_jsonb(), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+    __table_args__ = (
+        CheckConstraint("event_type IN ('firing','resolved')", name="outbox_event_type_chk"),
+        CheckConstraint(
+            "state IN ('pending','sending','sent','retry','failed','discarded')",
+            name="outbox_state_chk",
+        ),
+        Index(
+            "outbox_pending_idx",
+            "state",
+            "next_attempt_at",
+            postgresql_where=text("state IN ('pending','retry')"),
+        ),
+    )

server/monlet_server/redaction.py

@@ -0,0 +1,26 @@
+import re
+
+_PATTERNS: list[tuple[re.Pattern[str], object]] = [
+    (
+        re.compile(r"(?i)(authorization|set-cookie|cookie)([\"'=:]+\s*)([^\r\n]+)"),
+        lambda m: f"{m.group(1)}{m.group(2)}***",
+    ),
+    (
+        re.compile(r"(?i)(token|secret|password|api[_-]?key|credential)([\"'=:]+\s*)([^\s\"',;]+)"),
+        lambda m: f"{m.group(1)}{m.group(2)}***",
+    ),
+    (re.compile(r"(?i)(bearer)\s+\S+"), r"\1 ***"),
+]
+
+
+def redact(value: str | None) -> str | None:
+    if value is None:
+        return None
+    out = value
+    for pat, repl in _PATTERNS:
+        out = pat.sub(repl, out)
+    return out
+
+
+def redact_dict(d: dict) -> dict:
+    return {k: redact(v) if isinstance(v, str) else v for k, v in d.items()}

server/monlet_server/schemas.py

@@ -0,0 +1,201 @@
+import re
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+ID_PATTERN = r"^[A-Za-z0-9._:-]+$"
+UUIDV7_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+ID_RE = re.compile(ID_PATTERN)
+SENSITIVE_LABEL_WORDS = (
+    "token",
+    "secret",
+    "password",
+    "credential",
+    "authorization",
+    "cookie",
+    "api_key",
+    "apikey",
+)
+
+Status = Literal["ok", "warning", "critical", "unknown"]
+AgentStatus = Literal["alive", "stale", "dead"]
+IncidentState = Literal["open", "resolved"]
+Severity = Literal["warning", "critical", "unknown"]
+OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"]
+OutboxEventType = Literal["firing", "resolved"]
+ErrorCode = Literal[
+    "unauthorized",
+    "not_found",
+    "validation",
+    "payload_too_large",
+    "rate_limited",
+    "internal",
+    "not_ready",
+]
+
+
+class HealthResponse(BaseModel):
+    status: Literal["ok"] = "ok"
+
+
+class ErrorBody(BaseModel):
+    code: ErrorCode
+    message: str
+    request_id: str
+
+
+class ErrorResponse(BaseModel):
+    error: ErrorBody
+
+
+class AcceptedResponse(BaseModel):
+    accepted: bool = True
+
+
+class EventBatchAcceptedResponse(BaseModel):
+    accepted: int = Field(ge=0)
+    deduplicated: int = Field(ge=0)
+
+
+def _is_sensitive_label_key(key: str) -> bool:
+    normalized = key.lower().replace("-", "_").replace(".", "_").replace(":", "_")
+    return any(word in normalized for word in SENSITIVE_LABEL_WORDS)
+
+
+class AgentFeatures(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    push: bool
+    metrics: bool
+
+
+class HeartbeatRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    observed_at: datetime
+    hostname: str = Field(max_length=253)
+    features: AgentFeatures
+    labels: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("labels")
+    @classmethod
+    def _check_labels(cls, v: dict[str, str]) -> dict[str, str]:
+        if len(v) > 32:
+            raise ValueError("too many labels (max 32)")
+        for k, val in v.items():
+            if len(k) > 64 or not ID_RE.match(k):
+                raise ValueError(f"invalid label key: {k!r}")
+            if _is_sensitive_label_key(k):
+                raise ValueError(f"label key is not allowed: {k!r}")
+            if len(val) > 256:
+                raise ValueError(f"label value too long for key {k!r}")
+        return v
+
+
+class CheckResultEvent(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    event_id: str = Field(pattern=UUIDV7_PATTERN, min_length=36, max_length=36)
+    check_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    observed_at: datetime
+    status: Status
+    exit_code: int
+    duration_ms: int = Field(ge=0)
+    output: str | None = Field(default=None, max_length=8192)
+    output_truncated: bool = False
+    notifications_enabled: bool = True
+    incident_key: str | None = Field(default=None, max_length=256)
+
+    @field_validator("output")
+    @classmethod
+    def _check_output_bytes(cls, v: str | None) -> str | None:
+        if v is not None and len(v.encode("utf-8")) > 8192:
+            raise ValueError("output exceeds 8192 UTF-8 bytes")
+        return v
+
+
+class EventBatchRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    events: list[CheckResultEvent] = Field(min_length=1, max_length=200)
+
+
+class Agent(BaseModel):
+    agent_id: str
+    hostname: str
+    status: AgentStatus
+    last_seen_at: datetime
+    features: AgentFeatures
+    labels: dict[str, str] = Field(default_factory=dict)
+
+
+class CheckState(BaseModel):
+    agent_id: str
+    check_id: str
+    status: Status
+    last_observed_at: datetime
+    exit_code: int
+    incident_key: str
+
+
+class Incident(BaseModel):
+    id: str
+    incident_key: str
+    state: IncidentState
+    severity: Severity
+    agent_id: str
+    check_id: str
+    opened_at: datetime
+    resolved_at: datetime | None = None
+    summary: str | None = None
+
+
+class StoredCheckResultEvent(CheckResultEvent):
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    received_at: datetime
+
+
+class NotificationOutboxItem(BaseModel):
+    id: str
+    notifier: str
+    state: OutboxState
+    incident_id: str
+    event_type: OutboxEventType
+    attempts: int = Field(ge=0)
+    next_attempt_at: datetime | None = None
+    last_error: str | None = None
+    updated_at: datetime
+
+
+class PageEnvelope(BaseModel):
+    next_cursor: str | None = None
+
+
+class AgentsPage(PageEnvelope):
+    items: list[Agent]
+
+
+class ChecksPage(PageEnvelope):
+    items: list[CheckState]
+
+
+class IncidentsPage(PageEnvelope):
+    items: list[Incident]
+
+
+class EventsPage(PageEnvelope):
+    items: list[StoredCheckResultEvent]
+
+
+class OutboxPage(PageEnvelope):
+    items: list[NotificationOutboxItem]
+
+
+class UUIDStr(BaseModel):
+    """Helper to validate generated UUIDs."""
+
+    value: UUID

server/monlet_server/services/detector.py

@@ -0,0 +1,90 @@
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Event, Incident, NotificationOutbox
+from ..settings import get_settings
+
+log = get_logger("monlet.detector")
+
+OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
+
+
+async def _prune_history(session) -> None:
+    s = get_settings()
+    if s.events_retention_max_rows > 0:
+        keep_events = (
+            select(Event.event_id)
+            .order_by(Event.received_at.desc(), Event.event_id.desc())
+            .limit(s.events_retention_max_rows)
+        )
+        await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
+
+    if s.outbox_retention_max_rows > 0:
+        keep_outbox = (
+            select(NotificationOutbox.id)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
+            .limit(s.outbox_retention_max_rows)
+        )
+        await session.execute(
+            delete(NotificationOutbox)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .where(NotificationOutbox.id.not_in(keep_outbox))
+        )
+
+
+async def _tick(sm: async_sessionmaker) -> None:
+    s = get_settings()
+    now = datetime.now(UTC)
+    stale_cut = now - timedelta(seconds=s.stale_after_sec)
+    dead_cut = now - timedelta(seconds=s.dead_after_sec)
+    async with sm() as session:
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= dead_cut)
+            .where(Agent.status != "dead")
+            .values(status="dead")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= stale_cut)
+            .where(Agent.last_seen_at > dead_cut)
+            .where(Agent.status != "stale")
+            .values(status="stale")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at > stale_cut)
+            .where(Agent.status != "alive")
+            .values(status="alive")
+        )
+        await _prune_history(session)
+        await session.commit()
+
+        for status in ("alive", "stale", "dead"):
+            r = await session.execute(
+                select(func.count()).select_from(Agent).where(Agent.status == status)
+            )
+            metrics.agents_gauge.labels(status=status).set(r.scalar_one())
+        r = await session.execute(
+            select(func.count()).select_from(Incident).where(Incident.state == "open")
+        )
+        metrics.open_incidents_gauge.set(r.scalar_one())
+
+
+async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
+    tick = get_settings().detector_tick_sec
+    while not stop_event.is_set():
+        try:
+            await _tick(sm)
+        except Exception as exc:
+            log.warning("detector_tick_failed", error=str(exc))
+        try:
+            await asyncio.wait_for(stop_event.wait(), timeout=tick)
+        except TimeoutError:
+            continue

server/monlet_server/services/ingestion.py

@@ -0,0 +1,237 @@
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+from fastapi import HTTPException
+from sqlalchemy import select, text
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Check, Event, Incident, NotificationOutbox
+from ..redaction import redact
+from ..schemas import CheckResultEvent, HeartbeatRequest
+from ..settings import get_settings
+
+INCIDENT_KEY_MAX = 256
+
+log = get_logger("monlet.ingestion")
+
+_SEVERITY_RANK = {"unknown": 1, "warning": 2, "critical": 3}
+
+
+def _severity_for_status(status: str) -> str:
+    if status in ("warning", "critical", "unknown"):
+        return status
+    return "unknown"
+
+
+def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
+    key = ev.incident_key or f"{agent_id}:{ev.check_id}"
+    if len(key) > INCIDENT_KEY_MAX:
+        raise HTTPException(
+            status_code=400,
+            detail=f"incident_key exceeds {INCIDENT_KEY_MAX} chars",
+        )
+    return key
+
+
+async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
+    observed = (
+        hb.observed_at.astimezone(UTC)
+        if hb.observed_at.tzinfo
+        else hb.observed_at.replace(tzinfo=UTC)
+    )
+    stmt = pg_insert(Agent).values(
+        agent_id=hb.agent_id,
+        hostname=hb.hostname,
+        status="alive",
+        last_seen_at=observed,
+        features=hb.features.model_dump(),
+        labels=hb.labels,
+    )
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[Agent.agent_id],
+        set_={
+            "hostname": stmt.excluded.hostname,
+            "status": "alive",
+            "last_seen_at": stmt.excluded.last_seen_at,
+            "features": stmt.excluded.features,
+            "labels": stmt.excluded.labels,
+        },
+    )
+    await session.execute(stmt)
+    metrics.heartbeats_total.labels(result="ok").inc()
+
+
+async def _apply_incident(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+    incident_key: str,
+) -> tuple[bool, Incident | None]:
+    """Return (transition, incident). transition True if open->resolved or none->open."""
+    observed = ev.observed_at
+    if ev.status == "ok":
+        res = await session.execute(
+            select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+        )
+        inc = res.scalar_one_or_none()
+        if inc is None:
+            return False, None
+        inc.state = "resolved"
+        inc.resolved_at = observed
+        inc.last_event_id = UUID(ev.event_id)
+        metrics.incidents_resolved_total.inc()
+        return True, inc
+
+    severity = _severity_for_status(ev.status)
+    new_id = uuid4()
+    summary = (redact(ev.output) or "")[:200] if ev.output else None
+    stmt = pg_insert(Incident).values(
+        id=new_id,
+        incident_key=incident_key,
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        state="open",
+        severity=severity,
+        opened_at=observed,
+        last_event_id=UUID(ev.event_id),
+        summary=summary,
+    )
+    stmt = stmt.on_conflict_do_nothing(
+        index_elements=[Incident.incident_key],
+        index_where=text("state = 'open'"),
+    ).returning(Incident.id)
+    ins = await session.execute(stmt)
+    inserted_id = ins.scalar_one_or_none()
+    if inserted_id is not None:
+        await session.flush()
+        res = await session.execute(select(Incident).where(Incident.id == inserted_id))
+        metrics.incidents_opened_total.inc()
+        return True, res.scalar_one()
+
+    res = await session.execute(
+        select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+    )
+    inc = res.scalar_one()
+    if inc.agent_id != agent_id or inc.check_id != ev.check_id:
+        raise HTTPException(
+            status_code=400,
+            detail="incident_key already bound to another (agent_id, check_id)",
+        )
+    cur = _SEVERITY_RANK.get(inc.severity, 0)
+    new = _SEVERITY_RANK.get(severity, 0)
+    if new > cur:
+        inc.severity = severity
+    inc.last_event_id = UUID(ev.event_id)
+    return False, inc
+
+
+async def _enqueue_outbox(
+    session: AsyncSession,
+    incident: Incident,
+    event_type: str,
+    payload: dict,
+) -> None:
+    notifiers = get_settings().enabled_notifiers
+    if not notifiers:
+        return
+    now = datetime.now(UTC)
+    for name in notifiers:
+        session.add(
+            NotificationOutbox(
+                id=uuid4(),
+                incident_id=incident.id,
+                notifier=name,
+                event_type=event_type,
+                state="pending",
+                attempts=0,
+                next_attempt_at=now,
+                payload=payload,
+            )
+        )
+
+
+async def ingest_event(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+) -> bool:
+    """Return True if accepted, False if deduplicated."""
+    observed = ev.observed_at
+    output = redact(ev.output)
+    incident_key = _incident_key(agent_id, ev)
+
+    stmt = pg_insert(Event).values(
+        event_id=UUID(ev.event_id),
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        observed_at=observed,
+        status=ev.status,
+        exit_code=ev.exit_code,
+        duration_ms=ev.duration_ms,
+        output=output,
+        output_truncated=ev.output_truncated,
+        notifications_enabled=ev.notifications_enabled,
+        incident_key=incident_key,
+    )
+    stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
+    result = await session.execute(stmt)
+    inserted = result.scalar_one_or_none()
+    if inserted is None:
+        metrics.events_total.labels(result="deduplicated").inc()
+        return False
+
+    metrics.events_total.labels(result="accepted").inc()
+
+    chk_res = await session.execute(
+        select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
+    )
+    cur = chk_res.scalar_one_or_none()
+    is_late = cur is not None and cur.last_observed_at > observed
+    if is_late:
+        return True
+
+    if cur is None:
+        session.add(
+            Check(
+                agent_id=agent_id,
+                check_id=ev.check_id,
+                status=ev.status,
+                exit_code=ev.exit_code,
+                last_observed_at=observed,
+                last_event_id=UUID(ev.event_id),
+                incident_key=incident_key,
+            )
+        )
+    else:
+        cur.status = ev.status
+        cur.exit_code = ev.exit_code
+        cur.last_observed_at = observed
+        cur.last_event_id = UUID(ev.event_id)
+        cur.incident_key = incident_key
+
+    transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
+    if transition and inc is not None and ev.notifications_enabled:
+        event_type = "resolved" if ev.status == "ok" else "firing"
+        await _enqueue_outbox(
+            session,
+            inc,
+            event_type,
+            {
+                "incident_id": str(inc.id),
+                "agent_id": agent_id,
+                "check_id": ev.check_id,
+                "status": ev.status,
+                "severity": inc.severity,
+                "incident_key": incident_key,
+                "observed_at": observed.isoformat(),
+                "opened_at": inc.opened_at.isoformat() if inc.opened_at else None,
+                "resolved_at": inc.resolved_at.isoformat() if inc.resolved_at else None,
+                "summary": inc.summary,
+                "output": output,
+                "event_type": event_type,
+            },
+        )
+    return True

server/monlet_server/services/notifier_worker.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..redaction import redact
+from ..settings import Settings, get_settings
+from .notifiers import Notifier, build_registry
+
+log = get_logger("monlet.notifier_worker")
+
+
+# Per ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
+_BACKOFF_SCHEDULE_SEC = (5, 15, 60, 300, 1800, 3600, 7200, 14400)
+
+
+def _backoff(attempts: int) -> timedelta:
+    idx = max(0, min(attempts - 1, len(_BACKOFF_SCHEDULE_SEC) - 1))
+    return timedelta(seconds=_BACKOFF_SCHEDULE_SEC[idx])
+
+
+async def _claim_batch(session, limit: int, lease_sec: int) -> list[dict]:
+    res = await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = 'sending', updated_at = now()
+             WHERE id IN (
+                SELECT id FROM notification_outbox
+                 WHERE (
+                        state IN ('pending','retry')
+                        AND (next_attempt_at IS NULL OR next_attempt_at <= now())
+                       )
+                    OR (
+                        state = 'sending'
+                        AND updated_at < now() - make_interval(secs => :lease)
+                       )
+                 ORDER BY next_attempt_at NULLS FIRST
+                 LIMIT :lim
+                 FOR UPDATE SKIP LOCKED
+             )
+         RETURNING id, notifier, event_type, attempts, payload
+            """
+        ),
+        {"lim": limit, "lease": lease_sec},
+    )
+    rows = res.mappings().all()
+    return [dict(r) for r in rows]
+
+
+async def _finalize(
+    session,
+    row_id,
+    *,
+    state: str,
+    attempts: int,
+    next_attempt_at: datetime | None,
+    last_error: str | None,
+) -> None:
+    await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = :state,
+                   attempts = :attempts,
+                   next_attempt_at = :next_attempt_at,
+                   last_error = :last_error,
+                   updated_at = now()
+             WHERE id = :id
+            """
+        ),
+        {
+            "state": state,
+            "attempts": attempts,
+            "next_attempt_at": next_attempt_at,
+            "last_error": last_error,
+            "id": row_id,
+        },
+    )
+
+
+async def _process_row(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    row: dict,
+    max_attempts: int,
+) -> None:
+    notifier = registry.get(row["notifier"])
+    attempts = int(row["attempts"]) + 1
+    async with sm() as session:
+        if notifier is None:
+            await _finalize(
+                session,
+                row["id"],
+                state="discarded",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=f"unknown_notifier:{row['notifier']}",
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=row["notifier"], result="discarded").inc()
+            return
+
+        try:
+            result = await notifier.deliver(row["event_type"], row["payload"])
+        except Exception as exc:  # defensive: notifier should not raise
+            result_ok = False
+            result_perm = False
+            result_err = f"exception:{type(exc).__name__}"
+        else:
+            result_ok = result.ok
+            result_perm = result.permanent
+            result_err = redact(result.error) if result.error else None
+
+        if result_ok:
+            await _finalize(
+                session,
+                row["id"],
+                state="sent",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=None,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="sent").inc()
+            return
+
+        if result_perm or attempts >= max_attempts:
+            await _finalize(
+                session,
+                row["id"],
+                state="failed",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=result_err,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="failed").inc()
+            return
+
+        next_at = datetime.now(UTC) + _backoff(attempts)
+        await _finalize(
+            session,
+            row["id"],
+            state="retry",
+            attempts=attempts,
+            next_attempt_at=next_at,
+            last_error=result_err,
+        )
+        await session.commit()
+        metrics.notifications_total.labels(notifier=notifier.name, result="retry").inc()
+
+
+async def tick(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    settings: Settings,
+) -> int:
+    async with sm() as session:
+        rows = await _claim_batch(
+            session, settings.notifier_batch_size, settings.notifier_lease_sec
+        )
+        await session.commit()
+    if not rows:
+        return 0
+    await asyncio.gather(
+        *(_process_row(sm, registry, r, settings.notifier_max_attempts) for r in rows)
+    )
+    return len(rows)
+
+
+async def run_notifier_worker(
+    sm: async_sessionmaker,
+    stop_event: asyncio.Event,
+    client: httpx.AsyncClient | None = None,
+) -> None:
+    settings = get_settings()
+    own_client = False
+    if client is None:
+        client = httpx.AsyncClient(timeout=settings.notifier_http_timeout_sec)
+        own_client = True
+    registry = build_registry(settings, client)
+    if not registry:
+        log.info("notifier_worker_disabled_no_notifiers")
+        if own_client:
+            await client.aclose()
+        return
+    log.info("notifier_worker_started", notifiers=list(registry.keys()))
+    try:
+        while not stop_event.is_set():
+            try:
+                await tick(sm, registry, settings)
+            except Exception as exc:
+                log.warning("notifier_tick_failed", error=str(exc))
+            try:
+                await asyncio.wait_for(stop_event.wait(), timeout=settings.notifier_tick_sec)
+            except TimeoutError:
+                continue
+    finally:
+        if own_client:
+            await client.aclose()

server/monlet_server/services/notifiers/__init__.py

@@ -0,0 +1,4 @@
+from .base import DeliveryResult, Notifier
+from .registry import build_registry
+
+__all__ = ["DeliveryResult", "Notifier", "build_registry"]

server/monlet_server/services/notifiers/alertmanager.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class AlertmanagerNotifier:
+    name = "alertmanager"
+
+    def __init__(self, client: httpx.AsyncClient, url: str) -> None:
+        self._client = client
+        self._url = url.rstrip("/")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        labels = {
+            "alertname": "monlet_incident",
+            "agent_id": str(payload.get("agent_id", "")),
+            "check_id": str(payload.get("check_id", "")),
+            "incident_key": str(payload.get("incident_key", "")),
+        }
+        annotations = {
+            "severity": str(payload.get("severity", "unknown")),
+            "status": str(payload.get("status", "")),
+            "summary": redact(payload.get("summary") or "") or "",
+        }
+        if payload.get("output"):
+            annotations["output"] = (redact(payload["output"]) or "")[:1024]
+        alert = {
+            "labels": labels,
+            "annotations": annotations,
+            "startsAt": payload.get("opened_at") or payload.get("observed_at"),
+        }
+        if event_type == "resolved" and payload.get("resolved_at"):
+            alert["endsAt"] = payload["resolved_at"]
+        try:
+            resp = await self._client.post(f"{self._url}/api/v2/alerts", json=[alert])
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/base.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+    ok: bool
+    permanent: bool = False
+    error: str | None = None
+
+
+class Notifier(Protocol):
+    name: str
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult: ...

server/monlet_server/services/notifiers/debug.py

@@ -0,0 +1,14 @@
+from ...logging_config import get_logger
+from ...redaction import redact_dict
+from .base import DeliveryResult
+
+
+class DebugNotifier:
+    name = "debug"
+
+    def __init__(self) -> None:
+        self._log = get_logger("monlet.notifier.debug")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        self._log.info("notify", event_type=event_type, payload=redact_dict(payload))
+        return DeliveryResult(ok=True)

server/monlet_server/services/notifiers/http.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import httpx
+
+
+def classify_response(status: int) -> tuple[bool, bool]:
+    """Return (ok, permanent_failure)."""
+    if 200 <= status < 300:
+        return True, False
+    if 400 <= status < 500 and status not in (408, 425, 429):
+        return False, True
+    return False, False
+
+
+def classify_exception(exc: Exception) -> tuple[bool, bool]:
+    if isinstance(exc, httpx.TimeoutException | httpx.NetworkError):
+        return False, False
+    return False, False

server/monlet_server/services/notifiers/registry.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import httpx
+
+from ...settings import Settings
+from .alertmanager import AlertmanagerNotifier
+from .base import Notifier
+from .debug import DebugNotifier
+from .telegram import TelegramNotifier
+from .webhook import WebhookNotifier
+
+
+def build_registry(settings: Settings, client: httpx.AsyncClient) -> dict[str, Notifier]:
+    reg: dict[str, Notifier] = {}
+    if settings.notifier_debug_enabled:
+        reg["debug"] = DebugNotifier()
+    if (
+        settings.notifier_telegram_enabled
+        and settings.notifier_telegram_token
+        and settings.notifier_telegram_chat_id
+    ):
+        reg["telegram"] = TelegramNotifier(
+            client, settings.notifier_telegram_token, settings.notifier_telegram_chat_id
+        )
+    if settings.notifier_webhook_enabled and settings.notifier_webhook_url:
+        reg["webhook"] = WebhookNotifier(
+            client, settings.notifier_webhook_url, settings.notifier_webhook_token
+        )
+    if settings.notifier_alertmanager_enabled and settings.notifier_alertmanager_url:
+        reg["alertmanager"] = AlertmanagerNotifier(client, settings.notifier_alertmanager_url)
+    return reg

server/monlet_server/services/notifiers/telegram.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+def _format_message(event_type: str, payload: dict) -> str:
+    icon = "[FIRING]" if event_type == "firing" else "[RESOLVED]"
+    sev = payload.get("severity", "unknown")
+    agent = payload.get("agent_id", "?")
+    check = payload.get("check_id", "?")
+    status = payload.get("status", "?")
+    incident_key = payload.get("incident_key", "?")
+    output = payload.get("output") or ""
+    lines = [
+        f"{icon} {sev} {agent}/{check}",
+        f"status: {status}",
+        f"incident_key: {incident_key}",
+        f"observed_at: {payload.get('observed_at', '?')}",
+    ]
+    if output:
+        lines.append(f"output: {output[:300]}")
+    return redact("\n".join(lines)) or ""
+
+
+class TelegramNotifier:
+    name = "telegram"
+
+    def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str) -> None:
+        self._client = client
+        self._token = token
+        self._chat_id = chat_id
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        url = f"https://api.telegram.org/bot{self._token}/sendMessage"
+        body = {"chat_id": self._chat_id, "text": _format_message(event_type, payload)}
+        try:
+            resp = await self._client.post(url, json=body)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/webhook.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact_dict
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class WebhookNotifier:
+    name = "webhook"
+
+    def __init__(self, client: httpx.AsyncClient, url: str, token: str = "") -> None:
+        self._client = client
+        self._url = url
+        self._token = token
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        headers = {"Content-Type": "application/json"}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+        body = {"event_type": event_type, "incident": redact_dict(payload)}
+        try:
+            resp = await self._client.post(self._url, json=body, headers=headers)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/settings.py

@@ -0,0 +1,80 @@
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="MONLET_", env_file=".env", extra="ignore")
+
+    auth_token: str
+    database_url: str = "postgresql+asyncpg://monlet:monlet@127.0.0.1:5432/monlet"
+    log_level: str = "INFO"
+    body_limit_bytes: int = 1_048_576
+    output_max_bytes: int = 8192
+    max_batch_events: int = 200
+    stale_after_sec: int = 90
+    dead_after_sec: int = 300
+    detector_tick_sec: int = 15
+    host: str = "0.0.0.0"
+    port: int = 8000
+    enable_detector: bool = True
+    enable_notifier_worker: bool = True
+    events_retention_max_rows: int = 100_000
+    outbox_retention_max_rows: int = 50_000
+    notifier_tick_sec: int = 5
+    notifier_batch_size: int = 20
+    notifier_max_attempts: int = 8
+    notifier_lease_sec: int = 300
+    notifier_http_timeout_sec: float = 10.0
+
+    notifier_debug_enabled: bool = True
+    notifier_telegram_enabled: bool = False
+    notifier_telegram_token: str = ""
+    notifier_telegram_chat_id: str = ""
+    notifier_webhook_enabled: bool = False
+    notifier_webhook_url: str = ""
+    notifier_webhook_token: str = ""
+    notifier_alertmanager_enabled: bool = False
+    notifier_alertmanager_url: str = ""
+
+    @field_validator("auth_token")
+    @classmethod
+    def _validate_auth_token(cls, value: str) -> str:
+        if not value or value == "changeme":
+            raise ValueError("MONLET_AUTH_TOKEN must be set to a non-default value")
+        return value
+
+    @property
+    def enabled_notifiers(self) -> list[str]:
+        out: list[str] = []
+        if self.notifier_debug_enabled:
+            out.append("debug")
+        if (
+            self.notifier_telegram_enabled
+            and self.notifier_telegram_token
+            and self.notifier_telegram_chat_id
+        ):
+            out.append("telegram")
+        if self.notifier_webhook_enabled and self.notifier_webhook_url:
+            out.append("webhook")
+        if self.notifier_alertmanager_enabled and self.notifier_alertmanager_url:
+            out.append("alertmanager")
+        return out
+
+    @property
+    def sync_database_url(self) -> str:
+        return self.database_url.replace("+asyncpg", "+psycopg")
+
+
+_settings: Settings | None = None
+
+
+def get_settings() -> Settings:
+    global _settings
+    if _settings is None:
+        _settings = Settings()
+    return _settings
+
+
+def reset_settings_cache() -> None:
+    global _settings
+    _settings = None

server/pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "monlet-server"
+version = "0.1.0"
+description = "Monlet central server (ingestion, state, incidents, outbox)"
+requires-python = ">=3.14,<3.15"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.32",
+    "pydantic>=2.9",
+    "pydantic-settings>=2.6",
+    "sqlalchemy[asyncio]>=2.0.36",
+    "asyncpg>=0.30",
+    "psycopg[binary]>=3.2",
+    "alembic>=1.13",
+    "structlog>=24.4",
+    "prometheus-client>=0.21",
+    "python-multipart>=0.0.12",
+    "httpx>=0.27",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.3",
+    "pytest-asyncio>=0.24",
+    "httpx>=0.27",
+    "testcontainers[postgresql]>=4.8",
+    "ruff>=0.7",
+    "freezegun>=1.5",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["monlet_server"]
+
+[tool.uv]
+package = true
+
+[tool.ruff]
+line-length = 100
+target-version = "py314"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = ["E501", "B008"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

server/tests/_helpers.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import random
+import time
+import uuid
+from datetime import UTC, datetime
+
+
+def uuidv7() -> str:
+    ts_ms = int(time.time() * 1000)
+    rand_a = random.getrandbits(12)
+    rand_b = random.getrandbits(62)
+    high = (ts_ms & 0xFFFFFFFFFFFF) << 16 | 0x7000 | rand_a
+    low = (0b10 << 62) | rand_b
+    return str(uuid.UUID(int=(high << 64) | low))
+
+
+def now_iso(offset_sec: int = 0) -> str:
+    return datetime.now(UTC).replace(microsecond=0).isoformat()
+
+
+def make_event(
+    check_id: str = "disk_root",
+    status: str = "ok",
+    exit_code: int = 0,
+    observed_at: str | None = None,
+    output: str | None = None,
+    incident_key: str | None = None,
+    notifications_enabled: bool = True,
+) -> dict:
+    ev = {
+        "event_id": uuidv7(),
+        "check_id": check_id,
+        "observed_at": observed_at or now_iso(),
+        "status": status,
+        "exit_code": exit_code,
+        "duration_ms": 12,
+        "output_truncated": False,
+        "notifications_enabled": notifications_enabled,
+    }
+    if output is not None:
+        ev["output"] = output
+    if incident_key is not None:
+        ev["incident_key"] = incident_key
+    return ev
+
+
+def make_heartbeat(agent_id: str = "agent-1") -> dict:
+    return {
+        "agent_id": agent_id,
+        "observed_at": now_iso(),
+        "hostname": "host-1",
+        "features": {"push": True, "metrics": True},
+        "labels": {},
+    }

server/tests/conftest.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import AsyncIterator, Iterator
+
+import pytest
+import pytest_asyncio
+from alembic.config import Config
+from httpx import ASGITransport, AsyncClient
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from testcontainers.postgres import PostgresContainer
+
+from alembic import command
+
+os.environ.setdefault("MONLET_AUTH_TOKEN", "test-token")
+os.environ.setdefault("MONLET_ENABLE_DETECTOR", "false")
+os.environ.setdefault("MONLET_ENABLE_NOTIFIER_WORKER", "false")
+
+from monlet_server import db as db_module  # noqa: E402
+from monlet_server.main import create_app  # noqa: E402
+from monlet_server.settings import get_settings, reset_settings_cache  # noqa: E402
+
+HERE = os.path.dirname(__file__)
+SERVER_ROOT = os.path.dirname(HERE)
+
+
+@pytest.fixture(scope="session")
+def event_loop() -> Iterator[asyncio.AbstractEventLoop]:
+    loop = asyncio.new_event_loop()
+    yield loop
+    loop.close()
+
+
+@pytest.fixture(scope="session")
+def pg_container() -> Iterator[PostgresContainer]:
+    with PostgresContainer("postgres:16-alpine") as pg:
+        yield pg
+
+
+@pytest.fixture(scope="session")
+def database_url(pg_container: PostgresContainer) -> str:
+    raw = pg_container.get_connection_url()
+    if raw.startswith("postgresql+psycopg2://"):
+        async_url = raw.replace("postgresql+psycopg2://", "postgresql+asyncpg://")
+    elif raw.startswith("postgresql://"):
+        async_url = raw.replace("postgresql://", "postgresql+asyncpg://", 1)
+    else:
+        async_url = raw
+    return async_url
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _apply_migrations(database_url: str) -> Iterator[None]:
+    os.environ["MONLET_DATABASE_URL"] = database_url
+    reset_settings_cache()
+    cfg = Config(os.path.join(SERVER_ROOT, "alembic.ini"))
+    cfg.set_main_option("script_location", os.path.join(SERVER_ROOT, "alembic"))
+    cfg.set_main_option("sqlalchemy.url", get_settings().sync_database_url)
+    command.upgrade(cfg, "head")
+    yield
+
+
+@pytest_asyncio.fixture
+async def engine(database_url: str):
+    eng = create_async_engine(database_url, future=True)
+    yield eng
+    await eng.dispose()
+
+
+@pytest_asyncio.fixture
+async def session(engine) -> AsyncIterator[AsyncSession]:
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    async with sm() as s:
+        yield s
+
+
+@pytest_asyncio.fixture(autouse=True)
+async def _truncate_tables(engine) -> AsyncIterator[None]:
+    yield
+    from sqlalchemy import text
+
+    async with engine.begin() as conn:
+        await conn.execute(
+            text(
+                "TRUNCATE TABLE notification_outbox, incidents, events, checks, agents RESTART IDENTITY CASCADE"
+            )
+        )
+
+
+@pytest_asyncio.fixture
+async def app_client(database_url: str) -> AsyncIterator[AsyncClient]:
+    reset_settings_cache()
+    await db_module.dispose_engine()
+    app = create_app()
+    async with app.router.lifespan_context(app):
+        transport = ASGITransport(app=app)
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            yield client
+    await db_module.dispose_engine()
+
+
+@pytest.fixture
+def auth_headers() -> dict[str, str]:
+    return {"Authorization": "Bearer test-token"}

server/tests/test_auth.py

@@ -0,0 +1,35 @@
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_health_no_auth(app_client):
+    r = await app_client.get("/api/v1/health")
+    assert r.status_code == 200
+    assert r.json()["status"] == "ok"
+
+
+@pytest.mark.asyncio
+async def test_metrics_no_auth(app_client):
+    r = await app_client.get("/metrics")
+    assert r.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_missing_token_401(app_client):
+    r = await app_client.get("/api/v1/agents")
+    assert r.status_code == 401
+    body = r.json()
+    assert body["error"]["code"] == "unauthorized"
+    assert body["error"]["request_id"]
+
+
+@pytest.mark.asyncio
+async def test_wrong_token_401(app_client):
+    r = await app_client.get("/api/v1/agents", headers={"Authorization": "Bearer nope"})
+    assert r.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_valid_token_200(app_client, auth_headers):
+    r = await app_client.get("/api/v1/agents", headers=auth_headers)
+    assert r.status_code == 200

server/tests/test_body_limit.py

@@ -0,0 +1,30 @@
+import json
+import uuid
+
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_body_too_large_413(app_client, auth_headers):
+    big_event = make_event(output="x" * 8000)
+    payload = {"agent_id": "agent-1", "events": [big_event] * 200}
+    raw = json.dumps(payload) + " " * (1_048_577 - len(json.dumps(payload)))
+    rid_in = str(uuid.uuid4())
+    r = await app_client.post(
+        "/api/v1/events",
+        content=raw,
+        headers={**auth_headers, "Content-Type": "application/json", "X-Request-Id": rid_in},
+    )
+    assert r.status_code == 413
+    body = r.json()
+    assert body["error"]["code"] == "payload_too_large"
+    assert body["error"]["request_id"] == rid_in
+    assert r.headers.get("X-Request-Id") == rid_in
+
+
+@pytest.mark.asyncio
+async def test_small_body_ok(app_client, auth_headers):
+    r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    assert r.status_code == 202

server/tests/test_constraints.py

@@ -0,0 +1,70 @@
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.exc import IntegrityError
+
+from monlet_server.models import Incident
+
+
+@pytest.mark.asyncio
+async def test_event_id_conflict_do_nothing(engine):
+    eid = str(uuid4())
+    async with engine.begin() as conn:
+        await conn.execute(
+            text(
+                "INSERT INTO agents (agent_id, hostname, features, status, last_seen_at) "
+                "VALUES ('a','h',jsonb_build_object('push', true, 'metrics', false),'alive', now())"
+            )
+        )
+        await conn.execute(
+            text(
+                "INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
+                " VALUES (:e,'a','c', now(),'ok',0,1,'a:c')"
+            ),
+            {"e": eid},
+        )
+        r = await conn.execute(
+            text(
+                "INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
+                " VALUES (:e,'a','c', now(),'ok',0,1,'a:c') ON CONFLICT (event_id) DO NOTHING RETURNING event_id"
+            ),
+            {"e": eid},
+        )
+        assert r.scalar_one_or_none() is None
+
+
+@pytest.mark.asyncio
+async def test_partial_unique_open_incident(session):
+    key = "k1"
+    now = datetime.now(UTC)
+    eid = uuid4()
+    session.add(
+        Incident(
+            id=uuid4(),
+            incident_key=key,
+            agent_id="a",
+            check_id="c",
+            state="open",
+            severity="warning",
+            opened_at=now,
+            last_event_id=eid,
+        )
+    )
+    await session.commit()
+    session.add(
+        Incident(
+            id=uuid4(),
+            incident_key=key,
+            agent_id="a",
+            check_id="c",
+            state="open",
+            severity="critical",
+            opened_at=now,
+            last_event_id=eid,
+        )
+    )
+    with pytest.raises(IntegrityError):
+        await session.commit()
+    await session.rollback()

server/tests/test_detector.py

@@ -0,0 +1,109 @@
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import func, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from monlet_server.models import Agent, Event, Incident, NotificationOutbox
+from monlet_server.services.detector import _tick
+from monlet_server.settings import reset_settings_cache
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_detector_transitions(app_client, auth_headers, engine, session):
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
+    )
+
+    now = datetime.now(UTC)
+    await session.execute(
+        update(Agent)
+        .where(Agent.agent_id == "agent-stale")
+        .values(last_seen_at=now - timedelta(seconds=120))
+    )
+    await session.execute(
+        update(Agent)
+        .where(Agent.agent_id == "agent-dead")
+        .values(last_seen_at=now - timedelta(minutes=10))
+    )
+    await session.commit()
+
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    await _tick(sm)
+
+    statuses = {
+        r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
+    }
+    assert statuses["agent-alive"] == "alive"
+    assert statuses["agent-stale"] == "stale"
+    assert statuses["agent-dead"] == "dead"
+
+
+@pytest.mark.asyncio
+async def test_detector_prunes_bounded_history(
+    app_client, auth_headers, engine, session, monkeypatch
+):
+    monkeypatch.setenv("MONLET_EVENTS_RETENTION_MAX_ROWS", "2")
+    monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
+    reset_settings_cache()
+    try:
+        await app_client.post(
+            "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
+        )
+        events = [make_event(check_id="prune_check") for _ in range(5)]
+        await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-prune", "events": events},
+            headers=auth_headers,
+        )
+        critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
+        await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-prune", "events": [critical]},
+            headers=auth_headers,
+        )
+
+        inc = (
+            await session.execute(select(Incident).where(Incident.check_id == "critical_prune"))
+        ).scalar_one()
+        now = datetime.now(UTC)
+        for _ in range(5):
+            session.add(
+                NotificationOutbox(
+                    id=uuid4(),
+                    incident_id=inc.id,
+                    notifier="debug",
+                    event_type="firing",
+                    state="sent",
+                    attempts=1,
+                    next_attempt_at=None,
+                    payload={},
+                    created_at=now,
+                    updated_at=now,
+                )
+            )
+        await session.commit()
+
+        sm = async_sessionmaker(engine, expire_on_commit=False)
+        await _tick(sm)
+
+        assert (await session.execute(select(func.count()).select_from(Event))).scalar_one() == 2
+        terminal_outbox = (
+            await session.execute(
+                select(func.count())
+                .select_from(NotificationOutbox)
+                .where(NotificationOutbox.state == "sent")
+            )
+        ).scalar_one()
+        assert terminal_outbox == 2
+    finally:
+        reset_settings_cache()

server/tests/test_events_ingestion.py

@@ -0,0 +1,73 @@
+import pytest
+from sqlalchemy import func, select
+
+from monlet_server.models import Check, Event
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_event_accepted_and_state(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="ok")
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r.status_code == 202
+    assert r.json() == {"accepted": 1, "deduplicated": 0}
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 1
+    chk = (await session.execute(select(Check))).scalar_one()
+    assert chk.status == "ok"
+
+
+@pytest.mark.asyncio
+async def test_duplicate_event_deduplicated(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2)
+    r1 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    r2 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r1.json() == {"accepted": 1, "deduplicated": 0}
+    assert r2.json() == {"accepted": 0, "deduplicated": 1}
+
+
+@pytest.mark.asyncio
+async def test_empty_batch_400(app_client, auth_headers):
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": []}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_oversize_batch_400(app_client, auth_headers):
+    evs = [make_event() for _ in range(201)]
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_late_event_does_not_move_state(app_client, auth_headers, session):
+    from datetime import UTC, datetime, timedelta
+
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    newer = datetime.now(UTC).replace(microsecond=0)
+    older = (newer - timedelta(minutes=5)).isoformat()
+    ev_now = make_event(status="critical", exit_code=2, observed_at=newer.isoformat())
+    ev_old = make_event(status="ok", exit_code=0, observed_at=older)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev_now]}, headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev_old]}, headers=auth_headers
+    )
+    chk = (await session.execute(select(Check))).scalar_one()
+    assert chk.status == "critical"
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 2

server/tests/test_heartbeat.py

@@ -0,0 +1,61 @@
+import pytest
+from sqlalchemy import select
+
+from monlet_server.models import Agent
+
+from ._helpers import make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_creates_agent(app_client, auth_headers, session):
+    r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    assert r.status_code == 202
+    res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
+    a = res.scalar_one()
+    assert a.hostname == "host-1"
+    assert a.status == "alive"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_upsert(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    hb = make_heartbeat()
+    hb["features"] = {"push": True, "metrics": False}
+    await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
+    a = res.scalar_one()
+    assert a.features == {"push": True, "metrics": False}
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_validation_400(app_client, auth_headers):
+    r = await app_client.post(
+        "/api/v1/heartbeat",
+        json={
+            "agent_id": "bad id!",
+            "observed_at": "x",
+            "hostname": "h",
+            "features": {"push": True, "metrics": False},
+        },
+        headers=auth_headers,
+    )
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_rejects_extra_fields(app_client, auth_headers):
+    hb = make_heartbeat()
+    hb["mode"] = "hybrid"
+    r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_rejects_sensitive_label_keys(app_client, auth_headers):
+    hb = make_heartbeat()
+    hb["labels"] = {"api_key": "secret"}
+    r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"

server/tests/test_incident_key.py

@@ -0,0 +1,55 @@
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_incident_key_ownership_rejected(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-1"), headers=auth_headers)
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-2"), headers=auth_headers)
+
+    ev1 = make_event(status="critical", exit_code=2, incident_key="shared-key")
+    r1 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=auth_headers
+    )
+    assert r1.status_code == 202
+
+    ev2 = make_event(status="critical", exit_code=2, incident_key="shared-key")
+    r2 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=auth_headers
+    )
+    assert r2.status_code == 400
+    assert r2.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_incident_key_too_long(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2, incident_key="x" * 257)
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_events_query_cursor_pagination(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    evs = [make_event() for _ in range(5)]
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
+    )
+    r = await app_client.get("/api/v1/events/query?limit=2", headers=auth_headers)
+    assert r.status_code == 200
+    page1 = r.json()
+    assert len(page1["items"]) == 2
+    assert page1["next_cursor"]
+
+    r2 = await app_client.get(
+        f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=auth_headers
+    )
+    page2 = r2.json()
+    assert len(page2["items"]) == 2
+    ids1 = {i["event_id"] for i in page1["items"]}
+    ids2 = {i["event_id"] for i in page2["items"]}
+    assert ids1.isdisjoint(ids2)