Implement Monlet MVP stack and UI updates

2026-05-27 10:01:59 +04:00
parent dcea096327
commit edc51e9c59
145 changed files with 15618 additions and 248 deletions
--- a/agent/README.md
+++ b/agent/README.md
@@ -14,19 +14,36 @@ Do not use `go install` or `go env -w`. Module cache, build cache, and tool bina

 ## Configuration

-See `config.example.toml`. Required sections: top-level `state_dir`, `[server]` (`url`, `token`), at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`). `agent_id` is optional — if empty, the agent generates and persists a UUID under `state_dir/agent_id`.
+See `config.example.toml`. Required fields: top-level `state_dir`, at least one `[[checks]]` (`id`, `command`, `interval`, `timeout`), and at least one output enabled: `server.enabled = true` or `metrics.enabled = true`. `agent_id` is optional; if omitted, it defaults to `hostname`.

 | Key | Default | Notes |
 |---|---|---|
-| `mode` | `hybrid` | `prometheus_only` (only `/metrics`), `push_only` (only heartbeat+events), `hybrid` (both). `prometheus_only` does not require `server.url`/`server.token`. |
+| `server.enabled` | `false` | Enables heartbeat/event push. Requires `server.url` and `server.token` when true. |
+| `metrics.enabled` | `false` | Enables `/metrics`. |
+| `labels` | `{}` | Optional heartbeat labels. Max 31 custom labels because `monlet_agent_version` is generated by the agent. Secret-like keys are rejected. |
 | `server.heartbeat_interval` | `30s` | ADR-0005 |
 | `server.batch_interval` | `10s` | events flush cadence |
 | `metrics.listen` | `127.0.0.1:9465` | low-cardinality only |
-| `checks[].notification_owner` | `server` | `server`, `prometheus`, `none` |
+| `checks[].command` | required | Shell string executed as `/bin/sh -c`; TOML multi-line strings are supported. |
+| `checks[].notifications_enabled` | `true` | Set `false` for checks that must not create server notifications. |
+
+Example:
+
+```toml
+[[checks]]
+id = "disk_root"
+command = '''
+set -eu
+/usr/local/lib/monlet/check_disk_root.sh
+'''
+interval = "60s"
+timeout = "10s"
+```

 ## Behaviour

 - Per-check scheduler with no-overlap: a tick is skipped if the previous run is still in flight (`monlet_agent_check_skipped_total`).
+- Check commands run through `/bin/sh -c`; the old argv-array form is intentionally unsupported.
 - Timeout via `exec.CommandContext` with `Setpgid` so the process group is killed on expiry; timeout → `unknown`.
 - Exit code mapping: 0=ok, 1=warning, 2=critical, 3+=unknown (ADR-0005).
 - `output` (stdout+stderr) is truncated by **UTF-8 byte length** to 8 KiB with marker `...[truncated N bytes]`.
@@ -34,6 +51,8 @@ See `config.example.toml`. Required sections: top-level `state_dir`, `[server]`
 - Spool: per-event JSON files in `state_dir/spool/`, FIFO, limits 10 000 events / 50 MiB, drop oldest on overflow (ADR-0005).
 - Retry backoff for heartbeat and events: 1s → 30s, exp ×2, jitter ±20%. 4xx are dropped (not retried) to avoid hot loop.
 - Bearer auth header on both endpoints (ADR-0007).
+- Heartbeats include configured labels plus reserved `monlet_agent_version=<binary version>`.
+- Heartbeats include explicit feature flags: `push` from `server.enabled`, `metrics` from `metrics.enabled`.

 ## Prometheus metrics

@@ -52,7 +71,7 @@ Exposed on `metrics.listen` at `/metrics`. Labels are static (`check_id`, `statu
 - `monlet_agent_send_attempts_total{kind}`, `monlet_agent_send_failures_total{kind}` (`kind` ∈ `heartbeat`, `events`)
 - `monlet_agent_events_accepted_total`, `monlet_agent_events_deduplicated_total`
 - `monlet_agent_last_heartbeat_timestamp_seconds`
- `monlet_agent_build_info{version,mode}` (constant 1)
+- `monlet_agent_build_info{version,push,metrics}` (constant 1)

 ## systemd

--- a/agent/cmd/monlet-agent/main.go
+++ b/agent/cmd/monlet-agent/main.go
@@ -17,7 +17,7 @@ import (
 	"github.com/monlet/agent/internal/spool"
 )

-var version = "dev"
+var version = "0.1.0"

 func main() {
 	cfgPath := flag.String("config", "config.toml", "path to TOML config")
@@ -31,7 +31,7 @@ func main() {
 		os.Exit(1)
 	}

-	agentID, err := ids.LoadOrCreateAgentID(cfg.StateDir, cfg.AgentID)
+	agentID, err := ids.ResolveAgentID(cfg.AgentID, cfg.Hostname)
 	if err != nil {
 		log.Error("agent_id", "err", err)
 		os.Exit(1)
@@ -47,7 +47,7 @@ func main() {
 		Cfg:     cfg,
 		AgentID: agentID,
 		Version: version,
-		Client:  client.New(cfg.Server.URL, cfg.Server.Token, agentID, version),
+		Client:  client.New(cfg.Server.URL, cfg.Server.Token, agentID),
 		Spool:   sp,
 		Metrics: metrics.New(),
 		Log:     log,
--- a/agent/config.example.toml
+++ b/agent/config.example.toml
@@ -1,9 +1,12 @@
-agent_id = "example-host-01"
 hostname = "example-host-01"
-mode = "hybrid"
 state_dir = "/var/lib/monlet-agent"

+[labels]
+env = "prod"
+role = "api"
+
 [server]
+enabled = true
 url = "http://127.0.0.1:8000"
 token = "replace-me"
 heartbeat_interval = "30s"
@@ -16,7 +19,8 @@ listen = "127.0.0.1:9465"
 [[checks]]
 id = "disk_root"
 name = "Root disk usage"
-command = ["/usr/local/lib/monlet/check_disk_root.sh"]
+command = '''
+/usr/local/lib/monlet/check_disk_root.sh
+'''
 interval = "60s"
 timeout = "10s"
-notification_owner = "server"
--- a/agent/internal/app/app.go
+++ b/agent/internal/app/app.go
@@ -28,7 +28,7 @@ func (a *App) Run(ctx context.Context) error {
 	events := make(chan scheduler.Event, 256)

 	a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
-	a.Metrics.BuildInfo.WithLabelValues(a.Version, a.Cfg.Mode).Set(1)
+	a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(a.Cfg.PushesToServer()), boolLabel(a.Cfg.ExposesMetrics())).Set(1)
 	for _, c := range a.Cfg.Checks {
 		a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds())
 	}
@@ -122,8 +122,11 @@ func (a *App) heartbeatLoop(ctx context.Context) {
 			AgentID:    a.AgentID,
 			ObservedAt: time.Now().UTC(),
 			Hostname:   a.Cfg.Hostname,
-			Version:    a.Version,
-			Mode:       a.Cfg.Mode,
+			Features: client.AgentFeatures{
+				Push:    a.Cfg.PushesToServer(),
+				Metrics: a.Cfg.ExposesMetrics(),
+			},
+			Labels: a.Cfg.HeartbeatLabels(a.Version),
 		}
 		a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc()
 		err := a.Client.SendHeartbeat(ctx, hb)
@@ -210,3 +213,10 @@ func sleep(ctx context.Context, d time.Duration) bool {
 		return false
 	}
 }
+
+func boolLabel(v bool) string {
+	if v {
+		return "true"
+	}
+	return "false"
+}
--- a/agent/internal/app/app_test.go
+++ b/agent/internal/app/app_test.go
@@ -30,22 +30,20 @@ func TestPrometheusOnlyDoesNotPush(t *testing.T) {
 	dir := t.TempDir()
 	cfg := &config.Config{
 		AgentID:  "a1",
-		Mode:     "prometheus_only",
 		StateDir: dir,
-		Server:   config.ServerConfig{URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
-		Metrics:  config.MetricsConfig{Enabled: false},
+		Server:   config.ServerConfig{Enabled: boolPtr(false), URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
+		Metrics:  config.MetricsConfig{Enabled: true, Listen: "127.0.0.1:0"},
 		Checks: []config.CheckConfig{{
-			ID:                "c1",
-			Command:           []string{"sh", "-c", "echo hi"},
-			Interval:          config.Duration{Duration: 100 * time.Millisecond},
-			Timeout:           config.Duration{Duration: 50 * time.Millisecond},
-			NotificationOwner: "server",
+			ID:       "c1",
+			Command:  "echo hi",
+			Interval: config.Duration{Duration: 100 * time.Millisecond},
+			Timeout:  config.Duration{Duration: 50 * time.Millisecond},
 		}},
 	}
 	sp, _ := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
 	a := &App{
 		Cfg: cfg, AgentID: "a1", Version: "t",
-		Client: client.New(srv.URL, "t", "a1", "t"),
+		Client: client.New(srv.URL, "t", "a1"),
 		Spool:  sp, Metrics: metrics.New(),
 		Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
 	}
@@ -62,16 +60,22 @@ func TestPrometheusOnlyDoesNotPush(t *testing.T) {

 func TestEndToEndPushAndReplay(t *testing.T) {
 	var (
-		mu           sync.Mutex
-		seenEventIDs = make(map[string]int)
-		hbCount      int32
-		fail         atomic.Bool
+		mu            sync.Mutex
+		seenEventIDs  = make(map[string]int)
+		lastHeartbeat client.HeartbeatRequest
+		hbCount       int32
+		fail          atomic.Bool
 	)
 	fail.Store(true)

 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
 		case "/api/v1/heartbeat":
+			var req client.HeartbeatRequest
+			_ = json.NewDecoder(r.Body).Decode(&req)
+			mu.Lock()
+			lastHeartbeat = req
+			mu.Unlock()
 			atomic.AddInt32(&hbCount, 1)
 			w.WriteHeader(http.StatusAccepted)
 			_, _ = io.WriteString(w, `{"accepted":true}`)
@@ -97,9 +101,10 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 	cfg := &config.Config{
 		AgentID:  "a1",
 		Hostname: "h",
-		Mode:     "hybrid",
 		StateDir: dir,
+		Labels:   map[string]string{"env": "test"},
 		Server: config.ServerConfig{
+			Enabled:           boolPtr(true),
 			URL:               srv.URL,
 			Token:             "t",
 			HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond},
@@ -107,11 +112,10 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 		},
 		Metrics: config.MetricsConfig{Enabled: false},
 		Checks: []config.CheckConfig{{
-			ID:                "c1",
-			Command:           []string{"sh", "-c", "echo hi"},
-			Interval:          config.Duration{Duration: 150 * time.Millisecond},
-			Timeout:           config.Duration{Duration: 100 * time.Millisecond},
-			NotificationOwner: "server",
+			ID:       "c1",
+			Command:  "echo hi",
+			Interval: config.Duration{Duration: 150 * time.Millisecond},
+			Timeout:  config.Duration{Duration: 100 * time.Millisecond},
 		}},
 	}
 	sp, err := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
@@ -122,7 +126,7 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 		Cfg:     cfg,
 		AgentID: "a1",
 		Version: "test",
-		Client:  client.New(cfg.Server.URL, "t", "a1", "test"),
+		Client:  client.New(cfg.Server.URL, "t", "a1"),
 		Spool:   sp,
 		Metrics: metrics.New(),
 		Log:     slog.New(slog.NewTextHandler(io.Discard, nil)),
@@ -168,4 +172,17 @@ func TestEndToEndPushAndReplay(t *testing.T) {
 	if atomic.LoadInt32(&hbCount) == 0 {
 		t.Fatal("no heartbeats observed")
 	}
+	if lastHeartbeat.Labels["env"] != "test" {
+		t.Fatalf("configured heartbeat label missing: %#v", lastHeartbeat.Labels)
+	}
+	if lastHeartbeat.Labels[config.AgentVersionLabel] != "test" {
+		t.Fatalf("version heartbeat label missing: %#v", lastHeartbeat.Labels)
+	}
+	if !lastHeartbeat.Features.Push || lastHeartbeat.Features.Metrics {
+		t.Fatalf("unexpected heartbeat features: %+v", lastHeartbeat.Features)
+	}
+}
+
+func boolPtr(v bool) *bool {
+	return &v
 }
--- a/agent/internal/client/client.go
+++ b/agent/internal/client/client.go
@@ -21,26 +21,28 @@ type Client struct {
 	baseURL string
 	token   string
 	agentID string
-	version string
 	http    *http.Client
 }

-func New(baseURL, token, agentID, version string) *Client {
+func New(baseURL, token, agentID string) *Client {
 	return &Client{
 		baseURL: strings.TrimRight(baseURL, "/"),
 		token:   token,
 		agentID: agentID,
-		version: version,
 		http:    &http.Client{Timeout: 15 * time.Second},
 	}
 }

+type AgentFeatures struct {
+	Push    bool `json:"push"`
+	Metrics bool `json:"metrics"`
+}
+
 type HeartbeatRequest struct {
 	AgentID    string            `json:"agent_id"`
 	ObservedAt time.Time         `json:"observed_at"`
 	Hostname   string            `json:"hostname"`
-	Version    string            `json:"version"`
-	Mode       string            `json:"mode,omitempty"`
+	Features   AgentFeatures     `json:"features"`
 	Labels     map[string]string `json:"labels,omitempty"`
 }

@@ -65,7 +67,11 @@ func (c *Client) SendEvents(ctx context.Context, events []scheduler.Event) (*Eve
 	if len(events) > MaxBatchEvents {
 		return nil, fmt.Errorf("batch exceeds %d events", MaxBatchEvents)
 	}
-	req := EventBatchRequest{AgentID: c.agentID, Events: events}
+	wireEvents := make([]scheduler.Event, len(events))
+	for i, ev := range events {
+		wireEvents[i] = ev.WireEvent()
+	}
+	req := EventBatchRequest{AgentID: c.agentID, Events: wireEvents}
 	var resp EventBatchResponse
 	if err := c.post(ctx, "/api/v1/events", req, &resp); err != nil {
 		return nil, err
--- a/agent/internal/client/client_test.go
+++ b/agent/internal/client/client_test.go
@@ -24,8 +24,13 @@ func TestSendHeartbeatSuccess(t *testing.T) {
 		_, _ = io.WriteString(w, `{"accepted":true}`)
 	}))
 	defer srv.Close()
-	c := New(srv.URL, "tok", "a1", "v0")
-	err := c.SendHeartbeat(context.Background(), HeartbeatRequest{AgentID: "a1", ObservedAt: time.Now().UTC(), Hostname: "h", Version: "v0"})
+	c := New(srv.URL, "tok", "a1")
+	err := c.SendHeartbeat(context.Background(), HeartbeatRequest{
+		AgentID:    "a1",
+		ObservedAt: time.Now().UTC(),
+		Hostname:   "h",
+		Features:   AgentFeatures{Push: true, Metrics: false},
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -42,7 +47,7 @@ func TestSendEventsBatch(t *testing.T) {
 		_, _ = io.WriteString(w, `{"accepted":2,"deduplicated":0}`)
 	}))
 	defer srv.Close()
-	c := New(srv.URL, "tok", "a1", "v0")
+	c := New(srv.URL, "tok", "a1")
 	resp, err := c.SendEvents(context.Background(), []scheduler.Event{{EventID: "x"}, {EventID: "y"}})
 	if err != nil {
 		t.Fatal(err)
@@ -53,7 +58,7 @@ func TestSendEventsBatch(t *testing.T) {
 }

 func TestSendEventsTooBig(t *testing.T) {
-	c := New("http://x", "t", "a1", "v0")
+	c := New("http://x", "t", "a1")
 	big := make([]scheduler.Event, MaxBatchEvents+1)
 	_, err := c.SendEvents(context.Background(), big)
 	if err == nil {
--- a/agent/internal/config/config.go
+++ b/agent/internal/config/config.go
@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strings"
 	"time"
+	"unicode/utf8"

 	"github.com/BurntSushi/toml"
 )
@@ -13,22 +15,27 @@ var idPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]+$`)

 const (
 	maxIDLen           = 128
+	maxLabelKeyLen     = 64
+	maxLabelValueLen   = 256
+	maxLabels          = 32
+	AgentVersionLabel  = "monlet_agent_version"
 	defaultHeartbeat   = 30 * time.Second
 	defaultBatch       = 10 * time.Second
 	defaultMetricsAddr = "127.0.0.1:9465"
 )

 type Config struct {
-	AgentID  string        `toml:"agent_id"`
-	Hostname string        `toml:"hostname"`
-	Mode     string        `toml:"mode"`
-	StateDir string        `toml:"state_dir"`
-	Server   ServerConfig  `toml:"server"`
-	Metrics  MetricsConfig `toml:"metrics"`
-	Checks   []CheckConfig `toml:"checks"`
+	AgentID  string            `toml:"agent_id"`
+	Hostname string            `toml:"hostname"`
+	StateDir string            `toml:"state_dir"`
+	Labels   map[string]string `toml:"labels"`
+	Server   ServerConfig      `toml:"server"`
+	Metrics  MetricsConfig     `toml:"metrics"`
+	Checks   []CheckConfig     `toml:"checks"`
 }

 type ServerConfig struct {
+	Enabled           *bool    `toml:"enabled"`
 	URL               string   `toml:"url"`
 	Token             string   `toml:"token"`
 	HeartbeatInterval Duration `toml:"heartbeat_interval"`
@@ -41,13 +48,13 @@ type MetricsConfig struct {
 }

 type CheckConfig struct {
-	ID                string   `toml:"id"`
-	Name              string   `toml:"name"`
-	Command           []string `toml:"command"`
-	Interval          Duration `toml:"interval"`
-	Timeout           Duration `toml:"timeout"`
-	NotificationOwner string   `toml:"notification_owner"`
-	DedupeKey         string   `toml:"dedupe_key"`
+	ID                   string   `toml:"id"`
+	Name                 string   `toml:"name"`
+	Command              string   `toml:"command"`
+	Interval             Duration `toml:"interval"`
+	Timeout              Duration `toml:"timeout"`
+	NotificationsEnabled *bool    `toml:"notifications_enabled"`
+	DedupeKey            string   `toml:"dedupe_key"`
 }

 type Duration struct{ time.Duration }
@@ -63,9 +70,17 @@ func (d *Duration) UnmarshalText(b []byte) error {

 func Load(path string) (*Config, error) {
 	var c Config
-	if _, err := toml.DecodeFile(path, &c); err != nil {
+	md, err := toml.DecodeFile(path, &c)
+	if err != nil {
 		return nil, fmt.Errorf("read config: %w", err)
 	}
+	if undecoded := md.Undecoded(); len(undecoded) > 0 {
+		keys := make([]string, len(undecoded))
+		for i, key := range undecoded {
+			keys[i] = key.String()
+		}
+		return nil, fmt.Errorf("unknown config keys: %s", strings.Join(keys, ", "))
+	}
 	c.applyDefaults()
 	if err := c.Validate(); err != nil {
 		return nil, err
@@ -74,9 +89,6 @@ func Load(path string) (*Config, error) {
 }

 func (c *Config) applyDefaults() {
-	if c.Mode == "" {
-		c.Mode = "hybrid"
-	}
 	if c.Hostname == "" {
 		if h, err := os.Hostname(); err == nil {
 			c.Hostname = h
@@ -99,21 +111,27 @@ func (c *Config) Validate() error {
 			return err
 		}
 	}
-	switch c.Mode {
-	case "prometheus_only", "push_only", "hybrid":
-	default:
-		return fmt.Errorf("invalid mode %q", c.Mode)
+	if c.Hostname == "" {
+		return fmt.Errorf("hostname is required")
 	}
 	if c.StateDir == "" {
 		return fmt.Errorf("state_dir is required")
 	}
+	if err := validateLabels(c.Labels); err != nil {
+		return err
+	}
 	if c.PushesToServer() {
 		if c.Server.URL == "" {
-			return fmt.Errorf("server.url is required for mode %q", c.Mode)
+			return fmt.Errorf("server.url is required when server.enabled = true")
 		}
 		if c.Server.Token == "" {
-			return fmt.Errorf("server.token is required for mode %q", c.Mode)
+			return fmt.Errorf("server.token is required when server.enabled = true")
 		}
+	} else if c.Server.URL != "" || c.Server.Token != "" {
+		return fmt.Errorf("server.url/token require server.enabled = true")
+	}
+	if !c.PushesToServer() && !c.ExposesMetrics() {
+		return fmt.Errorf("server.enabled or metrics.enabled must be true")
 	}
 	if len(c.Checks) == 0 {
 		return fmt.Errorf("at least one check is required")
@@ -128,7 +146,7 @@ func (c *Config) Validate() error {
 			return fmt.Errorf("duplicate check id %q", ch.ID)
 		}
 		seen[ch.ID] = struct{}{}
-		if len(ch.Command) == 0 {
+		if strings.TrimSpace(ch.Command) == "" {
 			return fmt.Errorf("check %q: command is required", ch.ID)
 		}
 		if ch.Interval.Duration <= 0 {
@@ -140,14 +158,6 @@ func (c *Config) Validate() error {
 		if ch.Timeout.Duration > ch.Interval.Duration {
 			return fmt.Errorf("check %q: timeout must be <= interval", ch.ID)
 		}
-		switch ch.NotificationOwner {
-		case "", "server", "prometheus", "none":
-		default:
-			return fmt.Errorf("check %q: invalid notification_owner %q", ch.ID, ch.NotificationOwner)
-		}
-		if ch.NotificationOwner == "" {
-			ch.NotificationOwner = "server"
-		}
 		if len(ch.DedupeKey) > 256 {
 			return fmt.Errorf("check %q: dedupe_key too long", ch.ID)
 		}
@@ -155,15 +165,31 @@ func (c *Config) Validate() error {
 	return nil
 }

-// PushesToServer is true when the mode requires heartbeat/events push.
+// PushesToServer is true when heartbeat/events push is enabled.
 func (c *Config) PushesToServer() bool {
-	return c.Mode == "push_only" || c.Mode == "hybrid"
+	return c.Server.Enabled != nil && *c.Server.Enabled
 }

 // ExposesMetrics is true when the agent should serve /metrics.
-// `metrics.enabled` is an independent gate; both must agree.
 func (c *Config) ExposesMetrics() bool {
-	return c.Metrics.Enabled && (c.Mode == "prometheus_only" || c.Mode == "hybrid")
+	return c.Metrics.Enabled
+}
+
+func (c *Config) HeartbeatLabels(version string) map[string]string {
+	labels := make(map[string]string, len(c.Labels)+1)
+	for k, v := range c.Labels {
+		labels[k] = v
+	}
+	labels[AgentVersionLabel] = version
+	return labels
+}
+
+func (c CheckConfig) NotificationsOn() bool {
+	return c.NotificationsEnabled == nil || *c.NotificationsEnabled
+}
+
+func (c CheckConfig) Argv() []string {
+	return []string{"/bin/sh", "-c", c.Command}
 }

 func ValidateID(field, v string) error {
@@ -178,3 +204,40 @@ func ValidateID(field, v string) error {
 	}
 	return nil
 }
+
+func validateLabels(labels map[string]string) error {
+	if len(labels) > maxLabels {
+		return fmt.Errorf("labels exceed %d entries", maxLabels)
+	}
+	if _, hasVersion := labels[AgentVersionLabel]; !hasVersion && len(labels) >= maxLabels {
+		return fmt.Errorf("labels leave no room for reserved %q label", AgentVersionLabel)
+	}
+	for k, v := range labels {
+		if k == "" {
+			return fmt.Errorf("label key is empty")
+		}
+		if len(k) > maxLabelKeyLen {
+			return fmt.Errorf("label key %q exceeds %d chars", k, maxLabelKeyLen)
+		}
+		if !idPattern.MatchString(k) {
+			return fmt.Errorf("label key %q has invalid characters", k)
+		}
+		if isSensitiveLabelKey(k) {
+			return fmt.Errorf("label key %q is not allowed", k)
+		}
+		if utf8.RuneCountInString(v) > maxLabelValueLen {
+			return fmt.Errorf("label %q value exceeds %d chars", k, maxLabelValueLen)
+		}
+	}
+	return nil
+}
+
+func isSensitiveLabelKey(k string) bool {
+	normalized := strings.NewReplacer("-", "_", ".", "_", ":", "_").Replace(strings.ToLower(k))
+	for _, word := range []string{"token", "secret", "password", "credential", "authorization", "cookie", "api_key", "apikey"} {
+		if strings.Contains(normalized, word) {
+			return true
+		}
+	}
+	return false
+}
--- a/agent/internal/config/config_test.go
+++ b/agent/internal/config/config_test.go
@@ -1,8 +1,10 @@
 package config

 import (
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )

@@ -20,12 +22,13 @@ agent_id = "host-1"
 state_dir = "/tmp/x"

 [server]
+enabled = true
 url = "http://localhost"
 token = "t"

 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `
@@ -35,14 +38,105 @@ func TestLoadMinimal(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if c.Mode != "hybrid" {
-		t.Errorf("default mode: %q", c.Mode)
+	if !c.PushesToServer() || c.ExposesMetrics() {
+		t.Errorf("features: push=%v metrics=%v", c.PushesToServer(), c.ExposesMetrics())
 	}
 	if c.Server.HeartbeatInterval.Duration == 0 {
 		t.Error("default heartbeat not applied")
 	}
-	if c.Checks[0].NotificationOwner != "server" {
-		t.Errorf("default notification_owner: %q", c.Checks[0].NotificationOwner)
+	if !c.Checks[0].NotificationsOn() {
+		t.Error("notifications should default on")
+	}
+}
+
+func TestLoadMultilineCommand(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = '''
+set -eu
+echo ok
+'''
+interval = "10s"
+timeout = "5s"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(c.Checks[0].Command, "echo ok") {
+		t.Fatalf("command not loaded: %q", c.Checks[0].Command)
+	}
+	if got := c.Checks[0].Argv(); len(got) != 3 || got[0] != "/bin/sh" || got[1] != "-c" {
+		t.Fatalf("argv: %#v", got)
+	}
+}
+
+func TestCommandArrayRejected(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected command array to be rejected")
+	}
+}
+
+func TestLoadLabels(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[labels]
+env = "prod"
+role = "api"
+
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	labels := c.HeartbeatLabels("1.2.3")
+	if labels["env"] != "prod" || labels["role"] != "api" {
+		t.Fatalf("configured labels missing: %#v", labels)
+	}
+	if labels[AgentVersionLabel] != "1.2.3" {
+		t.Fatalf("version label missing: %#v", labels)
+	}
+}
+
+func TestHeartbeatLabelsVersionLabelWins(t *testing.T) {
+	c := &Config{Labels: map[string]string{AgentVersionLabel: "manual"}}
+	labels := c.HeartbeatLabels("real")
+	if labels[AgentVersionLabel] != "real" {
+		t.Fatalf("version label must be generated from binary version, got %q", labels[AgentVersionLabel])
 	}
 }

@@ -52,11 +146,12 @@ func TestValidateBadID(t *testing.T) {
 agent_id = "bad id!"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -70,11 +165,12 @@ func TestValidateTimeoutExceedsInterval(t *testing.T) {
 agent_id = "x"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "5s"
 timeout = "10s"
 `))
@@ -83,13 +179,69 @@ timeout = "10s"
 	}
 }

-func TestPrometheusOnlyAllowsMissingServer(t *testing.T) {
-	c, err := Load(writeTmp(t, `
-mode = "prometheus_only"
+func TestValidateBadLabelKey(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "x"
 state_dir = "/tmp/x"
+[labels]
+"bad key" = "x"
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected bad label key error")
+	}
+}
+
+func TestValidateSensitiveLabelKey(t *testing.T) {
+	for _, key := range []string{"token", "api_key", "api-key", "password", "credential", "authorization", "cookie"} {
+		if err := validateLabels(map[string]string{key: "x"}); err == nil {
+			t.Fatalf("expected sensitive label key error for %q", key)
+		}
+	}
+}
+
+func TestValidateLabelsReservedSlot(t *testing.T) {
+	labels := make(map[string]string, maxLabels)
+	for i := 0; i < maxLabels; i++ {
+		labels[fmt.Sprintf("k%d", i)] = "v"
+	}
+	if err := validateLabels(labels); err == nil {
+		t.Fatal("expected reserved label slot error")
+	}
+	delete(labels, "k0")
+	labels[AgentVersionLabel] = "manual"
+	if err := validateLabels(labels); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestValidateLabelValueMaxLength(t *testing.T) {
+	if err := validateLabels(map[string]string{"note": strings.Repeat("ю", maxLabelValueLen)}); err != nil {
+		t.Fatal(err)
+	}
+	if err := validateLabels(map[string]string{"note": strings.Repeat("x", maxLabelValueLen+1)}); err == nil {
+		t.Fatal("expected label value length error")
+	}
+}
+
+func TestPrometheusOnlyAllowsMissingServer(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+state_dir = "/tmp/x"
+[server]
+enabled = false
+[metrics]
+enabled = true
+[[checks]]
+id = "c1"
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -97,17 +249,21 @@ timeout = "5s"
 		t.Fatal(err)
 	}
 	if c.PushesToServer() {
-		t.Fatal("prometheus_only must not push")
+		t.Fatal("metrics-only config must not push")
+	}
+	if !c.ExposesMetrics() {
+		t.Fatal("metrics-only config must expose metrics")
 	}
 }

 func TestPushOnlyRequiresServer(t *testing.T) {
 	_, err := Load(writeTmp(t, `
-mode = "push_only"
 state_dir = "/tmp/x"
+[server]
+enabled = true
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
@@ -116,21 +272,52 @@ timeout = "5s"
 	}
 }

+func TestUnknownConfigKeyRejected(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+mode = "hybrid"
+state_dir = "/tmp/x"
+[server]
+enabled = true
+url = "http://localhost"
+token = "t"
+[[checks]]
+id = "c1"
+command = "true"
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected unknown key error")
+	}
+}
+
+func TestNotificationsEnabledCanDisable(t *testing.T) {
+	c, err := Load(writeTmp(t, minimal+`
+notifications_enabled = false
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.Checks[0].NotificationsOn() {
+		t.Fatal("notifications_enabled=false was not applied")
+	}
+}
+
 func TestExposesMetricsGate(t *testing.T) {
 	cases := []struct {
-		mode    string
-		enabled bool
+		server  bool
+		metrics bool
 		want    bool
 	}{
-		{"hybrid", true, true},
-		{"hybrid", false, false},
-		{"prometheus_only", true, true},
-		{"push_only", true, false},
+		{true, true, true},
+		{true, false, false},
+		{false, true, true},
+		{false, false, false},
 	}
 	for _, tc := range cases {
-		c := &Config{Mode: tc.mode, Metrics: MetricsConfig{Enabled: tc.enabled}}
+		c := &Config{Server: ServerConfig{Enabled: &tc.server}, Metrics: MetricsConfig{Enabled: tc.metrics}}
 		if got := c.ExposesMetrics(); got != tc.want {
-			t.Errorf("mode=%s enabled=%v want=%v got=%v", tc.mode, tc.enabled, tc.want, got)
+			t.Errorf("server=%v metrics=%v want=%v got=%v", tc.server, tc.metrics, tc.want, got)
 		}
 	}
 }
@@ -140,16 +327,17 @@ func TestValidateDuplicateCheckID(t *testing.T) {
 agent_id = "x"
 state_dir = "/tmp/x"
 [server]
+enabled = true
 url = "http://localhost"
 token = "t"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 [[checks]]
 id = "c1"
-command = ["true"]
+command = "true"
 interval = "10s"
 timeout = "5s"
 `))
--- a/agent/internal/ids/ids.go
+++ b/agent/internal/ids/ids.go
@@ -2,38 +2,24 @@ package ids

 import (
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"

-	"github.com/google/uuid"
-
 	"github.com/monlet/agent/internal/config"
 )

-// LoadOrCreateAgentID returns configured ID if non-empty, otherwise loads
-// state_dir/agent_id, generating and persisting it on first run.
-func LoadOrCreateAgentID(stateDir, configured string) (string, error) {
+func ResolveAgentID(configured, hostname string) (string, error) {
 	if configured != "" {
-		return configured, nil
-	}
-	if err := os.MkdirAll(stateDir, 0o755); err != nil {
-		return "", fmt.Errorf("create state dir: %w", err)
-	}
-	path := filepath.Join(stateDir, "agent_id")
-	b, err := os.ReadFile(path)
-	if err == nil {
-		id := strings.TrimSpace(string(b))
+		id := strings.TrimSpace(configured)
 		if err := config.ValidateID("agent_id", id); err != nil {
-			return "", fmt.Errorf("persisted agent_id invalid: %w", err)
+			return "", err
 		}
 		return id, nil
 	}
-	if !os.IsNotExist(err) {
-		return "", err
+	id := strings.TrimSpace(hostname)
+	if id == "" {
+		return "", fmt.Errorf("hostname is empty")
 	}
-	id := uuid.NewString()
-	if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil {
+	if err := config.ValidateID("hostname as agent_id", id); err != nil {
 		return "", err
 	}
 	return id, nil
--- a/agent/internal/ids/ids_test.go
+++ b/agent/internal/ids/ids_test.go
@@ -1,33 +1,17 @@
 package ids

-import (
-	"os"
-	"path/filepath"
-	"testing"
-)
+import "testing"

 func TestConfiguredWins(t *testing.T) {
-	id, err := LoadOrCreateAgentID(t.TempDir(), "fixed")
+	id, err := ResolveAgentID("fixed", "host")
 	if err != nil || id != "fixed" {
 		t.Fatalf("got %q %v", id, err)
 	}
 }

-func TestGenerateAndReuse(t *testing.T) {
-	dir := t.TempDir()
-	id1, err := LoadOrCreateAgentID(dir, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	id2, err := LoadOrCreateAgentID(dir, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if id1 != id2 {
-		t.Fatalf("not stable: %q vs %q", id1, id2)
-	}
-	b, _ := os.ReadFile(filepath.Join(dir, "agent_id"))
-	if len(b) == 0 {
-		t.Fatal("agent_id not persisted")
+func TestHostnameFallback(t *testing.T) {
+	id, err := ResolveAgentID("", "host-1")
+	if err != nil || id != "host-1" {
+		t.Fatalf("got %q %v", id, err)
 	}
 }
--- a/agent/internal/metrics/metrics.go
+++ b/agent/internal/metrics/metrics.go
@@ -117,7 +117,7 @@ func New() *Metrics {
 	m.BuildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "monlet_agent_build_info",
 		Help: "Constant 1 with build labels.",
-	}, []string{"version", "mode"})
+	}, []string{"version", "push", "metrics"})
 	reg.MustRegister(m.CheckRuns, m.CheckSkipped, m.CheckDuration,
 		m.CheckStatus, m.CheckExitCode, m.CheckLastRun, m.CheckLastSuccess,
 		m.CheckInterval, m.ChecksConfigured,
--- a/agent/internal/scheduler/scheduler.go
+++ b/agent/internal/scheduler/scheduler.go
@@ -12,16 +12,24 @@ import (

 // Event is a contract-shaped check result emitted by the scheduler.
 type Event struct {
-	EventID           string    `json:"event_id"`
-	CheckID           string    `json:"check_id"`
-	ObservedAt        time.Time `json:"observed_at"`
-	Status            string    `json:"status"`
-	ExitCode          int       `json:"exit_code"`
-	DurationMs        int64     `json:"duration_ms"`
-	Output            string    `json:"output,omitempty"`
-	OutputTruncated   bool      `json:"output_truncated,omitempty"`
-	NotificationOwner string    `json:"notification_owner,omitempty"`
-	IncidentKey       string    `json:"incident_key,omitempty"`
+	EventID              string    `json:"event_id"`
+	CheckID              string    `json:"check_id"`
+	ObservedAt           time.Time `json:"observed_at"`
+	Status               string    `json:"status"`
+	ExitCode             int       `json:"exit_code"`
+	DurationMs           int64     `json:"duration_ms"`
+	Output               string    `json:"output,omitempty"`
+	OutputTruncated      bool      `json:"output_truncated,omitempty"`
+	NotificationsEnabled *bool     `json:"notifications_enabled,omitempty"`
+	IncidentKey          string    `json:"incident_key,omitempty"`
+}
+
+func (e Event) WireEvent() Event {
+	if e.NotificationsEnabled == nil {
+		enabled := true
+		e.NotificationsEnabled = &enabled
+	}
+	return e
 }

 type Hooks struct {
@@ -66,22 +74,22 @@ func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out cha
 			mu.Unlock()
 		}()

-		res := runner.Run(ctx, c.Command, c.Timeout.Duration)
+		res := runner.Run(ctx, c.Argv(), c.Timeout.Duration)
 		id, err := eventid.New()
 		if err != nil {
 			return
 		}
 		ev := Event{
-			EventID:           id,
-			CheckID:           c.ID,
-			ObservedAt:        time.Now().UTC(),
-			Status:            string(res.Status),
-			ExitCode:          res.ExitCode,
-			DurationMs:        res.DurationMs,
-			Output:            res.Output,
-			OutputTruncated:   res.OutputTruncated,
-			NotificationOwner: c.NotificationOwner,
-			IncidentKey:       incidentKey(agentID, c),
+			EventID:              id,
+			CheckID:              c.ID,
+			ObservedAt:           time.Now().UTC(),
+			Status:               string(res.Status),
+			ExitCode:             res.ExitCode,
+			DurationMs:           res.DurationMs,
+			Output:               res.Output,
+			OutputTruncated:      res.OutputTruncated,
+			NotificationsEnabled: boolPtr(c.NotificationsOn()),
+			IncidentKey:          incidentKey(agentID, c),
 		}
 		if h.OnResult != nil {
 			h.OnResult(ev)
@@ -105,6 +113,10 @@ func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out cha
 	}
 }

+func boolPtr(v bool) *bool {
+	return &v
+}
+
 func incidentKey(agentID string, c config.CheckConfig) string {
 	if c.DedupeKey != "" {
 		return c.DedupeKey
--- a/agent/internal/scheduler/scheduler_test.go
+++ b/agent/internal/scheduler/scheduler_test.go
@@ -11,11 +11,10 @@ import (

 func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig {
 	return config.CheckConfig{
-		ID:                id,
-		Command:           []string{"sh", "-c", cmd},
-		Interval:          config.Duration{Duration: interval},
-		Timeout:           config.Duration{Duration: timeout},
-		NotificationOwner: "server",
+		ID:       id,
+		Command:  cmd,
+		Interval: config.Duration{Duration: interval},
+		Timeout:  config.Duration{Duration: timeout},
 	}
 }