init monlet repo with stage 0-2 (docs, contract, agent MVP)

2026-05-26 16:41:27 +04:00
commit dcea096327
42 changed files with 3183 additions and 0 deletions
--- a/agent/internal/app/app.go
+++ b/agent/internal/app/app.go
@@ -0,0 +1,212 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/monlet/agent/internal/client"
+	"github.com/monlet/agent/internal/config"
+	"github.com/monlet/agent/internal/metrics"
+	"github.com/monlet/agent/internal/scheduler"
+	"github.com/monlet/agent/internal/spool"
+)
+
+type App struct {
+	Cfg     *config.Config
+	AgentID string
+	Version string
+	Client  *client.Client
+	Spool   *spool.Spool
+	Metrics *metrics.Metrics
+	Log     *slog.Logger
+}
+
+func (a *App) Run(ctx context.Context) error {
+	events := make(chan scheduler.Event, 256)
+
+	a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
+	a.Metrics.BuildInfo.WithLabelValues(a.Version, a.Cfg.Mode).Set(1)
+	for _, c := range a.Cfg.Checks {
+		a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds())
+	}
+
+	var wg sync.WaitGroup
+
+	if a.Cfg.ExposesMetrics() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := a.Metrics.Serve(ctx, a.Cfg.Metrics.Listen); err != nil {
+				a.Log.Error("metrics server", "err", err)
+			}
+		}()
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		scheduler.Run(ctx, a.AgentID, a.Cfg.Checks, events, scheduler.Hooks{
+			OnSkipped: func(id string) { a.Metrics.CheckSkipped.WithLabelValues(id).Inc() },
+			OnResult: func(ev scheduler.Event) {
+				a.Metrics.CheckRuns.WithLabelValues(ev.CheckID, ev.Status).Inc()
+				a.Metrics.CheckDuration.WithLabelValues(ev.CheckID).Observe(float64(ev.DurationMs) / 1000.0)
+				a.Metrics.CheckStatus.WithLabelValues(ev.CheckID).Set(metrics.StatusCode(ev.Status))
+				a.Metrics.CheckExitCode.WithLabelValues(ev.CheckID).Set(float64(ev.ExitCode))
+				ts := float64(ev.ObservedAt.Unix())
+				a.Metrics.CheckLastRun.WithLabelValues(ev.CheckID).Set(ts)
+				if ev.Status == "ok" {
+					a.Metrics.CheckLastSuccess.WithLabelValues(ev.CheckID).Set(ts)
+				}
+			},
+		})
+	}()
+
+	if a.Cfg.PushesToServer() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.spoolWriter(ctx, events)
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.heartbeatLoop(ctx)
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.sendLoop(ctx)
+		}()
+	} else {
+		// drain scheduler events to avoid blocking it
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-events:
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case ev := <-events:
+			if err := a.Spool.Enqueue(ev); err != nil {
+				a.Log.Error("spool enqueue", "err", err)
+			}
+			a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
+			a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
+		}
+	}
+}
+
+func (a *App) heartbeatLoop(ctx context.Context) {
+	interval := a.Cfg.Server.HeartbeatInterval.Duration
+	attempt := 0
+	for {
+		hb := client.HeartbeatRequest{
+			AgentID:    a.AgentID,
+			ObservedAt: time.Now().UTC(),
+			Hostname:   a.Cfg.Hostname,
+			Version:    a.Version,
+			Mode:       a.Cfg.Mode,
+		}
+		a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc()
+		err := a.Client.SendHeartbeat(ctx, hb)
+		if err != nil {
+			a.Metrics.SendFailures.WithLabelValues("heartbeat").Inc()
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			a.Log.Warn("heartbeat send failed", "err", err)
+			if client.Retryable(err) {
+				attempt++
+				if sleep(ctx, client.Delay(attempt)) {
+					return
+				}
+				continue
+			}
+		} else {
+			a.Metrics.LastHeartbeat.Set(float64(time.Now().Unix()))
+		}
+		attempt = 0
+		if sleep(ctx, interval) {
+			return
+		}
+	}
+}
+
+func (a *App) sendLoop(ctx context.Context) {
+	interval := a.Cfg.Server.BatchInterval.Duration
+	attempt := 0
+	for {
+		items, err := a.Spool.Peek(client.MaxBatchEvents)
+		if err != nil {
+			a.Log.Error("spool peek", "err", err)
+		}
+		if len(items) == 0 {
+			if sleep(ctx, interval) {
+				return
+			}
+			continue
+		}
+		evs := make([]scheduler.Event, len(items))
+		seqs := make([]uint64, len(items))
+		for i, it := range items {
+			evs[i] = it.Event
+			seqs[i] = it.Seq
+		}
+		a.Metrics.SendAttempts.WithLabelValues("events").Inc()
+		resp, err := a.Client.SendEvents(ctx, evs)
+		if err != nil {
+			a.Metrics.SendFailures.WithLabelValues("events").Inc()
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			a.Log.Warn("events send failed", "err", err)
+			if client.Retryable(err) {
+				attempt++
+				if sleep(ctx, client.Delay(attempt)) {
+					return
+				}
+				continue
+			}
+			// non-retryable: drop to avoid hot loop
+			a.Spool.Commit(seqs)
+			attempt = 0
+			continue
+		}
+		attempt = 0
+		a.Spool.Commit(seqs)
+		a.Metrics.EventsAccepted.Add(float64(resp.Accepted))
+		a.Metrics.EventsDedup.Add(float64(resp.Deduplicated))
+		a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
+		a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
+	}
+}
+
+// sleep returns true if context was cancelled while waiting.
+func sleep(ctx context.Context, d time.Duration) bool {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return true
+	case <-t.C:
+		return false
+	}
+}
--- a/agent/internal/app/app_test.go
+++ b/agent/internal/app/app_test.go
@@ -0,0 +1,171 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/monlet/agent/internal/client"
+	"github.com/monlet/agent/internal/config"
+	"github.com/monlet/agent/internal/metrics"
+	"github.com/monlet/agent/internal/spool"
+)
+
+func TestPrometheusOnlyDoesNotPush(t *testing.T) {
+	var hits int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hits, 1)
+		w.WriteHeader(http.StatusAccepted)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	cfg := &config.Config{
+		AgentID:  "a1",
+		Mode:     "prometheus_only",
+		StateDir: dir,
+		Server:   config.ServerConfig{URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
+		Metrics:  config.MetricsConfig{Enabled: false},
+		Checks: []config.CheckConfig{{
+			ID:                "c1",
+			Command:           []string{"sh", "-c", "echo hi"},
+			Interval:          config.Duration{Duration: 100 * time.Millisecond},
+			Timeout:           config.Duration{Duration: 50 * time.Millisecond},
+			NotificationOwner: "server",
+		}},
+	}
+	sp, _ := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
+	a := &App{
+		Cfg: cfg, AgentID: "a1", Version: "t",
+		Client: client.New(srv.URL, "t", "a1", "t"),
+		Spool:  sp, Metrics: metrics.New(),
+		Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
+	defer cancel()
+	_ = a.Run(ctx)
+	if atomic.LoadInt32(&hits) != 0 {
+		t.Fatalf("prometheus_only must not hit server, got %d requests", atomic.LoadInt32(&hits))
+	}
+	if sp.Len() != 0 {
+		t.Fatalf("prometheus_only must not spool events, got %d", sp.Len())
+	}
+}
+
+func TestEndToEndPushAndReplay(t *testing.T) {
+	var (
+		mu           sync.Mutex
+		seenEventIDs = make(map[string]int)
+		hbCount      int32
+		fail         atomic.Bool
+	)
+	fail.Store(true)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/heartbeat":
+			atomic.AddInt32(&hbCount, 1)
+			w.WriteHeader(http.StatusAccepted)
+			_, _ = io.WriteString(w, `{"accepted":true}`)
+		case "/api/v1/events":
+			if fail.Load() {
+				http.Error(w, "down", http.StatusServiceUnavailable)
+				return
+			}
+			var req client.EventBatchRequest
+			_ = json.NewDecoder(r.Body).Decode(&req)
+			mu.Lock()
+			for _, e := range req.Events {
+				seenEventIDs[e.EventID]++
+			}
+			mu.Unlock()
+			w.WriteHeader(http.StatusAccepted)
+			_ = json.NewEncoder(w).Encode(client.EventBatchResponse{Accepted: len(req.Events)})
+		}
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	cfg := &config.Config{
+		AgentID:  "a1",
+		Hostname: "h",
+		Mode:     "hybrid",
+		StateDir: dir,
+		Server: config.ServerConfig{
+			URL:               srv.URL,
+			Token:             "t",
+			HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond},
+			BatchInterval:     config.Duration{Duration: 100 * time.Millisecond},
+		},
+		Metrics: config.MetricsConfig{Enabled: false},
+		Checks: []config.CheckConfig{{
+			ID:                "c1",
+			Command:           []string{"sh", "-c", "echo hi"},
+			Interval:          config.Duration{Duration: 150 * time.Millisecond},
+			Timeout:           config.Duration{Duration: 100 * time.Millisecond},
+			NotificationOwner: "server",
+		}},
+	}
+	sp, err := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := &App{
+		Cfg:     cfg,
+		AgentID: "a1",
+		Version: "test",
+		Client:  client.New(cfg.Server.URL, "t", "a1", "test"),
+		Spool:   sp,
+		Metrics: metrics.New(),
+		Log:     slog.New(slog.NewTextHandler(io.Discard, nil)),
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { _ = a.Run(ctx); close(done) }()
+
+	// Phase 1: server failing, events accumulate in spool.
+	time.Sleep(400 * time.Millisecond)
+	if sp.Len() == 0 {
+		cancel()
+		<-done
+		t.Fatal("expected spooled events while server failing")
+	}
+
+	// Phase 2: server recovers.
+	fail.Store(false)
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		mu.Lock()
+		got := len(seenEventIDs)
+		mu.Unlock()
+		if got > 0 && sp.Len() == 0 {
+			break
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	cancel()
+	<-done
+
+	mu.Lock()
+	defer mu.Unlock()
+	if len(seenEventIDs) == 0 {
+		t.Fatal("no events delivered after recovery")
+	}
+	for id, n := range seenEventIDs {
+		if n > 1 {
+			t.Fatalf("event_id %s sent %d times (should be deduped by server, but agent should commit on first 2xx)", id, n)
+		}
+	}
+	if atomic.LoadInt32(&hbCount) == 0 {
+		t.Fatal("no heartbeats observed")
+	}
+}
--- a/agent/internal/client/backoff.go
+++ b/agent/internal/client/backoff.go
@@ -0,0 +1,29 @@
+package client
+
+import (
+	"math/rand/v2"
+	"time"
+)
+
+const (
+	BackoffMin = 1 * time.Second
+	BackoffMax = 30 * time.Second
+)
+
+// Delay computes exponential backoff with ±20% jitter.
+// attempt starts at 1 for the first failed try.
+func Delay(attempt int) time.Duration {
+	if attempt < 1 {
+		attempt = 1
+	}
+	d := BackoffMin
+	for i := 1; i < attempt; i++ {
+		d *= 2
+		if d >= BackoffMax {
+			d = BackoffMax
+			break
+		}
+	}
+	jitter := 1.0 + (rand.Float64()*0.4 - 0.2)
+	return time.Duration(float64(d) * jitter)
+}
--- a/agent/internal/client/client.go
+++ b/agent/internal/client/client.go
@@ -0,0 +1,135 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/monlet/agent/internal/scheduler"
+)
+
+const (
+	MaxBatchEvents = 200
+)
+
+type Client struct {
+	baseURL string
+	token   string
+	agentID string
+	version string
+	http    *http.Client
+}
+
+func New(baseURL, token, agentID, version string) *Client {
+	return &Client{
+		baseURL: strings.TrimRight(baseURL, "/"),
+		token:   token,
+		agentID: agentID,
+		version: version,
+		http:    &http.Client{Timeout: 15 * time.Second},
+	}
+}
+
+type HeartbeatRequest struct {
+	AgentID    string            `json:"agent_id"`
+	ObservedAt time.Time         `json:"observed_at"`
+	Hostname   string            `json:"hostname"`
+	Version    string            `json:"version"`
+	Mode       string            `json:"mode,omitempty"`
+	Labels     map[string]string `json:"labels,omitempty"`
+}
+
+type EventBatchRequest struct {
+	AgentID string            `json:"agent_id"`
+	Events  []scheduler.Event `json:"events"`
+}
+
+type EventBatchResponse struct {
+	Accepted     int `json:"accepted"`
+	Deduplicated int `json:"deduplicated"`
+}
+
+func (c *Client) SendHeartbeat(ctx context.Context, hb HeartbeatRequest) error {
+	return c.post(ctx, "/api/v1/heartbeat", hb, nil)
+}
+
+func (c *Client) SendEvents(ctx context.Context, events []scheduler.Event) (*EventBatchResponse, error) {
+	if len(events) == 0 {
+		return &EventBatchResponse{}, nil
+	}
+	if len(events) > MaxBatchEvents {
+		return nil, fmt.Errorf("batch exceeds %d events", MaxBatchEvents)
+	}
+	req := EventBatchRequest{AgentID: c.agentID, Events: events}
+	var resp EventBatchResponse
+	if err := c.post(ctx, "/api/v1/events", req, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *Client) post(ctx context.Context, path string, body any, out any) error {
+	b, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
+		return &HTTPError{Status: resp.StatusCode, Body: string(msg)}
+	}
+	if out != nil {
+		return json.NewDecoder(resp.Body).Decode(out)
+	}
+	return nil
+}
+
+type HTTPError struct {
+	Status int
+	Body   string
+}
+
+func (e *HTTPError) Error() string { return fmt.Sprintf("http %d: %s", e.Status, e.Body) }
+
+// Retryable reports whether the error should trigger a retry (network errors
+// and 5xx are retryable; 4xx are dropped to avoid hot-loop on bad payload).
+func Retryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	var he *HTTPError
+	if asErr(err, &he) {
+		return he.Status >= 500
+	}
+	return true
+}
+
+func asErr(err error, target **HTTPError) bool {
+	for e := err; e != nil; {
+		if h, ok := e.(*HTTPError); ok {
+			*target = h
+			return true
+		}
+		u, ok := e.(interface{ Unwrap() error })
+		if !ok {
+			return false
+		}
+		e = u.Unwrap()
+	}
+	return false
+}
--- a/agent/internal/client/client_test.go
+++ b/agent/internal/client/client_test.go
@@ -0,0 +1,86 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/monlet/agent/internal/scheduler"
+)
+
+func TestSendHeartbeatSuccess(t *testing.T) {
+	var got HeartbeatRequest
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v1/heartbeat" || r.Header.Get("Authorization") != "Bearer tok" {
+			http.Error(w, "bad", http.StatusUnauthorized)
+			return
+		}
+		_ = json.NewDecoder(r.Body).Decode(&got)
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = io.WriteString(w, `{"accepted":true}`)
+	}))
+	defer srv.Close()
+	c := New(srv.URL, "tok", "a1", "v0")
+	err := c.SendHeartbeat(context.Background(), HeartbeatRequest{AgentID: "a1", ObservedAt: time.Now().UTC(), Hostname: "h", Version: "v0"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.AgentID != "a1" {
+		t.Fatalf("got %+v", got)
+	}
+}
+
+func TestSendEventsBatch(t *testing.T) {
+	var got EventBatchRequest
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewDecoder(r.Body).Decode(&got)
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = io.WriteString(w, `{"accepted":2,"deduplicated":0}`)
+	}))
+	defer srv.Close()
+	c := New(srv.URL, "tok", "a1", "v0")
+	resp, err := c.SendEvents(context.Background(), []scheduler.Event{{EventID: "x"}, {EventID: "y"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.Accepted != 2 || got.AgentID != "a1" || len(got.Events) != 2 {
+		t.Fatalf("resp=%+v got=%+v", resp, got)
+	}
+}
+
+func TestSendEventsTooBig(t *testing.T) {
+	c := New("http://x", "t", "a1", "v0")
+	big := make([]scheduler.Event, MaxBatchEvents+1)
+	_, err := c.SendEvents(context.Background(), big)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+}
+
+func TestRetryableClassification(t *testing.T) {
+	if Retryable(&HTTPError{Status: 400, Body: ""}) {
+		t.Fatal("4xx should not retry")
+	}
+	if !Retryable(&HTTPError{Status: 503}) {
+		t.Fatal("5xx should retry")
+	}
+	if !Retryable(context.DeadlineExceeded) {
+		t.Fatal("network error should retry")
+	}
+}
+
+func TestBackoffBounds(t *testing.T) {
+	for a := 1; a <= 10; a++ {
+		d := Delay(a)
+		if d < BackoffMin*4/5 {
+			t.Fatalf("attempt %d: delay %s below min", a, d)
+		}
+		if d > BackoffMax*6/5 {
+			t.Fatalf("attempt %d: delay %s above max", a, d)
+		}
+	}
+}
--- a/agent/internal/config/config.go
+++ b/agent/internal/config/config.go
@@ -0,0 +1,180 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var idPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]+$`)
+
+const (
+	maxIDLen           = 128
+	defaultHeartbeat   = 30 * time.Second
+	defaultBatch       = 10 * time.Second
+	defaultMetricsAddr = "127.0.0.1:9465"
+)
+
+type Config struct {
+	AgentID  string        `toml:"agent_id"`
+	Hostname string        `toml:"hostname"`
+	Mode     string        `toml:"mode"`
+	StateDir string        `toml:"state_dir"`
+	Server   ServerConfig  `toml:"server"`
+	Metrics  MetricsConfig `toml:"metrics"`
+	Checks   []CheckConfig `toml:"checks"`
+}
+
+type ServerConfig struct {
+	URL               string   `toml:"url"`
+	Token             string   `toml:"token"`
+	HeartbeatInterval Duration `toml:"heartbeat_interval"`
+	BatchInterval     Duration `toml:"batch_interval"`
+}
+
+type MetricsConfig struct {
+	Enabled bool   `toml:"enabled"`
+	Listen  string `toml:"listen"`
+}
+
+type CheckConfig struct {
+	ID                string   `toml:"id"`
+	Name              string   `toml:"name"`
+	Command           []string `toml:"command"`
+	Interval          Duration `toml:"interval"`
+	Timeout           Duration `toml:"timeout"`
+	NotificationOwner string   `toml:"notification_owner"`
+	DedupeKey         string   `toml:"dedupe_key"`
+}
+
+type Duration struct{ time.Duration }
+
+func (d *Duration) UnmarshalText(b []byte) error {
+	v, err := time.ParseDuration(string(b))
+	if err != nil {
+		return err
+	}
+	d.Duration = v
+	return nil
+}
+
+func Load(path string) (*Config, error) {
+	var c Config
+	if _, err := toml.DecodeFile(path, &c); err != nil {
+		return nil, fmt.Errorf("read config: %w", err)
+	}
+	c.applyDefaults()
+	if err := c.Validate(); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}
+
+func (c *Config) applyDefaults() {
+	if c.Mode == "" {
+		c.Mode = "hybrid"
+	}
+	if c.Hostname == "" {
+		if h, err := os.Hostname(); err == nil {
+			c.Hostname = h
+		}
+	}
+	if c.Server.HeartbeatInterval.Duration == 0 {
+		c.Server.HeartbeatInterval.Duration = defaultHeartbeat
+	}
+	if c.Server.BatchInterval.Duration == 0 {
+		c.Server.BatchInterval.Duration = defaultBatch
+	}
+	if c.Metrics.Listen == "" {
+		c.Metrics.Listen = defaultMetricsAddr
+	}
+}
+
+func (c *Config) Validate() error {
+	if c.AgentID != "" {
+		if err := ValidateID("agent_id", c.AgentID); err != nil {
+			return err
+		}
+	}
+	switch c.Mode {
+	case "prometheus_only", "push_only", "hybrid":
+	default:
+		return fmt.Errorf("invalid mode %q", c.Mode)
+	}
+	if c.StateDir == "" {
+		return fmt.Errorf("state_dir is required")
+	}
+	if c.PushesToServer() {
+		if c.Server.URL == "" {
+			return fmt.Errorf("server.url is required for mode %q", c.Mode)
+		}
+		if c.Server.Token == "" {
+			return fmt.Errorf("server.token is required for mode %q", c.Mode)
+		}
+	}
+	if len(c.Checks) == 0 {
+		return fmt.Errorf("at least one check is required")
+	}
+	seen := make(map[string]struct{}, len(c.Checks))
+	for i := range c.Checks {
+		ch := &c.Checks[i]
+		if err := ValidateID("checks[].id", ch.ID); err != nil {
+			return err
+		}
+		if _, dup := seen[ch.ID]; dup {
+			return fmt.Errorf("duplicate check id %q", ch.ID)
+		}
+		seen[ch.ID] = struct{}{}
+		if len(ch.Command) == 0 {
+			return fmt.Errorf("check %q: command is required", ch.ID)
+		}
+		if ch.Interval.Duration <= 0 {
+			return fmt.Errorf("check %q: interval must be > 0", ch.ID)
+		}
+		if ch.Timeout.Duration <= 0 {
+			return fmt.Errorf("check %q: timeout must be > 0", ch.ID)
+		}
+		if ch.Timeout.Duration > ch.Interval.Duration {
+			return fmt.Errorf("check %q: timeout must be <= interval", ch.ID)
+		}
+		switch ch.NotificationOwner {
+		case "", "server", "prometheus", "none":
+		default:
+			return fmt.Errorf("check %q: invalid notification_owner %q", ch.ID, ch.NotificationOwner)
+		}
+		if ch.NotificationOwner == "" {
+			ch.NotificationOwner = "server"
+		}
+		if len(ch.DedupeKey) > 256 {
+			return fmt.Errorf("check %q: dedupe_key too long", ch.ID)
+		}
+	}
+	return nil
+}
+
+// PushesToServer is true when the mode requires heartbeat/events push.
+func (c *Config) PushesToServer() bool {
+	return c.Mode == "push_only" || c.Mode == "hybrid"
+}
+
+// ExposesMetrics is true when the agent should serve /metrics.
+// `metrics.enabled` is an independent gate; both must agree.
+func (c *Config) ExposesMetrics() bool {
+	return c.Metrics.Enabled && (c.Mode == "prometheus_only" || c.Mode == "hybrid")
+}
+
+func ValidateID(field, v string) error {
+	if v == "" {
+		return fmt.Errorf("%s is empty", field)
+	}
+	if len(v) > maxIDLen {
+		return fmt.Errorf("%s exceeds %d chars", field, maxIDLen)
+	}
+	if !idPattern.MatchString(v) {
+		return fmt.Errorf("%s has invalid characters", field)
+	}
+	return nil
+}
--- a/agent/internal/config/config_test.go
+++ b/agent/internal/config/config_test.go
@@ -0,0 +1,159 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func writeTmp(t *testing.T, body string) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "c.toml")
+	if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+const minimal = `
+agent_id = "host-1"
+state_dir = "/tmp/x"
+
+[server]
+url = "http://localhost"
+token = "t"
+
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`
+
+func TestLoadMinimal(t *testing.T) {
+	c, err := Load(writeTmp(t, minimal))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.Mode != "hybrid" {
+		t.Errorf("default mode: %q", c.Mode)
+	}
+	if c.Server.HeartbeatInterval.Duration == 0 {
+		t.Error("default heartbeat not applied")
+	}
+	if c.Checks[0].NotificationOwner != "server" {
+		t.Errorf("default notification_owner: %q", c.Checks[0].NotificationOwner)
+	}
+}
+
+func TestValidateBadID(t *testing.T) {
+	body := minimal + "\n"
+	_, err := Load(writeTmp(t, body[:0]+`
+agent_id = "bad id!"
+state_dir = "/tmp/x"
+[server]
+url = "http://localhost"
+token = "t"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected error for bad agent_id")
+	}
+}
+
+func TestValidateTimeoutExceedsInterval(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "x"
+state_dir = "/tmp/x"
+[server]
+url = "http://localhost"
+token = "t"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "5s"
+timeout = "10s"
+`))
+	if err == nil {
+		t.Fatal("expected error for timeout > interval")
+	}
+}
+
+func TestPrometheusOnlyAllowsMissingServer(t *testing.T) {
+	c, err := Load(writeTmp(t, `
+mode = "prometheus_only"
+state_dir = "/tmp/x"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.PushesToServer() {
+		t.Fatal("prometheus_only must not push")
+	}
+}
+
+func TestPushOnlyRequiresServer(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+mode = "push_only"
+state_dir = "/tmp/x"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("push_only must require server.url/token")
+	}
+}
+
+func TestExposesMetricsGate(t *testing.T) {
+	cases := []struct {
+		mode    string
+		enabled bool
+		want    bool
+	}{
+		{"hybrid", true, true},
+		{"hybrid", false, false},
+		{"prometheus_only", true, true},
+		{"push_only", true, false},
+	}
+	for _, tc := range cases {
+		c := &Config{Mode: tc.mode, Metrics: MetricsConfig{Enabled: tc.enabled}}
+		if got := c.ExposesMetrics(); got != tc.want {
+			t.Errorf("mode=%s enabled=%v want=%v got=%v", tc.mode, tc.enabled, tc.want, got)
+		}
+	}
+}
+
+func TestValidateDuplicateCheckID(t *testing.T) {
+	_, err := Load(writeTmp(t, `
+agent_id = "x"
+state_dir = "/tmp/x"
+[server]
+url = "http://localhost"
+token = "t"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+[[checks]]
+id = "c1"
+command = ["true"]
+interval = "10s"
+timeout = "5s"
+`))
+	if err == nil {
+		t.Fatal("expected duplicate id error")
+	}
+}
--- a/agent/internal/eventid/eventid.go
+++ b/agent/internal/eventid/eventid.go
@@ -0,0 +1,12 @@
+package eventid
+
+import "github.com/google/uuid"
+
+// New returns a UUIDv7 in canonical lower-case form.
+func New() (string, error) {
+	id, err := uuid.NewV7()
+	if err != nil {
+		return "", err
+	}
+	return id.String(), nil
+}
--- a/agent/internal/eventid/eventid_test.go
+++ b/agent/internal/eventid/eventid_test.go
@@ -0,0 +1,20 @@
+package eventid
+
+import (
+	"regexp"
+	"testing"
+)
+
+var pattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+func TestNewMatchesContract(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		id, err := New()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !pattern.MatchString(id) {
+			t.Fatalf("invalid UUIDv7: %q", id)
+		}
+	}
+}
--- a/agent/internal/ids/ids.go
+++ b/agent/internal/ids/ids.go
@@ -0,0 +1,40 @@
+package ids
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"github.com/monlet/agent/internal/config"
+)
+
+// LoadOrCreateAgentID returns configured ID if non-empty, otherwise loads
+// state_dir/agent_id, generating and persisting it on first run.
+func LoadOrCreateAgentID(stateDir, configured string) (string, error) {
+	if configured != "" {
+		return configured, nil
+	}
+	if err := os.MkdirAll(stateDir, 0o755); err != nil {
+		return "", fmt.Errorf("create state dir: %w", err)
+	}
+	path := filepath.Join(stateDir, "agent_id")
+	b, err := os.ReadFile(path)
+	if err == nil {
+		id := strings.TrimSpace(string(b))
+		if err := config.ValidateID("agent_id", id); err != nil {
+			return "", fmt.Errorf("persisted agent_id invalid: %w", err)
+		}
+		return id, nil
+	}
+	if !os.IsNotExist(err) {
+		return "", err
+	}
+	id := uuid.NewString()
+	if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil {
+		return "", err
+	}
+	return id, nil
+}
--- a/agent/internal/ids/ids_test.go
+++ b/agent/internal/ids/ids_test.go
@@ -0,0 +1,33 @@
+package ids
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestConfiguredWins(t *testing.T) {
+	id, err := LoadOrCreateAgentID(t.TempDir(), "fixed")
+	if err != nil || id != "fixed" {
+		t.Fatalf("got %q %v", id, err)
+	}
+}
+
+func TestGenerateAndReuse(t *testing.T) {
+	dir := t.TempDir()
+	id1, err := LoadOrCreateAgentID(dir, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	id2, err := LoadOrCreateAgentID(dir, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if id1 != id2 {
+		t.Fatalf("not stable: %q vs %q", id1, id2)
+	}
+	b, _ := os.ReadFile(filepath.Join(dir, "agent_id"))
+	if len(b) == 0 {
+		t.Fatal("agent_id not persisted")
+	}
+}
--- a/agent/internal/metrics/metrics.go
+++ b/agent/internal/metrics/metrics.go
@@ -0,0 +1,152 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type Metrics struct {
+	Registry *prometheus.Registry
+
+	CheckRuns        *prometheus.CounterVec
+	CheckSkipped     *prometheus.CounterVec
+	CheckDuration    *prometheus.HistogramVec
+	CheckStatus      *prometheus.GaugeVec
+	CheckExitCode    *prometheus.GaugeVec
+	CheckLastRun     *prometheus.GaugeVec
+	CheckLastSuccess *prometheus.GaugeVec
+	CheckInterval    *prometheus.GaugeVec
+	ChecksConfigured prometheus.Gauge
+	SpoolDepth       prometheus.Gauge
+	SpoolBytes       prometheus.Gauge
+	SendAttempts     *prometheus.CounterVec
+	SendFailures     *prometheus.CounterVec
+	EventsAccepted   prometheus.Counter
+	EventsDedup      prometheus.Counter
+	LastHeartbeat    prometheus.Gauge
+	BuildInfo        *prometheus.GaugeVec
+}
+
+// StatusCode maps a CheckResultEvent status string to a numeric gauge value.
+func StatusCode(status string) float64 {
+	switch status {
+	case "ok":
+		return 0
+	case "warning":
+		return 1
+	case "critical":
+		return 2
+	default:
+		return 3
+	}
+}
+
+func New() *Metrics {
+	reg := prometheus.NewRegistry()
+	m := &Metrics{Registry: reg}
+	m.CheckRuns = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "monlet_agent_check_runs_total",
+		Help: "Number of check runs by id and status.",
+	}, []string{"check_id", "status"})
+	m.CheckSkipped = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "monlet_agent_check_skipped_total",
+		Help: "Number of ticks skipped due to overlap.",
+	}, []string{"check_id"})
+	m.CheckDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "monlet_agent_check_duration_seconds",
+		Help:    "Check execution duration.",
+		Buckets: prometheus.DefBuckets,
+	}, []string{"check_id"})
+	m.SpoolDepth = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "monlet_agent_spool_events",
+		Help: "Current spool depth in events.",
+	})
+	m.SpoolBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "monlet_agent_spool_bytes",
+		Help: "Current spool size in bytes.",
+	})
+	m.SendAttempts = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "monlet_agent_send_attempts_total",
+		Help: "Outbound send attempts by kind.",
+	}, []string{"kind"})
+	m.SendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "monlet_agent_send_failures_total",
+		Help: "Outbound send failures by kind.",
+	}, []string{"kind"})
+	m.EventsAccepted = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "monlet_agent_events_accepted_total",
+		Help: "Events accepted by the server.",
+	})
+	m.EventsDedup = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "monlet_agent_events_deduplicated_total",
+		Help: "Events the server reported as duplicates.",
+	})
+	m.CheckStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_check_status",
+		Help: "Last observed status per check: 0=ok,1=warning,2=critical,3=unknown.",
+	}, []string{"check_id"})
+	m.CheckExitCode = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_check_exit_code",
+		Help: "Last exit code per check.",
+	}, []string{"check_id"})
+	m.CheckLastRun = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_check_last_run_timestamp_seconds",
+		Help: "Unix timestamp of the last completed run per check.",
+	}, []string{"check_id"})
+	m.CheckLastSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_check_last_success_timestamp_seconds",
+		Help: "Unix timestamp of the last ok run per check.",
+	}, []string{"check_id"})
+	m.CheckInterval = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_check_interval_seconds",
+		Help: "Configured interval per check.",
+	}, []string{"check_id"})
+	m.ChecksConfigured = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "monlet_agent_checks_configured",
+		Help: "Number of checks configured.",
+	})
+	m.LastHeartbeat = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "monlet_agent_last_heartbeat_timestamp_seconds",
+		Help: "Unix timestamp of the last successful heartbeat ack.",
+	})
+	m.BuildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "monlet_agent_build_info",
+		Help: "Constant 1 with build labels.",
+	}, []string{"version", "mode"})
+	reg.MustRegister(m.CheckRuns, m.CheckSkipped, m.CheckDuration,
+		m.CheckStatus, m.CheckExitCode, m.CheckLastRun, m.CheckLastSuccess,
+		m.CheckInterval, m.ChecksConfigured,
+		m.SpoolDepth, m.SpoolBytes,
+		m.SendAttempts, m.SendFailures,
+		m.EventsAccepted, m.EventsDedup,
+		m.LastHeartbeat, m.BuildInfo)
+	return m
+}
+
+// Serve blocks until ctx is done; the listener is closed on shutdown.
+func (m *Metrics) Serve(ctx context.Context, addr string) error {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{}))
+	srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+
+	errCh := make(chan error, 1)
+	go func() { errCh <- srv.ListenAndServe() }()
+
+	select {
+	case <-ctx.Done():
+		shutCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(shutCtx)
+		return nil
+	case err := <-errCh:
+		if errors.Is(err, http.ErrServerClosed) {
+			return nil
+		}
+		return err
+	}
+}
--- a/agent/internal/metrics/metrics_test.go
+++ b/agent/internal/metrics/metrics_test.go
@@ -0,0 +1,34 @@
+package metrics
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func TestMetricsExposeRegistered(t *testing.T) {
+	m := New()
+	m.CheckRuns.WithLabelValues("c1", "ok").Inc()
+	m.SpoolDepth.Set(7)
+	srv := httptest.NewServer(promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{}))
+	defer srv.Close()
+	resp, err := http.Get(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	s := string(body)
+	for _, want := range []string{
+		`monlet_agent_check_runs_total{check_id="c1",status="ok"} 1`,
+		`monlet_agent_spool_events 7`,
+	} {
+		if !strings.Contains(s, want) {
+			t.Errorf("missing %q in output", want)
+		}
+	}
+}
--- a/agent/internal/runner/runner.go
+++ b/agent/internal/runner/runner.go
@@ -0,0 +1,154 @@
+package runner
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"syscall"
+	"time"
+	"unicode/utf8"
+)
+
+var replacementRune = []byte("<22>")
+
+const (
+	MaxOutputBytes = 8192
+	truncMarkerFmt = "...[truncated %d bytes]"
+)
+
+type Status string
+
+const (
+	StatusOK       Status = "ok"
+	StatusWarning  Status = "warning"
+	StatusCritical Status = "critical"
+	StatusUnknown  Status = "unknown"
+)
+
+type Result struct {
+	Status          Status
+	ExitCode        int
+	DurationMs      int64
+	Output          string
+	OutputTruncated bool
+}
+
+// Run executes argv with timeout. Stdout+stderr are captured into a single
+// buffer; both are truncated to 8 KiB UTF-8 bytes if exceeded.
+func Run(ctx context.Context, argv []string, timeout time.Duration) Result {
+	start := time.Now()
+	cctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(cctx, argv[0], argv[1:]...)
+	cmd.SysProcAttr = sysProcAttr()
+	cmd.Cancel = func() error { return killGroup(cmd) }
+	cmd.WaitDelay = 2 * time.Second
+
+	var buf bytes.Buffer
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+
+	err := cmd.Run()
+	dur := time.Since(start).Milliseconds()
+
+	out, truncated := TruncateUTF8(buf.Bytes(), MaxOutputBytes)
+	r := Result{
+		DurationMs:      dur,
+		Output:          out,
+		OutputTruncated: truncated,
+	}
+
+	if cctx.Err() == context.DeadlineExceeded {
+		r.Status = StatusUnknown
+		r.ExitCode = -1
+		return r
+	}
+	if err != nil {
+		var ee *exec.ExitError
+		if errors.As(err, &ee) {
+			r.ExitCode = ee.ExitCode()
+		} else {
+			r.Status = StatusUnknown
+			r.ExitCode = -1
+			return r
+		}
+	}
+	r.Status = MapExit(r.ExitCode)
+	return r
+}
+
+func MapExit(code int) Status {
+	switch code {
+	case 0:
+		return StatusOK
+	case 1:
+		return StatusWarning
+	case 2:
+		return StatusCritical
+	default:
+		return StatusUnknown
+	}
+}
+
+// TruncateUTF8 returns a string whose UTF-8 byte length is <= max and a
+// truncation flag. If truncated, appends "...[truncated N bytes]" marker.
+func TruncateUTF8(b []byte, max int) (string, bool) {
+	// Sanitize invalid bytes first; otherwise json.Marshal replaces them with
+	// U+FFFD (3 bytes each) after our size check and the wire payload can
+	// exceed the contract limit.
+	b = bytes.ToValidUTF8(b, replacementRune)
+	if len(b) <= max {
+		return string(b), false
+	}
+	budget := max - 32
+	if budget < 0 {
+		budget = 0
+	}
+	for i := 0; i < 5; i++ {
+		cut := safeBoundary(b, budget)
+		marker := fmt.Sprintf(truncMarkerFmt, len(b)-cut)
+		if cut+len(marker) <= max {
+			return string(b[:cut]) + marker, true
+		}
+		budget = max - len(marker) - 1
+		if budget < 0 {
+			budget = 0
+		}
+	}
+	cut := safeBoundary(b, max/2)
+	marker := fmt.Sprintf(truncMarkerFmt, len(b)-cut)
+	return string(b[:cut]) + marker, true
+}
+
+func safeBoundary(b []byte, limit int) int {
+	if limit >= len(b) {
+		return len(b)
+	}
+	for i := limit; i > 0; i-- {
+		if utf8.RuneStart(b[i]) {
+			if utf8.Valid(b[:i]) {
+				return i
+			}
+		}
+	}
+	return 0
+}
+
+func sysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{Setpgid: true}
+}
+
+func killGroup(cmd *exec.Cmd) error {
+	if cmd.Process == nil {
+		return nil
+	}
+	pgid, err := syscall.Getpgid(cmd.Process.Pid)
+	if err == nil {
+		_ = syscall.Kill(-pgid, syscall.SIGKILL)
+		return nil
+	}
+	return cmd.Process.Kill()
+}
--- a/agent/internal/runner/runner_test.go
+++ b/agent/internal/runner/runner_test.go
@@ -0,0 +1,113 @@
+package runner
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf8"
+)
+
+func TestMapExit(t *testing.T) {
+	cases := map[int]Status{0: StatusOK, 1: StatusWarning, 2: StatusCritical, 3: StatusUnknown, 255: StatusUnknown}
+	for code, want := range cases {
+		if got := MapExit(code); got != want {
+			t.Errorf("MapExit(%d)=%s want %s", code, got, want)
+		}
+	}
+}
+
+func TestTruncateUTF8Short(t *testing.T) {
+	out, tr := TruncateUTF8([]byte("hi"), 100)
+	if tr || out != "hi" {
+		t.Fatalf("got %q tr=%v", out, tr)
+	}
+}
+
+func TestTruncateUTF8Long(t *testing.T) {
+	in := strings.Repeat("a", 9000)
+	out, tr := TruncateUTF8([]byte(in), MaxOutputBytes)
+	if !tr {
+		t.Fatal("expected truncated")
+	}
+	if len([]byte(out)) > MaxOutputBytes {
+		t.Fatalf("byte len %d > %d", len([]byte(out)), MaxOutputBytes)
+	}
+	if !strings.Contains(out, "truncated") {
+		t.Fatalf("no marker: %q", out[len(out)-50:])
+	}
+}
+
+func TestTruncateUTF8SanitizesInvalidBytes(t *testing.T) {
+	// invalid UTF-8 bytes that json.Marshal would replace with U+FFFD (3 bytes each)
+	in := append([]byte("hello"), 0xff, 0xfe, 0xfd)
+	out, tr := TruncateUTF8(in, 100)
+	if tr {
+		t.Fatal("should fit")
+	}
+	if !utf8.ValidString(out) {
+		t.Fatalf("output must be valid UTF-8: %q", out)
+	}
+}
+
+func TestTruncateUTF8Multibyte(t *testing.T) {
+	in := strings.Repeat("ё", 5000) // 2 bytes per rune = 10000 bytes
+	out, tr := TruncateUTF8([]byte(in), MaxOutputBytes)
+	if !tr {
+		t.Fatal("expected truncated")
+	}
+	if !utf8.ValidString(out) {
+		t.Fatal("invalid UTF-8 after truncate")
+	}
+	if len([]byte(out)) > MaxOutputBytes {
+		t.Fatalf("byte len %d > %d", len([]byte(out)), MaxOutputBytes)
+	}
+}
+
+func TestRunOK(t *testing.T) {
+	r := Run(context.Background(), []string{"sh", "-c", "echo ok"}, 2*time.Second)
+	if r.Status != StatusOK || r.ExitCode != 0 {
+		t.Fatalf("got %+v", r)
+	}
+	if !strings.Contains(r.Output, "ok") {
+		t.Fatalf("output: %q", r.Output)
+	}
+}
+
+func TestRunWarning(t *testing.T) {
+	r := Run(context.Background(), []string{"sh", "-c", "exit 1"}, 2*time.Second)
+	if r.Status != StatusWarning || r.ExitCode != 1 {
+		t.Fatalf("got %+v", r)
+	}
+}
+
+func TestRunCritical(t *testing.T) {
+	r := Run(context.Background(), []string{"sh", "-c", "exit 2"}, 2*time.Second)
+	if r.Status != StatusCritical {
+		t.Fatalf("got %+v", r)
+	}
+}
+
+func TestRunUnknownExit(t *testing.T) {
+	r := Run(context.Background(), []string{"sh", "-c", "exit 7"}, 2*time.Second)
+	if r.Status != StatusUnknown || r.ExitCode != 7 {
+		t.Fatalf("got %+v", r)
+	}
+}
+
+func TestRunTimeout(t *testing.T) {
+	r := Run(context.Background(), []string{"sh", "-c", "sleep 5"}, 200*time.Millisecond)
+	if r.Status != StatusUnknown {
+		t.Fatalf("got %+v", r)
+	}
+	if r.DurationMs > 2000 {
+		t.Fatalf("duration too long: %d", r.DurationMs)
+	}
+}
+
+func TestRunNoExec(t *testing.T) {
+	r := Run(context.Background(), []string{"/does/not/exist/zzz"}, 1*time.Second)
+	if r.Status != StatusUnknown {
+		t.Fatalf("got %+v", r)
+	}
+}
--- a/agent/internal/scheduler/scheduler.go
+++ b/agent/internal/scheduler/scheduler.go
@@ -0,0 +1,113 @@
+package scheduler
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/monlet/agent/internal/config"
+	"github.com/monlet/agent/internal/eventid"
+	"github.com/monlet/agent/internal/runner"
+)
+
+// Event is a contract-shaped check result emitted by the scheduler.
+type Event struct {
+	EventID           string    `json:"event_id"`
+	CheckID           string    `json:"check_id"`
+	ObservedAt        time.Time `json:"observed_at"`
+	Status            string    `json:"status"`
+	ExitCode          int       `json:"exit_code"`
+	DurationMs        int64     `json:"duration_ms"`
+	Output            string    `json:"output,omitempty"`
+	OutputTruncated   bool      `json:"output_truncated,omitempty"`
+	NotificationOwner string    `json:"notification_owner,omitempty"`
+	IncidentKey       string    `json:"incident_key,omitempty"`
+}
+
+type Hooks struct {
+	OnSkipped func(checkID string)
+	OnResult  func(ev Event)
+}
+
+// Run starts one goroutine per check. Emits events to out. Blocks until ctx is done.
+func Run(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) {
+	var wg sync.WaitGroup
+	for i := range checks {
+		c := checks[i]
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			runCheck(ctx, agentID, c, out, h)
+		}()
+	}
+	wg.Wait()
+}
+
+func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out chan<- Event, h Hooks) {
+	t := time.NewTicker(c.Interval.Duration)
+	defer t.Stop()
+	var running bool
+	var mu sync.Mutex
+
+	exec := func() {
+		mu.Lock()
+		if running {
+			mu.Unlock()
+			if h.OnSkipped != nil {
+				h.OnSkipped(c.ID)
+			}
+			return
+		}
+		running = true
+		mu.Unlock()
+		defer func() {
+			mu.Lock()
+			running = false
+			mu.Unlock()
+		}()
+
+		res := runner.Run(ctx, c.Command, c.Timeout.Duration)
+		id, err := eventid.New()
+		if err != nil {
+			return
+		}
+		ev := Event{
+			EventID:           id,
+			CheckID:           c.ID,
+			ObservedAt:        time.Now().UTC(),
+			Status:            string(res.Status),
+			ExitCode:          res.ExitCode,
+			DurationMs:        res.DurationMs,
+			Output:            res.Output,
+			OutputTruncated:   res.OutputTruncated,
+			NotificationOwner: c.NotificationOwner,
+			IncidentKey:       incidentKey(agentID, c),
+		}
+		if h.OnResult != nil {
+			h.OnResult(ev)
+		}
+		select {
+		case out <- ev:
+		case <-ctx.Done():
+		}
+	}
+
+	// Fire first run immediately so smoke tests don't wait a full interval.
+	go exec()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			go exec()
+		}
+	}
+}
+
+func incidentKey(agentID string, c config.CheckConfig) string {
+	if c.DedupeKey != "" {
+		return c.DedupeKey
+	}
+	return agentID + ":" + c.ID
+}
--- a/agent/internal/scheduler/scheduler_test.go
+++ b/agent/internal/scheduler/scheduler_test.go
@@ -0,0 +1,52 @@
+package scheduler
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/monlet/agent/internal/config"
+)
+
+func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig {
+	return config.CheckConfig{
+		ID:                id,
+		Command:           []string{"sh", "-c", cmd},
+		Interval:          config.Duration{Duration: interval},
+		Timeout:           config.Duration{Duration: timeout},
+		NotificationOwner: "server",
+	}
+}
+
+func TestEmitsEvents(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel()
+	out := make(chan Event, 10)
+	go Run(ctx, "a1", []config.CheckConfig{mkCheck("c1", "echo hi", 200*time.Millisecond, 100*time.Millisecond)}, out, Hooks{})
+	select {
+	case ev := <-out:
+		if ev.CheckID != "c1" || ev.Status != "ok" {
+			t.Fatalf("unexpected event: %+v", ev)
+		}
+		if ev.IncidentKey != "a1:c1" {
+			t.Fatalf("incident_key: %q", ev.IncidentKey)
+		}
+	case <-ctx.Done():
+		t.Fatal("no event")
+	}
+}
+
+func TestNoOverlapSkips(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 600*time.Millisecond)
+	defer cancel()
+	out := make(chan Event, 10)
+	var skipped int32
+	hooks := Hooks{OnSkipped: func(string) { atomic.AddInt32(&skipped, 1) }}
+	c := mkCheck("slow", "sleep 1", 100*time.Millisecond, 900*time.Millisecond)
+	go Run(ctx, "a1", []config.CheckConfig{c}, out, hooks)
+	<-ctx.Done()
+	if atomic.LoadInt32(&skipped) == 0 {
+		t.Fatal("expected at least one skipped tick")
+	}
+}
--- a/agent/internal/spool/spool.go
+++ b/agent/internal/spool/spool.go
@@ -0,0 +1,213 @@
+package spool
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/monlet/agent/internal/scheduler"
+)
+
+const (
+	DefaultMaxEvents = 10_000
+	DefaultMaxBytes  = 50 * 1024 * 1024
+	fileExt          = ".json"
+)
+
+type Spool struct {
+	dir       string
+	maxEvents int
+	maxBytes  int64
+
+	mu      sync.Mutex
+	nextSeq uint64
+	files   []entry // sorted ascending by seq
+	bytes   int64
+}
+
+type entry struct {
+	seq  uint64
+	size int64
+	name string
+}
+
+type Item struct {
+	Seq   uint64
+	Event scheduler.Event
+}
+
+func Open(dir string, maxEvents int, maxBytes int64) (*Spool, error) {
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, err
+	}
+	if maxEvents <= 0 {
+		maxEvents = DefaultMaxEvents
+	}
+	if maxBytes <= 0 {
+		maxBytes = DefaultMaxBytes
+	}
+	s := &Spool{dir: dir, maxEvents: maxEvents, maxBytes: maxBytes}
+	if err := s.rescan(); err != nil {
+		return nil, err
+	}
+	s.mu.Lock()
+	s.dropOldestLocked()
+	s.mu.Unlock()
+	return s, nil
+}
+
+func (s *Spool) rescan() error {
+	ents, err := os.ReadDir(s.dir)
+	if err != nil {
+		return err
+	}
+	s.files = s.files[:0]
+	s.bytes = 0
+	for _, e := range ents {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), fileExt) {
+			continue
+		}
+		base := strings.TrimSuffix(e.Name(), fileExt)
+		seq, err := strconv.ParseUint(base, 10, 64)
+		if err != nil {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			return err
+		}
+		s.files = append(s.files, entry{seq: seq, size: info.Size(), name: e.Name()})
+		s.bytes += info.Size()
+		if seq >= s.nextSeq {
+			s.nextSeq = seq + 1
+		}
+	}
+	sort.Slice(s.files, func(i, j int) bool { return s.files[i].seq < s.files[j].seq })
+	return nil
+}
+
+func (s *Spool) Len() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return len(s.files)
+}
+
+func (s *Spool) Bytes() int64 {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.bytes
+}
+
+func (s *Spool) Enqueue(ev scheduler.Event) error {
+	data, err := json.Marshal(ev)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	seq := s.nextSeq
+	s.nextSeq++
+	name := fmt.Sprintf("%014d%s", seq, fileExt)
+	tmp := filepath.Join(s.dir, name+".tmp")
+	final := filepath.Join(s.dir, name)
+	if err := os.WriteFile(tmp, data, 0o600); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, final); err != nil {
+		_ = os.Remove(tmp)
+		return err
+	}
+	size := int64(len(data))
+	s.files = append(s.files, entry{seq: seq, size: size, name: name})
+	s.bytes += size
+	s.dropOldestLocked()
+	return nil
+}
+
+func (s *Spool) dropOldestLocked() {
+	for len(s.files) > s.maxEvents || s.bytes > s.maxBytes {
+		if len(s.files) == 0 {
+			return
+		}
+		old := s.files[0]
+		_ = os.Remove(filepath.Join(s.dir, old.name))
+		s.bytes -= old.size
+		s.files = s.files[1:]
+	}
+}
+
+// Peek returns up to limit oldest items. Does not remove them.
+func (s *Spool) Peek(limit int) ([]Item, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if limit > len(s.files) {
+		limit = len(s.files)
+	}
+	out := make([]Item, 0, limit)
+	corrupt := make(map[uint64]struct{})
+	for i := 0; i < limit; i++ {
+		e := s.files[i]
+		b, err := os.ReadFile(filepath.Join(s.dir, e.name))
+		if err != nil {
+			if os.IsNotExist(err) {
+				corrupt[e.seq] = struct{}{}
+				continue
+			}
+			s.dropCorruptLocked(corrupt)
+			return out, err
+		}
+		var ev scheduler.Event
+		if err := json.Unmarshal(b, &ev); err != nil {
+			_ = os.Remove(filepath.Join(s.dir, e.name))
+			corrupt[e.seq] = struct{}{}
+			continue
+		}
+		out = append(out, Item{Seq: e.seq, Event: ev})
+	}
+	s.dropCorruptLocked(corrupt)
+	return out, nil
+}
+
+func (s *Spool) dropCorruptLocked(seqs map[uint64]struct{}) {
+	if len(seqs) == 0 {
+		return
+	}
+	kept := s.files[:0]
+	for _, e := range s.files {
+		if _, bad := seqs[e.seq]; bad {
+			s.bytes -= e.size
+			continue
+		}
+		kept = append(kept, e)
+	}
+	s.files = kept
+}
+
+// Commit removes items with the given sequence numbers.
+func (s *Spool) Commit(seqs []uint64) {
+	if len(seqs) == 0 {
+		return
+	}
+	want := make(map[uint64]struct{}, len(seqs))
+	for _, x := range seqs {
+		want[x] = struct{}{}
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	kept := s.files[:0]
+	for _, e := range s.files {
+		if _, ok := want[e.seq]; ok {
+			_ = os.Remove(filepath.Join(s.dir, e.name))
+			s.bytes -= e.size
+			continue
+		}
+		kept = append(kept, e)
+	}
+	s.files = kept
+}
--- a/agent/internal/spool/spool_test.go
+++ b/agent/internal/spool/spool_test.go
@@ -0,0 +1,129 @@
+package spool
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/monlet/agent/internal/scheduler"
+)
+
+func mkEvent(id string) scheduler.Event {
+	return scheduler.Event{EventID: id, CheckID: "c1", Status: "ok"}
+}
+
+func TestEnqueuePeekCommit(t *testing.T) {
+	s, err := Open(t.TempDir(), 10, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 3; i++ {
+		if err := s.Enqueue(mkEvent(fmt.Sprintf("e%d", i))); err != nil {
+			t.Fatal(err)
+		}
+	}
+	items, err := s.Peek(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(items) != 3 {
+		t.Fatalf("got %d items", len(items))
+	}
+	if items[0].Event.EventID != "e0" {
+		t.Fatalf("FIFO violated: %q", items[0].Event.EventID)
+	}
+	s.Commit([]uint64{items[0].Seq, items[1].Seq})
+	if s.Len() != 1 {
+		t.Fatalf("len=%d", s.Len())
+	}
+}
+
+func TestDropOldestByCount(t *testing.T) {
+	s, err := Open(t.TempDir(), 3, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 5; i++ {
+		_ = s.Enqueue(mkEvent(fmt.Sprintf("e%d", i)))
+	}
+	if s.Len() != 3 {
+		t.Fatalf("len=%d", s.Len())
+	}
+	items, _ := s.Peek(10)
+	if items[0].Event.EventID != "e2" {
+		t.Fatalf("expected oldest dropped, got %q", items[0].Event.EventID)
+	}
+}
+
+func TestDropOldestByBytes(t *testing.T) {
+	big := strings.Repeat("x", 1000)
+	s, err := Open(t.TempDir(), 1_000_000, 3000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 10; i++ {
+		ev := mkEvent(fmt.Sprintf("e%d", i))
+		ev.Output = big
+		_ = s.Enqueue(ev)
+	}
+	if s.Bytes() > 3000 {
+		t.Fatalf("bytes=%d > 3000", s.Bytes())
+	}
+}
+
+func TestReopenAppliesLimits(t *testing.T) {
+	dir := t.TempDir()
+	s, _ := Open(dir, 10, 1<<20)
+	for i := 0; i < 5; i++ {
+		_ = s.Enqueue(mkEvent(fmt.Sprintf("e%d", i)))
+	}
+	// Reopen with tighter limit: should drop oldest down to maxEvents.
+	s2, err := Open(dir, 2, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s2.Len() != 2 {
+		t.Fatalf("expected limits applied on Open, len=%d", s2.Len())
+	}
+	items, _ := s2.Peek(10)
+	if items[0].Event.EventID != "e3" {
+		t.Fatalf("expected oldest dropped, got %q", items[0].Event.EventID)
+	}
+}
+
+func TestPeekSkipsCorruptAndCleansList(t *testing.T) {
+	dir := t.TempDir()
+	s, _ := Open(dir, 10, 1<<20)
+	_ = s.Enqueue(mkEvent("e0"))
+	_ = s.Enqueue(mkEvent("e1"))
+	// Corrupt the first file on disk.
+	items, _ := s.Peek(10)
+	if err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("%014d.json", items[0].Seq)), []byte("not-json"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	out, _ := s.Peek(10)
+	if len(out) != 1 || out[0].Event.EventID != "e1" {
+		t.Fatalf("got %+v", out)
+	}
+	if s.Len() != 1 {
+		t.Fatalf("corrupt entry not pruned from list, len=%d", s.Len())
+	}
+}
+
+func TestReopenPreservesOrder(t *testing.T) {
+	dir := t.TempDir()
+	s, _ := Open(dir, 100, 1<<20)
+	for i := 0; i < 3; i++ {
+		_ = s.Enqueue(mkEvent(fmt.Sprintf("e%d", i)))
+	}
+	s2, err := Open(dir, 100, 1<<20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	items, _ := s2.Peek(10)
+	if len(items) != 3 || items[0].Event.EventID != "e0" || items[2].Event.EventID != "e2" {
+		t.Fatalf("bad reopen: %+v", items)
+	}
+}