package app import ( "context" "encoding/json" "io" "log/slog" "net/http" "net/http/httptest" "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/monlet/agent/internal/client" "github.com/monlet/agent/internal/config" "github.com/monlet/agent/internal/metrics" "github.com/monlet/agent/internal/spool" dto "github.com/prometheus/client_model/go" ) func TestPrometheusOnlyDoesNotPush(t *testing.T) { var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) w.WriteHeader(http.StatusAccepted) })) defer srv.Close() dir := t.TempDir() cfg := &config.Config{ AgentID: "a1", StateDir: dir, Server: config.ServerConfig{Enabled: boolPtr(false), URL: srv.URL, HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}}, Metrics: config.MetricsConfig{Enabled: true, Listen: "127.0.0.1:0"}, Checks: []config.CheckConfig{{ ID: "c1", Command: "echo hi", Interval: config.Duration{Duration: 100 * time.Millisecond}, Timeout: config.Duration{Duration: 50 * time.Millisecond}, }}, } sp, _ := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20) a := &App{ Cfg: cfg, AgentID: "a1", Version: "t", Client: client.New(srv.URL, "t", "a1"), Spool: sp, Metrics: metrics.New(), Log: slog.New(slog.NewTextHandler(io.Discard, nil)), } ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) defer cancel() _ = a.Run(ctx) if atomic.LoadInt32(&hits) != 0 { t.Fatalf("prometheus_only must not hit server, got %d requests", atomic.LoadInt32(&hits)) } if sp.Len() != 0 { t.Fatalf("prometheus_only must not spool events, got %d", sp.Len()) } } func TestEndToEndPushAndReplay(t *testing.T) { var ( mu sync.Mutex seenEventIDs = make(map[string]int) lastHeartbeat client.HeartbeatRequest hbCount int32 fail atomic.Bool ) fail.Store(true) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/v1/heartbeat": var req client.HeartbeatRequest _ = json.NewDecoder(r.Body).Decode(&req) mu.Lock() lastHeartbeat = req mu.Unlock() atomic.AddInt32(&hbCount, 1) w.WriteHeader(http.StatusAccepted) _, _ = io.WriteString(w, `{"accepted":true}`) case "/api/v1/events": if fail.Load() { http.Error(w, "down", http.StatusServiceUnavailable) return } var req client.EventBatchRequest _ = json.NewDecoder(r.Body).Decode(&req) mu.Lock() for _, e := range req.Events { seenEventIDs[e.EventID]++ } mu.Unlock() w.WriteHeader(http.StatusAccepted) _ = json.NewEncoder(w).Encode(client.EventBatchResponse{Accepted: len(req.Events)}) } })) defer srv.Close() dir := t.TempDir() cfg := &config.Config{ AgentID: "a1", Hostname: "h", StateDir: dir, Labels: map[string]string{"env": "test"}, Server: config.ServerConfig{ Enabled: boolPtr(true), URL: srv.URL, HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond}, BatchInterval: config.Duration{Duration: 100 * time.Millisecond}, }, Metrics: config.MetricsConfig{Enabled: false}, Checks: []config.CheckConfig{{ ID: "c1", Command: "echo hi", Interval: config.Duration{Duration: 150 * time.Millisecond}, Timeout: config.Duration{Duration: 100 * time.Millisecond}, }}, } sp, err := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20) if err != nil { t.Fatal(err) } a := &App{ Cfg: cfg, AgentID: "a1", Version: "test", Client: client.New(cfg.Server.URL, "t", "a1"), Spool: sp, Metrics: metrics.New(), Log: slog.New(slog.NewTextHandler(io.Discard, nil)), } ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { _ = a.Run(ctx); close(done) }() // Phase 1: server failing, events accumulate in spool. time.Sleep(400 * time.Millisecond) if sp.Len() == 0 { cancel() <-done t.Fatal("expected spooled events while server failing") } // Phase 2: server recovers. fail.Store(false) deadline := time.Now().Add(3 * time.Second) for time.Now().Before(deadline) { mu.Lock() got := len(seenEventIDs) mu.Unlock() if got > 0 && sp.Len() == 0 { break } time.Sleep(50 * time.Millisecond) } cancel() <-done mu.Lock() defer mu.Unlock() if len(seenEventIDs) == 0 { t.Fatal("no events delivered after recovery") } for id, n := range seenEventIDs { if n > 1 { t.Fatalf("event_id %s sent %d times (should be deduped by server, but agent should commit on first 2xx)", id, n) } } if atomic.LoadInt32(&hbCount) == 0 { t.Fatal("no heartbeats observed") } if lastHeartbeat.Labels["env"] != "test" { t.Fatalf("configured heartbeat label missing: %#v", lastHeartbeat.Labels) } if lastHeartbeat.Labels[config.AgentVersionLabel] != "test" { t.Fatalf("version heartbeat label missing: %#v", lastHeartbeat.Labels) } if !lastHeartbeat.Features.Push || lastHeartbeat.Features.Metrics { t.Fatalf("unexpected heartbeat features: %+v", lastHeartbeat.Features) } } func TestReloadRejectsImmutableChangesAndKeepsOldConfig(t *testing.T) { oldCfg := &config.Config{ AgentID: "a1", Hostname: "h", StateDir: "/tmp/old", Server: config.ServerConfig{ Enabled: boolPtr(true), URL: "http://old", HeartbeatInterval: config.Duration{Duration: 10 * time.Second}, BatchInterval: config.Duration{Duration: 10 * time.Second}, }, Metrics: config.MetricsConfig{Enabled: false}, Checks: []config.CheckConfig{{ ID: "c1", Command: "true", Interval: config.Duration{Duration: time.Second}, Timeout: config.Duration{Duration: time.Second}, }}, } newCfg := *oldCfg newCfg.Server.URL = "http://new" a := &App{ Cfg: oldCfg, AgentID: "a1", Metrics: metrics.New(), } if err := a.Reload(&newCfg, "a1"); err == nil { t.Fatal("expected immutable reload rejection") } if a.config().Server.URL != "http://old" { t.Fatalf("old config was replaced: %+v", a.config().Server) } } func TestReloadAcceptsLabelsAndChecks(t *testing.T) { oldCfg := &config.Config{ AgentID: "a1", Hostname: "h", StateDir: "/tmp/old", Labels: map[string]string{"env": "old"}, Server: config.ServerConfig{ Enabled: boolPtr(true), URL: "http://server", HeartbeatInterval: config.Duration{Duration: 10 * time.Second}, BatchInterval: config.Duration{Duration: 10 * time.Second}, }, Metrics: config.MetricsConfig{Enabled: false}, Checks: []config.CheckConfig{{ ID: "c1", Command: "true", Interval: config.Duration{Duration: time.Second}, Timeout: config.Duration{Duration: time.Second}, }}, } newCfg := *oldCfg newCfg.Labels = map[string]string{"env": "new"} newCfg.Checks = []config.CheckConfig{{ ID: "c2", Command: "true", Interval: config.Duration{Duration: 2 * time.Second}, Timeout: config.Duration{Duration: time.Second}, }} a := &App{ Cfg: oldCfg, AgentID: "a1", Metrics: metrics.New(), } if err := a.Reload(&newCfg, "a1"); err != nil { t.Fatal(err) } if a.config().Labels["env"] != "new" || len(a.config().Checks) != 1 || a.config().Checks[0].ID != "c2" { t.Fatalf("reload did not apply: %+v", a.config()) } } func TestReloadUpdatesIntervalMetricForCron(t *testing.T) { oldCfg := &config.Config{ AgentID: "a1", Hostname: "h", StateDir: "/tmp/old", Server: config.ServerConfig{ Enabled: boolPtr(true), URL: "http://server", HeartbeatInterval: config.Duration{Duration: 10 * time.Second}, BatchInterval: config.Duration{Duration: 10 * time.Second}, }, Metrics: config.MetricsConfig{Enabled: false}, Checks: []config.CheckConfig{{ ID: "c1", Command: "true", Interval: config.Duration{Duration: time.Second}, Timeout: config.Duration{Duration: time.Second}, }}, } a := &App{ Cfg: oldCfg, AgentID: "a1", Metrics: metrics.New(), } a.recordCheckScheduleMetrics(oldCfg.Checks) if got, ok := checkIntervalMetric(t, a, "c1"); !ok || got != 1 { t.Fatalf("interval metric missing before reload: value=%v ok=%v", got, ok) } cronCfg := *oldCfg cronCfg.Checks = []config.CheckConfig{{ ID: "c1", Command: "true", Cron: "0 * * * *", Timeout: config.Duration{Duration: time.Second}, }} if err := a.Reload(&cronCfg, "a1"); err != nil { t.Fatal(err) } if got, ok := checkIntervalMetric(t, a, "c1"); ok { t.Fatalf("interval metric must be deleted for cron check, got %v", got) } intervalCfg := cronCfg intervalCfg.Checks = []config.CheckConfig{{ ID: "c1", Command: "true", Interval: config.Duration{Duration: 2 * time.Second}, Timeout: config.Duration{Duration: time.Second}, }} if err := a.Reload(&intervalCfg, "a1"); err != nil { t.Fatal(err) } if got, ok := checkIntervalMetric(t, a, "c1"); !ok || got != 2 { t.Fatalf("interval metric missing after interval reload: value=%v ok=%v", got, ok) } } func checkIntervalMetric(t *testing.T, a *App, checkID string) (float64, bool) { t.Helper() families, err := a.Metrics.Registry.Gather() if err != nil { t.Fatal(err) } for _, family := range families { if family.GetName() != "monlet_agent_check_interval_seconds" { continue } for _, metric := range family.GetMetric() { if labelValue(metric.GetLabel(), "check_id") == checkID && metric.GetGauge() != nil { return metric.GetGauge().GetValue(), true } } } return 0, false } func labelValue(labels []*dto.LabelPair, name string) string { for _, label := range labels { if label.GetName() == name { return label.GetValue() } } return "" } func boolPtr(v bool) *bool { return &v }