init monlet repo with stage 0-2 (docs, contract, agent MVP)
This commit is contained in:
Stanislav Rossovskii
212
agent/internal/app/app.go
Normal file
212
agent/internal/app/app.go
Normal file
@@ -0,0 +1,212 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/monlet/agent/internal/client"
|
||||
"github.com/monlet/agent/internal/config"
|
||||
"github.com/monlet/agent/internal/metrics"
|
||||
"github.com/monlet/agent/internal/scheduler"
|
||||
"github.com/monlet/agent/internal/spool"
|
||||
)
|
||||
|
||||
type App struct {
|
||||
Cfg *config.Config
|
||||
AgentID string
|
||||
Version string
|
||||
Client *client.Client
|
||||
Spool *spool.Spool
|
||||
Metrics *metrics.Metrics
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
func (a *App) Run(ctx context.Context) error {
|
||||
events := make(chan scheduler.Event, 256)
|
||||
|
||||
a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
|
||||
a.Metrics.BuildInfo.WithLabelValues(a.Version, a.Cfg.Mode).Set(1)
|
||||
for _, c := range a.Cfg.Checks {
|
||||
a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds())
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
if a.Cfg.ExposesMetrics() {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := a.Metrics.Serve(ctx, a.Cfg.Metrics.Listen); err != nil {
|
||||
a.Log.Error("metrics server", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
scheduler.Run(ctx, a.AgentID, a.Cfg.Checks, events, scheduler.Hooks{
|
||||
OnSkipped: func(id string) { a.Metrics.CheckSkipped.WithLabelValues(id).Inc() },
|
||||
OnResult: func(ev scheduler.Event) {
|
||||
a.Metrics.CheckRuns.WithLabelValues(ev.CheckID, ev.Status).Inc()
|
||||
a.Metrics.CheckDuration.WithLabelValues(ev.CheckID).Observe(float64(ev.DurationMs) / 1000.0)
|
||||
a.Metrics.CheckStatus.WithLabelValues(ev.CheckID).Set(metrics.StatusCode(ev.Status))
|
||||
a.Metrics.CheckExitCode.WithLabelValues(ev.CheckID).Set(float64(ev.ExitCode))
|
||||
ts := float64(ev.ObservedAt.Unix())
|
||||
a.Metrics.CheckLastRun.WithLabelValues(ev.CheckID).Set(ts)
|
||||
if ev.Status == "ok" {
|
||||
a.Metrics.CheckLastSuccess.WithLabelValues(ev.CheckID).Set(ts)
|
||||
}
|
||||
},
|
||||
})
|
||||
}()
|
||||
|
||||
if a.Cfg.PushesToServer() {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.spoolWriter(ctx, events)
|
||||
}()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.heartbeatLoop(ctx)
|
||||
}()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.sendLoop(ctx)
|
||||
}()
|
||||
} else {
|
||||
// drain scheduler events to avoid blocking it
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-events:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ev := <-events:
|
||||
if err := a.Spool.Enqueue(ev); err != nil {
|
||||
a.Log.Error("spool enqueue", "err", err)
|
||||
}
|
||||
a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
|
||||
a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) heartbeatLoop(ctx context.Context) {
|
||||
interval := a.Cfg.Server.HeartbeatInterval.Duration
|
||||
attempt := 0
|
||||
for {
|
||||
hb := client.HeartbeatRequest{
|
||||
AgentID: a.AgentID,
|
||||
ObservedAt: time.Now().UTC(),
|
||||
Hostname: a.Cfg.Hostname,
|
||||
Version: a.Version,
|
||||
Mode: a.Cfg.Mode,
|
||||
}
|
||||
a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc()
|
||||
err := a.Client.SendHeartbeat(ctx, hb)
|
||||
if err != nil {
|
||||
a.Metrics.SendFailures.WithLabelValues("heartbeat").Inc()
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
a.Log.Warn("heartbeat send failed", "err", err)
|
||||
if client.Retryable(err) {
|
||||
attempt++
|
||||
if sleep(ctx, client.Delay(attempt)) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
a.Metrics.LastHeartbeat.Set(float64(time.Now().Unix()))
|
||||
}
|
||||
attempt = 0
|
||||
if sleep(ctx, interval) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) sendLoop(ctx context.Context) {
|
||||
interval := a.Cfg.Server.BatchInterval.Duration
|
||||
attempt := 0
|
||||
for {
|
||||
items, err := a.Spool.Peek(client.MaxBatchEvents)
|
||||
if err != nil {
|
||||
a.Log.Error("spool peek", "err", err)
|
||||
}
|
||||
if len(items) == 0 {
|
||||
if sleep(ctx, interval) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
evs := make([]scheduler.Event, len(items))
|
||||
seqs := make([]uint64, len(items))
|
||||
for i, it := range items {
|
||||
evs[i] = it.Event
|
||||
seqs[i] = it.Seq
|
||||
}
|
||||
a.Metrics.SendAttempts.WithLabelValues("events").Inc()
|
||||
resp, err := a.Client.SendEvents(ctx, evs)
|
||||
if err != nil {
|
||||
a.Metrics.SendFailures.WithLabelValues("events").Inc()
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
a.Log.Warn("events send failed", "err", err)
|
||||
if client.Retryable(err) {
|
||||
attempt++
|
||||
if sleep(ctx, client.Delay(attempt)) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
// non-retryable: drop to avoid hot loop
|
||||
a.Spool.Commit(seqs)
|
||||
attempt = 0
|
||||
continue
|
||||
}
|
||||
attempt = 0
|
||||
a.Spool.Commit(seqs)
|
||||
a.Metrics.EventsAccepted.Add(float64(resp.Accepted))
|
||||
a.Metrics.EventsDedup.Add(float64(resp.Deduplicated))
|
||||
a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
|
||||
a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
|
||||
}
|
||||
}
|
||||
|
||||
// sleep returns true if context was cancelled while waiting.
|
||||
func sleep(ctx context.Context, d time.Duration) bool {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return true
|
||||
case <-t.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
171
agent/internal/app/app_test.go
Normal file
171
agent/internal/app/app_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/monlet/agent/internal/client"
|
||||
"github.com/monlet/agent/internal/config"
|
||||
"github.com/monlet/agent/internal/metrics"
|
||||
"github.com/monlet/agent/internal/spool"
|
||||
)
|
||||
|
||||
func TestPrometheusOnlyDoesNotPush(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
AgentID: "a1",
|
||||
Mode: "prometheus_only",
|
||||
StateDir: dir,
|
||||
Server: config.ServerConfig{URL: srv.URL, Token: "t", HeartbeatInterval: config.Duration{Duration: 50 * time.Millisecond}, BatchInterval: config.Duration{Duration: 50 * time.Millisecond}},
|
||||
Metrics: config.MetricsConfig{Enabled: false},
|
||||
Checks: []config.CheckConfig{{
|
||||
ID: "c1",
|
||||
Command: []string{"sh", "-c", "echo hi"},
|
||||
Interval: config.Duration{Duration: 100 * time.Millisecond},
|
||||
Timeout: config.Duration{Duration: 50 * time.Millisecond},
|
||||
NotificationOwner: "server",
|
||||
}},
|
||||
}
|
||||
sp, _ := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
|
||||
a := &App{
|
||||
Cfg: cfg, AgentID: "a1", Version: "t",
|
||||
Client: client.New(srv.URL, "t", "a1", "t"),
|
||||
Spool: sp, Metrics: metrics.New(),
|
||||
Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
|
||||
defer cancel()
|
||||
_ = a.Run(ctx)
|
||||
if atomic.LoadInt32(&hits) != 0 {
|
||||
t.Fatalf("prometheus_only must not hit server, got %d requests", atomic.LoadInt32(&hits))
|
||||
}
|
||||
if sp.Len() != 0 {
|
||||
t.Fatalf("prometheus_only must not spool events, got %d", sp.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndToEndPushAndReplay(t *testing.T) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seenEventIDs = make(map[string]int)
|
||||
hbCount int32
|
||||
fail atomic.Bool
|
||||
)
|
||||
fail.Store(true)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/heartbeat":
|
||||
atomic.AddInt32(&hbCount, 1)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_, _ = io.WriteString(w, `{"accepted":true}`)
|
||||
case "/api/v1/events":
|
||||
if fail.Load() {
|
||||
http.Error(w, "down", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
var req client.EventBatchRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
mu.Lock()
|
||||
for _, e := range req.Events {
|
||||
seenEventIDs[e.EventID]++
|
||||
}
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(client.EventBatchResponse{Accepted: len(req.Events)})
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
AgentID: "a1",
|
||||
Hostname: "h",
|
||||
Mode: "hybrid",
|
||||
StateDir: dir,
|
||||
Server: config.ServerConfig{
|
||||
URL: srv.URL,
|
||||
Token: "t",
|
||||
HeartbeatInterval: config.Duration{Duration: 100 * time.Millisecond},
|
||||
BatchInterval: config.Duration{Duration: 100 * time.Millisecond},
|
||||
},
|
||||
Metrics: config.MetricsConfig{Enabled: false},
|
||||
Checks: []config.CheckConfig{{
|
||||
ID: "c1",
|
||||
Command: []string{"sh", "-c", "echo hi"},
|
||||
Interval: config.Duration{Duration: 150 * time.Millisecond},
|
||||
Timeout: config.Duration{Duration: 100 * time.Millisecond},
|
||||
NotificationOwner: "server",
|
||||
}},
|
||||
}
|
||||
sp, err := spool.Open(filepath.Join(dir, "spool"), 100, 1<<20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &App{
|
||||
Cfg: cfg,
|
||||
AgentID: "a1",
|
||||
Version: "test",
|
||||
Client: client.New(cfg.Server.URL, "t", "a1", "test"),
|
||||
Spool: sp,
|
||||
Metrics: metrics.New(),
|
||||
Log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { _ = a.Run(ctx); close(done) }()
|
||||
|
||||
// Phase 1: server failing, events accumulate in spool.
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
if sp.Len() == 0 {
|
||||
cancel()
|
||||
<-done
|
||||
t.Fatal("expected spooled events while server failing")
|
||||
}
|
||||
|
||||
// Phase 2: server recovers.
|
||||
fail.Store(false)
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
mu.Lock()
|
||||
got := len(seenEventIDs)
|
||||
mu.Unlock()
|
||||
if got > 0 && sp.Len() == 0 {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(seenEventIDs) == 0 {
|
||||
t.Fatal("no events delivered after recovery")
|
||||
}
|
||||
for id, n := range seenEventIDs {
|
||||
if n > 1 {
|
||||
t.Fatalf("event_id %s sent %d times (should be deduped by server, but agent should commit on first 2xx)", id, n)
|
||||
}
|
||||
}
|
||||
if atomic.LoadInt32(&hbCount) == 0 {
|
||||
t.Fatal("no heartbeats observed")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user