init monlet repo with stage 0-2 (docs, contract, agent MVP)

agent/internal/app/app.go

@@ -0,0 +1,212 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/monlet/agent/internal/client"
+	"github.com/monlet/agent/internal/config"
+	"github.com/monlet/agent/internal/metrics"
+	"github.com/monlet/agent/internal/scheduler"
+	"github.com/monlet/agent/internal/spool"
+)
+
+type App struct {
+	Cfg     *config.Config
+	AgentID string
+	Version string
+	Client  *client.Client
+	Spool   *spool.Spool
+	Metrics *metrics.Metrics
+	Log     *slog.Logger
+}
+
+func (a *App) Run(ctx context.Context) error {
+	events := make(chan scheduler.Event, 256)
+
+	a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
+	a.Metrics.BuildInfo.WithLabelValues(a.Version, a.Cfg.Mode).Set(1)
+	for _, c := range a.Cfg.Checks {
+		a.Metrics.CheckInterval.WithLabelValues(c.ID).Set(c.Interval.Duration.Seconds())
+	}
+
+	var wg sync.WaitGroup
+
+	if a.Cfg.ExposesMetrics() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := a.Metrics.Serve(ctx, a.Cfg.Metrics.Listen); err != nil {
+				a.Log.Error("metrics server", "err", err)
+			}
+		}()
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		scheduler.Run(ctx, a.AgentID, a.Cfg.Checks, events, scheduler.Hooks{
+			OnSkipped: func(id string) { a.Metrics.CheckSkipped.WithLabelValues(id).Inc() },
+			OnResult: func(ev scheduler.Event) {
+				a.Metrics.CheckRuns.WithLabelValues(ev.CheckID, ev.Status).Inc()
+				a.Metrics.CheckDuration.WithLabelValues(ev.CheckID).Observe(float64(ev.DurationMs) / 1000.0)
+				a.Metrics.CheckStatus.WithLabelValues(ev.CheckID).Set(metrics.StatusCode(ev.Status))
+				a.Metrics.CheckExitCode.WithLabelValues(ev.CheckID).Set(float64(ev.ExitCode))
+				ts := float64(ev.ObservedAt.Unix())
+				a.Metrics.CheckLastRun.WithLabelValues(ev.CheckID).Set(ts)
+				if ev.Status == "ok" {
+					a.Metrics.CheckLastSuccess.WithLabelValues(ev.CheckID).Set(ts)
+				}
+			},
+		})
+	}()
+
+	if a.Cfg.PushesToServer() {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.spoolWriter(ctx, events)
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.heartbeatLoop(ctx)
+		}()
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a.sendLoop(ctx)
+		}()
+	} else {
+		// drain scheduler events to avoid blocking it
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case <-events:
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case ev := <-events:
+			if err := a.Spool.Enqueue(ev); err != nil {
+				a.Log.Error("spool enqueue", "err", err)
+			}
+			a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
+			a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
+		}
+	}
+}
+
+func (a *App) heartbeatLoop(ctx context.Context) {
+	interval := a.Cfg.Server.HeartbeatInterval.Duration
+	attempt := 0
+	for {
+		hb := client.HeartbeatRequest{
+			AgentID:    a.AgentID,
+			ObservedAt: time.Now().UTC(),
+			Hostname:   a.Cfg.Hostname,
+			Version:    a.Version,
+			Mode:       a.Cfg.Mode,
+		}
+		a.Metrics.SendAttempts.WithLabelValues("heartbeat").Inc()
+		err := a.Client.SendHeartbeat(ctx, hb)
+		if err != nil {
+			a.Metrics.SendFailures.WithLabelValues("heartbeat").Inc()
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			a.Log.Warn("heartbeat send failed", "err", err)
+			if client.Retryable(err) {
+				attempt++
+				if sleep(ctx, client.Delay(attempt)) {
+					return
+				}
+				continue
+			}
+		} else {
+			a.Metrics.LastHeartbeat.Set(float64(time.Now().Unix()))
+		}
+		attempt = 0
+		if sleep(ctx, interval) {
+			return
+		}
+	}
+}
+
+func (a *App) sendLoop(ctx context.Context) {
+	interval := a.Cfg.Server.BatchInterval.Duration
+	attempt := 0
+	for {
+		items, err := a.Spool.Peek(client.MaxBatchEvents)
+		if err != nil {
+			a.Log.Error("spool peek", "err", err)
+		}
+		if len(items) == 0 {
+			if sleep(ctx, interval) {
+				return
+			}
+			continue
+		}
+		evs := make([]scheduler.Event, len(items))
+		seqs := make([]uint64, len(items))
+		for i, it := range items {
+			evs[i] = it.Event
+			seqs[i] = it.Seq
+		}
+		a.Metrics.SendAttempts.WithLabelValues("events").Inc()
+		resp, err := a.Client.SendEvents(ctx, evs)
+		if err != nil {
+			a.Metrics.SendFailures.WithLabelValues("events").Inc()
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+			a.Log.Warn("events send failed", "err", err)
+			if client.Retryable(err) {
+				attempt++
+				if sleep(ctx, client.Delay(attempt)) {
+					return
+				}
+				continue
+			}
+			// non-retryable: drop to avoid hot loop
+			a.Spool.Commit(seqs)
+			attempt = 0
+			continue
+		}
+		attempt = 0
+		a.Spool.Commit(seqs)
+		a.Metrics.EventsAccepted.Add(float64(resp.Accepted))
+		a.Metrics.EventsDedup.Add(float64(resp.Deduplicated))
+		a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
+		a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
+	}
+}
+
+// sleep returns true if context was cancelled while waiting.
+func sleep(ctx context.Context, d time.Duration) bool {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return true
+	case <-t.C:
+		return false
+	}
+}