Harden production workflows and agent admission

agent/internal/app/app.go

@@ -22,11 +22,23 @@ type App struct {
 	Spool   *spool.Spool
 	Metrics *metrics.Metrics
 	Log     *slog.Logger
+
+	lastDropLog time.Time
 }
 
+const (
+	dropReasonOverflowEvents = "spool_overflow_events"
+	dropReasonOverflowBytes  = "spool_overflow_bytes"
+	dropReasonSendRejected   = "send_non_retryable"
+
+	dropLogThrottle = 30 * time.Second
+)
+
 func (a *App) Run(ctx context.Context) error {
 	events := make(chan scheduler.Event, 256)
 
+	// Account drops accumulated during spool Open (e.g. shrunk limits on restart).
+	a.recordSpoolDrops(a.Spool.Drops())
 	a.Metrics.ChecksConfigured.Set(float64(len(a.Cfg.Checks)))
 	a.Metrics.BuildInfo.WithLabelValues(a.Version, boolLabel(a.Cfg.PushesToServer()), boolLabel(a.Cfg.ExposesMetrics())).Set(1)
 	for _, c := range a.Cfg.Checks {
@@ -108,6 +120,7 @@ func (a *App) spoolWriter(ctx context.Context, events <-chan scheduler.Event) {
 			if err := a.Spool.Enqueue(ev); err != nil {
 				a.Log.Error("spool enqueue", "err", err)
 			}
+			a.recordSpoolDrops(a.Spool.Drops())
 			a.Metrics.SpoolDepth.Set(float64(a.Spool.Len()))
 			a.Metrics.SpoolBytes.Set(float64(a.Spool.Bytes()))
 		}
@@ -188,7 +201,9 @@ func (a *App) sendLoop(ctx context.Context) {
 				}
 				continue
 			}
-			// non-retryable: drop to avoid hot loop
+			// non-retryable: drop to avoid hot loop.
+			a.Metrics.EventsDropped.WithLabelValues(dropReasonSendRejected).Add(float64(len(seqs)))
+			a.Log.Warn("events dropped: server rejected batch", "count", len(seqs), "reason", dropReasonSendRejected)
 			a.Spool.Commit(seqs)
 			attempt = 0
 			continue
@@ -214,6 +229,26 @@ func sleep(ctx context.Context, d time.Duration) bool {
 	}
 }
 
+func (a *App) recordSpoolDrops(d spool.DropStats) {
+	if d.OverflowEvents == 0 && d.OverflowBytes == 0 {
+		return
+	}
+	if d.OverflowEvents > 0 {
+		a.Metrics.EventsDropped.WithLabelValues(dropReasonOverflowEvents).Add(float64(d.OverflowEvents))
+	}
+	if d.OverflowBytes > 0 {
+		a.Metrics.EventsDropped.WithLabelValues(dropReasonOverflowBytes).Add(float64(d.OverflowBytes))
+	}
+	now := time.Now()
+	if now.Sub(a.lastDropLog) >= dropLogThrottle {
+		a.Log.Warn("spool drops",
+			"events_overflow", d.OverflowEvents,
+			"bytes_overflow", d.OverflowBytes,
+		)
+		a.lastDropLog = now
+	}
+}
+
 func boolLabel(v bool) string {
 	if v {
 		return "true"