Harden production workflows and agent admission

server/monlet_server/services/detector.py

@@ -202,14 +202,16 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime
                 summary=summary,
             )
         )
-    else:
+    elif changed:
+        # PH-009: update liveness check row only on status transitions to avoid
+        # O(N) per-tick rewrites of every agent_liveness row. last_observed_at
+        # then reflects the meaningful transition timestamp, not detector ticks.
+        cur.status = liveness_status
+        cur.exit_code = exit_code
+        cur.last_event_id = event_id
+        cur.incident_key = incident_key
         cur.last_observed_at = now
         cur.summary = summary
-        if changed:
-            cur.status = liveness_status
-            cur.exit_code = exit_code
-            cur.last_event_id = event_id
-            cur.incident_key = incident_key
 
     if next_status in ("stale", "dead"):
         await _open_liveness_incident(session, agent, next_status, now, event_id)
@@ -219,6 +221,34 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime
 
 async def _prune_history(session) -> None:
     s = get_settings()
+    if s.pending_agent_ttl_days > 0:
+        cutoff = datetime.now(UTC) - timedelta(days=s.pending_agent_ttl_days)
+        rows = (
+            (
+                await session.execute(
+                    select(Agent)
+                    .where(
+                        Agent.pending_key_hash.is_not(None),
+                        Agent.pending_seen_at < cutoff,
+                    )
+                    .order_by(Agent.pending_seen_at)
+                    .limit(max(s.pending_agent_prune_batch_size, 1))
+                )
+            )
+            .scalars()
+            .all()
+        )
+        for agent in rows:
+            if agent.accepted_key_hash is None:
+                await session.delete(agent)
+            else:
+                agent.pending_key_hash = None
+                agent.pending_key_fingerprint = None
+                agent.pending_hostname = None
+                agent.pending_seen_at = None
+                agent.pending_features = None
+                agent.pending_labels = None
+
     if s.event_dedup_retention_days > 0:
         cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
         batch = max(s.event_dedup_prune_batch_size, 1)
@@ -233,17 +263,29 @@ async def _prune_history(session) -> None:
         await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
 
     if s.outbox_retention_max_rows > 0:
+        # PH-012: keep the N newest terminal rows. Delete in a bounded batch to
+        # avoid one huge DELETE under failure storms. Active rows ('pending',
+        # 'sending', 'retry') are never eligible — terminal-only filter
+        # protects them.
         keep_outbox = (
             select(NotificationOutbox.id)
             .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
             .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
             .limit(s.outbox_retention_max_rows)
         )
-        await session.execute(
-            delete(NotificationOutbox)
+        prune_ids = (
+            select(NotificationOutbox.id)
             .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
             .where(NotificationOutbox.id.not_in(keep_outbox))
+            .limit(max(s.outbox_prune_batch_size, 1))
+            .scalar_subquery()
         )
+        await session.execute(
+            delete(NotificationOutbox).where(NotificationOutbox.id.in_(prune_ids))
+        )
+
+
+DETECTOR_BATCH_SIZE = 500
 
 
 async def _tick(sm: async_sessionmaker) -> None:
@@ -252,23 +294,50 @@ async def _tick(sm: async_sessionmaker) -> None:
     stale_cut = now - timedelta(seconds=s.stale_after_sec)
     dead_cut = now - timedelta(seconds=s.dead_after_sec)
     async with sm() as session:
-        agents = (await session.execute(select(Agent))).scalars().all()
-        for agent in agents:
-            if agent.last_seen_at <= dead_cut:
-                next_status = "dead"
-            elif agent.last_seen_at <= stale_cut:
-                next_status = "stale"
-            else:
-                next_status = "alive"
-            if agent.status != next_status:
-                agent.status = next_status
-            await _apply_liveness(session, agent, next_status, now)
+        # PH-009: keyset-paginate agents by agent_id to bound per-tick memory
+        # and per-batch transaction work. Each batch is committed separately so
+        # large inventories do not produce one huge transaction.
+        # Race trade-off: a new agent inserted with agent_id < last_seen_id
+        # mid-tick is skipped this tick and processed next tick. That delay is
+        # bounded by detector_tick_sec (5s by default) and is acceptable —
+        # liveness detection has a stale_after_sec / dead_after_sec horizon
+        # measured in tens of seconds, so a one-tick lag is invisible.
+        last_seen_id: str | None = None
+        while True:
+            q = (
+                select(Agent)
+                .where(Agent.accepted_key_hash.is_not(None))
+                .order_by(Agent.agent_id)
+                .limit(DETECTOR_BATCH_SIZE)
+            )
+            if last_seen_id is not None:
+                q = q.where(Agent.agent_id > last_seen_id)
+            agents = (await session.execute(q)).scalars().all()
+            if not agents:
+                break
+            for agent in agents:
+                if agent.last_seen_at <= dead_cut:
+                    next_status = "dead"
+                elif agent.last_seen_at <= stale_cut:
+                    next_status = "stale"
+                else:
+                    next_status = "alive"
+                if agent.status != next_status:
+                    agent.status = next_status
+                await _apply_liveness(session, agent, next_status, now)
+            last_seen_id = agents[-1].agent_id
+            await session.commit()
+            if len(agents) < DETECTOR_BATCH_SIZE:
+                break
+
         await _prune_history(session)
         await session.commit()
 
         for status in ("alive", "stale", "dead"):
             r = await session.execute(
-                select(func.count()).select_from(Agent).where(Agent.status == status)
+                select(func.count())
+                .select_from(Agent)
+                .where(Agent.accepted_key_hash.is_not(None), Agent.status == status)
             )
             metrics.agents_gauge.labels(status=status).set(r.scalar_one())
         r = await session.execute(