Harden production workflows and agent admission

server/tests/test_detector.py

@@ -9,20 +9,14 @@ from monlet_server.models import Agent, Check, Event, Incident, NotificationOutb
 from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
 from monlet_server.settings import reset_settings_cache
 
-from ._helpers import make_event, make_heartbeat
+from ._helpers import make_event, register_agent
 
 
 @pytest.mark.asyncio
-async def test_detector_transitions(app_client, auth_headers, engine, session):
-    await app_client.post(
-        "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
-    )
-    await app_client.post(
-        "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
-    )
-    await app_client.post(
-        "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
-    )
+async def test_detector_transitions(app_client, ui_auth_headers, engine, session):
+    await register_agent(app_client, ui_auth_headers, "agent-alive")
+    await register_agent(app_client, ui_auth_headers, "agent-stale")
+    await register_agent(app_client, ui_auth_headers, "agent-dead")
 
     now = datetime.now(UTC)
     await session.execute(
@@ -82,10 +76,57 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
 
 
 @pytest.mark.asyncio
-async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session):
-    await app_client.post(
-        "/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers
+async def test_detector_noop_tick_does_not_rewrite_liveness(
+    app_client, ui_auth_headers, engine, session
+):
+    """PH-009: when no agent's liveness status changes, the detector must not
+    rewrite the liveness Check rows. We verify by tagging last_observed_at to a
+    sentinel before the tick and checking the row was untouched."""
+    await register_agent(app_client, ui_auth_headers, "agent-still")
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    # Force a transition first so a liveness Check row exists.
+    await session.execute(
+        update(Agent)
+        .where(Agent.agent_id == "agent-still")
+        .values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10))
     )
+    await session.commit()
+    await _tick(sm)
+    session.expire_all()
+    row_before = (
+        await session.execute(
+            select(Check).where(
+                Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID
+            )
+        )
+    ).scalar_one()
+    sentinel = datetime(2030, 1, 1, tzinfo=UTC)
+    await session.execute(
+        update(Check)
+        .where(Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID)
+        .values(last_observed_at=sentinel)
+    )
+    await session.commit()
+
+    # Second tick with no liveness change: row must NOT be rewritten.
+    await _tick(sm)
+    session.expire_all()
+    row_after = (
+        await session.execute(
+            select(Check).where(
+                Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID
+            )
+        )
+    ).scalar_one()
+    assert row_after.last_observed_at == sentinel
+    assert row_after.last_event_id == row_before.last_event_id
+
+
+@pytest.mark.asyncio
+async def test_detector_owns_liveness_check_and_events(
+    app_client, ui_auth_headers, engine, session
+):
+    await register_agent(app_client, ui_auth_headers, "agent-flap")
     sm = async_sessionmaker(engine, expire_on_commit=False)
 
     # Tick 1: alive → Check exists with status=ok, one event.
@@ -144,13 +185,11 @@ async def test_detector_owns_liveness_check_and_events(app_client, auth_headers,
 
 
 @pytest.mark.asyncio
-async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch):
+async def test_detector_prunes_outbox(app_client, ui_auth_headers, engine, session, monkeypatch):
     monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
     reset_settings_cache()
     try:
-        await app_client.post(
-            "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
-        )
+        auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune")
         critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
         await app_client.post(
             "/api/v1/events",
@@ -192,3 +231,73 @@ async def test_detector_prunes_outbox(app_client, auth_headers, engine, session,
         assert terminal_outbox == 2
     finally:
         reset_settings_cache()
+
+
+@pytest.mark.asyncio
+async def test_detector_prune_preserves_active_outbox(
+    app_client, ui_auth_headers, engine, session, monkeypatch
+):
+    """PH-012: prune must never delete active rows (pending/sending/retry),
+    even when retention is exceeded. The keep-N-newest list applies only to
+    terminal states; active rows are filtered out of the eligible set."""
+    monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "1")
+    reset_settings_cache()
+    try:
+        auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune-active")
+        critical = make_event(check_id="active_prune", status="critical", exit_code=2)
+        await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-prune-active", "events": [critical]},
+            headers=auth_headers,
+        )
+        inc = (
+            await session.execute(select(Incident).where(Incident.check_id == "active_prune"))
+        ).scalar_one()
+        now = datetime.now(UTC)
+        # Mix of active and terminal states.
+        for state, count in (("sent", 5), ("pending", 3), ("retry", 2), ("sending", 1)):
+            for _ in range(count):
+                session.add(
+                    NotificationOutbox(
+                        id=uuid4(),
+                        incident_id=inc.id,
+                        notifier="debug",
+                        event_type="firing",
+                        state=state,
+                        attempts=1,
+                        next_attempt_at=None,
+                        payload={},
+                        created_at=now,
+                        updated_at=now,
+                    )
+                )
+        await session.commit()
+
+        sm = async_sessionmaker(engine, expire_on_commit=False)
+        await _tick(sm)
+
+        # Active rows must be untouched. Ingestion may add extra pending rows
+        # for the debug notifier; we only assert ours are still present and
+        # that no active rows were deleted by the prune.
+        for state, injected in (("pending", 3), ("retry", 2), ("sending", 1)):
+            n = (
+                await session.execute(
+                    select(func.count())
+                    .select_from(NotificationOutbox)
+                    .where(NotificationOutbox.state == state)
+                )
+            ).scalar_one()
+            assert n >= injected, (state, n, injected)
+        # Terminal rows must be capped to retention=1 (or 2 after notifier worker
+        # produces extras from the critical event we ingested; but worker is
+        # disabled in tests so we just check terminal pruning).
+        sent = (
+            await session.execute(
+                select(func.count())
+                .select_from(NotificationOutbox)
+                .where(NotificationOutbox.state == "sent")
+            )
+        ).scalar_one()
+        assert sent <= 2  # 1 retained + at most 1 created by detector enqueue
+    finally:
+        reset_settings_cache()