store only status-change events; UI client-side polling refactor

server/tests/test_detector.py

@@ -6,7 +6,7 @@ from sqlalchemy import func, select, update
 from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from monlet_server.models import Agent, Event, Incident, NotificationOutbox
-from monlet_server.services.detector import _tick
+from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
 from monlet_server.settings import reset_settings_cache
 
 from ._helpers import make_event, make_heartbeat
@@ -39,6 +39,7 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
 
     sm = async_sessionmaker(engine, expire_on_commit=False)
     await _tick(sm)
+    session.expire_all()
 
     statuses = {
         r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
@@ -47,6 +48,38 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
     assert statuses["agent-stale"] == "stale"
     assert statuses["agent-dead"] == "dead"
 
+    incidents = (
+        (await session.execute(select(Incident).where(Incident.check_id == LIVENESS_CHECK_ID)))
+        .scalars()
+        .all()
+    )
+    by_agent = {i.agent_id: i for i in incidents}
+    assert "agent-alive" not in by_agent
+    assert by_agent["agent-stale"].state == "open"
+    assert by_agent["agent-stale"].severity == "warning"
+    assert by_agent["agent-stale"].summary == "agent is stale"
+    assert by_agent["agent-dead"].state == "open"
+    assert by_agent["agent-dead"].severity == "critical"
+    assert by_agent["agent-dead"].summary == "agent is dead"
+
+    await session.execute(
+        update(Agent).where(Agent.agent_id == "agent-dead").values(last_seen_at=datetime.now(UTC))
+    )
+    await session.commit()
+    await _tick(sm)
+    session.expire_all()
+
+    resolved = (
+        await session.execute(
+            select(Incident).where(
+                Incident.agent_id == "agent-dead",
+                Incident.check_id == LIVENESS_CHECK_ID,
+            )
+        )
+    ).scalar_one()
+    assert resolved.state == "resolved"
+    assert resolved.resolved_at is not None
+
 
 @pytest.mark.asyncio
 async def test_detector_prunes_bounded_history(