store only status-change events; UI client-side polling refactor

server/tests/test_detector.py

@@ -6,7 +6,7 @@ from sqlalchemy import func, select, update
 from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from monlet_server.models import Agent, Event, Incident, NotificationOutbox
-from monlet_server.services.detector import _tick
+from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
 from monlet_server.settings import reset_settings_cache
 
 from ._helpers import make_event, make_heartbeat
@@ -39,6 +39,7 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
 
     sm = async_sessionmaker(engine, expire_on_commit=False)
     await _tick(sm)
+    session.expire_all()
 
     statuses = {
         r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
@@ -47,6 +48,38 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
     assert statuses["agent-stale"] == "stale"
     assert statuses["agent-dead"] == "dead"
 
+    incidents = (
+        (await session.execute(select(Incident).where(Incident.check_id == LIVENESS_CHECK_ID)))
+        .scalars()
+        .all()
+    )
+    by_agent = {i.agent_id: i for i in incidents}
+    assert "agent-alive" not in by_agent
+    assert by_agent["agent-stale"].state == "open"
+    assert by_agent["agent-stale"].severity == "warning"
+    assert by_agent["agent-stale"].summary == "agent is stale"
+    assert by_agent["agent-dead"].state == "open"
+    assert by_agent["agent-dead"].severity == "critical"
+    assert by_agent["agent-dead"].summary == "agent is dead"
+
+    await session.execute(
+        update(Agent).where(Agent.agent_id == "agent-dead").values(last_seen_at=datetime.now(UTC))
+    )
+    await session.commit()
+    await _tick(sm)
+    session.expire_all()
+
+    resolved = (
+        await session.execute(
+            select(Incident).where(
+                Incident.agent_id == "agent-dead",
+                Incident.check_id == LIVENESS_CHECK_ID,
+            )
+        )
+    ).scalar_one()
+    assert resolved.state == "resolved"
+    assert resolved.resolved_at is not None
+
 
 @pytest.mark.asyncio
 async def test_detector_prunes_bounded_history(

server/tests/test_events_ingestion.py

@@ -9,7 +9,7 @@ from ._helpers import make_event, make_heartbeat
 @pytest.mark.asyncio
 async def test_event_accepted_and_state(app_client, auth_headers, session):
     await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
-    ev = make_event(status="ok")
+    ev = make_event(status="ok", output="disk ok")
     r = await app_client.post(
         "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
     )
@@ -19,6 +19,7 @@ async def test_event_accepted_and_state(app_client, auth_headers, session):
     assert n == 1
     chk = (await session.execute(select(Check))).scalar_one()
     assert chk.status == "ok"
+    assert chk.summary == ev["output"]
 
 
 @pytest.mark.asyncio
@@ -64,6 +65,53 @@ async def test_oversize_batch_400(app_client, auth_headers):
     assert r.status_code == 400
 
 
+@pytest.mark.asyncio
+async def test_repeated_same_status_not_stored(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    evs = [make_event(status="ok") for _ in range(5)]
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
+    )
+    assert r.status_code == 202
+    assert r.json() == {"accepted": 1, "deduplicated": 4}
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 1
+
+
+@pytest.mark.asyncio
+async def test_status_transitions_stored(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    seq = [
+        make_event(status="ok"),
+        make_event(status="warning", exit_code=1),
+        make_event(status="warning", exit_code=1),
+        make_event(status="critical", exit_code=2),
+        make_event(status="ok"),
+    ]
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": seq}, headers=auth_headers
+    )
+    assert r.status_code == 202
+    assert r.json() == {"accepted": 4, "deduplicated": 1}
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 4
+
+
+@pytest.mark.asyncio
+async def test_same_status_different_exit_code_stored(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    seq = [
+        make_event(status="warning", exit_code=1),
+        make_event(status="warning", exit_code=3),
+    ]
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": seq}, headers=auth_headers
+    )
+    assert r.json() == {"accepted": 2, "deduplicated": 0}
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 2
+
+
 @pytest.mark.asyncio
 async def test_late_event_does_not_move_state(app_client, auth_headers, session):
     from datetime import UTC, datetime, timedelta
@@ -71,7 +119,9 @@ async def test_late_event_does_not_move_state(app_client, auth_headers, session)
     await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
     newer = datetime.now(UTC).replace(microsecond=0)
     older = (newer - timedelta(minutes=5)).isoformat()
-    ev_now = make_event(status="critical", exit_code=2, observed_at=newer.isoformat())
+    ev_now = make_event(
+        status="critical", exit_code=2, observed_at=newer.isoformat(), output="current critical"
+    )
     ev_old = make_event(status="ok", exit_code=0, observed_at=older)
     await app_client.post(
         "/api/v1/events", json={"agent_id": "agent-1", "events": [ev_now]}, headers=auth_headers
@@ -81,5 +131,6 @@ async def test_late_event_does_not_move_state(app_client, auth_headers, session)
     )
     chk = (await session.execute(select(Check))).scalar_one()
     assert chk.status == "critical"
+    assert chk.summary == ev_now["output"]
     n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
-    assert n == 2
+    assert n == 1

server/tests/test_incident_key.py

@@ -35,7 +35,7 @@ async def test_incident_key_too_long(app_client, auth_headers):
 @pytest.mark.asyncio
 async def test_events_query_cursor_pagination(app_client, auth_headers):
     await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
-    evs = [make_event() for _ in range(5)]
+    evs = [make_event(check_id=f"chk-{i}") for i in range(5)]
     await app_client.post(
         "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
     )