Implement Monlet MVP stack and UI updates

server/tests/_helpers.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import random
+import time
+import uuid
+from datetime import UTC, datetime
+
+
+def uuidv7() -> str:
+    ts_ms = int(time.time() * 1000)
+    rand_a = random.getrandbits(12)
+    rand_b = random.getrandbits(62)
+    high = (ts_ms & 0xFFFFFFFFFFFF) << 16 | 0x7000 | rand_a
+    low = (0b10 << 62) | rand_b
+    return str(uuid.UUID(int=(high << 64) | low))
+
+
+def now_iso(offset_sec: int = 0) -> str:
+    return datetime.now(UTC).replace(microsecond=0).isoformat()
+
+
+def make_event(
+    check_id: str = "disk_root",
+    status: str = "ok",
+    exit_code: int = 0,
+    observed_at: str | None = None,
+    output: str | None = None,
+    incident_key: str | None = None,
+    notifications_enabled: bool = True,
+) -> dict:
+    ev = {
+        "event_id": uuidv7(),
+        "check_id": check_id,
+        "observed_at": observed_at or now_iso(),
+        "status": status,
+        "exit_code": exit_code,
+        "duration_ms": 12,
+        "output_truncated": False,
+        "notifications_enabled": notifications_enabled,
+    }
+    if output is not None:
+        ev["output"] = output
+    if incident_key is not None:
+        ev["incident_key"] = incident_key
+    return ev
+
+
+def make_heartbeat(agent_id: str = "agent-1") -> dict:
+    return {
+        "agent_id": agent_id,
+        "observed_at": now_iso(),
+        "hostname": "host-1",
+        "features": {"push": True, "metrics": True},
+        "labels": {},
+    }

server/tests/conftest.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import AsyncIterator, Iterator
+
+import pytest
+import pytest_asyncio
+from alembic.config import Config
+from httpx import ASGITransport, AsyncClient
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from testcontainers.postgres import PostgresContainer
+
+from alembic import command
+
+os.environ.setdefault("MONLET_AUTH_TOKEN", "test-token")
+os.environ.setdefault("MONLET_ENABLE_DETECTOR", "false")
+os.environ.setdefault("MONLET_ENABLE_NOTIFIER_WORKER", "false")
+
+from monlet_server import db as db_module  # noqa: E402
+from monlet_server.main import create_app  # noqa: E402
+from monlet_server.settings import get_settings, reset_settings_cache  # noqa: E402
+
+HERE = os.path.dirname(__file__)
+SERVER_ROOT = os.path.dirname(HERE)
+
+
+@pytest.fixture(scope="session")
+def event_loop() -> Iterator[asyncio.AbstractEventLoop]:
+    loop = asyncio.new_event_loop()
+    yield loop
+    loop.close()
+
+
+@pytest.fixture(scope="session")
+def pg_container() -> Iterator[PostgresContainer]:
+    with PostgresContainer("postgres:16-alpine") as pg:
+        yield pg
+
+
+@pytest.fixture(scope="session")
+def database_url(pg_container: PostgresContainer) -> str:
+    raw = pg_container.get_connection_url()
+    if raw.startswith("postgresql+psycopg2://"):
+        async_url = raw.replace("postgresql+psycopg2://", "postgresql+asyncpg://")
+    elif raw.startswith("postgresql://"):
+        async_url = raw.replace("postgresql://", "postgresql+asyncpg://", 1)
+    else:
+        async_url = raw
+    return async_url
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _apply_migrations(database_url: str) -> Iterator[None]:
+    os.environ["MONLET_DATABASE_URL"] = database_url
+    reset_settings_cache()
+    cfg = Config(os.path.join(SERVER_ROOT, "alembic.ini"))
+    cfg.set_main_option("script_location", os.path.join(SERVER_ROOT, "alembic"))
+    cfg.set_main_option("sqlalchemy.url", get_settings().sync_database_url)
+    command.upgrade(cfg, "head")
+    yield
+
+
+@pytest_asyncio.fixture
+async def engine(database_url: str):
+    eng = create_async_engine(database_url, future=True)
+    yield eng
+    await eng.dispose()
+
+
+@pytest_asyncio.fixture
+async def session(engine) -> AsyncIterator[AsyncSession]:
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    async with sm() as s:
+        yield s
+
+
+@pytest_asyncio.fixture(autouse=True)
+async def _truncate_tables(engine) -> AsyncIterator[None]:
+    yield
+    from sqlalchemy import text
+
+    async with engine.begin() as conn:
+        await conn.execute(
+            text(
+                "TRUNCATE TABLE notification_outbox, incidents, events, checks, agents RESTART IDENTITY CASCADE"
+            )
+        )
+
+
+@pytest_asyncio.fixture
+async def app_client(database_url: str) -> AsyncIterator[AsyncClient]:
+    reset_settings_cache()
+    await db_module.dispose_engine()
+    app = create_app()
+    async with app.router.lifespan_context(app):
+        transport = ASGITransport(app=app)
+        async with AsyncClient(transport=transport, base_url="http://test") as client:
+            yield client
+    await db_module.dispose_engine()
+
+
+@pytest.fixture
+def auth_headers() -> dict[str, str]:
+    return {"Authorization": "Bearer test-token"}

server/tests/test_auth.py

@@ -0,0 +1,35 @@
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_health_no_auth(app_client):
+    r = await app_client.get("/api/v1/health")
+    assert r.status_code == 200
+    assert r.json()["status"] == "ok"
+
+
+@pytest.mark.asyncio
+async def test_metrics_no_auth(app_client):
+    r = await app_client.get("/metrics")
+    assert r.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_missing_token_401(app_client):
+    r = await app_client.get("/api/v1/agents")
+    assert r.status_code == 401
+    body = r.json()
+    assert body["error"]["code"] == "unauthorized"
+    assert body["error"]["request_id"]
+
+
+@pytest.mark.asyncio
+async def test_wrong_token_401(app_client):
+    r = await app_client.get("/api/v1/agents", headers={"Authorization": "Bearer nope"})
+    assert r.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_valid_token_200(app_client, auth_headers):
+    r = await app_client.get("/api/v1/agents", headers=auth_headers)
+    assert r.status_code == 200

server/tests/test_body_limit.py

@@ -0,0 +1,30 @@
+import json
+import uuid
+
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_body_too_large_413(app_client, auth_headers):
+    big_event = make_event(output="x" * 8000)
+    payload = {"agent_id": "agent-1", "events": [big_event] * 200}
+    raw = json.dumps(payload) + " " * (1_048_577 - len(json.dumps(payload)))
+    rid_in = str(uuid.uuid4())
+    r = await app_client.post(
+        "/api/v1/events",
+        content=raw,
+        headers={**auth_headers, "Content-Type": "application/json", "X-Request-Id": rid_in},
+    )
+    assert r.status_code == 413
+    body = r.json()
+    assert body["error"]["code"] == "payload_too_large"
+    assert body["error"]["request_id"] == rid_in
+    assert r.headers.get("X-Request-Id") == rid_in
+
+
+@pytest.mark.asyncio
+async def test_small_body_ok(app_client, auth_headers):
+    r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    assert r.status_code == 202

server/tests/test_constraints.py

@@ -0,0 +1,70 @@
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.exc import IntegrityError
+
+from monlet_server.models import Incident
+
+
+@pytest.mark.asyncio
+async def test_event_id_conflict_do_nothing(engine):
+    eid = str(uuid4())
+    async with engine.begin() as conn:
+        await conn.execute(
+            text(
+                "INSERT INTO agents (agent_id, hostname, features, status, last_seen_at) "
+                "VALUES ('a','h',jsonb_build_object('push', true, 'metrics', false),'alive', now())"
+            )
+        )
+        await conn.execute(
+            text(
+                "INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
+                " VALUES (:e,'a','c', now(),'ok',0,1,'a:c')"
+            ),
+            {"e": eid},
+        )
+        r = await conn.execute(
+            text(
+                "INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
+                " VALUES (:e,'a','c', now(),'ok',0,1,'a:c') ON CONFLICT (event_id) DO NOTHING RETURNING event_id"
+            ),
+            {"e": eid},
+        )
+        assert r.scalar_one_or_none() is None
+
+
+@pytest.mark.asyncio
+async def test_partial_unique_open_incident(session):
+    key = "k1"
+    now = datetime.now(UTC)
+    eid = uuid4()
+    session.add(
+        Incident(
+            id=uuid4(),
+            incident_key=key,
+            agent_id="a",
+            check_id="c",
+            state="open",
+            severity="warning",
+            opened_at=now,
+            last_event_id=eid,
+        )
+    )
+    await session.commit()
+    session.add(
+        Incident(
+            id=uuid4(),
+            incident_key=key,
+            agent_id="a",
+            check_id="c",
+            state="open",
+            severity="critical",
+            opened_at=now,
+            last_event_id=eid,
+        )
+    )
+    with pytest.raises(IntegrityError):
+        await session.commit()
+    await session.rollback()

server/tests/test_detector.py

@@ -0,0 +1,109 @@
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import func, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from monlet_server.models import Agent, Event, Incident, NotificationOutbox
+from monlet_server.services.detector import _tick
+from monlet_server.settings import reset_settings_cache
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_detector_transitions(app_client, auth_headers, engine, session):
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
+    )
+
+    now = datetime.now(UTC)
+    await session.execute(
+        update(Agent)
+        .where(Agent.agent_id == "agent-stale")
+        .values(last_seen_at=now - timedelta(seconds=120))
+    )
+    await session.execute(
+        update(Agent)
+        .where(Agent.agent_id == "agent-dead")
+        .values(last_seen_at=now - timedelta(minutes=10))
+    )
+    await session.commit()
+
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    await _tick(sm)
+
+    statuses = {
+        r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
+    }
+    assert statuses["agent-alive"] == "alive"
+    assert statuses["agent-stale"] == "stale"
+    assert statuses["agent-dead"] == "dead"
+
+
+@pytest.mark.asyncio
+async def test_detector_prunes_bounded_history(
+    app_client, auth_headers, engine, session, monkeypatch
+):
+    monkeypatch.setenv("MONLET_EVENTS_RETENTION_MAX_ROWS", "2")
+    monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
+    reset_settings_cache()
+    try:
+        await app_client.post(
+            "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
+        )
+        events = [make_event(check_id="prune_check") for _ in range(5)]
+        await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-prune", "events": events},
+            headers=auth_headers,
+        )
+        critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
+        await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-prune", "events": [critical]},
+            headers=auth_headers,
+        )
+
+        inc = (
+            await session.execute(select(Incident).where(Incident.check_id == "critical_prune"))
+        ).scalar_one()
+        now = datetime.now(UTC)
+        for _ in range(5):
+            session.add(
+                NotificationOutbox(
+                    id=uuid4(),
+                    incident_id=inc.id,
+                    notifier="debug",
+                    event_type="firing",
+                    state="sent",
+                    attempts=1,
+                    next_attempt_at=None,
+                    payload={},
+                    created_at=now,
+                    updated_at=now,
+                )
+            )
+        await session.commit()
+
+        sm = async_sessionmaker(engine, expire_on_commit=False)
+        await _tick(sm)
+
+        assert (await session.execute(select(func.count()).select_from(Event))).scalar_one() == 2
+        terminal_outbox = (
+            await session.execute(
+                select(func.count())
+                .select_from(NotificationOutbox)
+                .where(NotificationOutbox.state == "sent")
+            )
+        ).scalar_one()
+        assert terminal_outbox == 2
+    finally:
+        reset_settings_cache()

server/tests/test_events_ingestion.py

@@ -0,0 +1,73 @@
+import pytest
+from sqlalchemy import func, select
+
+from monlet_server.models import Check, Event
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_event_accepted_and_state(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="ok")
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r.status_code == 202
+    assert r.json() == {"accepted": 1, "deduplicated": 0}
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 1
+    chk = (await session.execute(select(Check))).scalar_one()
+    assert chk.status == "ok"
+
+
+@pytest.mark.asyncio
+async def test_duplicate_event_deduplicated(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2)
+    r1 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    r2 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r1.json() == {"accepted": 1, "deduplicated": 0}
+    assert r2.json() == {"accepted": 0, "deduplicated": 1}
+
+
+@pytest.mark.asyncio
+async def test_empty_batch_400(app_client, auth_headers):
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": []}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_oversize_batch_400(app_client, auth_headers):
+    evs = [make_event() for _ in range(201)]
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_late_event_does_not_move_state(app_client, auth_headers, session):
+    from datetime import UTC, datetime, timedelta
+
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    newer = datetime.now(UTC).replace(microsecond=0)
+    older = (newer - timedelta(minutes=5)).isoformat()
+    ev_now = make_event(status="critical", exit_code=2, observed_at=newer.isoformat())
+    ev_old = make_event(status="ok", exit_code=0, observed_at=older)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev_now]}, headers=auth_headers
+    )
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev_old]}, headers=auth_headers
+    )
+    chk = (await session.execute(select(Check))).scalar_one()
+    assert chk.status == "critical"
+    n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
+    assert n == 2

server/tests/test_heartbeat.py

@@ -0,0 +1,61 @@
+import pytest
+from sqlalchemy import select
+
+from monlet_server.models import Agent
+
+from ._helpers import make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_creates_agent(app_client, auth_headers, session):
+    r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    assert r.status_code == 202
+    res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
+    a = res.scalar_one()
+    assert a.hostname == "host-1"
+    assert a.status == "alive"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_upsert(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    hb = make_heartbeat()
+    hb["features"] = {"push": True, "metrics": False}
+    await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
+    a = res.scalar_one()
+    assert a.features == {"push": True, "metrics": False}
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_validation_400(app_client, auth_headers):
+    r = await app_client.post(
+        "/api/v1/heartbeat",
+        json={
+            "agent_id": "bad id!",
+            "observed_at": "x",
+            "hostname": "h",
+            "features": {"push": True, "metrics": False},
+        },
+        headers=auth_headers,
+    )
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_rejects_extra_fields(app_client, auth_headers):
+    hb = make_heartbeat()
+    hb["mode"] = "hybrid"
+    r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_heartbeat_rejects_sensitive_label_keys(app_client, auth_headers):
+    hb = make_heartbeat()
+    hb["labels"] = {"api_key": "secret"}
+    r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"

server/tests/test_incident_key.py

@@ -0,0 +1,55 @@
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_incident_key_ownership_rejected(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-1"), headers=auth_headers)
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-2"), headers=auth_headers)
+
+    ev1 = make_event(status="critical", exit_code=2, incident_key="shared-key")
+    r1 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=auth_headers
+    )
+    assert r1.status_code == 202
+
+    ev2 = make_event(status="critical", exit_code=2, incident_key="shared-key")
+    r2 = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=auth_headers
+    )
+    assert r2.status_code == 400
+    assert r2.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_incident_key_too_long(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2, incident_key="x" * 257)
+    r = await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    assert r.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_events_query_cursor_pagination(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    evs = [make_event() for _ in range(5)]
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
+    )
+    r = await app_client.get("/api/v1/events/query?limit=2", headers=auth_headers)
+    assert r.status_code == 200
+    page1 = r.json()
+    assert len(page1["items"]) == 2
+    assert page1["next_cursor"]
+
+    r2 = await app_client.get(
+        f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=auth_headers
+    )
+    page2 = r2.json()
+    assert len(page2["items"]) == 2
+    ids1 = {i["event_id"] for i in page1["items"]}
+    ids2 = {i["event_id"] for i in page2["items"]}
+    assert ids1.isdisjoint(ids2)

server/tests/test_incidents.py

@@ -0,0 +1,79 @@
+import pytest
+from sqlalchemy import select
+
+from monlet_server.models import Incident, NotificationOutbox
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_open_escalate_resolve_reopen(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    a = "agent-1"
+
+    # warning -> open warning
+    ev1 = make_event(status="warning", exit_code=1)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev1]}, headers=auth_headers
+    )
+    inc = (await session.execute(select(Incident))).scalar_one()
+    assert inc.state == "open"
+    assert inc.severity == "warning"
+
+    # critical -> escalated, still single open
+    ev2 = make_event(status="critical", exit_code=2)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev2]}, headers=auth_headers
+    )
+    session.expire_all()
+    incs = (await session.execute(select(Incident))).scalars().all()
+    assert len(incs) == 1
+    assert incs[0].severity == "critical"
+
+    # ok -> resolved
+    ev3 = make_event(status="ok", exit_code=0)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev3]}, headers=auth_headers
+    )
+    session.expire_all()
+    incs = (await session.execute(select(Incident))).scalars().all()
+    assert len(incs) == 1
+    assert incs[0].state == "resolved"
+
+    # critical again -> new open
+    ev4 = make_event(status="critical", exit_code=2)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev4]}, headers=auth_headers
+    )
+    incs = (await session.execute(select(Incident).order_by(Incident.opened_at))).scalars().all()
+    assert len(incs) == 2
+    states = {i.state for i in incs}
+    assert states == {"open", "resolved"}
+
+
+@pytest.mark.asyncio
+async def test_outbox_skipped_when_notifications_disabled(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2, notifications_enabled=False)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    rows = (await session.execute(select(NotificationOutbox))).scalars().all()
+    assert rows == []
+
+
+@pytest.mark.asyncio
+async def test_outbox_firing_and_resolved(app_client, auth_headers, session):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    a = "agent-1"
+    ev_open = make_event(status="critical", exit_code=2)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev_open]}, headers=auth_headers
+    )
+    ev_res = make_event(status="ok", exit_code=0)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": a, "events": [ev_res]}, headers=auth_headers
+    )
+    rows = (await session.execute(select(NotificationOutbox))).scalars().all()
+    types = sorted(r.event_type for r in rows)
+    assert types == ["firing", "resolved"]

server/tests/test_integration_agent.py

@@ -0,0 +1,33 @@
+import json
+import os
+
+import pytest
+from sqlalchemy import select
+
+from monlet_server.models import Agent, Check, Incident
+
+EXAMPLES = os.path.join(os.path.dirname(__file__), "..", "..", "examples")
+
+
+@pytest.mark.asyncio
+async def test_examples_payload(app_client, auth_headers, session):
+    with open(os.path.join(EXAMPLES, "heartbeat.json")) as f:
+        hb = json.load(f)
+    with open(os.path.join(EXAMPLES, "event-batch.json")) as f:
+        batch = json.load(f)
+
+    r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
+    assert r.status_code == 202
+
+    r = await app_client.post("/api/v1/events", json=batch, headers=auth_headers)
+    assert r.status_code == 202
+
+    agents = (await session.execute(select(Agent))).scalars().all()
+    assert len(agents) == 1
+    checks = (await session.execute(select(Check))).scalars().all()
+    assert {c.check_id for c in checks} == {"disk_root", "postgres_up"}
+    open_inc = (
+        (await session.execute(select(Incident).where(Incident.state == "open"))).scalars().all()
+    )
+    assert len(open_inc) == 1
+    assert open_inc[0].check_id == "postgres_up"

server/tests/test_metrics.py

@@ -0,0 +1,18 @@
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_metrics_endpoint(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    ev = make_event(status="critical", exit_code=2)
+    await app_client.post(
+        "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+    )
+    r = await app_client.get("/metrics")
+    assert r.status_code == 200
+    body = r.text
+    assert "monlet_server_heartbeats_total" in body
+    assert "monlet_server_events_total" in body
+    assert "monlet_server_incidents_opened_total" in body

server/tests/test_notifiers.py

@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import httpx
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from monlet_server.models import Incident, NotificationOutbox
+from monlet_server.services.notifier_worker import _backoff, tick
+from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
+from monlet_server.services.notifiers.base import DeliveryResult
+from monlet_server.services.notifiers.debug import DebugNotifier
+from monlet_server.services.notifiers.telegram import TelegramNotifier
+from monlet_server.services.notifiers.webhook import WebhookNotifier
+from monlet_server.settings import Settings
+
+
+def _payload() -> dict:
+    return {
+        "incident_id": str(uuid4()),
+        "agent_id": "agent-1",
+        "check_id": "chk-1",
+        "status": "critical",
+        "severity": "critical",
+        "incident_key": "agent-1:chk-1",
+        "observed_at": datetime.now(UTC).isoformat(),
+        "opened_at": datetime.now(UTC).isoformat(),
+        "summary": "boom",
+        "output": "stderr: boom token=abc123",
+    }
+
+
+class _StubTransport(httpx.AsyncBaseTransport):
+    def __init__(self, handler):
+        self._handler = handler
+        self.requests: list[httpx.Request] = []
+
+    async def handle_async_request(self, request):
+        self.requests.append(request)
+        return self._handler(request)
+
+
+def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
+    t = _StubTransport(handler)
+    return httpx.AsyncClient(transport=t, base_url="http://x"), t
+
+
+@pytest.mark.asyncio
+async def test_debug_notifier_always_ok():
+    res = await DebugNotifier().deliver("firing", _payload())
+    assert res.ok and not res.permanent
+
+
+@pytest.mark.asyncio
+async def test_telegram_success_and_redaction():
+    def handler(req):
+        body = req.content.decode()
+        assert "token=abc123" not in body
+        assert "***" in body
+        return httpx.Response(200, json={"ok": True})
+
+    client, t = _client(handler)
+    try:
+        n = TelegramNotifier(client, "TOK", "123")
+        res = await n.deliver("firing", _payload())
+        assert res.ok
+        assert "/botTOK/sendMessage" in str(t.requests[0].url)
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_4xx_permanent():
+    client, _ = _client(lambda r: httpx.Response(400, json={}))
+    try:
+        n = TelegramNotifier(client, "TOK", "123")
+        res = await n.deliver("firing", _payload())
+        assert not res.ok and res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_5xx_transient():
+    client, _ = _client(lambda r: httpx.Response(503, json={}))
+    try:
+        n = TelegramNotifier(client, "TOK", "123")
+        res = await n.deliver("firing", _payload())
+        assert not res.ok and not res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_network_error_transient():
+    def handler(req):
+        raise httpx.ConnectError("nope")
+
+    client, _ = _client(handler)
+    try:
+        n = TelegramNotifier(client, "TOK", "123")
+        res = await n.deliver("firing", _payload())
+        assert not res.ok and not res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_webhook_sends_bearer_and_redacts_payload():
+    captured: dict = {}
+
+    def handler(req):
+        captured["auth"] = req.headers.get("authorization")
+        captured["body"] = req.content.decode()
+        return httpx.Response(202)
+
+    client, _ = _client(handler)
+    try:
+        n = WebhookNotifier(client, "http://x/hook", token="WTOK")
+        res = await n.deliver("firing", _payload())
+        assert res.ok
+        assert captured["auth"] == "Bearer WTOK"
+        assert "token=abc123" not in captured["body"]
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_alertmanager_payload_shape():
+    captured: dict = {}
+
+    def handler(req):
+        captured["url"] = str(req.url)
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        n = AlertmanagerNotifier(client, "http://am/")
+        res = await n.deliver("firing", _payload())
+        assert res.ok
+        assert captured["url"].endswith("/api/v2/alerts")
+        assert '"alertname":"monlet_incident"' in captured["body"]
+    finally:
+        await client.aclose()
+
+
+def test_backoff_matches_adr_0005():
+    # ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
+    expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
+    for i, sec in enumerate(expected, start=1):
+        assert _backoff(i).total_seconds() == sec
+    # Caps at last entry.
+    assert _backoff(99).total_seconds() == 14400
+
+
+# ---- Worker integration with real PG ----
+
+
+@pytest.fixture
+def settings_full() -> Settings:
+    s = Settings()
+    s.notifier_max_attempts = 3
+    s.notifier_batch_size = 50
+    return s
+
+
+async def _seed_incident_and_outbox(
+    sm: async_sessionmaker, notifier: str
+) -> tuple[NotificationOutbox, Incident]:
+    async with sm() as session:
+        # Need agent for FK on incidents.agent_id? No FK on incidents.agent_id per schema.
+        inc = Incident(
+            id=uuid4(),
+            incident_key=f"k-{uuid4()}",
+            agent_id="agent-1",
+            check_id="chk-1",
+            state="open",
+            severity="critical",
+            opened_at=datetime.now(UTC),
+            last_event_id=uuid4(),
+            summary="boom",
+        )
+        session.add(inc)
+        await session.flush()
+        row = NotificationOutbox(
+            id=uuid4(),
+            incident_id=inc.id,
+            notifier=notifier,
+            event_type="firing",
+            state="pending",
+            attempts=0,
+            next_attempt_at=datetime.now(UTC),
+            payload=_payload(),
+        )
+        session.add(row)
+        await session.commit()
+        return row, inc
+
+
+class _StubNotifier:
+    def __init__(self, name: str, results: list[DeliveryResult]) -> None:
+        self.name = name
+        self._results = list(results)
+        self.calls = 0
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        self.calls += 1
+        if self._results:
+            return self._results.pop(0)
+        return DeliveryResult(ok=True)
+
+
+@pytest.mark.asyncio
+async def test_worker_marks_sent(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "stub")
+    registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])}
+    n = await tick(sm, registry, settings_full)
+    assert n == 1
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "sent"
+        assert fresh.attempts == 1
+        assert fresh.last_error is None
+
+
+@pytest.mark.asyncio
+async def test_worker_retries_then_succeeds(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "stub")
+    notifier = _StubNotifier(
+        "stub",
+        [DeliveryResult(ok=False, permanent=False, error="net"), DeliveryResult(ok=True)],
+    )
+    registry = {"stub": notifier}
+    await tick(sm, registry, settings_full)
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "retry"
+        assert fresh.attempts == 1
+        assert fresh.next_attempt_at is not None
+        # Force eligibility for second tick.
+        fresh.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
+        await session.commit()
+    await tick(sm, registry, settings_full)
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "sent"
+        assert fresh.attempts == 2
+
+
+@pytest.mark.asyncio
+async def test_worker_permanent_failure_marked_failed(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "stub")
+    registry = {
+        "stub": _StubNotifier("stub", [DeliveryResult(ok=False, permanent=True, error="http_400")])
+    }
+    await tick(sm, registry, settings_full)
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "failed"
+        assert fresh.last_error == "http_400"
+
+
+@pytest.mark.asyncio
+async def test_worker_unknown_notifier_discarded(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "ghost")
+    await tick(sm, {}, settings_full)
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "discarded"
+
+
+@pytest.mark.asyncio
+async def test_worker_exhausts_attempts(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "stub")
+    notifier = _StubNotifier(
+        "stub",
+        [DeliveryResult(ok=False, permanent=False, error="boom")] * 10,
+    )
+    registry = {"stub": notifier}
+    for _ in range(settings_full.notifier_max_attempts):
+        async with sm() as session:
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+            # Force ready.
+            await session.execute(
+                NotificationOutbox.__table__.update()
+                .where(NotificationOutbox.id == row.id)
+                .values(next_attempt_at=datetime.now(UTC) - timedelta(seconds=1))
+            )
+            await session.commit()
+        await tick(sm, registry, settings_full)
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "failed"
+        assert fresh.attempts == settings_full.notifier_max_attempts
+
+
+@pytest.mark.asyncio
+async def test_worker_recovers_stuck_sending(engine, settings_full):
+    sm = async_sessionmaker(engine, expire_on_commit=False)
+    row, _ = await _seed_incident_and_outbox(sm, "stub")
+    async with sm() as session:
+        await session.execute(
+            NotificationOutbox.__table__.update()
+            .where(NotificationOutbox.id == row.id)
+            .values(state="sending", updated_at=datetime.now(UTC) - timedelta(hours=1))
+        )
+        await session.commit()
+    settings_full.notifier_lease_sec = 60
+    registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])}
+    n = await tick(sm, registry, settings_full)
+    assert n == 1
+    async with sm() as session:
+        fresh = (
+            await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
+        ).scalar_one()
+        assert fresh.state == "sent"
+
+
+@pytest.mark.asyncio
+async def test_alertmanager_severity_not_in_labels():
+    captured: dict = {}
+
+    def handler(req):
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        n = AlertmanagerNotifier(client, "http://am")
+        await n.deliver("firing", _payload())
+        import json as _json
+
+        alerts = _json.loads(captured["body"])
+        assert "severity" not in alerts[0]["labels"]
+        assert alerts[0]["annotations"]["severity"] == "critical"
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_redacts_secret_in_incident_key():
+    captured: dict = {}
+
+    def handler(req):
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        p = _payload()
+        p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
+        n = TelegramNotifier(client, "TOK", "1")
+        await n.deliver("firing", p)
+        assert "leakedsecret" not in captured["body"]
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_fanout_creates_row_per_enabled_notifier(
+    app_client, auth_headers, session, monkeypatch
+):
+    # Enable telegram (token+chat_id satisfied) and webhook in addition to debug.
+    os.environ["MONLET_NOTIFIER_TELEGRAM_ENABLED"] = "true"
+    os.environ["MONLET_NOTIFIER_TELEGRAM_TOKEN"] = "tok"
+    os.environ["MONLET_NOTIFIER_TELEGRAM_CHAT_ID"] = "1"
+    os.environ["MONLET_NOTIFIER_WEBHOOK_ENABLED"] = "true"
+    os.environ["MONLET_NOTIFIER_WEBHOOK_URL"] = "http://x/hook"
+    from monlet_server.settings import reset_settings_cache
+
+    reset_settings_cache()
+    try:
+        from tests._helpers import make_event, make_heartbeat
+
+        await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+        ev = make_event(status="critical", exit_code=2)
+        r = await app_client.post(
+            "/api/v1/events",
+            json={"agent_id": "agent-1", "events": [ev]},
+            headers=auth_headers,
+        )
+        assert r.status_code == 202
+        rows = (await session.execute(select(NotificationOutbox))).scalars().all()
+        names = sorted(r.notifier for r in rows)
+        assert names == ["debug", "telegram", "webhook"]
+    finally:
+        for k in (
+            "MONLET_NOTIFIER_TELEGRAM_ENABLED",
+            "MONLET_NOTIFIER_TELEGRAM_TOKEN",
+            "MONLET_NOTIFIER_TELEGRAM_CHAT_ID",
+            "MONLET_NOTIFIER_WEBHOOK_ENABLED",
+            "MONLET_NOTIFIER_WEBHOOK_URL",
+        ):
+            os.environ.pop(k, None)
+        reset_settings_cache()

server/tests/test_pagination.py

@@ -0,0 +1,100 @@
+import pytest
+
+from ._helpers import make_event, make_heartbeat
+
+
+@pytest.mark.asyncio
+async def test_agents_cursor(app_client, auth_headers):
+    for i in range(5):
+        await app_client.post(
+            "/api/v1/heartbeat", json=make_heartbeat(f"a-{i}"), headers=auth_headers
+        )
+    r1 = await app_client.get("/api/v1/agents?limit=2", headers=auth_headers)
+    p1 = r1.json()
+    assert len(p1["items"]) == 2
+    assert p1["next_cursor"]
+
+    r2 = await app_client.get(
+        f"/api/v1/agents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
+    )
+    p2 = r2.json()
+    assert len(p2["items"]) == 2
+    ids1 = {x["agent_id"] for x in p1["items"]}
+    ids2 = {x["agent_id"] for x in p2["items"]}
+    assert ids1.isdisjoint(ids2)
+
+
+@pytest.mark.asyncio
+async def test_checks_cursor(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    for i in range(4):
+        ev = make_event(check_id=f"chk-{i}")
+        await app_client.post(
+            "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+        )
+    r1 = await app_client.get("/api/v1/checks?limit=2", headers=auth_headers)
+    p1 = r1.json()
+    assert len(p1["items"]) == 2
+    assert p1["next_cursor"]
+    r2 = await app_client.get(
+        f"/api/v1/checks?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
+    )
+    p2 = r2.json()
+    assert len(p2["items"]) == 2
+
+
+@pytest.mark.asyncio
+async def test_checks_rejects_invalid_agent_id_query(app_client, auth_headers):
+    r = await app_client.get("/api/v1/checks?agent_id=bad%20id", headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_events_query_rejects_invalid_id_queries(app_client, auth_headers):
+    r = await app_client.get("/api/v1/events/query?agent_id=bad%20id", headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+    r = await app_client.get("/api/v1/events/query?check_id=bad%20id", headers=auth_headers)
+    assert r.status_code == 400
+    assert r.json()["error"]["code"] == "validation"
+
+
+@pytest.mark.asyncio
+async def test_incidents_cursor(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    for i in range(3):
+        ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
+        await app_client.post(
+            "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+        )
+    r1 = await app_client.get("/api/v1/incidents?limit=2", headers=auth_headers)
+    p1 = r1.json()
+    assert len(p1["items"]) == 2
+    assert p1["next_cursor"]
+    r2 = await app_client.get(
+        f"/api/v1/incidents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
+    )
+    p2 = r2.json()
+    assert len(p2["items"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_outbox_cursor(app_client, auth_headers):
+    await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
+    for i in range(3):
+        ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
+        await app_client.post(
+            "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
+        )
+    r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=auth_headers)
+    p1 = r1.json()
+    assert len(p1["items"]) == 2
+    assert p1["next_cursor"]
+
+
+@pytest.mark.asyncio
+async def test_invalid_cursor_400(app_client, auth_headers):
+    r = await app_client.get("/api/v1/agents?cursor=$$$bad", headers=auth_headers)
+    assert r.status_code == 400

server/tests/test_redaction.py

@@ -0,0 +1,47 @@
+from monlet_server.redaction import redact
+
+
+def test_redact_authorization_header_full_value():
+    # Per security.md the entire Authorization value is redacted.
+    assert redact("Authorization: Bearer abcdef123") == "Authorization: ***"
+
+
+def test_redact_bearer_outside_header():
+    assert redact("token sent as Bearer abcdef123 to peer") == "token sent as Bearer *** to peer"
+
+
+def test_redact_token_kv():
+    out = redact('token="abc123secret"')
+    assert "abc123secret" not in out
+    assert "***" in out
+
+
+def test_redact_password_kv():
+    out = redact("password=qwerty12345")
+    assert "qwerty12345" not in out
+
+
+def test_redact_authorization_header_value():
+    out = redact("authorization: deadbeefcafe")
+    assert "deadbeefcafe" not in out
+    assert "***" in out
+
+
+def test_redact_cookie_header():
+    out = redact("Cookie: session=abc; user=u1")
+    assert "abc" not in out
+    assert "***" in out
+
+
+def test_redact_set_cookie_header():
+    out = redact("Set-Cookie: sid=verysecret123; Path=/")
+    assert "verysecret123" not in out
+
+
+def test_redact_credential_env():
+    out = redact("MY_CREDENTIAL=topsecret")
+    assert "topsecret" not in out
+
+
+def test_none_passthrough():
+    assert redact(None) is None

server/tests/test_request_id.py

@@ -0,0 +1,26 @@
+import uuid
+
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_generated_request_id(app_client):
+    r = await app_client.get("/api/v1/health")
+    rid = r.headers.get("X-Request-Id")
+    assert rid
+    uuid.UUID(rid)
+
+
+@pytest.mark.asyncio
+async def test_incoming_request_id_preserved(app_client):
+    rid_in = str(uuid.uuid4())
+    r = await app_client.get("/api/v1/health", headers={"X-Request-Id": rid_in})
+    assert r.headers.get("X-Request-Id") == rid_in
+
+
+@pytest.mark.asyncio
+async def test_error_includes_request_id(app_client):
+    rid_in = str(uuid.uuid4())
+    r = await app_client.get("/api/v1/agents", headers={"X-Request-Id": rid_in})
+    assert r.status_code == 401
+    assert r.json()["error"]["request_id"] == rid_in