monlet/server/tests/test_detector.py

from datetime import UTC, datetime, timedelta
from uuid import uuid4

import pytest
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import async_sessionmaker

from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox
from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
from monlet_server.settings import reset_settings_cache

from ._helpers import make_event, make_heartbeat


@pytest.mark.asyncio
async def test_detector_transitions(app_client, auth_headers, engine, session):
    await app_client.post(
        "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
    )
    await app_client.post(
        "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
    )
    await app_client.post(
        "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
    )

    now = datetime.now(UTC)
    await session.execute(
        update(Agent)
        .where(Agent.agent_id == "agent-stale")
        .values(last_seen_at=now - timedelta(seconds=120))
    )
    await session.execute(
        update(Agent)
        .where(Agent.agent_id == "agent-dead")
        .values(last_seen_at=now - timedelta(minutes=10))
    )
    await session.commit()

    sm = async_sessionmaker(engine, expire_on_commit=False)
    await _tick(sm)
    session.expire_all()

    statuses = {
        r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
    }
    assert statuses["agent-alive"] == "alive"
    assert statuses["agent-stale"] == "stale"
    assert statuses["agent-dead"] == "dead"

    incidents = (
        (await session.execute(select(Incident).where(Incident.check_id == LIVENESS_CHECK_ID)))
        .scalars()
        .all()
    )
    by_agent = {i.agent_id: i for i in incidents}
    assert "agent-alive" not in by_agent
    assert by_agent["agent-stale"].state == "open"
    assert by_agent["agent-stale"].severity == "warning"
    assert by_agent["agent-stale"].summary == "agent is stale"
    assert by_agent["agent-dead"].state == "open"
    assert by_agent["agent-dead"].severity == "critical"
    assert by_agent["agent-dead"].summary == "agent is dead"

    await session.execute(
        update(Agent).where(Agent.agent_id == "agent-dead").values(last_seen_at=datetime.now(UTC))
    )
    await session.commit()
    await _tick(sm)
    session.expire_all()

    resolved = (
        await session.execute(
            select(Incident).where(
                Incident.agent_id == "agent-dead",
                Incident.check_id == LIVENESS_CHECK_ID,
            )
        )
    ).scalar_one()
    assert resolved.state == "resolved"
    assert resolved.resolved_at is not None


@pytest.mark.asyncio
async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session):
    await app_client.post(
        "/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers
    )
    sm = async_sessionmaker(engine, expire_on_commit=False)

    # Tick 1: alive → Check exists with status=ok, one event.
    await _tick(sm)
    session.expire_all()
    check = (
        await session.execute(
            select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID)
        )
    ).scalar_one()
    assert check.status == "ok"
    n_events = (
        await session.execute(
            select(func.count())
            .select_from(Event)
            .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
        )
    ).scalar_one()
    assert n_events == 1

    # Tick 2: still alive → no new event.
    await _tick(sm)
    session.expire_all()
    n_events = (
        await session.execute(
            select(func.count())
            .select_from(Event)
            .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
        )
    ).scalar_one()
    assert n_events == 1

    # Push agent into dead → transition event written, Check updates.
    await session.execute(
        update(Agent)
        .where(Agent.agent_id == "agent-flap")
        .values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10))
    )
    await session.commit()
    await _tick(sm)
    session.expire_all()
    check = (
        await session.execute(
            select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID)
        )
    ).scalar_one()
    assert check.status == "critical"
    n_events = (
        await session.execute(
            select(func.count())
            .select_from(Event)
            .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
        )
    ).scalar_one()
    assert n_events == 2


@pytest.mark.asyncio
async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch):
    monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
    reset_settings_cache()
    try:
        await app_client.post(
            "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
        )
        critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
        await app_client.post(
            "/api/v1/events",
            json={"agent_id": "agent-prune", "events": [critical]},
            headers=auth_headers,
        )

        inc = (
            await session.execute(select(Incident).where(Incident.check_id == "critical_prune"))
        ).scalar_one()
        now = datetime.now(UTC)
        for _ in range(5):
            session.add(
                NotificationOutbox(
                    id=uuid4(),
                    incident_id=inc.id,
                    notifier="debug",
                    event_type="firing",
                    state="sent",
                    attempts=1,
                    next_attempt_at=None,
                    payload={},
                    created_at=now,
                    updated_at=now,
                )
            )
        await session.commit()

        sm = async_sessionmaker(engine, expire_on_commit=False)
        await _tick(sm)

        terminal_outbox = (
            await session.execute(
                select(func.count())
                .select_from(NotificationOutbox)
                .where(NotificationOutbox.state == "sent")
            )
        ).scalar_one()
        assert terminal_outbox == 2
    finally:
        reset_settings_cache()