from datetime import UTC, datetime, timedelta from uuid import uuid4 import pytest from sqlalchemy import func, select, update from sqlalchemy.ext.asyncio import async_sessionmaker from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick from monlet_server.settings import reset_settings_cache from ._helpers import make_event, make_heartbeat @pytest.mark.asyncio async def test_detector_transitions(app_client, auth_headers, engine, session): await app_client.post( "/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers ) await app_client.post( "/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers ) await app_client.post( "/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers ) now = datetime.now(UTC) await session.execute( update(Agent) .where(Agent.agent_id == "agent-stale") .values(last_seen_at=now - timedelta(seconds=120)) ) await session.execute( update(Agent) .where(Agent.agent_id == "agent-dead") .values(last_seen_at=now - timedelta(minutes=10)) ) await session.commit() sm = async_sessionmaker(engine, expire_on_commit=False) await _tick(sm) session.expire_all() statuses = { r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all() } assert statuses["agent-alive"] == "alive" assert statuses["agent-stale"] == "stale" assert statuses["agent-dead"] == "dead" incidents = ( (await session.execute(select(Incident).where(Incident.check_id == LIVENESS_CHECK_ID))) .scalars() .all() ) by_agent = {i.agent_id: i for i in incidents} assert "agent-alive" not in by_agent assert by_agent["agent-stale"].state == "open" assert by_agent["agent-stale"].severity == "warning" assert by_agent["agent-stale"].summary == "agent is stale" assert by_agent["agent-dead"].state == "open" assert by_agent["agent-dead"].severity == "critical" assert by_agent["agent-dead"].summary == "agent is dead" await session.execute( update(Agent).where(Agent.agent_id == "agent-dead").values(last_seen_at=datetime.now(UTC)) ) await session.commit() await _tick(sm) session.expire_all() resolved = ( await session.execute( select(Incident).where( Incident.agent_id == "agent-dead", Incident.check_id == LIVENESS_CHECK_ID, ) ) ).scalar_one() assert resolved.state == "resolved" assert resolved.resolved_at is not None @pytest.mark.asyncio async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session): await app_client.post( "/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers ) sm = async_sessionmaker(engine, expire_on_commit=False) # Tick 1: alive → Check exists with status=ok, one event. await _tick(sm) session.expire_all() check = ( await session.execute( select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID) ) ).scalar_one() assert check.status == "ok" n_events = ( await session.execute( select(func.count()) .select_from(Event) .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID) ) ).scalar_one() assert n_events == 1 # Tick 2: still alive → no new event. await _tick(sm) session.expire_all() n_events = ( await session.execute( select(func.count()) .select_from(Event) .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID) ) ).scalar_one() assert n_events == 1 # Push agent into dead → transition event written, Check updates. await session.execute( update(Agent) .where(Agent.agent_id == "agent-flap") .values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10)) ) await session.commit() await _tick(sm) session.expire_all() check = ( await session.execute( select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID) ) ).scalar_one() assert check.status == "critical" n_events = ( await session.execute( select(func.count()) .select_from(Event) .where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID) ) ).scalar_one() assert n_events == 2 @pytest.mark.asyncio async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch): monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2") reset_settings_cache() try: await app_client.post( "/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers ) critical = make_event(check_id="critical_prune", status="critical", exit_code=2) await app_client.post( "/api/v1/events", json={"agent_id": "agent-prune", "events": [critical]}, headers=auth_headers, ) inc = ( await session.execute(select(Incident).where(Incident.check_id == "critical_prune")) ).scalar_one() now = datetime.now(UTC) for _ in range(5): session.add( NotificationOutbox( id=uuid4(), incident_id=inc.id, notifier="debug", event_type="firing", state="sent", attempts=1, next_attempt_at=None, payload={}, created_at=now, updated_at=now, ) ) await session.commit() sm = async_sessionmaker(engine, expire_on_commit=False) await _tick(sm) terminal_outbox = ( await session.execute( select(func.count()) .select_from(NotificationOutbox) .where(NotificationOutbox.state == "sent") ) ).scalar_one() assert terminal_outbox == 2 finally: reset_settings_cache()