import asyncio from datetime import UTC, datetime, timedelta from uuid import uuid4 from sqlalchemy import delete, func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import async_sessionmaker from .. import metrics from ..logging_config import get_logger from ..models import Agent, Event, Incident, NotificationOutbox from ..settings import get_settings log = get_logger("monlet.detector") OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded") LIVENESS_CHECK_ID = "agent_liveness" def _liveness_key(agent_id: str) -> str: return f"agent:{agent_id}:liveness" def _liveness_severity(status: str) -> str: return "critical" if status == "dead" else "warning" async def _enqueue_liveness_outbox( session, incident: Incident, event_type: str, agent: Agent, status: str, now: datetime, summary: str, severity: str, ) -> None: notifiers = get_settings().enabled_notifiers if not notifiers: return for name in notifiers: session.add( NotificationOutbox( id=uuid4(), incident_id=incident.id, notifier=name, event_type=event_type, state="pending", attempts=0, next_attempt_at=now, payload={ "incident_id": str(incident.id), "agent_id": agent.agent_id, "check_id": LIVENESS_CHECK_ID, "status": status, "severity": severity, "incident_key": incident.incident_key, "observed_at": now.isoformat(), "opened_at": incident.opened_at.isoformat() if incident.opened_at else None, "resolved_at": incident.resolved_at.isoformat() if incident.resolved_at else None, "summary": summary, "output": summary, "event_type": event_type, }, ) ) async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None: severity = _liveness_severity(status) summary = f"agent is {status}" incident_key = _liveness_key(agent.agent_id) new_id = uuid4() stmt = ( pg_insert(Incident) .values( id=new_id, incident_key=incident_key, agent_id=agent.agent_id, check_id=LIVENESS_CHECK_ID, state="open", severity=severity, opened_at=now, summary=summary, last_event_id=uuid4(), ) .on_conflict_do_nothing( index_elements=[Incident.incident_key], index_where=text("state = 'open'"), ) .returning(Incident.id) ) inserted_id = (await session.execute(stmt)).scalar_one_or_none() if inserted_id is not None: await session.flush() inc = ( await session.execute(select(Incident).where(Incident.id == inserted_id)) ).scalar_one() metrics.incidents_opened_total.inc() await _enqueue_liveness_outbox( session, inc, "firing", agent, status, now, summary=summary, severity=severity ) return inc = ( await session.execute( select(Incident).where( Incident.incident_key == incident_key, Incident.state == "open", ) ) ).scalar_one() if status == "dead" and inc.severity != "critical": inc.severity = "critical" inc.summary = summary inc.last_event_id = uuid4() await _enqueue_liveness_outbox( session, inc, "firing", agent, status, now, summary=summary, severity="critical" ) async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None: inc = ( await session.execute( select(Incident).where( Incident.incident_key == _liveness_key(agent.agent_id), Incident.state == "open", ) ) ).scalar_one_or_none() if inc is None: return inc.state = "resolved" inc.resolved_at = now inc.last_event_id = uuid4() metrics.incidents_resolved_total.inc() await _enqueue_liveness_outbox( session, inc, "resolved", agent, "ok", now, summary="agent is alive", severity="ok", ) async def _prune_history(session) -> None: s = get_settings() if s.events_retention_max_rows > 0: keep_events = ( select(Event.event_id) .order_by(Event.received_at.desc(), Event.event_id.desc()) .limit(s.events_retention_max_rows) ) await session.execute(delete(Event).where(Event.event_id.not_in(keep_events))) if s.outbox_retention_max_rows > 0: keep_outbox = ( select(NotificationOutbox.id) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()) .limit(s.outbox_retention_max_rows) ) await session.execute( delete(NotificationOutbox) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .where(NotificationOutbox.id.not_in(keep_outbox)) ) async def _tick(sm: async_sessionmaker) -> None: s = get_settings() now = datetime.now(UTC) stale_cut = now - timedelta(seconds=s.stale_after_sec) dead_cut = now - timedelta(seconds=s.dead_after_sec) async with sm() as session: agents = (await session.execute(select(Agent))).scalars().all() for agent in agents: if agent.last_seen_at <= dead_cut: next_status = "dead" elif agent.last_seen_at <= stale_cut: next_status = "stale" else: next_status = "alive" if agent.status != next_status: agent.status = next_status if next_status in ("stale", "dead"): await _open_liveness_incident(session, agent, next_status, now) else: await _resolve_liveness_incident(session, agent, now) await _prune_history(session) await session.commit() for status in ("alive", "stale", "dead"): r = await session.execute( select(func.count()).select_from(Agent).where(Agent.status == status) ) metrics.agents_gauge.labels(status=status).set(r.scalar_one()) r = await session.execute( select(func.count()).select_from(Incident).where(Incident.state == "open") ) metrics.open_incidents_gauge.set(r.scalar_one()) async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None: tick = get_settings().detector_tick_sec while not stop_event.is_set(): try: await _tick(sm) except Exception as exc: log.warning("detector_tick_failed", error=str(exc)) try: await asyncio.wait_for(stop_event.wait(), timeout=tick) except TimeoutError: continue