import asyncio from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4, uuid7 from sqlalchemy import delete, func, select, text from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import async_sessionmaker from .. import metrics from ..logging_config import get_logger from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox from ..settings import get_settings log = get_logger("monlet.detector") OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded") LIVENESS_CHECK_ID = "agent_liveness" def _liveness_key(agent_id: str) -> str: return f"agent:{agent_id}:liveness" def _liveness_severity(status: str) -> str: return "critical" if status == "dead" else "warning" _LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"} _LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2} async def _enqueue_liveness_outbox( session, incident: Incident, event_type: str, agent: Agent, status: str, now: datetime, summary: str, severity: str, ) -> None: notifiers = get_settings().enabled_notifiers if not notifiers: return for name in notifiers: session.add( NotificationOutbox( id=uuid4(), incident_id=incident.id, notifier=name, event_type=event_type, state="pending", attempts=0, next_attempt_at=now, payload={ "incident_id": str(incident.id), "agent_id": agent.agent_id, "check_id": LIVENESS_CHECK_ID, "status": status, "severity": severity, "incident_key": incident.incident_key, "observed_at": now.isoformat(), "opened_at": incident.opened_at.isoformat() if incident.opened_at else None, "resolved_at": incident.resolved_at.isoformat() if incident.resolved_at else None, "summary": summary, "output": summary, "event_type": event_type, }, ) ) async def _open_liveness_incident( session, agent: Agent, status: str, now: datetime, event_id: UUID ) -> None: severity = _liveness_severity(status) summary = f"agent is {status}" incident_key = _liveness_key(agent.agent_id) new_id = uuid4() stmt = ( pg_insert(Incident) .values( id=new_id, incident_key=incident_key, agent_id=agent.agent_id, check_id=LIVENESS_CHECK_ID, state="open", severity=severity, opened_at=now, summary=summary, last_event_id=event_id, ) .on_conflict_do_nothing( index_elements=[Incident.incident_key], index_where=text("state = 'open'"), ) .returning(Incident.id) ) inserted_id = (await session.execute(stmt)).scalar_one_or_none() if inserted_id is not None: await session.flush() inc = ( await session.execute(select(Incident).where(Incident.id == inserted_id)) ).scalar_one() metrics.incidents_opened_total.inc() await _enqueue_liveness_outbox( session, inc, "firing", agent, status, now, summary=summary, severity=severity ) return inc = ( await session.execute( select(Incident).where( Incident.incident_key == incident_key, Incident.state == "open", ) ) ).scalar_one() if status == "dead" and inc.severity != "critical": inc.severity = "critical" inc.summary = summary inc.last_event_id = event_id await _enqueue_liveness_outbox( session, inc, "firing", agent, status, now, summary=summary, severity="critical" ) async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None: inc = ( await session.execute( select(Incident).where( Incident.incident_key == _liveness_key(agent.agent_id), Incident.state == "open", ) ) ).scalar_one_or_none() if inc is None: return inc.state = "resolved" inc.resolved_at = now inc.last_event_id = event_id metrics.incidents_resolved_total.inc() await _enqueue_liveness_outbox( session, inc, "resolved", agent, "ok", now, summary="agent is alive", severity="ok", ) async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None: """Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle.""" liveness_status = _LIVENESS_CHECK_STATUS[next_status] exit_code = _LIVENESS_EXIT_CODE[liveness_status] summary = f"agent is {next_status}" incident_key = _liveness_key(agent.agent_id) cur = ( await session.execute( select(Check).where( Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID ) ) ).scalar_one_or_none() changed = cur is None or cur.status != liveness_status # Liveness events flow through /events/query which validates event_id as UUIDv7. event_id = cur.last_event_id if cur and not changed else uuid7() if changed: session.add( Event( event_id=event_id, agent_id=agent.agent_id, check_id=LIVENESS_CHECK_ID, observed_at=now, status=liveness_status, exit_code=exit_code, duration_ms=0, output=summary, output_truncated=False, notifications_enabled=True, incident_key=incident_key, ) ) if cur is None: session.add( Check( agent_id=agent.agent_id, check_id=LIVENESS_CHECK_ID, status=liveness_status, exit_code=exit_code, last_observed_at=now, last_event_id=event_id, incident_key=incident_key, summary=summary, ) ) else: cur.last_observed_at = now cur.summary = summary if changed: cur.status = liveness_status cur.exit_code = exit_code cur.last_event_id = event_id cur.incident_key = incident_key if next_status in ("stale", "dead"): await _open_liveness_incident(session, agent, next_status, now, event_id) else: await _resolve_liveness_incident(session, agent, now, event_id) async def _prune_history(session) -> None: s = get_settings() if s.event_dedup_retention_days > 0: cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days) batch = max(s.event_dedup_prune_batch_size, 1) # Bounded DELETE: keep each tick's prune transaction short to avoid bloat and # long-running locks on a large dedup table. sub = ( select(EventIngestDedup.event_id) .where(EventIngestDedup.received_at < cutoff) .limit(batch) .scalar_subquery() ) await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub))) if s.outbox_retention_max_rows > 0: keep_outbox = ( select(NotificationOutbox.id) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()) .limit(s.outbox_retention_max_rows) ) await session.execute( delete(NotificationOutbox) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .where(NotificationOutbox.id.not_in(keep_outbox)) ) async def _tick(sm: async_sessionmaker) -> None: s = get_settings() now = datetime.now(UTC) stale_cut = now - timedelta(seconds=s.stale_after_sec) dead_cut = now - timedelta(seconds=s.dead_after_sec) async with sm() as session: agents = (await session.execute(select(Agent))).scalars().all() for agent in agents: if agent.last_seen_at <= dead_cut: next_status = "dead" elif agent.last_seen_at <= stale_cut: next_status = "stale" else: next_status = "alive" if agent.status != next_status: agent.status = next_status await _apply_liveness(session, agent, next_status, now) await _prune_history(session) await session.commit() for status in ("alive", "stale", "dead"): r = await session.execute( select(func.count()).select_from(Agent).where(Agent.status == status) ) metrics.agents_gauge.labels(status=status).set(r.scalar_one()) r = await session.execute( select(func.count()).select_from(Incident).where(Incident.state == "open") ) metrics.open_incidents_gauge.set(r.scalar_one()) async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None: tick = get_settings().detector_tick_sec while not stop_event.is_set(): try: await _tick(sm) except Exception as exc: log.warning("detector_tick_failed", error=str(exc)) try: await asyncio.wait_for(stop_event.wait(), timeout=tick) except TimeoutError: continue