import asyncio from datetime import UTC, datetime, timedelta from sqlalchemy import delete, func, select, update from sqlalchemy.ext.asyncio import async_sessionmaker from .. import metrics from ..logging_config import get_logger from ..models import Agent, Event, Incident, NotificationOutbox from ..settings import get_settings log = get_logger("monlet.detector") OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded") async def _prune_history(session) -> None: s = get_settings() if s.events_retention_max_rows > 0: keep_events = ( select(Event.event_id) .order_by(Event.received_at.desc(), Event.event_id.desc()) .limit(s.events_retention_max_rows) ) await session.execute(delete(Event).where(Event.event_id.not_in(keep_events))) if s.outbox_retention_max_rows > 0: keep_outbox = ( select(NotificationOutbox.id) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()) .limit(s.outbox_retention_max_rows) ) await session.execute( delete(NotificationOutbox) .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES)) .where(NotificationOutbox.id.not_in(keep_outbox)) ) async def _tick(sm: async_sessionmaker) -> None: s = get_settings() now = datetime.now(UTC) stale_cut = now - timedelta(seconds=s.stale_after_sec) dead_cut = now - timedelta(seconds=s.dead_after_sec) async with sm() as session: await session.execute( update(Agent) .where(Agent.last_seen_at <= dead_cut) .where(Agent.status != "dead") .values(status="dead") ) await session.execute( update(Agent) .where(Agent.last_seen_at <= stale_cut) .where(Agent.last_seen_at > dead_cut) .where(Agent.status != "stale") .values(status="stale") ) await session.execute( update(Agent) .where(Agent.last_seen_at > stale_cut) .where(Agent.status != "alive") .values(status="alive") ) await _prune_history(session) await session.commit() for status in ("alive", "stale", "dead"): r = await session.execute( select(func.count()).select_from(Agent).where(Agent.status == status) ) metrics.agents_gauge.labels(status=status).set(r.scalar_one()) r = await session.execute( select(func.count()).select_from(Incident).where(Incident.state == "open") ) metrics.open_incidents_gauge.set(r.scalar_one()) async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None: tick = get_settings().detector_tick_sec while not stop_event.is_set(): try: await _tick(sm) except Exception as exc: log.warning("detector_tick_failed", error=str(exc)) try: await asyncio.wait_for(stop_event.wait(), timeout=tick) except TimeoutError: continue