91 lines
3.1 KiB
Python
91 lines
3.1 KiB
Python
import asyncio
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import delete, func, select, update
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
from .. import metrics
|
|
from ..logging_config import get_logger
|
|
from ..models import Agent, Event, Incident, NotificationOutbox
|
|
from ..settings import get_settings
|
|
|
|
log = get_logger("monlet.detector")
|
|
|
|
OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
|
|
|
|
|
|
async def _prune_history(session) -> None:
|
|
s = get_settings()
|
|
if s.events_retention_max_rows > 0:
|
|
keep_events = (
|
|
select(Event.event_id)
|
|
.order_by(Event.received_at.desc(), Event.event_id.desc())
|
|
.limit(s.events_retention_max_rows)
|
|
)
|
|
await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
|
|
|
|
if s.outbox_retention_max_rows > 0:
|
|
keep_outbox = (
|
|
select(NotificationOutbox.id)
|
|
.where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
|
|
.order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
|
|
.limit(s.outbox_retention_max_rows)
|
|
)
|
|
await session.execute(
|
|
delete(NotificationOutbox)
|
|
.where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
|
|
.where(NotificationOutbox.id.not_in(keep_outbox))
|
|
)
|
|
|
|
|
|
async def _tick(sm: async_sessionmaker) -> None:
|
|
s = get_settings()
|
|
now = datetime.now(UTC)
|
|
stale_cut = now - timedelta(seconds=s.stale_after_sec)
|
|
dead_cut = now - timedelta(seconds=s.dead_after_sec)
|
|
async with sm() as session:
|
|
await session.execute(
|
|
update(Agent)
|
|
.where(Agent.last_seen_at <= dead_cut)
|
|
.where(Agent.status != "dead")
|
|
.values(status="dead")
|
|
)
|
|
await session.execute(
|
|
update(Agent)
|
|
.where(Agent.last_seen_at <= stale_cut)
|
|
.where(Agent.last_seen_at > dead_cut)
|
|
.where(Agent.status != "stale")
|
|
.values(status="stale")
|
|
)
|
|
await session.execute(
|
|
update(Agent)
|
|
.where(Agent.last_seen_at > stale_cut)
|
|
.where(Agent.status != "alive")
|
|
.values(status="alive")
|
|
)
|
|
await _prune_history(session)
|
|
await session.commit()
|
|
|
|
for status in ("alive", "stale", "dead"):
|
|
r = await session.execute(
|
|
select(func.count()).select_from(Agent).where(Agent.status == status)
|
|
)
|
|
metrics.agents_gauge.labels(status=status).set(r.scalar_one())
|
|
r = await session.execute(
|
|
select(func.count()).select_from(Incident).where(Incident.state == "open")
|
|
)
|
|
metrics.open_incidents_gauge.set(r.scalar_one())
|
|
|
|
|
|
async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
|
|
tick = get_settings().detector_tick_sec
|
|
while not stop_event.is_set():
|
|
try:
|
|
await _tick(sm)
|
|
except Exception as exc:
|
|
log.warning("detector_tick_failed", error=str(exc))
|
|
try:
|
|
await asyncio.wait_for(stop_event.wait(), timeout=tick)
|
|
except TimeoutError:
|
|
continue
|