291 lines
9.6 KiB
Python
291 lines
9.6 KiB
Python
import asyncio
|
|
from datetime import UTC, datetime, timedelta
|
|
from uuid import UUID, uuid4, uuid7
|
|
|
|
from sqlalchemy import delete, func, select, text
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
from .. import metrics
|
|
from ..logging_config import get_logger
|
|
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
|
|
from ..settings import get_settings
|
|
|
|
log = get_logger("monlet.detector")
|
|
|
|
OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
|
|
LIVENESS_CHECK_ID = "agent_liveness"
|
|
|
|
|
|
def _liveness_key(agent_id: str) -> str:
|
|
return f"agent:{agent_id}:liveness"
|
|
|
|
|
|
def _liveness_severity(status: str) -> str:
|
|
return "critical" if status == "dead" else "warning"
|
|
|
|
|
|
_LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"}
|
|
_LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2}
|
|
|
|
|
|
async def _enqueue_liveness_outbox(
|
|
session,
|
|
incident: Incident,
|
|
event_type: str,
|
|
agent: Agent,
|
|
status: str,
|
|
now: datetime,
|
|
summary: str,
|
|
severity: str,
|
|
) -> None:
|
|
notifiers = get_settings().enabled_notifiers
|
|
if not notifiers:
|
|
return
|
|
for name in notifiers:
|
|
session.add(
|
|
NotificationOutbox(
|
|
id=uuid4(),
|
|
incident_id=incident.id,
|
|
notifier=name,
|
|
event_type=event_type,
|
|
state="pending",
|
|
attempts=0,
|
|
next_attempt_at=now,
|
|
payload={
|
|
"incident_id": str(incident.id),
|
|
"agent_id": agent.agent_id,
|
|
"check_id": LIVENESS_CHECK_ID,
|
|
"status": status,
|
|
"severity": severity,
|
|
"incident_key": incident.incident_key,
|
|
"observed_at": now.isoformat(),
|
|
"opened_at": incident.opened_at.isoformat() if incident.opened_at else None,
|
|
"resolved_at": incident.resolved_at.isoformat()
|
|
if incident.resolved_at
|
|
else None,
|
|
"summary": summary,
|
|
"output": summary,
|
|
"event_type": event_type,
|
|
},
|
|
)
|
|
)
|
|
|
|
|
|
async def _open_liveness_incident(
|
|
session, agent: Agent, status: str, now: datetime, event_id: UUID
|
|
) -> None:
|
|
severity = _liveness_severity(status)
|
|
summary = f"agent is {status}"
|
|
incident_key = _liveness_key(agent.agent_id)
|
|
new_id = uuid4()
|
|
stmt = (
|
|
pg_insert(Incident)
|
|
.values(
|
|
id=new_id,
|
|
incident_key=incident_key,
|
|
agent_id=agent.agent_id,
|
|
check_id=LIVENESS_CHECK_ID,
|
|
state="open",
|
|
severity=severity,
|
|
opened_at=now,
|
|
summary=summary,
|
|
last_event_id=event_id,
|
|
)
|
|
.on_conflict_do_nothing(
|
|
index_elements=[Incident.incident_key],
|
|
index_where=text("state = 'open'"),
|
|
)
|
|
.returning(Incident.id)
|
|
)
|
|
inserted_id = (await session.execute(stmt)).scalar_one_or_none()
|
|
if inserted_id is not None:
|
|
await session.flush()
|
|
inc = (
|
|
await session.execute(select(Incident).where(Incident.id == inserted_id))
|
|
).scalar_one()
|
|
metrics.incidents_opened_total.inc()
|
|
await _enqueue_liveness_outbox(
|
|
session, inc, "firing", agent, status, now, summary=summary, severity=severity
|
|
)
|
|
return
|
|
|
|
inc = (
|
|
await session.execute(
|
|
select(Incident).where(
|
|
Incident.incident_key == incident_key,
|
|
Incident.state == "open",
|
|
)
|
|
)
|
|
).scalar_one()
|
|
if status == "dead" and inc.severity != "critical":
|
|
inc.severity = "critical"
|
|
inc.summary = summary
|
|
inc.last_event_id = event_id
|
|
await _enqueue_liveness_outbox(
|
|
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
|
|
)
|
|
|
|
|
|
async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None:
|
|
inc = (
|
|
await session.execute(
|
|
select(Incident).where(
|
|
Incident.incident_key == _liveness_key(agent.agent_id),
|
|
Incident.state == "open",
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if inc is None:
|
|
return
|
|
inc.state = "resolved"
|
|
inc.resolved_at = now
|
|
inc.last_event_id = event_id
|
|
metrics.incidents_resolved_total.inc()
|
|
await _enqueue_liveness_outbox(
|
|
session,
|
|
inc,
|
|
"resolved",
|
|
agent,
|
|
"ok",
|
|
now,
|
|
summary="agent is alive",
|
|
severity="ok",
|
|
)
|
|
|
|
|
|
async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None:
|
|
"""Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle."""
|
|
liveness_status = _LIVENESS_CHECK_STATUS[next_status]
|
|
exit_code = _LIVENESS_EXIT_CODE[liveness_status]
|
|
summary = f"agent is {next_status}"
|
|
incident_key = _liveness_key(agent.agent_id)
|
|
|
|
cur = (
|
|
await session.execute(
|
|
select(Check).where(
|
|
Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
changed = cur is None or cur.status != liveness_status
|
|
|
|
# Liveness events flow through /events/query which validates event_id as UUIDv7.
|
|
event_id = cur.last_event_id if cur and not changed else uuid7()
|
|
if changed:
|
|
session.add(
|
|
Event(
|
|
event_id=event_id,
|
|
agent_id=agent.agent_id,
|
|
check_id=LIVENESS_CHECK_ID,
|
|
observed_at=now,
|
|
status=liveness_status,
|
|
exit_code=exit_code,
|
|
duration_ms=0,
|
|
output=summary,
|
|
output_truncated=False,
|
|
notifications_enabled=True,
|
|
incident_key=incident_key,
|
|
)
|
|
)
|
|
|
|
if cur is None:
|
|
session.add(
|
|
Check(
|
|
agent_id=agent.agent_id,
|
|
check_id=LIVENESS_CHECK_ID,
|
|
status=liveness_status,
|
|
exit_code=exit_code,
|
|
last_observed_at=now,
|
|
last_event_id=event_id,
|
|
incident_key=incident_key,
|
|
summary=summary,
|
|
)
|
|
)
|
|
else:
|
|
cur.last_observed_at = now
|
|
cur.summary = summary
|
|
if changed:
|
|
cur.status = liveness_status
|
|
cur.exit_code = exit_code
|
|
cur.last_event_id = event_id
|
|
cur.incident_key = incident_key
|
|
|
|
if next_status in ("stale", "dead"):
|
|
await _open_liveness_incident(session, agent, next_status, now, event_id)
|
|
else:
|
|
await _resolve_liveness_incident(session, agent, now, event_id)
|
|
|
|
|
|
async def _prune_history(session) -> None:
|
|
s = get_settings()
|
|
if s.event_dedup_retention_days > 0:
|
|
cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
|
|
batch = max(s.event_dedup_prune_batch_size, 1)
|
|
# Bounded DELETE: keep each tick's prune transaction short to avoid bloat and
|
|
# long-running locks on a large dedup table.
|
|
sub = (
|
|
select(EventIngestDedup.event_id)
|
|
.where(EventIngestDedup.received_at < cutoff)
|
|
.limit(batch)
|
|
.scalar_subquery()
|
|
)
|
|
await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
|
|
|
|
if s.outbox_retention_max_rows > 0:
|
|
keep_outbox = (
|
|
select(NotificationOutbox.id)
|
|
.where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
|
|
.order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
|
|
.limit(s.outbox_retention_max_rows)
|
|
)
|
|
await session.execute(
|
|
delete(NotificationOutbox)
|
|
.where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
|
|
.where(NotificationOutbox.id.not_in(keep_outbox))
|
|
)
|
|
|
|
|
|
async def _tick(sm: async_sessionmaker) -> None:
|
|
s = get_settings()
|
|
now = datetime.now(UTC)
|
|
stale_cut = now - timedelta(seconds=s.stale_after_sec)
|
|
dead_cut = now - timedelta(seconds=s.dead_after_sec)
|
|
async with sm() as session:
|
|
agents = (await session.execute(select(Agent))).scalars().all()
|
|
for agent in agents:
|
|
if agent.last_seen_at <= dead_cut:
|
|
next_status = "dead"
|
|
elif agent.last_seen_at <= stale_cut:
|
|
next_status = "stale"
|
|
else:
|
|
next_status = "alive"
|
|
if agent.status != next_status:
|
|
agent.status = next_status
|
|
await _apply_liveness(session, agent, next_status, now)
|
|
await _prune_history(session)
|
|
await session.commit()
|
|
|
|
for status in ("alive", "stale", "dead"):
|
|
r = await session.execute(
|
|
select(func.count()).select_from(Agent).where(Agent.status == status)
|
|
)
|
|
metrics.agents_gauge.labels(status=status).set(r.scalar_one())
|
|
r = await session.execute(
|
|
select(func.count()).select_from(Incident).where(Incident.state == "open")
|
|
)
|
|
metrics.open_incidents_gauge.set(r.scalar_one())
|
|
|
|
|
|
async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
|
|
tick = get_settings().detector_tick_sec
|
|
while not stop_event.is_set():
|
|
try:
|
|
await _tick(sm)
|
|
except Exception as exc:
|
|
log.warning("detector_tick_failed", error=str(exc))
|
|
try:
|
|
await asyncio.wait_for(stop_event.wait(), timeout=tick)
|
|
except TimeoutError:
|
|
continue
|