creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor web pagination and add partitioned events, flap/timeout showcase agents

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 18:23:31 +04:00
parent d5f312f200
commit a2e88b4e76
46 changed files with 1600 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
server/monlet_server/services/detector.py
View File

@@ -1,6 +1,6 @@
import asyncio
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from uuid import UUID, uuid4, uuid7
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..settings import get_settings
log = get_logger("monlet.detector")
@@ -25,6 +25,10 @@ def _liveness_severity(status: str) -> str:
return "critical" if status == "dead" else "warning"
_LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"}
_LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2}
async def _enqueue_liveness_outbox(
session,
incident: Incident,
@@ -68,7 +72,9 @@ async def _enqueue_liveness_outbox(
)
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
async def _open_liveness_incident(
session, agent: Agent, status: str, now: datetime, event_id: UUID
) -> None:
severity = _liveness_severity(status)
summary = f"agent is {status}"
incident_key = _liveness_key(agent.agent_id)
@@ -84,7 +90,7 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
severity=severity,
opened_at=now,
summary=summary,
last_event_id=uuid4(),
last_event_id=event_id,
)
.on_conflict_do_nothing(
index_elements=[Incident.incident_key],
@@ -115,13 +121,13 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
if status == "dead" and inc.severity != "critical":
inc.severity = "critical"
inc.summary = summary
inc.last_event_id = uuid4()
inc.last_event_id = event_id
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
)
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None:
inc = (
await session.execute(
select(Incident).where(
@@ -134,7 +140,7 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
return
inc.state = "resolved"
inc.resolved_at = now
inc.last_event_id = uuid4()
inc.last_event_id = event_id
metrics.incidents_resolved_total.inc()
await _enqueue_liveness_outbox(
session,
@@ -148,15 +154,83 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
)
async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None:
"""Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle."""
liveness_status = _LIVENESS_CHECK_STATUS[next_status]
exit_code = _LIVENESS_EXIT_CODE[liveness_status]
summary = f"agent is {next_status}"
incident_key = _liveness_key(agent.agent_id)
cur = (
await session.execute(
select(Check).where(
Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID
)
)
).scalar_one_or_none()
changed = cur is None or cur.status != liveness_status
# Liveness events flow through /events/query which validates event_id as UUIDv7.
event_id = cur.last_event_id if cur and not changed else uuid7()
if changed:
session.add(
Event(
event_id=event_id,
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
observed_at=now,
status=liveness_status,
exit_code=exit_code,
duration_ms=0,
output=summary,
output_truncated=False,
notifications_enabled=True,
incident_key=incident_key,
)
)
if cur is None:
session.add(
Check(
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
status=liveness_status,
exit_code=exit_code,
last_observed_at=now,
last_event_id=event_id,
incident_key=incident_key,
summary=summary,
)
)
else:
cur.last_observed_at = now
cur.summary = summary
if changed:
cur.status = liveness_status
cur.exit_code = exit_code
cur.last_event_id = event_id
cur.incident_key = incident_key
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now, event_id)
else:
await _resolve_liveness_incident(session, agent, now, event_id)
async def _prune_history(session) -> None:
s = get_settings()
if s.events_retention_max_rows > 0:
keep_events = (
select(Event.event_id)
.order_by(Event.received_at.desc(), Event.event_id.desc())
.limit(s.events_retention_max_rows)
if s.event_dedup_retention_days > 0:
cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
batch = max(s.event_dedup_prune_batch_size, 1)
# Bounded DELETE: keep each tick's prune transaction short to avoid bloat and
# long-running locks on a large dedup table.
sub = (
select(EventIngestDedup.event_id)
.where(EventIngestDedup.received_at < cutoff)
.limit(batch)
.scalar_subquery()
)
await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
if s.outbox_retention_max_rows > 0:
keep_outbox = (
@@ -188,10 +262,7 @@ async def _tick(sm: async_sessionmaker) -> None:
next_status = "alive"
if agent.status != next_status:
agent.status = next_status
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now)
else:
await _resolve_liveness_incident(session, agent, now)
await _apply_liveness(session, agent, next_status, now)
await _prune_history(session)
await session.commit()