creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor web pagination and add partitioned events, flap/timeout showcase agents

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 18:23:31 +04:00
parent d5f312f200
commit a2e88b4e76
46 changed files with 1600 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
server/monlet_server/services/detector.py
View File

@@ -1,6 +1,6 @@
import asyncio
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from uuid import UUID, uuid4, uuid7
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..settings import get_settings
log = get_logger("monlet.detector")
@@ -25,6 +25,10 @@ def _liveness_severity(status: str) -> str:
return "critical" if status == "dead" else "warning"
_LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"}
_LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2}
async def _enqueue_liveness_outbox(
session,
incident: Incident,
@@ -68,7 +72,9 @@ async def _enqueue_liveness_outbox(
)
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
async def _open_liveness_incident(
session, agent: Agent, status: str, now: datetime, event_id: UUID
) -> None:
severity = _liveness_severity(status)
summary = f"agent is {status}"
incident_key = _liveness_key(agent.agent_id)
@@ -84,7 +90,7 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
severity=severity,
opened_at=now,
summary=summary,
last_event_id=uuid4(),
last_event_id=event_id,
)
.on_conflict_do_nothing(
index_elements=[Incident.incident_key],
@@ -115,13 +121,13 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
if status == "dead" and inc.severity != "critical":
inc.severity = "critical"
inc.summary = summary
inc.last_event_id = uuid4()
inc.last_event_id = event_id
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
)
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None:
inc = (
await session.execute(
select(Incident).where(
@@ -134,7 +140,7 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
return
inc.state = "resolved"
inc.resolved_at = now
inc.last_event_id = uuid4()
inc.last_event_id = event_id
metrics.incidents_resolved_total.inc()
await _enqueue_liveness_outbox(
session,
@@ -148,15 +154,83 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
)
async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None:
"""Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle."""
liveness_status = _LIVENESS_CHECK_STATUS[next_status]
exit_code = _LIVENESS_EXIT_CODE[liveness_status]
summary = f"agent is {next_status}"
incident_key = _liveness_key(agent.agent_id)
cur = (
await session.execute(
select(Check).where(
Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID
)
)
).scalar_one_or_none()
changed = cur is None or cur.status != liveness_status
# Liveness events flow through /events/query which validates event_id as UUIDv7.
event_id = cur.last_event_id if cur and not changed else uuid7()
if changed:
session.add(
Event(
event_id=event_id,
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
observed_at=now,
status=liveness_status,
exit_code=exit_code,
duration_ms=0,
output=summary,
output_truncated=False,
notifications_enabled=True,
incident_key=incident_key,
)
)
if cur is None:
session.add(
Check(
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
status=liveness_status,
exit_code=exit_code,
last_observed_at=now,
last_event_id=event_id,
incident_key=incident_key,
summary=summary,
)
)
else:
cur.last_observed_at = now
cur.summary = summary
if changed:
cur.status = liveness_status
cur.exit_code = exit_code
cur.last_event_id = event_id
cur.incident_key = incident_key
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now, event_id)
else:
await _resolve_liveness_incident(session, agent, now, event_id)
async def _prune_history(session) -> None:
s = get_settings()
if s.events_retention_max_rows > 0:
keep_events = (
select(Event.event_id)
.order_by(Event.received_at.desc(), Event.event_id.desc())
.limit(s.events_retention_max_rows)
if s.event_dedup_retention_days > 0:
cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
batch = max(s.event_dedup_prune_batch_size, 1)
# Bounded DELETE: keep each tick's prune transaction short to avoid bloat and
# long-running locks on a large dedup table.
sub = (
select(EventIngestDedup.event_id)
.where(EventIngestDedup.received_at < cutoff)
.limit(batch)
.scalar_subquery()
)
await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
if s.outbox_retention_max_rows > 0:
keep_outbox = (
@@ -188,10 +262,7 @@ async def _tick(sm: async_sessionmaker) -> None:
next_status = "alive"
if agent.status != next_status:
agent.status = next_status
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now)
else:
await _resolve_liveness_incident(session, agent, now)
await _apply_liveness(session, agent, next_status, now)
await _prune_history(session)
await session.commit()

40
server/monlet_server/services/ingestion.py
View File

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Check, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..redaction import redact
from ..schemas import CheckResultEvent, HeartbeatRequest
from ..settings import get_settings
@@ -155,26 +155,45 @@ async def ingest_event(
agent_id: str,
ev: CheckResultEvent,
) -> bool:
"""Return True if a new change row was written, False if not (duplicate/unchanged/late)."""
"""Return True if a new change row was written, False if not (duplicate/unchanged/late).
Idempotency is enforced first via the standalone event_ingest_dedup table — a
duplicate event_id never touches checks/incidents/outbox/events. The full pipeline
(dedup + events + check + incident + outbox) commits as a single transaction by
the caller, so a partial failure cannot leave a dedup row without its side-effects.
"""
observed = to_utc(ev.observed_at)
output = redact(ev.output)
check_summary = (output or "")[:200] if output else None
incident_key = _incident_key(agent_id, ev)
event_uuid = UUID(ev.event_id)
# Order matters: late check before dedup insert so that repeated late events stay
# observable as `late`, not silently collapsed into `deduplicated`.
chk_res = await session.execute(
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
)
cur = chk_res.scalar_one_or_none()
is_late = cur is not None and cur.last_observed_at > observed
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if is_late:
metrics.events_total.labels(result="late").inc()
return False
dedup_stmt = (
pg_insert(EventIngestDedup)
.values(event_id=event_uuid, received_at=datetime.now(UTC))
.on_conflict_do_nothing(index_elements=[EventIngestDedup.event_id])
.returning(EventIngestDedup.event_id)
)
if (await session.execute(dedup_stmt)).scalar_one_or_none() is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if changed:
stmt = pg_insert(Event).values(
event_id=UUID(ev.event_id),
event_id=event_uuid,
agent_id=agent_id,
check_id=ev.check_id,
observed_at=observed,
@@ -186,14 +205,7 @@ async def ingest_event(
notifications_enabled=ev.notifications_enabled,
incident_key=incident_key,
)
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(
Event.event_id
)
result = await session.execute(stmt)
inserted = result.scalar_one_or_none()
if inserted is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
await session.execute(stmt)
metrics.events_total.labels(result="accepted").inc()
else:
metrics.events_total.labels(result="unchanged").inc()
@@ -218,7 +230,7 @@ async def ingest_event(
"summary": check_summary,
}
if changed:
update_set["last_event_id"] = UUID(ev.event_id)
update_set["last_event_id"] = event_uuid
update_set["incident_key"] = incident_key
chk_stmt = pg_insert(Check).values(**check_values)
chk_stmt = chk_stmt.on_conflict_do_update(

143
server/monlet_server/services/partitioning.py Normal file
View File

@@ -0,0 +1,143 @@
"""Monthly partition maintenance for `events` (PARTITION BY RANGE(received_at)).
Idempotent: creates current + N future month partitions and drops old ones beyond
the retention window. Names are derived from year/month, so they cannot inject SQL.
"""
from __future__ import annotations
import asyncio
from datetime import UTC, date, datetime
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
from ..logging_config import get_logger
from ..settings import get_settings
log = get_logger("monlet.partitioning")
def _month_start(d: date) -> date:
return date(d.year, d.month, 1)
def _next_month(d: date) -> date:
year = d.year + (1 if d.month == 12 else 0)
month = 1 if d.month == 12 else d.month + 1
return date(year, month, 1)
def _shift_months(d: date, n: int) -> date:
y, m = d.year, d.month
total = y * 12 + (m - 1) + n
return date(total // 12, (total % 12) + 1, 1)
def _partition_name(d: date) -> str:
return f"events_p{d.year:04d}_{d.month:02d}"
async def ensure_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Create current + N future month partitions for events. Idempotent."""
s = get_settings()
today = _month_start(datetime.now(UTC).date())
months = [_shift_months(today, i) for i in range(0, max(s.events_future_partitions, 0) + 1)]
statements: list[str] = []
for start in months:
end = _next_month(start)
# Explicit UTC qualifier — Postgres otherwise interprets bare date literals via
# the session TimeZone, which would shift partition bounds on non-UTC servers.
statements.append(
f"CREATE TABLE IF NOT EXISTS {_partition_name(start)} "
f"PARTITION OF events FOR VALUES FROM ('{start.isoformat()} UTC') "
f"TO ('{end.isoformat()} UTC');"
)
async with _connect(engine_or_sm) as conn:
for stmt in statements:
await conn.execute(text(stmt))
async def drop_old_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Drop month partitions older than events_retention_months.
Each DROP runs in its own transaction so the ACCESS EXCLUSIVE lock on parent
`events` is held only for one partition at a time and ingest is not blocked
by a long maintenance batch.
"""
s = get_settings()
if s.events_retention_months <= 0:
return
cutoff = _shift_months(_month_start(datetime.now(UTC).date()), -s.events_retention_months)
engine = _resolve_engine(engine_or_sm)
async with engine.connect() as conn:
rows = (
await conn.execute(
text(
"""
SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
WHERE p.relname = 'events'
"""
)
)
).all()
to_drop: list[str] = []
for (name,) in rows:
if not name.startswith("events_p"):
continue
try:
y = int(name[8:12])
m = int(name[13:15])
except ValueError:
continue
if date(y, m, 1) < cutoff:
to_drop.append(name)
for name in to_drop:
async with engine.begin() as conn:
await conn.execute(text(f"DROP TABLE IF EXISTS {name};"))
log.info("event_partition_dropped", name=name)
def _resolve_engine(engine_or_sm: AsyncEngine | async_sessionmaker) -> AsyncEngine:
if isinstance(engine_or_sm, async_sessionmaker):
return engine_or_sm.kw["bind"]
return engine_or_sm
def _connect(engine_or_sm):
"""Return an async ctx for a transactional connection from either engine or sessionmaker."""
return _resolve_engine(engine_or_sm).begin()
async def run_partition_maintenance(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
"""Independent worker — runs even when MONLET_ENABLE_DETECTOR=false."""
interval = max(get_settings().partition_maintenance_interval_sec, 60)
while not stop_event.is_set():
try:
await ensure_event_partitions(sm)
await drop_old_event_partitions(sm)
except Exception as exc:
log.warning("partition_maintenance_failed", error=str(exc))
try:
await asyncio.wait_for(stop_event.wait(), timeout=interval)
except TimeoutError:
continue
# Re-export helpers for tests.
__all__ = [
"run_partition_maintenance",
"ensure_event_partitions",
"drop_old_event_partitions",
"_partition_name",
"_shift_months",
"_month_start",
]