creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor web pagination and add partitioned events, flap/timeout showcase agents

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 18:23:31 +04:00
parent d5f312f200
commit a2e88b4e76
46 changed files with 1600 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
server/monlet_server/main.py
View File

@@ -14,19 +14,43 @@ from .middleware.body_limit import BodyLimitMiddleware
from .middleware.request_id import RequestIdMiddleware
from .services.detector import run_detector
from .services.notifier_worker import run_notifier_worker
from .services.partitioning import ensure_event_partitions, run_partition_maintenance
from .settings import get_settings
log = get_logger("monlet.main")
async def _ensure_partitions_with_retry(engine, attempts: int = 6, base_delay: float = 1.0) -> None:
last_exc: Exception | None = None
for i in range(attempts):
try:
await ensure_event_partitions(engine)
return
except Exception as exc:
last_exc = exc
log.warning("ensure_event_partitions_retry", attempt=i + 1, error=str(exc))
await asyncio.sleep(min(base_delay * (2**i), 10.0))
assert last_exc is not None
raise last_exc
@asynccontextmanager
async def lifespan(app: FastAPI):
s = get_settings()
configure_logging(s.log_level)
get_engine()
engine = get_engine()
sm = get_sessionmaker()
# Mandatory: current month partition must exist before accepting writes.
# Retry briefly so a transient DB unavailability at startup (compose race)
# doesn't crash-loop the server.
await _ensure_partitions_with_retry(engine)
stop_event = asyncio.Event()
tasks: list[asyncio.Task] = []
# Partition maintenance is independent of detector — must run even when detector is off,
# otherwise events_future_partitions runs out and ingest fails.
tasks.append(
asyncio.create_task(run_partition_maintenance(sm, stop_event), name="partition_maintenance")
)
if s.enable_detector:
tasks.append(asyncio.create_task(run_detector(sm, stop_event), name="detector"))
if s.enable_notifier_worker:

16
server/monlet_server/models.py
View File

@@ -85,10 +85,11 @@ class Check(Base):
class Event(Base):
__tablename__ = "events"
event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
agent_id: Mapped[str] = mapped_column(Text, nullable=False)
check_id: Mapped[str] = mapped_column(Text, nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
# Partition key (RANGE) on Postgres; composite PK = (event_id, received_at).
received_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
@@ -103,6 +104,7 @@ class Event(Base):
incident_key: Mapped[str] = mapped_column(Text, nullable=False)
__table_args__ = (
PrimaryKeyConstraint("event_id", "received_at", name="events_pkey"),
CheckConstraint(
"status IN ('ok','warning','critical','unknown')", name="events_status_chk"
),
@@ -114,9 +116,21 @@ class Event(Base):
text("observed_at DESC"),
),
Index("events_observed_at_idx", text("observed_at DESC")),
Index("events_received_at_idx", text("received_at DESC")),
)
class EventIngestDedup(Base):
__tablename__ = "event_ingest_dedup"
event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
received_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
__table_args__ = (Index("event_ingest_dedup_received_at_idx", "received_at"),)
class Incident(Base):
__tablename__ = "incidents"

105
server/monlet_server/services/detector.py
View File

@@ -1,6 +1,6 @@
import asyncio
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from uuid import UUID, uuid4, uuid7
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..settings import get_settings
log = get_logger("monlet.detector")
@@ -25,6 +25,10 @@ def _liveness_severity(status: str) -> str:
return "critical" if status == "dead" else "warning"
_LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"}
_LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2}
async def _enqueue_liveness_outbox(
session,
incident: Incident,
@@ -68,7 +72,9 @@ async def _enqueue_liveness_outbox(
)
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
async def _open_liveness_incident(
session, agent: Agent, status: str, now: datetime, event_id: UUID
) -> None:
severity = _liveness_severity(status)
summary = f"agent is {status}"
incident_key = _liveness_key(agent.agent_id)
@@ -84,7 +90,7 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
severity=severity,
opened_at=now,
summary=summary,
last_event_id=uuid4(),
last_event_id=event_id,
)
.on_conflict_do_nothing(
index_elements=[Incident.incident_key],
@@ -115,13 +121,13 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
if status == "dead" and inc.severity != "critical":
inc.severity = "critical"
inc.summary = summary
inc.last_event_id = uuid4()
inc.last_event_id = event_id
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
)
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None:
inc = (
await session.execute(
select(Incident).where(
@@ -134,7 +140,7 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
return
inc.state = "resolved"
inc.resolved_at = now
inc.last_event_id = uuid4()
inc.last_event_id = event_id
metrics.incidents_resolved_total.inc()
await _enqueue_liveness_outbox(
session,
@@ -148,15 +154,83 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
)
async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None:
"""Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle."""
liveness_status = _LIVENESS_CHECK_STATUS[next_status]
exit_code = _LIVENESS_EXIT_CODE[liveness_status]
summary = f"agent is {next_status}"
incident_key = _liveness_key(agent.agent_id)
cur = (
await session.execute(
select(Check).where(
Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID
)
)
).scalar_one_or_none()
changed = cur is None or cur.status != liveness_status
# Liveness events flow through /events/query which validates event_id as UUIDv7.
event_id = cur.last_event_id if cur and not changed else uuid7()
if changed:
session.add(
Event(
event_id=event_id,
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
observed_at=now,
status=liveness_status,
exit_code=exit_code,
duration_ms=0,
output=summary,
output_truncated=False,
notifications_enabled=True,
incident_key=incident_key,
)
)
if cur is None:
session.add(
Check(
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
status=liveness_status,
exit_code=exit_code,
last_observed_at=now,
last_event_id=event_id,
incident_key=incident_key,
summary=summary,
)
)
else:
cur.last_observed_at = now
cur.summary = summary
if changed:
cur.status = liveness_status
cur.exit_code = exit_code
cur.last_event_id = event_id
cur.incident_key = incident_key
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now, event_id)
else:
await _resolve_liveness_incident(session, agent, now, event_id)
async def _prune_history(session) -> None:
s = get_settings()
if s.events_retention_max_rows > 0:
keep_events = (
select(Event.event_id)
.order_by(Event.received_at.desc(), Event.event_id.desc())
.limit(s.events_retention_max_rows)
if s.event_dedup_retention_days > 0:
cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
batch = max(s.event_dedup_prune_batch_size, 1)
# Bounded DELETE: keep each tick's prune transaction short to avoid bloat and
# long-running locks on a large dedup table.
sub = (
select(EventIngestDedup.event_id)
.where(EventIngestDedup.received_at < cutoff)
.limit(batch)
.scalar_subquery()
)
await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
if s.outbox_retention_max_rows > 0:
keep_outbox = (
@@ -188,10 +262,7 @@ async def _tick(sm: async_sessionmaker) -> None:
next_status = "alive"
if agent.status != next_status:
agent.status = next_status
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now)
else:
await _resolve_liveness_incident(session, agent, now)
await _apply_liveness(session, agent, next_status, now)
await _prune_history(session)
await session.commit()

40
server/monlet_server/services/ingestion.py
View File

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Check, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..redaction import redact
from ..schemas import CheckResultEvent, HeartbeatRequest
from ..settings import get_settings
@@ -155,26 +155,45 @@ async def ingest_event(
agent_id: str,
ev: CheckResultEvent,
) -> bool:
"""Return True if a new change row was written, False if not (duplicate/unchanged/late)."""
"""Return True if a new change row was written, False if not (duplicate/unchanged/late).
Idempotency is enforced first via the standalone event_ingest_dedup table — a
duplicate event_id never touches checks/incidents/outbox/events. The full pipeline
(dedup + events + check + incident + outbox) commits as a single transaction by
the caller, so a partial failure cannot leave a dedup row without its side-effects.
"""
observed = to_utc(ev.observed_at)
output = redact(ev.output)
check_summary = (output or "")[:200] if output else None
incident_key = _incident_key(agent_id, ev)
event_uuid = UUID(ev.event_id)
# Order matters: late check before dedup insert so that repeated late events stay
# observable as `late`, not silently collapsed into `deduplicated`.
chk_res = await session.execute(
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
)
cur = chk_res.scalar_one_or_none()
is_late = cur is not None and cur.last_observed_at > observed
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if is_late:
metrics.events_total.labels(result="late").inc()
return False
dedup_stmt = (
pg_insert(EventIngestDedup)
.values(event_id=event_uuid, received_at=datetime.now(UTC))
.on_conflict_do_nothing(index_elements=[EventIngestDedup.event_id])
.returning(EventIngestDedup.event_id)
)
if (await session.execute(dedup_stmt)).scalar_one_or_none() is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if changed:
stmt = pg_insert(Event).values(
event_id=UUID(ev.event_id),
event_id=event_uuid,
agent_id=agent_id,
check_id=ev.check_id,
observed_at=observed,
@@ -186,14 +205,7 @@ async def ingest_event(
notifications_enabled=ev.notifications_enabled,
incident_key=incident_key,
)
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(
Event.event_id
)
result = await session.execute(stmt)
inserted = result.scalar_one_or_none()
if inserted is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
await session.execute(stmt)
metrics.events_total.labels(result="accepted").inc()
else:
metrics.events_total.labels(result="unchanged").inc()
@@ -218,7 +230,7 @@ async def ingest_event(
"summary": check_summary,
}
if changed:
update_set["last_event_id"] = UUID(ev.event_id)
update_set["last_event_id"] = event_uuid
update_set["incident_key"] = incident_key
chk_stmt = pg_insert(Check).values(**check_values)
chk_stmt = chk_stmt.on_conflict_do_update(

143
server/monlet_server/services/partitioning.py Normal file
View File

@@ -0,0 +1,143 @@
"""Monthly partition maintenance for `events` (PARTITION BY RANGE(received_at)).
Idempotent: creates current + N future month partitions and drops old ones beyond
the retention window. Names are derived from year/month, so they cannot inject SQL.
"""
from __future__ import annotations
import asyncio
from datetime import UTC, date, datetime
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
from ..logging_config import get_logger
from ..settings import get_settings
log = get_logger("monlet.partitioning")
def _month_start(d: date) -> date:
return date(d.year, d.month, 1)
def _next_month(d: date) -> date:
year = d.year + (1 if d.month == 12 else 0)
month = 1 if d.month == 12 else d.month + 1
return date(year, month, 1)
def _shift_months(d: date, n: int) -> date:
y, m = d.year, d.month
total = y * 12 + (m - 1) + n
return date(total // 12, (total % 12) + 1, 1)
def _partition_name(d: date) -> str:
return f"events_p{d.year:04d}_{d.month:02d}"
async def ensure_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Create current + N future month partitions for events. Idempotent."""
s = get_settings()
today = _month_start(datetime.now(UTC).date())
months = [_shift_months(today, i) for i in range(0, max(s.events_future_partitions, 0) + 1)]
statements: list[str] = []
for start in months:
end = _next_month(start)
# Explicit UTC qualifier — Postgres otherwise interprets bare date literals via
# the session TimeZone, which would shift partition bounds on non-UTC servers.
statements.append(
f"CREATE TABLE IF NOT EXISTS {_partition_name(start)} "
f"PARTITION OF events FOR VALUES FROM ('{start.isoformat()} UTC') "
f"TO ('{end.isoformat()} UTC');"
)
async with _connect(engine_or_sm) as conn:
for stmt in statements:
await conn.execute(text(stmt))
async def drop_old_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Drop month partitions older than events_retention_months.
Each DROP runs in its own transaction so the ACCESS EXCLUSIVE lock on parent
`events` is held only for one partition at a time and ingest is not blocked
by a long maintenance batch.
"""
s = get_settings()
if s.events_retention_months <= 0:
return
cutoff = _shift_months(_month_start(datetime.now(UTC).date()), -s.events_retention_months)
engine = _resolve_engine(engine_or_sm)
async with engine.connect() as conn:
rows = (
await conn.execute(
text(
"""
SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
WHERE p.relname = 'events'
"""
)
)
).all()
to_drop: list[str] = []
for (name,) in rows:
if not name.startswith("events_p"):
continue
try:
y = int(name[8:12])
m = int(name[13:15])
except ValueError:
continue
if date(y, m, 1) < cutoff:
to_drop.append(name)
for name in to_drop:
async with engine.begin() as conn:
await conn.execute(text(f"DROP TABLE IF EXISTS {name};"))
log.info("event_partition_dropped", name=name)
def _resolve_engine(engine_or_sm: AsyncEngine | async_sessionmaker) -> AsyncEngine:
if isinstance(engine_or_sm, async_sessionmaker):
return engine_or_sm.kw["bind"]
return engine_or_sm
def _connect(engine_or_sm):
"""Return an async ctx for a transactional connection from either engine or sessionmaker."""
return _resolve_engine(engine_or_sm).begin()
async def run_partition_maintenance(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
"""Independent worker — runs even when MONLET_ENABLE_DETECTOR=false."""
interval = max(get_settings().partition_maintenance_interval_sec, 60)
while not stop_event.is_set():
try:
await ensure_event_partitions(sm)
await drop_old_event_partitions(sm)
except Exception as exc:
log.warning("partition_maintenance_failed", error=str(exc))
try:
await asyncio.wait_for(stop_event.wait(), timeout=interval)
except TimeoutError:
continue
# Re-export helpers for tests.
__all__ = [
"run_partition_maintenance",
"ensure_event_partitions",
"drop_old_event_partitions",
"_partition_name",
"_shift_months",
"_month_start",
]

6
server/monlet_server/settings.py
View File

@@ -20,7 +20,11 @@ class Settings(BaseSettings):
port: int = 8000
enable_detector: bool = True
enable_notifier_worker: bool = True
events_retention_max_rows: int = 20_000
events_retention_months: int = 36
events_future_partitions: int = 3
event_dedup_retention_days: int = 30
event_dedup_prune_batch_size: int = 5000
partition_maintenance_interval_sec: int = 3600
outbox_retention_max_rows: int = 50_000
notifier_tick_sec: int = 5
notifier_batch_size: int = 20