creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor web pagination and add partitioned events, flap/timeout showcase agents

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 18:23:31 +04:00
parent d5f312f200
commit a2e88b4e76
46 changed files with 1600 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/.env.example
View File

@@ -7,7 +7,7 @@ MONLET_STALE_AFTER_SEC=90
MONLET_DEAD_AFTER_SEC=300
MONLET_DETECTOR_TICK_SEC=15
MONLET_ENABLE_DETECTOR=true
MONLET_EVENTS_RETENTION_MAX_ROWS=100000
MONLET_EVENTS_RETENTION_MONTHS=36
MONLET_OUTBOX_RETENTION_MAX_ROWS=50000
MONLET_ENABLE_NOTIFIER_WORKER=true

5
server/README.md
View File

@@ -39,7 +39,10 @@ UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run fastapi dev monlet
| `MONLET_DEAD_AFTER_SEC` | `300` | Dead threshold |
| `MONLET_DETECTOR_TICK_SEC` | `15` | Detector tick |
| `MONLET_ENABLE_DETECTOR` | `true` | Toggle detector task |
| `MONLET_EVENTS_RETENTION_MAX_ROWS` | `100000` | Maximum retained check-result events; `0` disables pruning |
| `MONLET_EVENTS_RETENTION_MONTHS` | `36` | Months of `events` partitions to keep; `0` disables drop |
| `MONLET_EVENTS_FUTURE_PARTITIONS` | `3` | Future month partitions to pre-create |
| `MONLET_EVENT_DEDUP_RETENTION_DAYS` | `30` | Idempotency window for `event_id` |
| `MONLET_PARTITION_MAINTENANCE_INTERVAL_SEC` | `3600` | Partition create/drop worker interval |
| `MONLET_OUTBOX_RETENTION_MAX_ROWS` | `50000` | Maximum retained terminal outbox rows; `0` disables pruning |
| `MONLET_ENABLE_NOTIFIER_WORKER` | `true` | Toggle outbox worker |
| `MONLET_NOTIFIER_TICK_SEC` | `5` | Worker poll interval |

25
server/alembic/versions/0001_baseline.py
View File

@@ -50,10 +50,12 @@ def upgrade() -> None:
)
op.execute("CREATE INDEX checks_status_idx ON checks (status);")
# events: partitioned by RANGE(received_at). Partition key is part of PK per Postgres.
# Global idempotency by event_id is enforced via event_ingest_dedup, not this table.
op.execute(
"""
CREATE TABLE events (
event_id UUID PRIMARY KEY,
event_id UUID NOT NULL,
agent_id TEXT NOT NULL,
check_id TEXT NOT NULL,
observed_at TIMESTAMPTZ NOT NULL,
@@ -64,14 +66,30 @@ def upgrade() -> None:
output TEXT,
output_truncated BOOLEAN NOT NULL DEFAULT FALSE,
notifications_enabled BOOLEAN NOT NULL DEFAULT TRUE,
incident_key TEXT NOT NULL
);
incident_key TEXT NOT NULL,
PRIMARY KEY (event_id, received_at)
) PARTITION BY RANGE (received_at);
"""
)
op.execute(
"CREATE INDEX events_agent_check_observed_idx ON events (agent_id, check_id, observed_at DESC);"
)
op.execute("CREATE INDEX events_observed_at_idx ON events (observed_at DESC);")
op.execute("CREATE INDEX events_received_at_idx ON events (received_at DESC);")
# Global idempotency table: server enforces event_id uniqueness here, not on the
# partitioned events table. Pruned by retention TTL (event_dedup_retention_days).
op.execute(
"""
CREATE TABLE event_ingest_dedup (
event_id UUID PRIMARY KEY,
received_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
"""
)
op.execute(
"CREATE INDEX event_ingest_dedup_received_at_idx ON event_ingest_dedup (received_at);"
)
op.execute(
"""
@@ -119,6 +137,7 @@ def upgrade() -> None:
def downgrade() -> None:
op.execute("DROP TABLE IF EXISTS notification_outbox CASCADE;")
op.execute("DROP TABLE IF EXISTS incidents CASCADE;")
op.execute("DROP TABLE IF EXISTS event_ingest_dedup CASCADE;")
op.execute("DROP TABLE IF EXISTS events CASCADE;")
op.execute("DROP TABLE IF EXISTS checks CASCADE;")
op.execute("DROP TABLE IF EXISTS agents CASCADE;")

26
server/monlet_server/main.py
View File

@@ -14,19 +14,43 @@ from .middleware.body_limit import BodyLimitMiddleware
from .middleware.request_id import RequestIdMiddleware
from .services.detector import run_detector
from .services.notifier_worker import run_notifier_worker
from .services.partitioning import ensure_event_partitions, run_partition_maintenance
from .settings import get_settings
log = get_logger("monlet.main")
async def _ensure_partitions_with_retry(engine, attempts: int = 6, base_delay: float = 1.0) -> None:
last_exc: Exception | None = None
for i in range(attempts):
try:
await ensure_event_partitions(engine)
return
except Exception as exc:
last_exc = exc
log.warning("ensure_event_partitions_retry", attempt=i + 1, error=str(exc))
await asyncio.sleep(min(base_delay * (2**i), 10.0))
assert last_exc is not None
raise last_exc
@asynccontextmanager
async def lifespan(app: FastAPI):
s = get_settings()
configure_logging(s.log_level)
get_engine()
engine = get_engine()
sm = get_sessionmaker()
# Mandatory: current month partition must exist before accepting writes.
# Retry briefly so a transient DB unavailability at startup (compose race)
# doesn't crash-loop the server.
await _ensure_partitions_with_retry(engine)
stop_event = asyncio.Event()
tasks: list[asyncio.Task] = []
# Partition maintenance is independent of detector — must run even when detector is off,
# otherwise events_future_partitions runs out and ingest fails.
tasks.append(
asyncio.create_task(run_partition_maintenance(sm, stop_event), name="partition_maintenance")
)
if s.enable_detector:
tasks.append(asyncio.create_task(run_detector(sm, stop_event), name="detector"))
if s.enable_notifier_worker:

16
server/monlet_server/models.py
View File

@@ -85,10 +85,11 @@ class Check(Base):
class Event(Base):
__tablename__ = "events"
event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
agent_id: Mapped[str] = mapped_column(Text, nullable=False)
check_id: Mapped[str] = mapped_column(Text, nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
# Partition key (RANGE) on Postgres; composite PK = (event_id, received_at).
received_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
@@ -103,6 +104,7 @@ class Event(Base):
incident_key: Mapped[str] = mapped_column(Text, nullable=False)
__table_args__ = (
PrimaryKeyConstraint("event_id", "received_at", name="events_pkey"),
CheckConstraint(
"status IN ('ok','warning','critical','unknown')", name="events_status_chk"
),
@@ -114,9 +116,21 @@ class Event(Base):
text("observed_at DESC"),
),
Index("events_observed_at_idx", text("observed_at DESC")),
Index("events_received_at_idx", text("received_at DESC")),
)
class EventIngestDedup(Base):
__tablename__ = "event_ingest_dedup"
event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
received_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
__table_args__ = (Index("event_ingest_dedup_received_at_idx", "received_at"),)
class Incident(Base):
__tablename__ = "incidents"

105
server/monlet_server/services/detector.py
View File

@@ -1,6 +1,6 @@
import asyncio
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from uuid import UUID, uuid4, uuid7
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..settings import get_settings
log = get_logger("monlet.detector")
@@ -25,6 +25,10 @@ def _liveness_severity(status: str) -> str:
return "critical" if status == "dead" else "warning"
_LIVENESS_CHECK_STATUS = {"alive": "ok", "stale": "warning", "dead": "critical"}
_LIVENESS_EXIT_CODE = {"ok": 0, "warning": 1, "critical": 2}
async def _enqueue_liveness_outbox(
session,
incident: Incident,
@@ -68,7 +72,9 @@ async def _enqueue_liveness_outbox(
)
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
async def _open_liveness_incident(
session, agent: Agent, status: str, now: datetime, event_id: UUID
) -> None:
severity = _liveness_severity(status)
summary = f"agent is {status}"
incident_key = _liveness_key(agent.agent_id)
@@ -84,7 +90,7 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
severity=severity,
opened_at=now,
summary=summary,
last_event_id=uuid4(),
last_event_id=event_id,
)
.on_conflict_do_nothing(
index_elements=[Incident.incident_key],
@@ -115,13 +121,13 @@ async def _open_liveness_incident(session, agent: Agent, status: str, now: datet
if status == "dead" and inc.severity != "critical":
inc.severity = "critical"
inc.summary = summary
inc.last_event_id = uuid4()
inc.last_event_id = event_id
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
)
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
async def _resolve_liveness_incident(session, agent: Agent, now: datetime, event_id: UUID) -> None:
inc = (
await session.execute(
select(Incident).where(
@@ -134,7 +140,7 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
return
inc.state = "resolved"
inc.resolved_at = now
inc.last_event_id = uuid4()
inc.last_event_id = event_id
metrics.incidents_resolved_total.inc()
await _enqueue_liveness_outbox(
session,
@@ -148,15 +154,83 @@ async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> No
)
async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime) -> None:
"""Server-owned liveness check: maintain Check + Event-on-transition + Incident lifecycle."""
liveness_status = _LIVENESS_CHECK_STATUS[next_status]
exit_code = _LIVENESS_EXIT_CODE[liveness_status]
summary = f"agent is {next_status}"
incident_key = _liveness_key(agent.agent_id)
cur = (
await session.execute(
select(Check).where(
Check.agent_id == agent.agent_id, Check.check_id == LIVENESS_CHECK_ID
)
)
).scalar_one_or_none()
changed = cur is None or cur.status != liveness_status
# Liveness events flow through /events/query which validates event_id as UUIDv7.
event_id = cur.last_event_id if cur and not changed else uuid7()
if changed:
session.add(
Event(
event_id=event_id,
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
observed_at=now,
status=liveness_status,
exit_code=exit_code,
duration_ms=0,
output=summary,
output_truncated=False,
notifications_enabled=True,
incident_key=incident_key,
)
)
if cur is None:
session.add(
Check(
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
status=liveness_status,
exit_code=exit_code,
last_observed_at=now,
last_event_id=event_id,
incident_key=incident_key,
summary=summary,
)
)
else:
cur.last_observed_at = now
cur.summary = summary
if changed:
cur.status = liveness_status
cur.exit_code = exit_code
cur.last_event_id = event_id
cur.incident_key = incident_key
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now, event_id)
else:
await _resolve_liveness_incident(session, agent, now, event_id)
async def _prune_history(session) -> None:
s = get_settings()
if s.events_retention_max_rows > 0:
keep_events = (
select(Event.event_id)
.order_by(Event.received_at.desc(), Event.event_id.desc())
.limit(s.events_retention_max_rows)
if s.event_dedup_retention_days > 0:
cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
batch = max(s.event_dedup_prune_batch_size, 1)
# Bounded DELETE: keep each tick's prune transaction short to avoid bloat and
# long-running locks on a large dedup table.
sub = (
select(EventIngestDedup.event_id)
.where(EventIngestDedup.received_at < cutoff)
.limit(batch)
.scalar_subquery()
)
await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
if s.outbox_retention_max_rows > 0:
keep_outbox = (
@@ -188,10 +262,7 @@ async def _tick(sm: async_sessionmaker) -> None:
next_status = "alive"
if agent.status != next_status:
agent.status = next_status
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now)
else:
await _resolve_liveness_incident(session, agent, now)
await _apply_liveness(session, agent, next_status, now)
await _prune_history(session)
await session.commit()

40
server/monlet_server/services/ingestion.py
View File

@@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import metrics
from ..logging_config import get_logger
from ..models import Agent, Check, Event, Incident, NotificationOutbox
from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
from ..redaction import redact
from ..schemas import CheckResultEvent, HeartbeatRequest
from ..settings import get_settings
@@ -155,26 +155,45 @@ async def ingest_event(
agent_id: str,
ev: CheckResultEvent,
) -> bool:
"""Return True if a new change row was written, False if not (duplicate/unchanged/late)."""
"""Return True if a new change row was written, False if not (duplicate/unchanged/late).
Idempotency is enforced first via the standalone event_ingest_dedup table — a
duplicate event_id never touches checks/incidents/outbox/events. The full pipeline
(dedup + events + check + incident + outbox) commits as a single transaction by
the caller, so a partial failure cannot leave a dedup row without its side-effects.
"""
observed = to_utc(ev.observed_at)
output = redact(ev.output)
check_summary = (output or "")[:200] if output else None
incident_key = _incident_key(agent_id, ev)
event_uuid = UUID(ev.event_id)
# Order matters: late check before dedup insert so that repeated late events stay
# observable as `late`, not silently collapsed into `deduplicated`.
chk_res = await session.execute(
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
)
cur = chk_res.scalar_one_or_none()
is_late = cur is not None and cur.last_observed_at > observed
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if is_late:
metrics.events_total.labels(result="late").inc()
return False
dedup_stmt = (
pg_insert(EventIngestDedup)
.values(event_id=event_uuid, received_at=datetime.now(UTC))
.on_conflict_do_nothing(index_elements=[EventIngestDedup.event_id])
.returning(EventIngestDedup.event_id)
)
if (await session.execute(dedup_stmt)).scalar_one_or_none() is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if changed:
stmt = pg_insert(Event).values(
event_id=UUID(ev.event_id),
event_id=event_uuid,
agent_id=agent_id,
check_id=ev.check_id,
observed_at=observed,
@@ -186,14 +205,7 @@ async def ingest_event(
notifications_enabled=ev.notifications_enabled,
incident_key=incident_key,
)
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(
Event.event_id
)
result = await session.execute(stmt)
inserted = result.scalar_one_or_none()
if inserted is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
await session.execute(stmt)
metrics.events_total.labels(result="accepted").inc()
else:
metrics.events_total.labels(result="unchanged").inc()
@@ -218,7 +230,7 @@ async def ingest_event(
"summary": check_summary,
}
if changed:
update_set["last_event_id"] = UUID(ev.event_id)
update_set["last_event_id"] = event_uuid
update_set["incident_key"] = incident_key
chk_stmt = pg_insert(Check).values(**check_values)
chk_stmt = chk_stmt.on_conflict_do_update(

143
server/monlet_server/services/partitioning.py Normal file
View File

@@ -0,0 +1,143 @@
"""Monthly partition maintenance for `events` (PARTITION BY RANGE(received_at)).
Idempotent: creates current + N future month partitions and drops old ones beyond
the retention window. Names are derived from year/month, so they cannot inject SQL.
"""
from __future__ import annotations
import asyncio
from datetime import UTC, date, datetime
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
from ..logging_config import get_logger
from ..settings import get_settings
log = get_logger("monlet.partitioning")
def _month_start(d: date) -> date:
return date(d.year, d.month, 1)
def _next_month(d: date) -> date:
year = d.year + (1 if d.month == 12 else 0)
month = 1 if d.month == 12 else d.month + 1
return date(year, month, 1)
def _shift_months(d: date, n: int) -> date:
y, m = d.year, d.month
total = y * 12 + (m - 1) + n
return date(total // 12, (total % 12) + 1, 1)
def _partition_name(d: date) -> str:
return f"events_p{d.year:04d}_{d.month:02d}"
async def ensure_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Create current + N future month partitions for events. Idempotent."""
s = get_settings()
today = _month_start(datetime.now(UTC).date())
months = [_shift_months(today, i) for i in range(0, max(s.events_future_partitions, 0) + 1)]
statements: list[str] = []
for start in months:
end = _next_month(start)
# Explicit UTC qualifier — Postgres otherwise interprets bare date literals via
# the session TimeZone, which would shift partition bounds on non-UTC servers.
statements.append(
f"CREATE TABLE IF NOT EXISTS {_partition_name(start)} "
f"PARTITION OF events FOR VALUES FROM ('{start.isoformat()} UTC') "
f"TO ('{end.isoformat()} UTC');"
)
async with _connect(engine_or_sm) as conn:
for stmt in statements:
await conn.execute(text(stmt))
async def drop_old_event_partitions(engine_or_sm: AsyncEngine | async_sessionmaker) -> None:
"""Drop month partitions older than events_retention_months.
Each DROP runs in its own transaction so the ACCESS EXCLUSIVE lock on parent
`events` is held only for one partition at a time and ingest is not blocked
by a long maintenance batch.
"""
s = get_settings()
if s.events_retention_months <= 0:
return
cutoff = _shift_months(_month_start(datetime.now(UTC).date()), -s.events_retention_months)
engine = _resolve_engine(engine_or_sm)
async with engine.connect() as conn:
rows = (
await conn.execute(
text(
"""
SELECT c.relname
FROM pg_inherits i
JOIN pg_class c ON c.oid = i.inhrelid
JOIN pg_class p ON p.oid = i.inhparent
WHERE p.relname = 'events'
"""
)
)
).all()
to_drop: list[str] = []
for (name,) in rows:
if not name.startswith("events_p"):
continue
try:
y = int(name[8:12])
m = int(name[13:15])
except ValueError:
continue
if date(y, m, 1) < cutoff:
to_drop.append(name)
for name in to_drop:
async with engine.begin() as conn:
await conn.execute(text(f"DROP TABLE IF EXISTS {name};"))
log.info("event_partition_dropped", name=name)
def _resolve_engine(engine_or_sm: AsyncEngine | async_sessionmaker) -> AsyncEngine:
if isinstance(engine_or_sm, async_sessionmaker):
return engine_or_sm.kw["bind"]
return engine_or_sm
def _connect(engine_or_sm):
"""Return an async ctx for a transactional connection from either engine or sessionmaker."""
return _resolve_engine(engine_or_sm).begin()
async def run_partition_maintenance(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
"""Independent worker — runs even when MONLET_ENABLE_DETECTOR=false."""
interval = max(get_settings().partition_maintenance_interval_sec, 60)
while not stop_event.is_set():
try:
await ensure_event_partitions(sm)
await drop_old_event_partitions(sm)
except Exception as exc:
log.warning("partition_maintenance_failed", error=str(exc))
try:
await asyncio.wait_for(stop_event.wait(), timeout=interval)
except TimeoutError:
continue
# Re-export helpers for tests.
__all__ = [
"run_partition_maintenance",
"ensure_event_partitions",
"drop_old_event_partitions",
"_partition_name",
"_shift_months",
"_month_start",
]

6
server/monlet_server/settings.py
View File

@@ -20,7 +20,11 @@ class Settings(BaseSettings):
port: int = 8000
enable_detector: bool = True
enable_notifier_worker: bool = True
events_retention_max_rows: int = 20_000
events_retention_months: int = 36
events_future_partitions: int = 3
event_dedup_retention_days: int = 30
event_dedup_prune_batch_size: int = 5000
partition_maintenance_interval_sec: int = 3600
outbox_retention_max_rows: int = 50_000
notifier_tick_sec: int = 5
notifier_batch_size: int = 20

16
server/tests/conftest.py
View File

@@ -58,6 +58,20 @@ def _apply_migrations(database_url: str) -> Iterator[None]:
cfg.set_main_option("script_location", os.path.join(SERVER_ROOT, "alembic"))
cfg.set_main_option("sqlalchemy.url", get_settings().sync_database_url)
command.upgrade(cfg, "head")
# Partitions for `events` are created at runtime, not by the migration.
async def _bootstrap() -> None:
from sqlalchemy.ext.asyncio import create_async_engine
from monlet_server.services.partitioning import ensure_event_partitions
eng = create_async_engine(database_url, future=True)
try:
await ensure_event_partitions(eng)
finally:
await eng.dispose()
asyncio.run(_bootstrap())
yield
@@ -83,7 +97,7 @@ async def _truncate_tables(engine) -> AsyncIterator[None]:
async with engine.begin() as conn:
await conn.execute(
text(
"TRUNCATE TABLE notification_outbox, incidents, events, checks, agents RESTART IDENTITY CASCADE"
"TRUNCATE TABLE notification_outbox, incidents, events, event_ingest_dedup, checks, agents RESTART IDENTITY CASCADE"
)
)

18
server/tests/test_constraints.py
View File

@@ -9,26 +9,18 @@ from monlet_server.models import Incident
@pytest.mark.asyncio
async def test_event_id_conflict_do_nothing(engine):
async def test_event_id_dedup_conflict_do_nothing(engine):
"""Global idempotency is enforced via event_ingest_dedup, not the partitioned events table."""
eid = str(uuid4())
async with engine.begin() as conn:
await conn.execute(
text(
"INSERT INTO agents (agent_id, hostname, features, status, last_seen_at) "
"VALUES ('a','h',jsonb_build_object('push', true, 'metrics', false),'alive', now())"
)
)
await conn.execute(
text(
"INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
" VALUES (:e,'a','c', now(),'ok',0,1,'a:c')"
),
text("INSERT INTO event_ingest_dedup (event_id) VALUES (:e)"),
{"e": eid},
)
r = await conn.execute(
text(
"INSERT INTO events (event_id, agent_id, check_id, observed_at, status, exit_code, duration_ms, incident_key)"
" VALUES (:e,'a','c', now(),'ok',0,1,'a:c') ON CONFLICT (event_id) DO NOTHING RETURNING event_id"
"INSERT INTO event_ingest_dedup (event_id) VALUES (:e) "
"ON CONFLICT (event_id) DO NOTHING RETURNING event_id"
),
{"e": eid},
)

76
server/tests/test_detector.py
View File

@@ -5,7 +5,7 @@ import pytest
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import async_sessionmaker
from monlet_server.models import Agent, Event, Incident, NotificationOutbox
from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox
from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
from monlet_server.settings import reset_settings_cache
@@ -82,22 +82,75 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
@pytest.mark.asyncio
async def test_detector_prunes_bounded_history(
app_client, auth_headers, engine, session, monkeypatch
):
monkeypatch.setenv("MONLET_EVENTS_RETENTION_MAX_ROWS", "2")
async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session):
await app_client.post(
"/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers
)
sm = async_sessionmaker(engine, expire_on_commit=False)
# Tick 1: alive → Check exists with status=ok, one event.
await _tick(sm)
session.expire_all()
check = (
await session.execute(
select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID)
)
).scalar_one()
assert check.status == "ok"
n_events = (
await session.execute(
select(func.count())
.select_from(Event)
.where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
)
).scalar_one()
assert n_events == 1
# Tick 2: still alive → no new event.
await _tick(sm)
session.expire_all()
n_events = (
await session.execute(
select(func.count())
.select_from(Event)
.where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
)
).scalar_one()
assert n_events == 1
# Push agent into dead → transition event written, Check updates.
await session.execute(
update(Agent)
.where(Agent.agent_id == "agent-flap")
.values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10))
)
await session.commit()
await _tick(sm)
session.expire_all()
check = (
await session.execute(
select(Check).where(Check.agent_id == "agent-flap", Check.check_id == LIVENESS_CHECK_ID)
)
).scalar_one()
assert check.status == "critical"
n_events = (
await session.execute(
select(func.count())
.select_from(Event)
.where(Event.agent_id == "agent-flap", Event.check_id == LIVENESS_CHECK_ID)
)
).scalar_one()
assert n_events == 2
@pytest.mark.asyncio
async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch):
monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
reset_settings_cache()
try:
await app_client.post(
"/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
)
events = [make_event(check_id="prune_check") for _ in range(5)]
await app_client.post(
"/api/v1/events",
json={"agent_id": "agent-prune", "events": events},
headers=auth_headers,
)
critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
await app_client.post(
"/api/v1/events",
@@ -129,7 +182,6 @@ async def test_detector_prunes_bounded_history(
sm = async_sessionmaker(engine, expire_on_commit=False)
await _tick(sm)
assert (await session.execute(select(func.count()).select_from(Event))).scalar_one() == 2
terminal_outbox = (
await session.execute(
select(func.count())

125
server/tests/test_partitioning.py Normal file
View File

@@ -0,0 +1,125 @@
from __future__ import annotations
from datetime import UTC, date, datetime, timedelta
import pytest
from sqlalchemy import text
from monlet_server.services.partitioning import (
_month_start,
_partition_name,
_shift_months,
drop_old_event_partitions,
ensure_event_partitions,
)
from monlet_server.settings import reset_settings_cache
async def _partitions(engine) -> set[str]:
async with engine.begin() as conn:
rows = (
await conn.execute(
text(
"SELECT c.relname FROM pg_inherits i "
"JOIN pg_class c ON c.oid=i.inhrelid "
"JOIN pg_class p ON p.oid=i.inhparent "
"WHERE p.relname='events'"
)
)
).all()
return {r[0] for r in rows}
@pytest.mark.asyncio
async def test_ensure_creates_current_and_future_partitions(engine, monkeypatch):
monkeypatch.setenv("MONLET_EVENTS_FUTURE_PARTITIONS", "2")
reset_settings_cache()
try:
await ensure_event_partitions(engine)
today = _month_start(datetime.now(UTC).date())
for i in range(0, 3):
assert _partition_name(_shift_months(today, i)) in await _partitions(engine)
finally:
reset_settings_cache()
@pytest.mark.asyncio
async def test_drop_old_partitions_uses_retention(engine, monkeypatch):
monkeypatch.setenv("MONLET_EVENTS_RETENTION_MONTHS", "1")
reset_settings_cache()
try:
# Create an old partition directly.
old = date(2020, 1, 1)
name = _partition_name(old)
end = _shift_months(old, 1)
async with engine.begin() as conn:
await conn.execute(
text(
f"CREATE TABLE IF NOT EXISTS {name} PARTITION OF events "
f"FOR VALUES FROM ('{old.isoformat()} UTC') TO ('{end.isoformat()} UTC');"
)
)
assert name in await _partitions(engine)
await drop_old_event_partitions(engine)
assert name not in await _partitions(engine)
finally:
reset_settings_cache()
@pytest.mark.asyncio
async def test_dedup_prevents_replay_via_ingest(app_client, auth_headers, session):
from sqlalchemy import func, select
from monlet_server.models import Event
from ._helpers import make_event, make_heartbeat
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
ev = make_event(status="critical", exit_code=2)
r1 = await app_client.post(
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
)
assert r1.json() == {"accepted": 1, "deduplicated": 0}
# Replay the exact same event_id long after — dedup must block ingestion entirely,
# never touching checks/incidents/outbox/events.
r2 = await app_client.post(
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
)
assert r2.json() == {"accepted": 0, "deduplicated": 1}
n = (await session.execute(select(func.count()).select_from(Event))).scalar_one()
assert n == 1
@pytest.mark.asyncio
async def test_late_dedup_does_not_keep_replay_doors_open(engine, monkeypatch):
"""Dedup retention TTL must be at least documented agent spool window."""
from uuid import uuid4
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from monlet_server.models import EventIngestDedup
from monlet_server.services.detector import _prune_history
monkeypatch.setenv("MONLET_EVENT_DEDUP_RETENTION_DAYS", "1")
reset_settings_cache()
try:
sm = async_sessionmaker(engine, expire_on_commit=False)
async with sm() as s:
old = datetime.now(UTC) - timedelta(days=5)
s.add(EventIngestDedup(event_id=uuid4(), received_at=old))
s.add(EventIngestDedup(event_id=uuid4(), received_at=datetime.now(UTC)))
await s.commit()
async with sm() as s:
await _prune_history(s)
await s.commit()
async with sm() as s:
n = (await s.execute(select(func.count()).select_from(EventIngestDedup))).scalar_one()
assert n == 1
finally:
reset_settings_cache()