Implement Monlet MVP stack and UI updates

server/monlet_server/__init__.py

@@ -0,0 +1 @@
+"""Monlet server package."""

server/monlet_server/api/agents.py

@@ -0,0 +1,48 @@
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, encode
+from ..db import get_session
+from ..models import Agent as AgentModel
+from ..schemas import Agent, AgentsPage
+
+router = APIRouter()
+
+
+def _to_schema(a: AgentModel) -> Agent:
+    return Agent(
+        agent_id=a.agent_id,
+        hostname=a.hostname,
+        status=a.status,
+        last_seen_at=a.last_seen_at,
+        features=a.features or {"push": False, "metrics": False},
+        labels=a.labels or {},
+    )
+
+
+@router.get("/agents", response_model=AgentsPage)
+async def list_agents(
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> AgentsPage:
+    stmt = select(AgentModel).order_by(AgentModel.agent_id)
+    if cursor is not None:
+        (last_id,) = decode(cursor, 1)
+        stmt = stmt.where(AgentModel.agent_id > last_id)
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].agent_id)
+    return AgentsPage(items=[_to_schema(a) for a in rows], next_cursor=next_cursor)
+
+
+@router.get("/agents/{agent_id}", response_model=Agent)
+async def get_agent(agent_id: str, session: AsyncSession = Depends(get_session)) -> Agent:
+    res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id))
+    a = res.scalar_one_or_none()
+    if a is None:
+        raise HTTPException(status_code=404, detail="agent not found")
+    return _to_schema(a)

server/monlet_server/api/checks.py

@@ -0,0 +1,47 @@
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, encode
+from ..db import get_session
+from ..models import Check
+from ..schemas import ID_PATTERN, ChecksPage, CheckState
+
+router = APIRouter()
+
+
+@router.get("/checks", response_model=ChecksPage)
+async def list_checks(
+    agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> ChecksPage:
+    stmt = select(Check).order_by(Check.agent_id, Check.check_id)
+    if agent_id is not None:
+        stmt = stmt.where(Check.agent_id == agent_id)
+    if cursor is not None:
+        last_agent, last_check = decode(cursor, 2)
+        stmt = stmt.where(
+            or_(
+                Check.agent_id > last_agent,
+                and_(Check.agent_id == last_agent, Check.check_id > last_check),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].agent_id, rows[-1].check_id)
+    items = [
+        CheckState(
+            agent_id=r.agent_id,
+            check_id=r.check_id,
+            status=r.status,
+            last_observed_at=r.last_observed_at,
+            exit_code=r.exit_code,
+            incident_key=r.incident_key,
+        )
+        for r in rows
+    ]
+    return ChecksPage(items=items, next_cursor=next_cursor)

server/monlet_server/api/events.py

@@ -0,0 +1,102 @@
+import base64
+import binascii
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Query, status
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..models import Event
+from ..schemas import (
+    ID_PATTERN,
+    EventBatchAcceptedResponse,
+    EventBatchRequest,
+    EventsPage,
+    StoredCheckResultEvent,
+)
+from ..services.ingestion import ingest_event
+
+router = APIRouter()
+
+
+def _encode_cursor(received_at: datetime, event_id) -> str:
+    raw = f"{received_at.isoformat()}|{event_id}".encode()
+    return base64.urlsafe_b64encode(raw).decode().rstrip("=")
+
+
+def _decode_cursor(cursor: str) -> tuple[datetime, str]:
+    pad = "=" * (-len(cursor) % 4)
+    try:
+        raw = base64.urlsafe_b64decode(cursor + pad).decode()
+        ts_str, eid = raw.split("|", 1)
+        return datetime.fromisoformat(ts_str), eid
+    except (binascii.Error, ValueError) as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc
+
+
+@router.post(
+    "/events",
+    response_model=EventBatchAcceptedResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+async def events(
+    body: EventBatchRequest, session: AsyncSession = Depends(get_session)
+) -> EventBatchAcceptedResponse:
+    accepted = 0
+    deduplicated = 0
+    for ev in body.events:
+        ok = await ingest_event(session, body.agent_id, ev)
+        if ok:
+            accepted += 1
+        else:
+            deduplicated += 1
+    await session.commit()
+    return EventBatchAcceptedResponse(accepted=accepted, deduplicated=deduplicated)
+
+
+@router.get("/events/query", response_model=EventsPage)
+async def query_events(
+    agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> EventsPage:
+    stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc())
+    if agent_id is not None:
+        stmt = stmt.where(Event.agent_id == agent_id)
+    if check_id is not None:
+        stmt = stmt.where(Event.check_id == check_id)
+    if cursor is not None:
+        ts, eid = _decode_cursor(cursor)
+        stmt = stmt.where(
+            or_(
+                Event.received_at < ts,
+                and_(Event.received_at == ts, Event.event_id < eid),
+            )
+        )
+    stmt = stmt.limit(limit + 1)
+    rows = (await session.execute(stmt)).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id)
+    items = [
+        StoredCheckResultEvent(
+            event_id=str(r.event_id),
+            agent_id=r.agent_id,
+            check_id=r.check_id,
+            observed_at=r.observed_at,
+            received_at=r.received_at,
+            status=r.status,
+            exit_code=r.exit_code,
+            duration_ms=r.duration_ms,
+            output=r.output,
+            output_truncated=r.output_truncated,
+            notifications_enabled=r.notifications_enabled,
+            incident_key=r.incident_key,
+        )
+        for r in rows
+    ]
+    return EventsPage(items=items, next_cursor=next_cursor)

server/monlet_server/api/health.py

@@ -0,0 +1,22 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..schemas import HealthResponse
+
+router = APIRouter()
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health() -> HealthResponse:
+    return HealthResponse()
+
+
+@router.get("/ready", response_model=HealthResponse)
+async def ready(session: AsyncSession = Depends(get_session)) -> HealthResponse:
+    try:
+        await session.execute(text("SELECT 1"))
+    except Exception as exc:
+        raise HTTPException(status_code=503, detail="database not reachable") from exc
+    return HealthResponse()

server/monlet_server/api/heartbeat.py

@@ -0,0 +1,21 @@
+from fastapi import APIRouter, Depends, status
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..schemas import AcceptedResponse, HeartbeatRequest
+from ..services.ingestion import upsert_agent_heartbeat
+
+router = APIRouter()
+
+
+@router.post(
+    "/heartbeat",
+    response_model=AcceptedResponse,
+    status_code=status.HTTP_202_ACCEPTED,
+)
+async def heartbeat(
+    body: HeartbeatRequest, session: AsyncSession = Depends(get_session)
+) -> AcceptedResponse:
+    await upsert_agent_heartbeat(session, body)
+    await session.commit()
+    return AcceptedResponse(accepted=True)

server/monlet_server/api/incidents.py

@@ -0,0 +1,53 @@
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, decode_datetime, encode
+from ..db import get_session
+from ..models import Incident as IncidentModel
+from ..schemas import Incident, IncidentsPage
+
+router = APIRouter()
+
+
+def _to_schema(i: IncidentModel) -> Incident:
+    return Incident(
+        id=str(i.id),
+        incident_key=i.incident_key,
+        state=i.state,
+        severity=i.severity,
+        agent_id=i.agent_id,
+        check_id=i.check_id,
+        opened_at=i.opened_at,
+        resolved_at=i.resolved_at,
+        summary=i.summary,
+    )
+
+
+@router.get("/incidents", response_model=IncidentsPage)
+async def list_incidents(
+    state: str | None = Query(default=None),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> IncidentsPage:
+    if state is not None and state not in ("open", "resolved"):
+        raise HTTPException(status_code=400, detail="invalid state")
+    stmt = select(IncidentModel).order_by(IncidentModel.opened_at.desc(), IncidentModel.id.desc())
+    if state is not None:
+        stmt = stmt.where(IncidentModel.state == state)
+    if cursor is not None:
+        ts_str, last_id = decode(cursor, 2)
+        ts = decode_datetime(ts_str)
+        stmt = stmt.where(
+            or_(
+                IncidentModel.opened_at < ts,
+                and_(IncidentModel.opened_at == ts, IncidentModel.id < last_id),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].opened_at.isoformat(), rows[-1].id)
+    return IncidentsPage(items=[_to_schema(r) for r in rows], next_cursor=next_cursor)

server/monlet_server/api/outbox.py

@@ -0,0 +1,58 @@
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..cursors import decode, decode_datetime, encode
+from ..db import get_session
+from ..models import NotificationOutbox
+from ..schemas import NotificationOutboxItem, OutboxPage
+
+router = APIRouter()
+
+
+@router.get("/notifiers/outbox", response_model=OutboxPage)
+async def list_outbox(
+    state: str | None = Query(default=None),
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> OutboxPage:
+    stmt = select(NotificationOutbox).order_by(
+        NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()
+    )
+    if state is not None:
+        stmt = stmt.where(NotificationOutbox.state == state)
+    if cursor is not None:
+        ts_str, last_id = decode(cursor, 2)
+        ts = decode_datetime(ts_str)
+        stmt = stmt.where(
+            or_(
+                NotificationOutbox.created_at < ts,
+                and_(
+                    NotificationOutbox.created_at == ts,
+                    NotificationOutbox.id < last_id,
+                ),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    next_cursor = None
+    if len(rows) > limit:
+        rows = rows[:limit]
+        next_cursor = encode(rows[-1].created_at.isoformat(), rows[-1].id)
+    return OutboxPage(
+        items=[
+            NotificationOutboxItem(
+                id=str(r.id),
+                notifier=r.notifier,
+                state=r.state,
+                incident_id=str(r.incident_id),
+                event_type=r.event_type,
+                attempts=r.attempts,
+                next_attempt_at=r.next_attempt_at,
+                last_error=r.last_error,
+                updated_at=r.updated_at,
+            )
+            for r in rows
+        ],
+        next_cursor=next_cursor,
+    )

server/monlet_server/cursors.py

@@ -0,0 +1,29 @@
+import base64
+import binascii
+from datetime import datetime
+
+from fastapi import HTTPException
+
+
+def encode(*parts: object) -> str:
+    raw = "|".join(str(p) for p in parts).encode()
+    return base64.urlsafe_b64encode(raw).decode().rstrip("=")
+
+
+def decode(cursor: str, count: int) -> list[str]:
+    pad = "=" * (-len(cursor) % 4)
+    try:
+        raw = base64.urlsafe_b64decode(cursor + pad).decode()
+    except (binascii.Error, ValueError) as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc
+    parts = raw.split("|", count - 1)
+    if len(parts) != count:
+        raise HTTPException(status_code=400, detail="invalid cursor")
+    return parts
+
+
+def decode_datetime(value: str) -> datetime:
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc

server/monlet_server/db.py

@@ -0,0 +1,43 @@
+from collections.abc import AsyncIterator
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from .settings import get_settings
+
+_engine: AsyncEngine | None = None
+_sessionmaker: async_sessionmaker[AsyncSession] | None = None
+
+
+def get_engine() -> AsyncEngine:
+    global _engine, _sessionmaker
+    if _engine is None:
+        s = get_settings()
+        _engine = create_async_engine(s.database_url, pool_pre_ping=True, future=True)
+        _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False)
+    return _engine
+
+
+def get_sessionmaker() -> async_sessionmaker[AsyncSession]:
+    if _sessionmaker is None:
+        get_engine()
+    assert _sessionmaker is not None
+    return _sessionmaker
+
+
+async def dispose_engine() -> None:
+    global _engine, _sessionmaker
+    if _engine is not None:
+        await _engine.dispose()
+    _engine = None
+    _sessionmaker = None
+
+
+async def get_session() -> AsyncIterator[AsyncSession]:
+    sm = get_sessionmaker()
+    async with sm() as session:
+        yield session

server/monlet_server/errors.py

@@ -0,0 +1,54 @@
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from .logging_config import get_logger, request_id_var
+from .schemas import ErrorBody, ErrorCode, ErrorResponse
+
+log = get_logger("monlet.errors")
+
+
+def _request_id(request: Request) -> str:
+    rid = getattr(request.state, "request_id", None)
+    if rid is None:
+        rid = request_id_var.get() or ""
+    return rid
+
+
+def error_response(
+    request: Request, status_code: int, code: ErrorCode, message: str
+) -> JSONResponse:
+    body = ErrorResponse(
+        error=ErrorBody(code=code, message=message, request_id=_request_id(request))
+    )
+    return JSONResponse(status_code=status_code, content=body.model_dump())
+
+
+_STATUS_TO_CODE: dict[int, ErrorCode] = {
+    400: "validation",
+    401: "unauthorized",
+    404: "not_found",
+    413: "payload_too_large",
+    429: "rate_limited",
+    500: "internal",
+    503: "not_ready",
+}
+
+
+def install_error_handlers(app: FastAPI) -> None:
+    @app.exception_handler(RequestValidationError)
+    async def _validation(request: Request, exc: RequestValidationError):
+        message = "request validation failed"
+        return error_response(request, 400, "validation", message)
+
+    @app.exception_handler(StarletteHTTPException)
+    async def _http(request: Request, exc: StarletteHTTPException):
+        code = _STATUS_TO_CODE.get(exc.status_code, "internal")
+        msg = exc.detail if isinstance(exc.detail, str) else code
+        return error_response(request, exc.status_code, code, msg)
+
+    @app.exception_handler(Exception)
+    async def _unhandled(request: Request, exc: Exception):
+        log.exception("unhandled_exception", error=type(exc).__name__)
+        return error_response(request, 500, "internal", "internal server error")

server/monlet_server/logging_config.py

@@ -0,0 +1,36 @@
+import logging
+from contextvars import ContextVar
+
+import structlog
+
+request_id_var: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def _add_request_id(_logger, _name, event_dict):
+    rid = request_id_var.get()
+    if rid is not None:
+        event_dict["request_id"] = rid
+    return event_dict
+
+
+def configure_logging(level: str = "INFO") -> None:
+    logging.basicConfig(format="%(message)s", level=getattr(logging, level.upper(), logging.INFO))
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            _add_request_id,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso", utc=True),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.JSONRenderer(),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(
+            getattr(logging, level.upper(), logging.INFO)
+        ),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str = "monlet"):
+    return structlog.get_logger(name)

server/monlet_server/main.py

@@ -0,0 +1,74 @@
+import asyncio
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.responses import Response
+
+from . import metrics
+from .api import agents, checks, events, health, heartbeat, incidents, outbox
+from .db import dispose_engine, get_engine, get_sessionmaker
+from .errors import install_error_handlers
+from .logging_config import configure_logging, get_logger
+from .middleware.auth import BearerAuthMiddleware
+from .middleware.body_limit import BodyLimitMiddleware
+from .middleware.request_id import RequestIdMiddleware
+from .services.detector import run_detector
+from .services.notifier_worker import run_notifier_worker
+from .settings import get_settings
+
+log = get_logger("monlet.main")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    s = get_settings()
+    configure_logging(s.log_level)
+    get_engine()
+    sm = get_sessionmaker()
+    stop_event = asyncio.Event()
+    tasks: list[asyncio.Task] = []
+    if s.enable_detector:
+        tasks.append(asyncio.create_task(run_detector(sm, stop_event), name="detector"))
+    if s.enable_notifier_worker:
+        tasks.append(
+            asyncio.create_task(run_notifier_worker(sm, stop_event), name="notifier_worker")
+        )
+    log.info("server_started", host=s.host, port=s.port)
+    try:
+        yield
+    finally:
+        stop_event.set()
+        for t in tasks:
+            try:
+                await asyncio.wait_for(t, timeout=5.0)
+            except TimeoutError:
+                t.cancel()
+        await dispose_engine()
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(title="Monlet Server", version="1.0.0", lifespan=lifespan)
+
+    app.add_middleware(BearerAuthMiddleware)
+    app.add_middleware(BodyLimitMiddleware)
+    app.add_middleware(RequestIdMiddleware)
+
+    install_error_handlers(app)
+
+    app.include_router(health.router, prefix="/api/v1")
+    app.include_router(heartbeat.router, prefix="/api/v1")
+    app.include_router(events.router, prefix="/api/v1")
+    app.include_router(agents.router, prefix="/api/v1")
+    app.include_router(checks.router, prefix="/api/v1")
+    app.include_router(incidents.router, prefix="/api/v1")
+    app.include_router(outbox.router, prefix="/api/v1")
+
+    @app.get("/metrics")
+    async def metrics_endpoint() -> Response:
+        body, ct = metrics.render()
+        return Response(content=body, media_type=ct)
+
+    return app
+
+
+app = create_app()

server/monlet_server/metrics.py

@@ -0,0 +1,54 @@
+from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, generate_latest
+from prometheus_client.exposition import CONTENT_TYPE_LATEST
+
+REGISTRY = CollectorRegistry()
+
+heartbeats_total = Counter(
+    "monlet_server_heartbeats_total",
+    "Total heartbeats received.",
+    ["result"],
+    registry=REGISTRY,
+)
+events_total = Counter(
+    "monlet_server_events_total",
+    "Total events ingested.",
+    ["result"],
+    registry=REGISTRY,
+)
+incidents_opened_total = Counter(
+    "monlet_server_incidents_opened_total",
+    "Total incidents opened.",
+    registry=REGISTRY,
+)
+incidents_resolved_total = Counter(
+    "monlet_server_incidents_resolved_total",
+    "Total incidents resolved.",
+    registry=REGISTRY,
+)
+agents_gauge = Gauge(
+    "monlet_server_agents",
+    "Agents by status.",
+    ["status"],
+    registry=REGISTRY,
+)
+open_incidents_gauge = Gauge(
+    "monlet_server_open_incidents",
+    "Number of open incidents.",
+    registry=REGISTRY,
+)
+notifications_total = Counter(
+    "monlet_server_notifications_total",
+    "Notification delivery attempts by notifier and result.",
+    ["notifier", "result"],
+    registry=REGISTRY,
+)
+request_duration = Histogram(
+    "monlet_server_request_duration_seconds",
+    "Request duration seconds.",
+    ["path"],
+    registry=REGISTRY,
+)
+
+
+def render() -> tuple[bytes, str]:
+    return generate_latest(REGISTRY), CONTENT_TYPE_LATEST

server/monlet_server/middleware/auth.py

@@ -0,0 +1,22 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..errors import error_response
+from ..settings import get_settings
+
+NO_AUTH_PATHS = {"/api/v1/health", "/api/v1/ready", "/metrics"}
+
+
+class BearerAuthMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        if request.url.path in NO_AUTH_PATHS:
+            return await call_next(request)
+
+        expected = get_settings().auth_token
+        header = request.headers.get("authorization")
+        if not header or not header.lower().startswith("bearer "):
+            return error_response(request, 401, "unauthorized", "authentication required")
+        token = header.split(" ", 1)[1].strip()
+        if token != expected:
+            return error_response(request, 401, "unauthorized", "authentication required")
+        return await call_next(request)

server/monlet_server/middleware/body_limit.py

@@ -0,0 +1,32 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..errors import error_response
+from ..settings import get_settings
+
+
+class BodyLimitMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        limit = get_settings().body_limit_bytes
+        cl = request.headers.get("content-length")
+        if cl is not None:
+            try:
+                if int(cl) > limit:
+                    return error_response(request, 413, "payload_too_large", "body too large")
+            except ValueError:
+                return error_response(request, 400, "validation", "invalid content-length")
+            return await call_next(request)
+
+        total = 0
+        chunks: list[bytes] = []
+        async for chunk in request.stream():
+            total += len(chunk)
+            if total > limit:
+                return error_response(request, 413, "payload_too_large", "body too large")
+            chunks.append(chunk)
+
+        async def receive():
+            return {"type": "http.request", "body": b"".join(chunks), "more_body": False}
+
+        request._receive = receive  # type: ignore[attr-defined]
+        return await call_next(request)

server/monlet_server/middleware/request_id.py

@@ -0,0 +1,23 @@
+import re
+import uuid
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+
+from ..logging_config import request_id_var
+
+_UUID_RE = re.compile(r"^[0-9a-fA-F-]{8,64}$")
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        incoming = request.headers.get("X-Request-Id")
+        rid = incoming if incoming and _UUID_RE.match(incoming) else str(uuid.uuid4())
+        request.state.request_id = rid
+        token = request_id_var.set(rid)
+        try:
+            response = await call_next(request)
+        finally:
+            request_id_var.reset(token)
+        response.headers["X-Request-Id"] = rid
+        return response

server/monlet_server/models.py

@@ -0,0 +1,181 @@
+from datetime import datetime
+from uuid import UUID
+
+from sqlalchemy import (
+    JSON,
+    Boolean,
+    CheckConstraint,
+    DateTime,
+    ForeignKey,
+    Index,
+    Integer,
+    PrimaryKeyConstraint,
+    String,
+    Text,
+    func,
+    text,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import UUID as PG_UUID
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+def _jsonb():
+    return JSONB().with_variant(JSON(), "sqlite")
+
+
+def _uuid():
+    return PG_UUID(as_uuid=True).with_variant(String(36), "sqlite")
+
+
+class Agent(Base):
+    __tablename__ = "agents"
+
+    agent_id: Mapped[str] = mapped_column(Text, primary_key=True)
+    hostname: Mapped[str] = mapped_column(Text, nullable=False)
+    status: Mapped[str] = mapped_column(Text, nullable=False, default="alive")
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    features: Mapped[dict] = mapped_column(
+        _jsonb(),
+        nullable=False,
+        default=lambda: {"push": False, "metrics": False},
+        server_default=text("jsonb_build_object('push', false, 'metrics', false)"),
+    )
+    labels: Mapped[dict] = mapped_column(
+        _jsonb(), nullable=False, default=dict, server_default=text("'{}'::jsonb")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+    __table_args__ = (
+        CheckConstraint("status IN ('alive','stale','dead')", name="agents_status_chk"),
+        Index("agents_status_idx", "status"),
+        Index("agents_last_seen_idx", "last_seen_at"),
+    )
+
+
+class Check(Base):
+    __tablename__ = "checks"
+
+    agent_id: Mapped[str] = mapped_column(
+        Text, ForeignKey("agents.agent_id", ondelete="CASCADE"), nullable=False
+    )
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    exit_code: Mapped[int] = mapped_column(Integer, nullable=False)
+    last_observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+
+    __table_args__ = (
+        PrimaryKeyConstraint("agent_id", "check_id", name="checks_pkey"),
+        CheckConstraint(
+            "status IN ('ok','warning','critical','unknown')", name="checks_status_chk"
+        ),
+        Index("checks_status_idx", "status"),
+    )
+
+
+class Event(Base):
+    __tablename__ = "events"
+
+    event_id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    agent_id: Mapped[str] = mapped_column(Text, nullable=False)
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    received_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    exit_code: Mapped[int] = mapped_column(Integer, nullable=False)
+    duration_ms: Mapped[int] = mapped_column(Integer, nullable=False)
+    output: Mapped[str | None] = mapped_column(Text, nullable=True)
+    output_truncated: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    notifications_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, default=True, server_default=text("TRUE")
+    )
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+
+    __table_args__ = (
+        CheckConstraint(
+            "status IN ('ok','warning','critical','unknown')", name="events_status_chk"
+        ),
+        CheckConstraint("duration_ms >= 0", name="events_duration_chk"),
+        Index(
+            "events_agent_check_observed_idx",
+            "agent_id",
+            "check_id",
+            text("observed_at DESC"),
+        ),
+        Index("events_observed_at_idx", text("observed_at DESC")),
+    )
+
+
+class Incident(Base):
+    __tablename__ = "incidents"
+
+    id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    incident_key: Mapped[str] = mapped_column(Text, nullable=False)
+    agent_id: Mapped[str] = mapped_column(Text, nullable=False)
+    check_id: Mapped[str] = mapped_column(Text, nullable=False)
+    state: Mapped[str] = mapped_column(Text, nullable=False)
+    severity: Mapped[str] = mapped_column(Text, nullable=False)
+    opened_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+    last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
+
+    __table_args__ = (
+        CheckConstraint("state IN ('open','resolved')", name="incidents_state_chk"),
+        CheckConstraint(
+            "severity IN ('warning','critical','unknown')", name="incidents_severity_chk"
+        ),
+        Index(
+            "incidents_open_key_idx",
+            "incident_key",
+            unique=True,
+            postgresql_where=text("state = 'open'"),
+        ),
+        Index("incidents_state_opened_idx", "state", text("opened_at DESC")),
+    )
+
+
+class NotificationOutbox(Base):
+    __tablename__ = "notification_outbox"
+
+    id: Mapped[UUID] = mapped_column(_uuid(), primary_key=True)
+    incident_id: Mapped[UUID] = mapped_column(
+        _uuid(), ForeignKey("incidents.id", ondelete="CASCADE"), nullable=False
+    )
+    notifier: Mapped[str] = mapped_column(Text, nullable=False)
+    event_type: Mapped[str] = mapped_column(Text, nullable=False)
+    state: Mapped[str] = mapped_column(Text, nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    payload: Mapped[dict] = mapped_column(_jsonb(), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+    __table_args__ = (
+        CheckConstraint("event_type IN ('firing','resolved')", name="outbox_event_type_chk"),
+        CheckConstraint(
+            "state IN ('pending','sending','sent','retry','failed','discarded')",
+            name="outbox_state_chk",
+        ),
+        Index(
+            "outbox_pending_idx",
+            "state",
+            "next_attempt_at",
+            postgresql_where=text("state IN ('pending','retry')"),
+        ),
+    )

server/monlet_server/redaction.py

@@ -0,0 +1,26 @@
+import re
+
+_PATTERNS: list[tuple[re.Pattern[str], object]] = [
+    (
+        re.compile(r"(?i)(authorization|set-cookie|cookie)([\"'=:]+\s*)([^\r\n]+)"),
+        lambda m: f"{m.group(1)}{m.group(2)}***",
+    ),
+    (
+        re.compile(r"(?i)(token|secret|password|api[_-]?key|credential)([\"'=:]+\s*)([^\s\"',;]+)"),
+        lambda m: f"{m.group(1)}{m.group(2)}***",
+    ),
+    (re.compile(r"(?i)(bearer)\s+\S+"), r"\1 ***"),
+]
+
+
+def redact(value: str | None) -> str | None:
+    if value is None:
+        return None
+    out = value
+    for pat, repl in _PATTERNS:
+        out = pat.sub(repl, out)
+    return out
+
+
+def redact_dict(d: dict) -> dict:
+    return {k: redact(v) if isinstance(v, str) else v for k, v in d.items()}

server/monlet_server/schemas.py

@@ -0,0 +1,201 @@
+import re
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+ID_PATTERN = r"^[A-Za-z0-9._:-]+$"
+UUIDV7_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
+ID_RE = re.compile(ID_PATTERN)
+SENSITIVE_LABEL_WORDS = (
+    "token",
+    "secret",
+    "password",
+    "credential",
+    "authorization",
+    "cookie",
+    "api_key",
+    "apikey",
+)
+
+Status = Literal["ok", "warning", "critical", "unknown"]
+AgentStatus = Literal["alive", "stale", "dead"]
+IncidentState = Literal["open", "resolved"]
+Severity = Literal["warning", "critical", "unknown"]
+OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"]
+OutboxEventType = Literal["firing", "resolved"]
+ErrorCode = Literal[
+    "unauthorized",
+    "not_found",
+    "validation",
+    "payload_too_large",
+    "rate_limited",
+    "internal",
+    "not_ready",
+]
+
+
+class HealthResponse(BaseModel):
+    status: Literal["ok"] = "ok"
+
+
+class ErrorBody(BaseModel):
+    code: ErrorCode
+    message: str
+    request_id: str
+
+
+class ErrorResponse(BaseModel):
+    error: ErrorBody
+
+
+class AcceptedResponse(BaseModel):
+    accepted: bool = True
+
+
+class EventBatchAcceptedResponse(BaseModel):
+    accepted: int = Field(ge=0)
+    deduplicated: int = Field(ge=0)
+
+
+def _is_sensitive_label_key(key: str) -> bool:
+    normalized = key.lower().replace("-", "_").replace(".", "_").replace(":", "_")
+    return any(word in normalized for word in SENSITIVE_LABEL_WORDS)
+
+
+class AgentFeatures(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    push: bool
+    metrics: bool
+
+
+class HeartbeatRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    observed_at: datetime
+    hostname: str = Field(max_length=253)
+    features: AgentFeatures
+    labels: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("labels")
+    @classmethod
+    def _check_labels(cls, v: dict[str, str]) -> dict[str, str]:
+        if len(v) > 32:
+            raise ValueError("too many labels (max 32)")
+        for k, val in v.items():
+            if len(k) > 64 or not ID_RE.match(k):
+                raise ValueError(f"invalid label key: {k!r}")
+            if _is_sensitive_label_key(k):
+                raise ValueError(f"label key is not allowed: {k!r}")
+            if len(val) > 256:
+                raise ValueError(f"label value too long for key {k!r}")
+        return v
+
+
+class CheckResultEvent(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    event_id: str = Field(pattern=UUIDV7_PATTERN, min_length=36, max_length=36)
+    check_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    observed_at: datetime
+    status: Status
+    exit_code: int
+    duration_ms: int = Field(ge=0)
+    output: str | None = Field(default=None, max_length=8192)
+    output_truncated: bool = False
+    notifications_enabled: bool = True
+    incident_key: str | None = Field(default=None, max_length=256)
+
+    @field_validator("output")
+    @classmethod
+    def _check_output_bytes(cls, v: str | None) -> str | None:
+        if v is not None and len(v.encode("utf-8")) > 8192:
+            raise ValueError("output exceeds 8192 UTF-8 bytes")
+        return v
+
+
+class EventBatchRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    events: list[CheckResultEvent] = Field(min_length=1, max_length=200)
+
+
+class Agent(BaseModel):
+    agent_id: str
+    hostname: str
+    status: AgentStatus
+    last_seen_at: datetime
+    features: AgentFeatures
+    labels: dict[str, str] = Field(default_factory=dict)
+
+
+class CheckState(BaseModel):
+    agent_id: str
+    check_id: str
+    status: Status
+    last_observed_at: datetime
+    exit_code: int
+    incident_key: str
+
+
+class Incident(BaseModel):
+    id: str
+    incident_key: str
+    state: IncidentState
+    severity: Severity
+    agent_id: str
+    check_id: str
+    opened_at: datetime
+    resolved_at: datetime | None = None
+    summary: str | None = None
+
+
+class StoredCheckResultEvent(CheckResultEvent):
+    agent_id: str = Field(max_length=128, pattern=ID_PATTERN)
+    received_at: datetime
+
+
+class NotificationOutboxItem(BaseModel):
+    id: str
+    notifier: str
+    state: OutboxState
+    incident_id: str
+    event_type: OutboxEventType
+    attempts: int = Field(ge=0)
+    next_attempt_at: datetime | None = None
+    last_error: str | None = None
+    updated_at: datetime
+
+
+class PageEnvelope(BaseModel):
+    next_cursor: str | None = None
+
+
+class AgentsPage(PageEnvelope):
+    items: list[Agent]
+
+
+class ChecksPage(PageEnvelope):
+    items: list[CheckState]
+
+
+class IncidentsPage(PageEnvelope):
+    items: list[Incident]
+
+
+class EventsPage(PageEnvelope):
+    items: list[StoredCheckResultEvent]
+
+
+class OutboxPage(PageEnvelope):
+    items: list[NotificationOutboxItem]
+
+
+class UUIDStr(BaseModel):
+    """Helper to validate generated UUIDs."""
+
+    value: UUID

server/monlet_server/services/detector.py

@@ -0,0 +1,90 @@
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Event, Incident, NotificationOutbox
+from ..settings import get_settings
+
+log = get_logger("monlet.detector")
+
+OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
+
+
+async def _prune_history(session) -> None:
+    s = get_settings()
+    if s.events_retention_max_rows > 0:
+        keep_events = (
+            select(Event.event_id)
+            .order_by(Event.received_at.desc(), Event.event_id.desc())
+            .limit(s.events_retention_max_rows)
+        )
+        await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
+
+    if s.outbox_retention_max_rows > 0:
+        keep_outbox = (
+            select(NotificationOutbox.id)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
+            .limit(s.outbox_retention_max_rows)
+        )
+        await session.execute(
+            delete(NotificationOutbox)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .where(NotificationOutbox.id.not_in(keep_outbox))
+        )
+
+
+async def _tick(sm: async_sessionmaker) -> None:
+    s = get_settings()
+    now = datetime.now(UTC)
+    stale_cut = now - timedelta(seconds=s.stale_after_sec)
+    dead_cut = now - timedelta(seconds=s.dead_after_sec)
+    async with sm() as session:
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= dead_cut)
+            .where(Agent.status != "dead")
+            .values(status="dead")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= stale_cut)
+            .where(Agent.last_seen_at > dead_cut)
+            .where(Agent.status != "stale")
+            .values(status="stale")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at > stale_cut)
+            .where(Agent.status != "alive")
+            .values(status="alive")
+        )
+        await _prune_history(session)
+        await session.commit()
+
+        for status in ("alive", "stale", "dead"):
+            r = await session.execute(
+                select(func.count()).select_from(Agent).where(Agent.status == status)
+            )
+            metrics.agents_gauge.labels(status=status).set(r.scalar_one())
+        r = await session.execute(
+            select(func.count()).select_from(Incident).where(Incident.state == "open")
+        )
+        metrics.open_incidents_gauge.set(r.scalar_one())
+
+
+async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
+    tick = get_settings().detector_tick_sec
+    while not stop_event.is_set():
+        try:
+            await _tick(sm)
+        except Exception as exc:
+            log.warning("detector_tick_failed", error=str(exc))
+        try:
+            await asyncio.wait_for(stop_event.wait(), timeout=tick)
+        except TimeoutError:
+            continue

server/monlet_server/services/ingestion.py

@@ -0,0 +1,237 @@
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+from fastapi import HTTPException
+from sqlalchemy import select, text
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Check, Event, Incident, NotificationOutbox
+from ..redaction import redact
+from ..schemas import CheckResultEvent, HeartbeatRequest
+from ..settings import get_settings
+
+INCIDENT_KEY_MAX = 256
+
+log = get_logger("monlet.ingestion")
+
+_SEVERITY_RANK = {"unknown": 1, "warning": 2, "critical": 3}
+
+
+def _severity_for_status(status: str) -> str:
+    if status in ("warning", "critical", "unknown"):
+        return status
+    return "unknown"
+
+
+def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
+    key = ev.incident_key or f"{agent_id}:{ev.check_id}"
+    if len(key) > INCIDENT_KEY_MAX:
+        raise HTTPException(
+            status_code=400,
+            detail=f"incident_key exceeds {INCIDENT_KEY_MAX} chars",
+        )
+    return key
+
+
+async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
+    observed = (
+        hb.observed_at.astimezone(UTC)
+        if hb.observed_at.tzinfo
+        else hb.observed_at.replace(tzinfo=UTC)
+    )
+    stmt = pg_insert(Agent).values(
+        agent_id=hb.agent_id,
+        hostname=hb.hostname,
+        status="alive",
+        last_seen_at=observed,
+        features=hb.features.model_dump(),
+        labels=hb.labels,
+    )
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[Agent.agent_id],
+        set_={
+            "hostname": stmt.excluded.hostname,
+            "status": "alive",
+            "last_seen_at": stmt.excluded.last_seen_at,
+            "features": stmt.excluded.features,
+            "labels": stmt.excluded.labels,
+        },
+    )
+    await session.execute(stmt)
+    metrics.heartbeats_total.labels(result="ok").inc()
+
+
+async def _apply_incident(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+    incident_key: str,
+) -> tuple[bool, Incident | None]:
+    """Return (transition, incident). transition True if open->resolved or none->open."""
+    observed = ev.observed_at
+    if ev.status == "ok":
+        res = await session.execute(
+            select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+        )
+        inc = res.scalar_one_or_none()
+        if inc is None:
+            return False, None
+        inc.state = "resolved"
+        inc.resolved_at = observed
+        inc.last_event_id = UUID(ev.event_id)
+        metrics.incidents_resolved_total.inc()
+        return True, inc
+
+    severity = _severity_for_status(ev.status)
+    new_id = uuid4()
+    summary = (redact(ev.output) or "")[:200] if ev.output else None
+    stmt = pg_insert(Incident).values(
+        id=new_id,
+        incident_key=incident_key,
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        state="open",
+        severity=severity,
+        opened_at=observed,
+        last_event_id=UUID(ev.event_id),
+        summary=summary,
+    )
+    stmt = stmt.on_conflict_do_nothing(
+        index_elements=[Incident.incident_key],
+        index_where=text("state = 'open'"),
+    ).returning(Incident.id)
+    ins = await session.execute(stmt)
+    inserted_id = ins.scalar_one_or_none()
+    if inserted_id is not None:
+        await session.flush()
+        res = await session.execute(select(Incident).where(Incident.id == inserted_id))
+        metrics.incidents_opened_total.inc()
+        return True, res.scalar_one()
+
+    res = await session.execute(
+        select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+    )
+    inc = res.scalar_one()
+    if inc.agent_id != agent_id or inc.check_id != ev.check_id:
+        raise HTTPException(
+            status_code=400,
+            detail="incident_key already bound to another (agent_id, check_id)",
+        )
+    cur = _SEVERITY_RANK.get(inc.severity, 0)
+    new = _SEVERITY_RANK.get(severity, 0)
+    if new > cur:
+        inc.severity = severity
+    inc.last_event_id = UUID(ev.event_id)
+    return False, inc
+
+
+async def _enqueue_outbox(
+    session: AsyncSession,
+    incident: Incident,
+    event_type: str,
+    payload: dict,
+) -> None:
+    notifiers = get_settings().enabled_notifiers
+    if not notifiers:
+        return
+    now = datetime.now(UTC)
+    for name in notifiers:
+        session.add(
+            NotificationOutbox(
+                id=uuid4(),
+                incident_id=incident.id,
+                notifier=name,
+                event_type=event_type,
+                state="pending",
+                attempts=0,
+                next_attempt_at=now,
+                payload=payload,
+            )
+        )
+
+
+async def ingest_event(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+) -> bool:
+    """Return True if accepted, False if deduplicated."""
+    observed = ev.observed_at
+    output = redact(ev.output)
+    incident_key = _incident_key(agent_id, ev)
+
+    stmt = pg_insert(Event).values(
+        event_id=UUID(ev.event_id),
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        observed_at=observed,
+        status=ev.status,
+        exit_code=ev.exit_code,
+        duration_ms=ev.duration_ms,
+        output=output,
+        output_truncated=ev.output_truncated,
+        notifications_enabled=ev.notifications_enabled,
+        incident_key=incident_key,
+    )
+    stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
+    result = await session.execute(stmt)
+    inserted = result.scalar_one_or_none()
+    if inserted is None:
+        metrics.events_total.labels(result="deduplicated").inc()
+        return False
+
+    metrics.events_total.labels(result="accepted").inc()
+
+    chk_res = await session.execute(
+        select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
+    )
+    cur = chk_res.scalar_one_or_none()
+    is_late = cur is not None and cur.last_observed_at > observed
+    if is_late:
+        return True
+
+    if cur is None:
+        session.add(
+            Check(
+                agent_id=agent_id,
+                check_id=ev.check_id,
+                status=ev.status,
+                exit_code=ev.exit_code,
+                last_observed_at=observed,
+                last_event_id=UUID(ev.event_id),
+                incident_key=incident_key,
+            )
+        )
+    else:
+        cur.status = ev.status
+        cur.exit_code = ev.exit_code
+        cur.last_observed_at = observed
+        cur.last_event_id = UUID(ev.event_id)
+        cur.incident_key = incident_key
+
+    transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
+    if transition and inc is not None and ev.notifications_enabled:
+        event_type = "resolved" if ev.status == "ok" else "firing"
+        await _enqueue_outbox(
+            session,
+            inc,
+            event_type,
+            {
+                "incident_id": str(inc.id),
+                "agent_id": agent_id,
+                "check_id": ev.check_id,
+                "status": ev.status,
+                "severity": inc.severity,
+                "incident_key": incident_key,
+                "observed_at": observed.isoformat(),
+                "opened_at": inc.opened_at.isoformat() if inc.opened_at else None,
+                "resolved_at": inc.resolved_at.isoformat() if inc.resolved_at else None,
+                "summary": inc.summary,
+                "output": output,
+                "event_type": event_type,
+            },
+        )
+    return True

server/monlet_server/services/notifier_worker.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..redaction import redact
+from ..settings import Settings, get_settings
+from .notifiers import Notifier, build_registry
+
+log = get_logger("monlet.notifier_worker")
+
+
+# Per ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
+_BACKOFF_SCHEDULE_SEC = (5, 15, 60, 300, 1800, 3600, 7200, 14400)
+
+
+def _backoff(attempts: int) -> timedelta:
+    idx = max(0, min(attempts - 1, len(_BACKOFF_SCHEDULE_SEC) - 1))
+    return timedelta(seconds=_BACKOFF_SCHEDULE_SEC[idx])
+
+
+async def _claim_batch(session, limit: int, lease_sec: int) -> list[dict]:
+    res = await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = 'sending', updated_at = now()
+             WHERE id IN (
+                SELECT id FROM notification_outbox
+                 WHERE (
+                        state IN ('pending','retry')
+                        AND (next_attempt_at IS NULL OR next_attempt_at <= now())
+                       )
+                    OR (
+                        state = 'sending'
+                        AND updated_at < now() - make_interval(secs => :lease)
+                       )
+                 ORDER BY next_attempt_at NULLS FIRST
+                 LIMIT :lim
+                 FOR UPDATE SKIP LOCKED
+             )
+         RETURNING id, notifier, event_type, attempts, payload
+            """
+        ),
+        {"lim": limit, "lease": lease_sec},
+    )
+    rows = res.mappings().all()
+    return [dict(r) for r in rows]
+
+
+async def _finalize(
+    session,
+    row_id,
+    *,
+    state: str,
+    attempts: int,
+    next_attempt_at: datetime | None,
+    last_error: str | None,
+) -> None:
+    await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = :state,
+                   attempts = :attempts,
+                   next_attempt_at = :next_attempt_at,
+                   last_error = :last_error,
+                   updated_at = now()
+             WHERE id = :id
+            """
+        ),
+        {
+            "state": state,
+            "attempts": attempts,
+            "next_attempt_at": next_attempt_at,
+            "last_error": last_error,
+            "id": row_id,
+        },
+    )
+
+
+async def _process_row(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    row: dict,
+    max_attempts: int,
+) -> None:
+    notifier = registry.get(row["notifier"])
+    attempts = int(row["attempts"]) + 1
+    async with sm() as session:
+        if notifier is None:
+            await _finalize(
+                session,
+                row["id"],
+                state="discarded",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=f"unknown_notifier:{row['notifier']}",
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=row["notifier"], result="discarded").inc()
+            return
+
+        try:
+            result = await notifier.deliver(row["event_type"], row["payload"])
+        except Exception as exc:  # defensive: notifier should not raise
+            result_ok = False
+            result_perm = False
+            result_err = f"exception:{type(exc).__name__}"
+        else:
+            result_ok = result.ok
+            result_perm = result.permanent
+            result_err = redact(result.error) if result.error else None
+
+        if result_ok:
+            await _finalize(
+                session,
+                row["id"],
+                state="sent",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=None,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="sent").inc()
+            return
+
+        if result_perm or attempts >= max_attempts:
+            await _finalize(
+                session,
+                row["id"],
+                state="failed",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=result_err,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="failed").inc()
+            return
+
+        next_at = datetime.now(UTC) + _backoff(attempts)
+        await _finalize(
+            session,
+            row["id"],
+            state="retry",
+            attempts=attempts,
+            next_attempt_at=next_at,
+            last_error=result_err,
+        )
+        await session.commit()
+        metrics.notifications_total.labels(notifier=notifier.name, result="retry").inc()
+
+
+async def tick(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    settings: Settings,
+) -> int:
+    async with sm() as session:
+        rows = await _claim_batch(
+            session, settings.notifier_batch_size, settings.notifier_lease_sec
+        )
+        await session.commit()
+    if not rows:
+        return 0
+    await asyncio.gather(
+        *(_process_row(sm, registry, r, settings.notifier_max_attempts) for r in rows)
+    )
+    return len(rows)
+
+
+async def run_notifier_worker(
+    sm: async_sessionmaker,
+    stop_event: asyncio.Event,
+    client: httpx.AsyncClient | None = None,
+) -> None:
+    settings = get_settings()
+    own_client = False
+    if client is None:
+        client = httpx.AsyncClient(timeout=settings.notifier_http_timeout_sec)
+        own_client = True
+    registry = build_registry(settings, client)
+    if not registry:
+        log.info("notifier_worker_disabled_no_notifiers")
+        if own_client:
+            await client.aclose()
+        return
+    log.info("notifier_worker_started", notifiers=list(registry.keys()))
+    try:
+        while not stop_event.is_set():
+            try:
+                await tick(sm, registry, settings)
+            except Exception as exc:
+                log.warning("notifier_tick_failed", error=str(exc))
+            try:
+                await asyncio.wait_for(stop_event.wait(), timeout=settings.notifier_tick_sec)
+            except TimeoutError:
+                continue
+    finally:
+        if own_client:
+            await client.aclose()

server/monlet_server/services/notifiers/__init__.py

@@ -0,0 +1,4 @@
+from .base import DeliveryResult, Notifier
+from .registry import build_registry
+
+__all__ = ["DeliveryResult", "Notifier", "build_registry"]

server/monlet_server/services/notifiers/alertmanager.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class AlertmanagerNotifier:
+    name = "alertmanager"
+
+    def __init__(self, client: httpx.AsyncClient, url: str) -> None:
+        self._client = client
+        self._url = url.rstrip("/")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        labels = {
+            "alertname": "monlet_incident",
+            "agent_id": str(payload.get("agent_id", "")),
+            "check_id": str(payload.get("check_id", "")),
+            "incident_key": str(payload.get("incident_key", "")),
+        }
+        annotations = {
+            "severity": str(payload.get("severity", "unknown")),
+            "status": str(payload.get("status", "")),
+            "summary": redact(payload.get("summary") or "") or "",
+        }
+        if payload.get("output"):
+            annotations["output"] = (redact(payload["output"]) or "")[:1024]
+        alert = {
+            "labels": labels,
+            "annotations": annotations,
+            "startsAt": payload.get("opened_at") or payload.get("observed_at"),
+        }
+        if event_type == "resolved" and payload.get("resolved_at"):
+            alert["endsAt"] = payload["resolved_at"]
+        try:
+            resp = await self._client.post(f"{self._url}/api/v2/alerts", json=[alert])
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/base.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+    ok: bool
+    permanent: bool = False
+    error: str | None = None
+
+
+class Notifier(Protocol):
+    name: str
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult: ...

server/monlet_server/services/notifiers/debug.py

@@ -0,0 +1,14 @@
+from ...logging_config import get_logger
+from ...redaction import redact_dict
+from .base import DeliveryResult
+
+
+class DebugNotifier:
+    name = "debug"
+
+    def __init__(self) -> None:
+        self._log = get_logger("monlet.notifier.debug")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        self._log.info("notify", event_type=event_type, payload=redact_dict(payload))
+        return DeliveryResult(ok=True)

server/monlet_server/services/notifiers/http.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import httpx
+
+
+def classify_response(status: int) -> tuple[bool, bool]:
+    """Return (ok, permanent_failure)."""
+    if 200 <= status < 300:
+        return True, False
+    if 400 <= status < 500 and status not in (408, 425, 429):
+        return False, True
+    return False, False
+
+
+def classify_exception(exc: Exception) -> tuple[bool, bool]:
+    if isinstance(exc, httpx.TimeoutException | httpx.NetworkError):
+        return False, False
+    return False, False

server/monlet_server/services/notifiers/registry.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import httpx
+
+from ...settings import Settings
+from .alertmanager import AlertmanagerNotifier
+from .base import Notifier
+from .debug import DebugNotifier
+from .telegram import TelegramNotifier
+from .webhook import WebhookNotifier
+
+
+def build_registry(settings: Settings, client: httpx.AsyncClient) -> dict[str, Notifier]:
+    reg: dict[str, Notifier] = {}
+    if settings.notifier_debug_enabled:
+        reg["debug"] = DebugNotifier()
+    if (
+        settings.notifier_telegram_enabled
+        and settings.notifier_telegram_token
+        and settings.notifier_telegram_chat_id
+    ):
+        reg["telegram"] = TelegramNotifier(
+            client, settings.notifier_telegram_token, settings.notifier_telegram_chat_id
+        )
+    if settings.notifier_webhook_enabled and settings.notifier_webhook_url:
+        reg["webhook"] = WebhookNotifier(
+            client, settings.notifier_webhook_url, settings.notifier_webhook_token
+        )
+    if settings.notifier_alertmanager_enabled and settings.notifier_alertmanager_url:
+        reg["alertmanager"] = AlertmanagerNotifier(client, settings.notifier_alertmanager_url)
+    return reg

server/monlet_server/services/notifiers/telegram.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+def _format_message(event_type: str, payload: dict) -> str:
+    icon = "[FIRING]" if event_type == "firing" else "[RESOLVED]"
+    sev = payload.get("severity", "unknown")
+    agent = payload.get("agent_id", "?")
+    check = payload.get("check_id", "?")
+    status = payload.get("status", "?")
+    incident_key = payload.get("incident_key", "?")
+    output = payload.get("output") or ""
+    lines = [
+        f"{icon} {sev} {agent}/{check}",
+        f"status: {status}",
+        f"incident_key: {incident_key}",
+        f"observed_at: {payload.get('observed_at', '?')}",
+    ]
+    if output:
+        lines.append(f"output: {output[:300]}")
+    return redact("\n".join(lines)) or ""
+
+
+class TelegramNotifier:
+    name = "telegram"
+
+    def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str) -> None:
+        self._client = client
+        self._token = token
+        self._chat_id = chat_id
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        url = f"https://api.telegram.org/bot{self._token}/sendMessage"
+        body = {"chat_id": self._chat_id, "text": _format_message(event_type, payload)}
+        try:
+            resp = await self._client.post(url, json=body)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/webhook.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact_dict
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class WebhookNotifier:
+    name = "webhook"
+
+    def __init__(self, client: httpx.AsyncClient, url: str, token: str = "") -> None:
+        self._client = client
+        self._url = url
+        self._token = token
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        headers = {"Content-Type": "application/json"}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+        body = {"event_type": event_type, "incident": redact_dict(payload)}
+        try:
+            resp = await self._client.post(self._url, json=body, headers=headers)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/settings.py

@@ -0,0 +1,80 @@
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="MONLET_", env_file=".env", extra="ignore")
+
+    auth_token: str
+    database_url: str = "postgresql+asyncpg://monlet:monlet@127.0.0.1:5432/monlet"
+    log_level: str = "INFO"
+    body_limit_bytes: int = 1_048_576
+    output_max_bytes: int = 8192
+    max_batch_events: int = 200
+    stale_after_sec: int = 90
+    dead_after_sec: int = 300
+    detector_tick_sec: int = 15
+    host: str = "0.0.0.0"
+    port: int = 8000
+    enable_detector: bool = True
+    enable_notifier_worker: bool = True
+    events_retention_max_rows: int = 100_000
+    outbox_retention_max_rows: int = 50_000
+    notifier_tick_sec: int = 5
+    notifier_batch_size: int = 20
+    notifier_max_attempts: int = 8
+    notifier_lease_sec: int = 300
+    notifier_http_timeout_sec: float = 10.0
+
+    notifier_debug_enabled: bool = True
+    notifier_telegram_enabled: bool = False
+    notifier_telegram_token: str = ""
+    notifier_telegram_chat_id: str = ""
+    notifier_webhook_enabled: bool = False
+    notifier_webhook_url: str = ""
+    notifier_webhook_token: str = ""
+    notifier_alertmanager_enabled: bool = False
+    notifier_alertmanager_url: str = ""
+
+    @field_validator("auth_token")
+    @classmethod
+    def _validate_auth_token(cls, value: str) -> str:
+        if not value or value == "changeme":
+            raise ValueError("MONLET_AUTH_TOKEN must be set to a non-default value")
+        return value
+
+    @property
+    def enabled_notifiers(self) -> list[str]:
+        out: list[str] = []
+        if self.notifier_debug_enabled:
+            out.append("debug")
+        if (
+            self.notifier_telegram_enabled
+            and self.notifier_telegram_token
+            and self.notifier_telegram_chat_id
+        ):
+            out.append("telegram")
+        if self.notifier_webhook_enabled and self.notifier_webhook_url:
+            out.append("webhook")
+        if self.notifier_alertmanager_enabled and self.notifier_alertmanager_url:
+            out.append("alertmanager")
+        return out
+
+    @property
+    def sync_database_url(self) -> str:
+        return self.database_url.replace("+asyncpg", "+psycopg")
+
+
+_settings: Settings | None = None
+
+
+def get_settings() -> Settings:
+    global _settings
+    if _settings is None:
+        _settings = Settings()
+    return _settings
+
+
+def reset_settings_cache() -> None:
+    global _settings
+    _settings = None