Harden production workflows and agent admission

server/monlet_server/agent_auth.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from hashlib import sha256
+
+from fastapi import Request
+
+
+@dataclass(frozen=True)
+class AgentCredential:
+    key_hash: str
+    key_fingerprint: str
+
+
+def hash_agent_key(token: str) -> str:
+    return sha256(token.encode("utf-8")).hexdigest()
+
+
+def fingerprint_from_hash(key_hash: str) -> str:
+    return key_hash[:16]
+
+
+def credential_from_token(token: str) -> AgentCredential:
+    key_hash = hash_agent_key(token)
+    return AgentCredential(key_hash=key_hash, key_fingerprint=fingerprint_from_hash(key_hash))
+
+
+def get_agent_credential(request: Request) -> AgentCredential:
+    cred = getattr(request.state, "agent_credential", None)
+    if not isinstance(cred, AgentCredential):
+        raise RuntimeError("agent credential missing")
+    return cred

server/monlet_server/api/agents.py

@@ -1,49 +1,341 @@
+from datetime import UTC, datetime
+
 from fastapi import APIRouter, Depends, HTTPException, Query
-from sqlalchemy import select
+from sqlalchemy import and_, delete, or_, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from ..cursors import decode, encode
 from ..db import get_session
 from ..models import Agent as AgentModel
-from ..schemas import Agent, AgentsPage
+from ..models import AgentBlacklist, Event, Incident, NotificationOutbox
+from ..schemas import (
+    ActionCountResponse,
+    Agent,
+    AgentBlacklistItem,
+    AgentBlacklistPage,
+    AgentsPage,
+)
 from ..timeutil import to_utc
 
 router = APIRouter()
 
 
+def _is_pending(a: AgentModel) -> bool:
+    return a.pending_key_hash is not None
+
+
 def _to_schema(a: AgentModel) -> Agent:
+    pending = _is_pending(a)
+    hostname = a.pending_hostname if pending and a.pending_hostname is not None else a.hostname
+    last_seen_at = (
+        a.pending_seen_at if pending and a.pending_seen_at is not None else a.last_seen_at
+    )
+    features = a.pending_features if pending and a.pending_features is not None else a.features
+    labels = a.pending_labels if pending and a.pending_labels is not None else a.labels
     return Agent(
         agent_id=a.agent_id,
-        hostname=a.hostname,
+        hostname=hostname,
         status=a.status,
-        last_seen_at=to_utc(a.last_seen_at),
-        features=a.features or {"push": False, "metrics": False},
-        labels=a.labels or {},
+        admission_state="pending" if pending or a.accepted_key_hash is None else "accepted",
+        rotation_pending=pending and a.accepted_key_hash is not None,
+        key_fingerprint=(a.pending_key_fingerprint if pending else a.accepted_key_fingerprint),
+        last_seen_at=to_utc(last_seen_at),
+        features=features or {"push": False, "metrics": False},
+        labels=labels or {},
     )
 
 
+def _blacklist_schema(row: AgentBlacklist) -> AgentBlacklistItem:
+    return AgentBlacklistItem(
+        id=row.key_hash,
+        key_fingerprint=row.key_fingerprint,
+        agent_id=row.agent_id,
+        hostname=row.hostname,
+        created_at=to_utc(row.created_at),
+        last_seen_at=to_utc(row.last_seen_at),
+    )
+
+
+async def _get_agent(session: AsyncSession, agent_id: str) -> AgentModel:
+    res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id))
+    a = res.scalar_one_or_none()
+    if a is None:
+        raise HTTPException(status_code=404, detail="agent not found")
+    return a
+
+
+def _clear_pending(a: AgentModel) -> None:
+    a.pending_key_hash = None
+    a.pending_key_fingerprint = None
+    a.pending_hostname = None
+    a.pending_seen_at = None
+    a.pending_features = None
+    a.pending_labels = None
+
+
+def _accept_pending(a: AgentModel) -> bool:
+    if not _is_pending(a):
+        return False
+    now = datetime.now(UTC)
+    a.accepted_key_hash = a.pending_key_hash
+    a.accepted_key_fingerprint = a.pending_key_fingerprint
+    a.accepted_at = now
+    a.hostname = a.pending_hostname or a.hostname
+    a.last_seen_at = a.pending_seen_at or a.last_seen_at
+    if a.pending_features is not None:
+        a.features = a.pending_features
+    if a.pending_labels is not None:
+        a.labels = a.pending_labels
+    a.status = "alive"
+    _clear_pending(a)
+    return True
+
+
+async def _block_pending(session: AsyncSession, a: AgentModel) -> bool:
+    if not _is_pending(a):
+        return False
+    now = datetime.now(UTC)
+    stmt = pg_insert(AgentBlacklist).values(
+        key_hash=a.pending_key_hash,
+        key_fingerprint=a.pending_key_fingerprint,
+        agent_id=a.agent_id,
+        hostname=a.pending_hostname or a.hostname,
+        last_seen_at=a.pending_seen_at or now,
+    )
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[AgentBlacklist.key_hash],
+        set_={
+            "agent_id": stmt.excluded.agent_id,
+            "hostname": stmt.excluded.hostname,
+            "last_seen_at": stmt.excluded.last_seen_at,
+        },
+    )
+    await session.execute(stmt)
+    _clear_pending(a)
+    if a.accepted_key_hash is None:
+        await _delete_agent_data(session, a.agent_id)
+    return True
+
+
+async def _delete_agent_data(session: AsyncSession, agent_id: str) -> None:
+    incident_ids = select(Incident.id).where(Incident.agent_id == agent_id)
+    await session.execute(
+        delete(NotificationOutbox).where(NotificationOutbox.incident_id.in_(incident_ids))
+    )
+    await session.execute(delete(Incident).where(Incident.agent_id == agent_id))
+    await session.execute(delete(Event).where(Event.agent_id == agent_id))
+    await session.execute(delete(AgentModel).where(AgentModel.agent_id == agent_id))
+
+
+async def _remove_pending_or_agent(session: AsyncSession, a: AgentModel) -> None:
+    if _is_pending(a) and a.accepted_key_hash is not None:
+        _clear_pending(a)
+        return
+    await _delete_agent_data(session, a.agent_id)
+
+
+def _agent_rank(a: AgentModel) -> int:
+    return a.admission_rank
+
+
+def _decode_agent_cursor(cursor: str) -> tuple[int, str]:
+    rank_raw, agent_id = decode(cursor, 2)
+    try:
+        rank = int(rank_raw)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail="invalid cursor") from exc
+    if rank not in (0, 1):
+        raise HTTPException(status_code=400, detail="invalid cursor")
+    return rank, agent_id
+
+
 @router.get("/agents", response_model=AgentsPage)
 async def list_agents(
     cursor: str | None = Query(default=None, max_length=512),
+    back: bool = Query(default=False),
     limit: int = Query(default=100, ge=1, le=500),
     session: AsyncSession = Depends(get_session),
 ) -> AgentsPage:
-    stmt = select(AgentModel).order_by(AgentModel.agent_id)
+    if back:
+        stmt = select(AgentModel).order_by(
+            AgentModel.admission_rank.desc(), AgentModel.agent_id.desc()
+        )
+    else:
+        stmt = select(AgentModel).order_by(
+            AgentModel.admission_rank.asc(), AgentModel.agent_id.asc()
+        )
     if cursor is not None:
-        (last_id,) = decode(cursor, 1)
-        stmt = stmt.where(AgentModel.agent_id > last_id)
+        last_rank, last_id = _decode_agent_cursor(cursor)
+        if back:
+            stmt = stmt.where(
+                or_(
+                    AgentModel.admission_rank < last_rank,
+                    and_(
+                        AgentModel.admission_rank == last_rank,
+                        AgentModel.agent_id < last_id,
+                    ),
+                )
+            )
+        else:
+            stmt = stmt.where(
+                or_(
+                    AgentModel.admission_rank > last_rank,
+                    and_(
+                        AgentModel.admission_rank == last_rank,
+                        AgentModel.agent_id > last_id,
+                    ),
+                )
+            )
     rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
-    next_cursor = None
-    if len(rows) > limit:
-        rows = rows[:limit]
-        next_cursor = encode(rows[-1].agent_id)
-    return AgentsPage(items=[_to_schema(a) for a in rows], next_cursor=next_cursor)
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    if back:
+        rows = list(reversed(rows))
+    next_cursor: str | None = None
+    prev_cursor: str | None = None
+    if rows:
+        if back:
+            next_cursor = encode(_agent_rank(rows[-1]), rows[-1].agent_id)
+            if has_more:
+                prev_cursor = encode(_agent_rank(rows[0]), rows[0].agent_id)
+        else:
+            if has_more:
+                next_cursor = encode(_agent_rank(rows[-1]), rows[-1].agent_id)
+            if cursor is not None:
+                prev_cursor = encode(_agent_rank(rows[0]), rows[0].agent_id)
+    return AgentsPage(
+        items=[_to_schema(a) for a in rows],
+        next_cursor=next_cursor,
+        prev_cursor=prev_cursor,
+    )
+
+
+@router.get("/agents/admission/blacklist", response_model=AgentBlacklistPage)
+async def list_agent_blacklist(
+    cursor: str | None = Query(default=None, max_length=512),
+    limit: int = Query(default=100, ge=1, le=500),
+    session: AsyncSession = Depends(get_session),
+) -> AgentBlacklistPage:
+    stmt = select(AgentBlacklist).order_by(AgentBlacklist.key_fingerprint, AgentBlacklist.key_hash)
+    if cursor is not None:
+        last_fingerprint, last_hash = decode(cursor, 2)
+        stmt = stmt.where(
+            or_(
+                AgentBlacklist.key_fingerprint > last_fingerprint,
+                and_(
+                    AgentBlacklist.key_fingerprint == last_fingerprint,
+                    AgentBlacklist.key_hash > last_hash,
+                ),
+            )
+        )
+    rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    return AgentBlacklistPage(
+        items=[_blacklist_schema(r) for r in rows],
+        next_cursor=encode(rows[-1].key_fingerprint, rows[-1].key_hash)
+        if rows and has_more
+        else None,
+        prev_cursor=None,
+    )
+
+
+@router.delete("/agents/admission/blacklist", response_model=ActionCountResponse)
+async def clear_agent_blacklist(
+    session: AsyncSession = Depends(get_session),
+) -> ActionCountResponse:
+    res = await session.execute(delete(AgentBlacklist))
+    await session.commit()
+    return ActionCountResponse(count=res.rowcount or 0)
+
+
+@router.delete("/agents/admission/blacklist/{key_id}", response_model=ActionCountResponse)
+async def delete_agent_blacklist_item(
+    key_id: str,
+    session: AsyncSession = Depends(get_session),
+) -> ActionCountResponse:
+    res = await session.execute(delete(AgentBlacklist).where(AgentBlacklist.key_hash == key_id))
+    await session.commit()
+    return ActionCountResponse(count=res.rowcount or 0)
+
+
+@router.post("/agents/admission/accept-all", response_model=ActionCountResponse)
+async def accept_all_pending_agents(
+    include_rotation: bool = Query(default=False),
+    session: AsyncSession = Depends(get_session),
+) -> ActionCountResponse:
+    stmt = select(AgentModel).where(AgentModel.pending_key_hash.is_not(None))
+    if not include_rotation:
+        stmt = stmt.where(AgentModel.accepted_key_hash.is_(None))
+    rows = (await session.execute(stmt)).scalars().all()
+    count = sum(1 for a in rows if _accept_pending(a))
+    await session.commit()
+    return ActionCountResponse(count=count)
+
+
+@router.post("/agents/admission/block-all", response_model=ActionCountResponse)
+async def block_all_pending_agents(
+    session: AsyncSession = Depends(get_session),
+) -> ActionCountResponse:
+    rows = (
+        (await session.execute(select(AgentModel).where(AgentModel.pending_key_hash.is_not(None))))
+        .scalars()
+        .all()
+    )
+    count = 0
+    for a in rows:
+        if await _block_pending(session, a):
+            count += 1
+    await session.commit()
+    return ActionCountResponse(count=count)
+
+
+@router.post("/agents/admission/remove-all", response_model=ActionCountResponse)
+async def remove_all_pending_agents(
+    session: AsyncSession = Depends(get_session),
+) -> ActionCountResponse:
+    rows = (
+        (await session.execute(select(AgentModel).where(AgentModel.pending_key_hash.is_not(None))))
+        .scalars()
+        .all()
+    )
+    for agent in rows:
+        await _remove_pending_or_agent(session, agent)
+    await session.commit()
+    return ActionCountResponse(count=len(rows))
+
+
+@router.post("/agents/{agent_id}/accept", response_model=ActionCountResponse)
+async def accept_agent(
+    agent_id: str, session: AsyncSession = Depends(get_session)
+) -> ActionCountResponse:
+    agent = await _get_agent(session, agent_id)
+    count = 1 if _accept_pending(agent) else 0
+    await session.commit()
+    return ActionCountResponse(count=count)
+
+
+@router.post("/agents/{agent_id}/block", response_model=ActionCountResponse)
+async def block_agent(
+    agent_id: str, session: AsyncSession = Depends(get_session)
+) -> ActionCountResponse:
+    agent = await _get_agent(session, agent_id)
+    count = 1 if await _block_pending(session, agent) else 0
+    await session.commit()
+    return ActionCountResponse(count=count)
+
+
+@router.delete("/agents/{agent_id}", response_model=ActionCountResponse)
+async def delete_agent(
+    agent_id: str, session: AsyncSession = Depends(get_session)
+) -> ActionCountResponse:
+    agent = await _get_agent(session, agent_id)
+    await _remove_pending_or_agent(session, agent)
+    await session.commit()
+    return ActionCountResponse(count=1)
 
 
 @router.get("/agents/{agent_id}", response_model=Agent)
 async def get_agent(agent_id: str, session: AsyncSession = Depends(get_session)) -> Agent:
-    res = await session.execute(select(AgentModel).where(AgentModel.agent_id == agent_id))
-    a = res.scalar_one_or_none()
-    if a is None:
-        raise HTTPException(status_code=404, detail="agent not found")
-    return _to_schema(a)
+    return _to_schema(await _get_agent(session, agent_id))

server/monlet_server/api/checks.py

@@ -15,25 +15,50 @@ router = APIRouter()
 async def list_checks(
     agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
     cursor: str | None = Query(default=None, max_length=512),
+    back: bool = Query(default=False),
     limit: int = Query(default=100, ge=1, le=500),
     session: AsyncSession = Depends(get_session),
 ) -> ChecksPage:
-    stmt = select(Check).order_by(Check.agent_id, Check.check_id)
+    # PH-review: bidirectional cursor. Natural sort is ASC (agent_id, check_id).
+    if back:
+        stmt = select(Check).order_by(Check.agent_id.desc(), Check.check_id.desc())
+    else:
+        stmt = select(Check).order_by(Check.agent_id, Check.check_id)
     if agent_id is not None:
         stmt = stmt.where(Check.agent_id == agent_id)
     if cursor is not None:
         last_agent, last_check = decode(cursor, 2)
-        stmt = stmt.where(
-            or_(
-                Check.agent_id > last_agent,
-                and_(Check.agent_id == last_agent, Check.check_id > last_check),
+        if back:
+            stmt = stmt.where(
+                or_(
+                    Check.agent_id < last_agent,
+                    and_(Check.agent_id == last_agent, Check.check_id < last_check),
+                )
+            )
+        else:
+            stmt = stmt.where(
+                or_(
+                    Check.agent_id > last_agent,
+                    and_(Check.agent_id == last_agent, Check.check_id > last_check),
+                )
             )
-        )
     rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
-    next_cursor = None
-    if len(rows) > limit:
-        rows = rows[:limit]
-        next_cursor = encode(rows[-1].agent_id, rows[-1].check_id)
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    if back:
+        rows = list(reversed(rows))
+    next_cursor: str | None = None
+    prev_cursor: str | None = None
+    if rows:
+        if back:
+            next_cursor = encode(rows[-1].agent_id, rows[-1].check_id)
+            if has_more:
+                prev_cursor = encode(rows[0].agent_id, rows[0].check_id)
+        else:
+            if has_more:
+                next_cursor = encode(rows[-1].agent_id, rows[-1].check_id)
+            if cursor is not None:
+                prev_cursor = encode(rows[0].agent_id, rows[0].check_id)
     items = [
         CheckState(
             agent_id=r.agent_id,
@@ -46,4 +71,4 @@ async def list_checks(
         )
         for r in rows
     ]
-    return ChecksPage(items=items, next_cursor=next_cursor)
+    return ChecksPage(items=items, next_cursor=next_cursor, prev_cursor=prev_cursor)

server/monlet_server/api/events.py

@@ -1,11 +1,13 @@
 import base64
 import binascii
 from datetime import datetime
+from uuid import UUID
 
 from fastapi import APIRouter, Depends, HTTPException, Query, status
 from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from ..agent_auth import AgentCredential, get_agent_credential
 from ..db import get_session
 from ..models import Event
 from ..schemas import (
@@ -15,7 +17,7 @@ from ..schemas import (
     EventsPage,
     StoredCheckResultEvent,
 )
-from ..services.ingestion import ingest_event
+from ..services.ingestion import accepted_agent_for_key, ingest_event
 from ..timeutil import to_utc
 
 router = APIRouter()
@@ -26,12 +28,14 @@ def _encode_cursor(received_at: datetime, event_id) -> str:
     return base64.urlsafe_b64encode(raw).decode().rstrip("=")
 
 
-def _decode_cursor(cursor: str) -> tuple[datetime, str]:
+def _decode_cursor(cursor: str) -> tuple[datetime, UUID]:
     pad = "=" * (-len(cursor) % 4)
     try:
         raw = base64.urlsafe_b64decode(cursor + pad).decode()
         ts_str, eid = raw.split("|", 1)
-        return to_utc(datetime.fromisoformat(ts_str)), eid
+        # PH-010: decode event_id to UUID before comparing to the UUID column
+        # to avoid silent type-coercion behavior across DB drivers.
+        return to_utc(datetime.fromisoformat(ts_str)), UUID(eid)
     except (binascii.Error, ValueError) as exc:
         raise HTTPException(status_code=400, detail="invalid cursor") from exc
 
@@ -42,12 +46,18 @@ def _decode_cursor(cursor: str) -> tuple[datetime, str]:
     status_code=status.HTTP_202_ACCEPTED,
 )
 async def events(
-    body: EventBatchRequest, session: AsyncSession = Depends(get_session)
+    body: EventBatchRequest,
+    credential: AgentCredential = Depends(get_agent_credential),
+    session: AsyncSession = Depends(get_session),
 ) -> EventBatchAcceptedResponse:
+    agent = await accepted_agent_for_key(session, credential)
+    if agent is None:
+        await session.commit()
+        return EventBatchAcceptedResponse(accepted=0, deduplicated=0)
     accepted = 0
     deduplicated = 0
     for ev in body.events:
-        ok = await ingest_event(session, body.agent_id, ev)
+        ok = await ingest_event(session, agent.agent_id, ev)
         if ok:
             accepted += 1
         else:
@@ -61,28 +71,55 @@ async def query_events(
     agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
     check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
     cursor: str | None = Query(default=None, max_length=512),
+    back: bool = Query(default=False),
     limit: int = Query(default=100, ge=1, le=500),
     session: AsyncSession = Depends(get_session),
 ) -> EventsPage:
-    stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc())
+    # PH-review: bidirectional cursor. The natural sort is DESC on
+    # (received_at, event_id); `back=True` reverses the query so we can return
+    # the previous page without an unbounded cursor stack in the URL.
+    if back:
+        stmt = select(Event).order_by(Event.received_at.asc(), Event.event_id.asc())
+    else:
+        stmt = select(Event).order_by(Event.received_at.desc(), Event.event_id.desc())
     if agent_id is not None:
         stmt = stmt.where(Event.agent_id == agent_id)
     if check_id is not None:
         stmt = stmt.where(Event.check_id == check_id)
     if cursor is not None:
         ts, eid = _decode_cursor(cursor)
-        stmt = stmt.where(
-            or_(
-                Event.received_at < ts,
-                and_(Event.received_at == ts, Event.event_id < eid),
+        if back:
+            stmt = stmt.where(
+                or_(
+                    Event.received_at > ts,
+                    and_(Event.received_at == ts, Event.event_id > eid),
+                )
+            )
+        else:
+            stmt = stmt.where(
+                or_(
+                    Event.received_at < ts,
+                    and_(Event.received_at == ts, Event.event_id < eid),
+                )
             )
-        )
     stmt = stmt.limit(limit + 1)
     rows = (await session.execute(stmt)).scalars().all()
-    next_cursor = None
-    if len(rows) > limit:
-        rows = rows[:limit]
-        next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id)
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    if back:
+        rows = list(reversed(rows))
+    next_cursor: str | None = None
+    prev_cursor: str | None = None
+    if rows:
+        if back:
+            next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id)
+            if has_more:
+                prev_cursor = _encode_cursor(rows[0].received_at, rows[0].event_id)
+        else:
+            if has_more:
+                next_cursor = _encode_cursor(rows[-1].received_at, rows[-1].event_id)
+            if cursor is not None:
+                prev_cursor = _encode_cursor(rows[0].received_at, rows[0].event_id)
     items = [
         StoredCheckResultEvent(
             event_id=str(r.event_id),
@@ -100,4 +137,4 @@ async def query_events(
         )
         for r in rows
     ]
-    return EventsPage(items=items, next_cursor=next_cursor)
+    return EventsPage(items=items, next_cursor=next_cursor, prev_cursor=prev_cursor)

server/monlet_server/api/heartbeat.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends, status
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from ..agent_auth import AgentCredential, get_agent_credential
 from ..db import get_session
 from ..schemas import AcceptedResponse, HeartbeatRequest
 from ..services.ingestion import upsert_agent_heartbeat
@@ -14,8 +15,10 @@ router = APIRouter()
     status_code=status.HTTP_202_ACCEPTED,
 )
 async def heartbeat(
-    body: HeartbeatRequest, session: AsyncSession = Depends(get_session)
+    body: HeartbeatRequest,
+    credential: AgentCredential = Depends(get_agent_credential),
+    session: AsyncSession = Depends(get_session),
 ) -> AcceptedResponse:
-    await upsert_agent_heartbeat(session, body)
+    await upsert_agent_heartbeat(session, body, credential)
     await session.commit()
     return AcceptedResponse(accepted=True)

server/monlet_server/api/incidents.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from fastapi import APIRouter, Depends, HTTPException, Query
 from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -5,11 +7,29 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from ..cursors import decode, decode_datetime, encode
 from ..db import get_session
 from ..models import Incident as IncidentModel
-from ..schemas import Incident, IncidentsPage
+from ..schemas import ID_PATTERN, Incident, IncidentsPage
 from ..timeutil import to_utc
 
 router = APIRouter()
 
+_SORT_COLUMNS = {
+    "opened_at": IncidentModel.opened_at,
+    "resolved_at": IncidentModel.resolved_at,
+}
+SortField = Literal["status", "opened_at", "resolved_at"]
+
+# PH-review: the rank is materialized as a STORED generated column
+# `incidents.status_rank` (see models.py / 0001_baseline.py). This map mirrors
+# the SQL CASE for Python-side cursor encoding only.
+_STATUS_RANK: dict[tuple[str, str], int] = {
+    ("open", "critical"): 5,
+    ("open", "warning"): 4,
+    ("open", "unknown"): 3,
+    ("resolved", "critical"): 2,
+    ("resolved", "warning"): 1,
+    ("resolved", "unknown"): 0,
+}
+
 
 def _to_schema(i: IncidentModel) -> Incident:
     return Incident(
@@ -25,30 +45,105 @@ def _to_schema(i: IncidentModel) -> Incident:
     )
 
 
+def _encode_incident_cursor(i: IncidentModel, sort: SortField) -> str:
+    if sort == "status":
+        return encode(i.status_rank, to_utc(i.opened_at).isoformat(), i.id)
+    return encode(to_utc(getattr(i, sort)).isoformat(), i.id)
+
+
 @router.get("/incidents", response_model=IncidentsPage)
 async def list_incidents(
-    state: str | None = Query(default=None),
+    state: Literal["open", "resolved"] | None = Query(default=None),
+    severity: Literal["warning", "critical", "unknown"] | None = Query(default=None),
+    agent_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    check_id: str | None = Query(default=None, max_length=128, pattern=ID_PATTERN),
+    sort: SortField = Query(default="status"),
+    direction: Literal["asc", "desc"] = Query(default="desc"),
     cursor: str | None = Query(default=None, max_length=512),
+    back: bool = Query(default=False),
     limit: int = Query(default=100, ge=1, le=500),
     session: AsyncSession = Depends(get_session),
 ) -> IncidentsPage:
-    if state is not None and state not in ("open", "resolved"):
-        raise HTTPException(status_code=400, detail="invalid state")
-    stmt = select(IncidentModel).order_by(IncidentModel.opened_at.desc(), IncidentModel.id.desc())
+    # PH-001 finding-4: sort=resolved_at must skip open incidents — their
+    # resolved_at is NULL and would otherwise break to_utc() on the cursor.
+    if sort == "resolved_at" and state is None:
+        state = "resolved"
+    if sort == "resolved_at" and state == "open":
+        raise HTTPException(status_code=400, detail="sort=resolved_at requires state=resolved")
+    # PH-review: `back=True` flips the effective sort to walk to the previous
+    # page. We reverse the rows before returning so the response order matches
+    # the canonical (user-requested) direction.
+    effective_desc = (direction == "desc") ^ back
+    if sort == "status":
+        rank_col = IncidentModel.status_rank
+        rank_order = rank_col.desc() if effective_desc else rank_col.asc()
+        opened_order = (
+            IncidentModel.opened_at.desc() if effective_desc else IncidentModel.opened_at.asc()
+        )
+        id_order = IncidentModel.id.desc() if effective_desc else IncidentModel.id.asc()
+        stmt = select(IncidentModel).order_by(rank_order, opened_order, id_order)
+    else:
+        col = _SORT_COLUMNS[sort]
+        order_col = col.desc() if effective_desc else col.asc()
+        tiebreak = IncidentModel.id.desc() if effective_desc else IncidentModel.id.asc()
+        stmt = select(IncidentModel).order_by(order_col, tiebreak)
+    if sort == "resolved_at":
+        stmt = stmt.where(IncidentModel.resolved_at.is_not(None))
     if state is not None:
         stmt = stmt.where(IncidentModel.state == state)
+    if severity is not None:
+        stmt = stmt.where(IncidentModel.severity == severity)
+    if agent_id is not None:
+        stmt = stmt.where(IncidentModel.agent_id == agent_id)
+    if check_id is not None:
+        stmt = stmt.where(IncidentModel.check_id == check_id)
     if cursor is not None:
-        ts_str, last_id = decode(cursor, 2)
-        ts = decode_datetime(ts_str)
-        stmt = stmt.where(
-            or_(
-                IncidentModel.opened_at < ts,
-                and_(IncidentModel.opened_at == ts, IncidentModel.id < last_id),
+        if sort == "status":
+            rank_str, ts_str, last_id = decode(cursor, 3)
+            try:
+                rank = int(rank_str)
+            except ValueError as exc:
+                raise HTTPException(status_code=400, detail="invalid cursor") from exc
+            ts = decode_datetime(ts_str)
+            rank_col = IncidentModel.status_rank
+            rank_cmp = rank_col < rank if effective_desc else rank_col > rank
+            opened_cmp = (
+                IncidentModel.opened_at < ts if effective_desc else IncidentModel.opened_at > ts
             )
-        )
+            id_cmp = IncidentModel.id < last_id if effective_desc else IncidentModel.id > last_id
+            stmt = stmt.where(
+                or_(
+                    rank_cmp,
+                    and_(rank_col == rank, opened_cmp),
+                    and_(rank_col == rank, IncidentModel.opened_at == ts, id_cmp),
+                )
+            )
+        else:
+            col = _SORT_COLUMNS[sort]
+            ts_str, last_id = decode(cursor, 2)
+            ts = decode_datetime(ts_str)
+            cmp = col < ts if effective_desc else col > ts
+            tie = IncidentModel.id < last_id if effective_desc else IncidentModel.id > last_id
+            stmt = stmt.where(or_(cmp, and_(col == ts, tie)))
     rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
-    next_cursor = None
-    if len(rows) > limit:
-        rows = rows[:limit]
-        next_cursor = encode(to_utc(rows[-1].opened_at).isoformat(), rows[-1].id)
-    return IncidentsPage(items=[_to_schema(r) for r in rows], next_cursor=next_cursor)
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    if back:
+        rows = list(reversed(rows))
+    next_cursor: str | None = None
+    prev_cursor: str | None = None
+    if rows:
+        if back:
+            next_cursor = _encode_incident_cursor(rows[-1], sort)
+            if has_more:
+                prev_cursor = _encode_incident_cursor(rows[0], sort)
+        else:
+            if has_more:
+                next_cursor = _encode_incident_cursor(rows[-1], sort)
+            if cursor is not None:
+                prev_cursor = _encode_incident_cursor(rows[0], sort)
+    return IncidentsPage(
+        items=[_to_schema(r) for r in rows],
+        next_cursor=next_cursor,
+        prev_cursor=prev_cursor,
+    )

server/monlet_server/api/outbox.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from fastapi import APIRouter, Depends, Query
 from sqlalchemy import and_, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -10,36 +12,73 @@ from ..timeutil import to_utc
 
 router = APIRouter()
 
+# PH-002/PH-011/PH-018: explicit sort + filter contracts.
+OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"]
+OutboxEventType = Literal["firing", "resolved"]
+OutboxNotifier = Literal["debug", "telegram", "webhook", "alertmanager"]
+_SORT_COLUMNS = {
+    "created_at": NotificationOutbox.created_at,
+    "updated_at": NotificationOutbox.updated_at,
+}
+SortField = Literal["created_at", "updated_at"]
+
+
+def _encode_outbox_cursor(row: NotificationOutbox, sort: SortField) -> str:
+    return encode(to_utc(getattr(row, sort)).isoformat(), row.id)
+
 
 @router.get("/notifiers/outbox", response_model=OutboxPage)
 async def list_outbox(
-    state: str | None = Query(default=None),
+    state: OutboxState | None = Query(default=None),
+    notifier: OutboxNotifier | None = Query(default=None),
+    event_type: OutboxEventType | None = Query(default=None),
+    sort: SortField = Query(default="updated_at"),
+    direction: Literal["asc", "desc"] = Query(default="desc"),
     cursor: str | None = Query(default=None, max_length=512),
+    back: bool = Query(default=False),
     limit: int = Query(default=100, ge=1, le=500),
     session: AsyncSession = Depends(get_session),
 ) -> OutboxPage:
-    stmt = select(NotificationOutbox).order_by(
-        NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc()
-    )
+    col = _SORT_COLUMNS[sort]
+    # PH-review: bidirectional cursor — for `back=True` we run the query in the
+    # opposite direction relative to the requested sort and reverse the result
+    # client-side. This avoids the unbounded cursor_stack in the URL.
+    effective_desc = (direction == "desc") ^ back
+    order_col = col.desc() if effective_desc else col.asc()
+    tiebreak = NotificationOutbox.id.desc() if effective_desc else NotificationOutbox.id.asc()
+    stmt = select(NotificationOutbox).order_by(order_col, tiebreak)
     if state is not None:
         stmt = stmt.where(NotificationOutbox.state == state)
+    if notifier is not None:
+        stmt = stmt.where(NotificationOutbox.notifier == notifier)
+    if event_type is not None:
+        stmt = stmt.where(NotificationOutbox.event_type == event_type)
     if cursor is not None:
         ts_str, last_id = decode(cursor, 2)
         ts = decode_datetime(ts_str)
-        stmt = stmt.where(
-            or_(
-                NotificationOutbox.created_at < ts,
-                and_(
-                    NotificationOutbox.created_at == ts,
-                    NotificationOutbox.id < last_id,
-                ),
-            )
-        )
+        cmp = col < ts if effective_desc else col > ts
+        tie = NotificationOutbox.id < last_id if effective_desc else NotificationOutbox.id > last_id
+        stmt = stmt.where(or_(cmp, and_(col == ts, tie)))
     rows = (await session.execute(stmt.limit(limit + 1))).scalars().all()
-    next_cursor = None
-    if len(rows) > limit:
-        rows = rows[:limit]
-        next_cursor = encode(to_utc(rows[-1].created_at).isoformat(), rows[-1].id)
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    if back:
+        rows = list(reversed(rows))
+    next_cursor: str | None = None
+    prev_cursor: str | None = None
+    if rows:
+        if back:
+            # Came from a forward page → there is always a next page (the one
+            # we just came from). `has_more` here means there are still earlier
+            # pages remaining.
+            next_cursor = _encode_outbox_cursor(rows[-1], sort)
+            if has_more:
+                prev_cursor = _encode_outbox_cursor(rows[0], sort)
+        else:
+            if has_more:
+                next_cursor = _encode_outbox_cursor(rows[-1], sort)
+            if cursor is not None:
+                prev_cursor = _encode_outbox_cursor(rows[0], sort)
     return OutboxPage(
         items=[
             NotificationOutboxItem(
@@ -56,4 +95,5 @@ async def list_outbox(
             for r in rows
         ],
         next_cursor=next_cursor,
+        prev_cursor=prev_cursor,
     )

server/monlet_server/api/system_summary.py

@@ -0,0 +1,43 @@
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..db import get_session
+from ..models import Agent, Check, Incident
+from ..schemas import SystemSummaryResponse
+
+router = APIRouter()
+
+
+@router.get("/system-summary", response_model=SystemSummaryResponse)
+async def system_summary(session: AsyncSession = Depends(get_session)) -> SystemSummaryResponse:
+    agents_rows = (
+        await session.execute(
+            select(Agent.status, func.count())
+            .where(Agent.accepted_key_hash.is_not(None))
+            .group_by(Agent.status)
+        )
+    ).all()
+    checks_rows = (
+        await session.execute(select(Check.status, func.count()).group_by(Check.status))
+    ).all()
+    open_incidents = (
+        await session.execute(
+            select(func.count()).select_from(Incident).where(Incident.state == "open")
+        )
+    ).scalar_one()
+
+    agents = {status: count for status, count in agents_rows}
+    checks = {status: count for status, count in checks_rows}
+    return SystemSummaryResponse(
+        agents_online=agents.get("alive", 0),
+        agents_offline=agents.get("stale", 0) + agents.get("dead", 0),
+        checks_ok=checks.get("ok", 0),
+        checks_warning=checks.get("warning", 0),
+        checks_critical=checks.get("critical", 0),
+        checks_unknown=checks.get("unknown", 0),
+        open_incidents=open_incidents,
+        generated_at=datetime.now(UTC),
+    )

server/monlet_server/db.py

@@ -17,7 +17,17 @@ def get_engine() -> AsyncEngine:
     global _engine, _sessionmaker
     if _engine is None:
         s = get_settings()
-        _engine = create_async_engine(s.database_url, pool_pre_ping=True, future=True)
+        # PH-016: explicit pool sizing. pool_pre_ping protects against stale
+        # connections after PG restarts; pool_recycle bounds connection age.
+        _engine = create_async_engine(
+            s.database_url,
+            pool_pre_ping=True,
+            pool_size=s.db_pool_size,
+            max_overflow=s.db_max_overflow,
+            pool_timeout=s.db_pool_timeout_sec,
+            pool_recycle=s.db_pool_recycle_sec,
+            future=True,
+        )
         _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False)
     return _engine
 

server/monlet_server/errors.py

@@ -28,6 +28,8 @@ def error_response(
 _STATUS_TO_CODE: dict[int, ErrorCode] = {
     400: "validation",
     401: "unauthorized",
+    403: "forbidden",
+    409: "conflict",
     404: "not_found",
     413: "payload_too_large",
     429: "rate_limited",
@@ -44,7 +46,11 @@ def install_error_handlers(app: FastAPI) -> None:
 
     @app.exception_handler(StarletteHTTPException)
     async def _http(request: Request, exc: StarletteHTTPException):
-        code = _STATUS_TO_CODE.get(exc.status_code, "internal")
+        code = (
+            "agent_blocked"
+            if exc.status_code == 403 and exc.detail == "agent blocked"
+            else _STATUS_TO_CODE.get(exc.status_code, "internal")
+        )
         msg = exc.detail if isinstance(exc.detail, str) else code
         return error_response(request, exc.status_code, code, msg)
 

server/monlet_server/main.py

@@ -5,7 +5,7 @@ from fastapi import FastAPI
 from fastapi.responses import Response
 
 from . import metrics
-from .api import agents, checks, events, health, heartbeat, incidents, outbox
+from .api import agents, checks, events, health, heartbeat, incidents, outbox, system_summary
 from .db import dispose_engine, get_engine, get_sessionmaker
 from .errors import install_error_handlers
 from .logging_config import configure_logging, get_logger
@@ -86,6 +86,7 @@ def create_app() -> FastAPI:
     app.include_router(checks.router, prefix="/api/v1")
     app.include_router(incidents.router, prefix="/api/v1")
     app.include_router(outbox.router, prefix="/api/v1")
+    app.include_router(system_summary.router, prefix="/api/v1")
 
     @app.get("/metrics")
     async def metrics_endpoint() -> Response:

server/monlet_server/middleware/auth.py

@@ -1,22 +1,42 @@
+import hmac
+
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
 
+from ..agent_auth import credential_from_token
 from ..errors import error_response
 from ..settings import get_settings
 
 NO_AUTH_PATHS = {"/api/v1/health", "/api/v1/ready", "/metrics"}
 
+AGENT_PATHS = {"/api/v1/heartbeat", "/api/v1/events"}
+
+
+def _match_token(presented: str, accepted: list[str]) -> bool:
+    matched = False
+    # Constant-time across all accepted tokens to avoid timing oracle.
+    for t in accepted:
+        if hmac.compare_digest(presented, t):
+            matched = True
+    return matched
+
 
 class BearerAuthMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next):
-        if request.url.path in NO_AUTH_PATHS:
+        path = request.url.path
+        if path in NO_AUTH_PATHS:
             return await call_next(request)
 
-        expected = get_settings().auth_token
         header = request.headers.get("authorization")
         if not header or not header.lower().startswith("bearer "):
             return error_response(request, 401, "unauthorized", "authentication required")
         token = header.split(" ", 1)[1].strip()
-        if token != expected:
-            return error_response(request, 401, "unauthorized", "authentication required")
+
+        if path in AGENT_PATHS:
+            if len(token) < 32 or len(token) > 512:
+                return error_response(request, 401, "unauthorized", "authentication required")
+            request.state.agent_credential = credential_from_token(token)
+        else:
+            if not _match_token(token, get_settings().auth_tokens):
+                return error_response(request, 401, "unauthorized", "authentication required")
         return await call_next(request)

server/monlet_server/models.py

@@ -5,6 +5,7 @@ from sqlalchemy import (
     JSON,
     Boolean,
     CheckConstraint,
+    Computed,
     DateTime,
     ForeignKey,
     Index,
@@ -51,11 +52,57 @@ class Agent(Base):
     created_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False, server_default=func.now()
     )
+    accepted_key_hash: Mapped[str | None] = mapped_column(Text, nullable=True)
+    accepted_key_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True)
+    accepted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    pending_key_hash: Mapped[str | None] = mapped_column(Text, nullable=True)
+    pending_key_fingerprint: Mapped[str | None] = mapped_column(Text, nullable=True)
+    pending_hostname: Mapped[str | None] = mapped_column(Text, nullable=True)
+    pending_seen_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    pending_features: Mapped[dict | None] = mapped_column(_jsonb(), nullable=True)
+    pending_labels: Mapped[dict | None] = mapped_column(_jsonb(), nullable=True)
+    admission_rank: Mapped[int] = mapped_column(
+        Integer,
+        Computed("CASE WHEN pending_key_hash IS NOT NULL THEN 0 ELSE 1 END", persisted=True),
+        nullable=False,
+    )
 
     __table_args__ = (
         CheckConstraint("status IN ('alive','stale','dead')", name="agents_status_chk"),
         Index("agents_status_idx", "status"),
         Index("agents_last_seen_idx", "last_seen_at"),
+        Index("agents_admission_rank_idx", "admission_rank", "agent_id"),
+        Index("agents_hostname_idx", "hostname", unique=True),
+        Index(
+            "agents_accepted_key_hash_idx",
+            "accepted_key_hash",
+            unique=True,
+            postgresql_where=text("accepted_key_hash IS NOT NULL"),
+        ),
+        Index(
+            "agents_pending_key_hash_idx",
+            "pending_key_hash",
+            unique=True,
+            postgresql_where=text("pending_key_hash IS NOT NULL"),
+        ),
+    )
+
+
+class AgentBlacklist(Base):
+    __tablename__ = "agent_blacklist"
+
+    key_hash: Mapped[str] = mapped_column(Text, primary_key=True)
+    key_fingerprint: Mapped[str] = mapped_column(Text, nullable=False)
+    agent_id: Mapped[str] = mapped_column(Text, nullable=False)
+    hostname: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+    __table_args__ = (
+        Index("agent_blacklist_fingerprint_idx", "key_fingerprint"),
+        Index("agent_blacklist_last_seen_idx", "last_seen_at"),
     )
 
 
@@ -117,6 +164,19 @@ class Event(Base):
         ),
         Index("events_observed_at_idx", text("observed_at DESC")),
         Index("events_received_at_idx", text("received_at DESC")),
+        Index("events_received_event_idx", text("received_at DESC"), text("event_id DESC")),
+        Index(
+            "events_agent_received_event_idx",
+            "agent_id",
+            text("received_at DESC"),
+            text("event_id DESC"),
+        ),
+        Index(
+            "events_check_received_event_idx",
+            "check_id",
+            text("received_at DESC"),
+            text("event_id DESC"),
+        ),
     )
 
 
@@ -144,6 +204,25 @@ class Incident(Base):
     resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
     summary: Mapped[str | None] = mapped_column(Text, nullable=True)
     last_event_id: Mapped[UUID] = mapped_column(_uuid(), nullable=False)
+    # PH-review: stored generated column so the default UI sort
+    # (status_rank, opened_at, id) is served by a plain B-tree index without
+    # planner mismatches between query-time and index-time CASE expressions.
+    # Keep aligned with `_STATUS_RANK` in api/incidents.py.
+    status_rank: Mapped[int] = mapped_column(
+        Integer,
+        Computed(
+            "CASE "
+            "WHEN state = 'open' AND severity = 'critical' THEN 5 "
+            "WHEN state = 'open' AND severity = 'warning' THEN 4 "
+            "WHEN state = 'open' AND severity = 'unknown' THEN 3 "
+            "WHEN state = 'resolved' AND severity = 'critical' THEN 2 "
+            "WHEN state = 'resolved' AND severity = 'warning' THEN 1 "
+            "WHEN state = 'resolved' AND severity = 'unknown' THEN 0 "
+            "END",
+            persisted=True,
+        ),
+        nullable=False,
+    )
 
     __table_args__ = (
         CheckConstraint("state IN ('open','resolved')", name="incidents_state_chk"),
@@ -156,7 +235,12 @@ class Incident(Base):
             unique=True,
             postgresql_where=text("state = 'open'"),
         ),
-        Index("incidents_state_opened_idx", "state", text("opened_at DESC")),
+        Index(
+            "incidents_status_rank_idx",
+            text("status_rank DESC"),
+            text("opened_at DESC"),
+            text("id DESC"),
+        ),
     )
 
 
@@ -193,4 +277,6 @@ class NotificationOutbox(Base):
             "next_attempt_at",
             postgresql_where=text("state IN ('pending','retry')"),
         ),
+        Index("outbox_created_at_idx", text("created_at DESC"), text("id DESC")),
+        Index("outbox_updated_at_idx", text("updated_at DESC"), text("id DESC")),
     )

server/monlet_server/schemas.py

@@ -23,18 +23,22 @@ SENSITIVE_LABEL_WORDS = (
 
 Status = Literal["ok", "warning", "critical", "unknown"]
 AgentStatus = Literal["alive", "stale", "dead"]
+AgentAdmissionState = Literal["pending", "accepted"]
 IncidentState = Literal["open", "resolved"]
 Severity = Literal["warning", "critical", "unknown"]
 OutboxState = Literal["pending", "sending", "sent", "retry", "failed", "discarded"]
 OutboxEventType = Literal["firing", "resolved"]
 ErrorCode = Literal[
     "unauthorized",
+    "forbidden",
+    "conflict",
     "not_found",
     "validation",
     "payload_too_large",
     "rate_limited",
     "internal",
     "not_ready",
+    "agent_blocked",
 ]
 
 
@@ -140,6 +144,9 @@ class Agent(BaseModel):
     agent_id: str
     hostname: str
     status: AgentStatus
+    admission_state: AgentAdmissionState
+    rotation_pending: bool = False
+    key_fingerprint: str | None = None
     last_seen_at: datetime
     features: AgentFeatures
     labels: dict[str, str] = Field(default_factory=dict)
@@ -186,12 +193,30 @@ class NotificationOutboxItem(BaseModel):
 
 class PageEnvelope(BaseModel):
     next_cursor: str | None = None
+    prev_cursor: str | None = None
 
 
 class AgentsPage(PageEnvelope):
     items: list[Agent]
 
 
+class AgentBlacklistItem(BaseModel):
+    id: str
+    key_fingerprint: str
+    agent_id: str
+    hostname: str
+    created_at: datetime
+    last_seen_at: datetime
+
+
+class AgentBlacklistPage(PageEnvelope):
+    items: list[AgentBlacklistItem]
+
+
+class ActionCountResponse(BaseModel):
+    count: int = Field(ge=0)
+
+
 class ChecksPage(PageEnvelope):
     items: list[CheckState]
 
@@ -212,3 +237,14 @@ class UUIDStr(BaseModel):
     """Helper to validate generated UUIDs."""
 
     value: UUID
+
+
+class SystemSummaryResponse(BaseModel):
+    agents_online: int
+    agents_offline: int
+    checks_ok: int
+    checks_warning: int
+    checks_critical: int
+    checks_unknown: int
+    open_incidents: int
+    generated_at: datetime

server/monlet_server/services/detector.py

@@ -202,14 +202,16 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime
                 summary=summary,
             )
         )
-    else:
+    elif changed:
+        # PH-009: update liveness check row only on status transitions to avoid
+        # O(N) per-tick rewrites of every agent_liveness row. last_observed_at
+        # then reflects the meaningful transition timestamp, not detector ticks.
+        cur.status = liveness_status
+        cur.exit_code = exit_code
+        cur.last_event_id = event_id
+        cur.incident_key = incident_key
         cur.last_observed_at = now
         cur.summary = summary
-        if changed:
-            cur.status = liveness_status
-            cur.exit_code = exit_code
-            cur.last_event_id = event_id
-            cur.incident_key = incident_key
 
     if next_status in ("stale", "dead"):
         await _open_liveness_incident(session, agent, next_status, now, event_id)
@@ -219,6 +221,34 @@ async def _apply_liveness(session, agent: Agent, next_status: str, now: datetime
 
 async def _prune_history(session) -> None:
     s = get_settings()
+    if s.pending_agent_ttl_days > 0:
+        cutoff = datetime.now(UTC) - timedelta(days=s.pending_agent_ttl_days)
+        rows = (
+            (
+                await session.execute(
+                    select(Agent)
+                    .where(
+                        Agent.pending_key_hash.is_not(None),
+                        Agent.pending_seen_at < cutoff,
+                    )
+                    .order_by(Agent.pending_seen_at)
+                    .limit(max(s.pending_agent_prune_batch_size, 1))
+                )
+            )
+            .scalars()
+            .all()
+        )
+        for agent in rows:
+            if agent.accepted_key_hash is None:
+                await session.delete(agent)
+            else:
+                agent.pending_key_hash = None
+                agent.pending_key_fingerprint = None
+                agent.pending_hostname = None
+                agent.pending_seen_at = None
+                agent.pending_features = None
+                agent.pending_labels = None
+
     if s.event_dedup_retention_days > 0:
         cutoff = datetime.now(UTC) - timedelta(days=s.event_dedup_retention_days)
         batch = max(s.event_dedup_prune_batch_size, 1)
@@ -233,17 +263,29 @@ async def _prune_history(session) -> None:
         await session.execute(delete(EventIngestDedup).where(EventIngestDedup.event_id.in_(sub)))
 
     if s.outbox_retention_max_rows > 0:
+        # PH-012: keep the N newest terminal rows. Delete in a bounded batch to
+        # avoid one huge DELETE under failure storms. Active rows ('pending',
+        # 'sending', 'retry') are never eligible — terminal-only filter
+        # protects them.
         keep_outbox = (
             select(NotificationOutbox.id)
             .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
             .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
             .limit(s.outbox_retention_max_rows)
         )
-        await session.execute(
-            delete(NotificationOutbox)
+        prune_ids = (
+            select(NotificationOutbox.id)
             .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
             .where(NotificationOutbox.id.not_in(keep_outbox))
+            .limit(max(s.outbox_prune_batch_size, 1))
+            .scalar_subquery()
         )
+        await session.execute(
+            delete(NotificationOutbox).where(NotificationOutbox.id.in_(prune_ids))
+        )
+
+
+DETECTOR_BATCH_SIZE = 500
 
 
 async def _tick(sm: async_sessionmaker) -> None:
@@ -252,23 +294,50 @@ async def _tick(sm: async_sessionmaker) -> None:
     stale_cut = now - timedelta(seconds=s.stale_after_sec)
     dead_cut = now - timedelta(seconds=s.dead_after_sec)
     async with sm() as session:
-        agents = (await session.execute(select(Agent))).scalars().all()
-        for agent in agents:
-            if agent.last_seen_at <= dead_cut:
-                next_status = "dead"
-            elif agent.last_seen_at <= stale_cut:
-                next_status = "stale"
-            else:
-                next_status = "alive"
-            if agent.status != next_status:
-                agent.status = next_status
-            await _apply_liveness(session, agent, next_status, now)
+        # PH-009: keyset-paginate agents by agent_id to bound per-tick memory
+        # and per-batch transaction work. Each batch is committed separately so
+        # large inventories do not produce one huge transaction.
+        # Race trade-off: a new agent inserted with agent_id < last_seen_id
+        # mid-tick is skipped this tick and processed next tick. That delay is
+        # bounded by detector_tick_sec (5s by default) and is acceptable —
+        # liveness detection has a stale_after_sec / dead_after_sec horizon
+        # measured in tens of seconds, so a one-tick lag is invisible.
+        last_seen_id: str | None = None
+        while True:
+            q = (
+                select(Agent)
+                .where(Agent.accepted_key_hash.is_not(None))
+                .order_by(Agent.agent_id)
+                .limit(DETECTOR_BATCH_SIZE)
+            )
+            if last_seen_id is not None:
+                q = q.where(Agent.agent_id > last_seen_id)
+            agents = (await session.execute(q)).scalars().all()
+            if not agents:
+                break
+            for agent in agents:
+                if agent.last_seen_at <= dead_cut:
+                    next_status = "dead"
+                elif agent.last_seen_at <= stale_cut:
+                    next_status = "stale"
+                else:
+                    next_status = "alive"
+                if agent.status != next_status:
+                    agent.status = next_status
+                await _apply_liveness(session, agent, next_status, now)
+            last_seen_id = agents[-1].agent_id
+            await session.commit()
+            if len(agents) < DETECTOR_BATCH_SIZE:
+                break
+
         await _prune_history(session)
         await session.commit()
 
         for status in ("alive", "stale", "dead"):
             r = await session.execute(
-                select(func.count()).select_from(Agent).where(Agent.status == status)
+                select(func.count())
+                .select_from(Agent)
+                .where(Agent.accepted_key_hash.is_not(None), Agent.status == status)
             )
             metrics.agents_gauge.labels(status=status).set(r.scalar_one())
         r = await session.execute(

server/monlet_server/services/ingestion.py

@@ -2,13 +2,22 @@ from datetime import UTC, datetime
 from uuid import UUID, uuid4
 
 from fastapi import HTTPException
-from sqlalchemy import select, text
+from sqlalchemy import func, select, text
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from .. import metrics
+from ..agent_auth import AgentCredential
 from ..logging_config import get_logger
-from ..models import Agent, Check, Event, EventIngestDedup, Incident, NotificationOutbox
+from ..models import (
+    Agent,
+    AgentBlacklist,
+    Check,
+    Event,
+    EventIngestDedup,
+    Incident,
+    NotificationOutbox,
+)
 from ..redaction import redact
 from ..schemas import CheckResultEvent, HeartbeatRequest
 from ..settings import get_settings
@@ -37,28 +46,143 @@ def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
     return key
 
 
-async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
+async def _blacklist_row(session: AsyncSession, cred: AgentCredential) -> AgentBlacklist | None:
+    res = await session.execute(
+        select(AgentBlacklist).where(AgentBlacklist.key_hash == cred.key_hash)
+    )
+    return res.scalar_one_or_none()
+
+
+async def reject_if_blocked(
+    session: AsyncSession,
+    cred: AgentCredential,
+    agent_id: str,
+    hostname: str,
+    observed: datetime | None = None,
+) -> None:
+    row = await _blacklist_row(session, cred)
+    if row is None:
+        return
+    raise HTTPException(status_code=403, detail="agent blocked")
+
+
+async def _ensure_pending_capacity(session: AsyncSession) -> None:
+    max_pending = get_settings().max_pending_agents
+    if max_pending <= 0:
+        return
+    count = (
+        await session.execute(
+            select(func.count()).select_from(Agent).where(Agent.pending_key_hash.is_not(None))
+        )
+    ).scalar_one()
+    if count >= max_pending:
+        raise HTTPException(status_code=429, detail="too many pending agents")
+
+
+async def _reject_hostname_collision(session: AsyncSession, agent_id: str, hostname: str) -> None:
+    row = (
+        await session.execute(
+            select(Agent).where(Agent.hostname == hostname, Agent.agent_id != agent_id)
+        )
+    ).scalar_one_or_none()
+    if row is not None:
+        raise HTTPException(status_code=409, detail="hostname already belongs to another agent")
+
+
+def _set_pending(
+    agent: Agent,
+    hb: HeartbeatRequest,
+    cred: AgentCredential,
+    observed: datetime,
+    features: dict,
+) -> None:
+    agent.pending_key_hash = cred.key_hash
+    agent.pending_key_fingerprint = cred.key_fingerprint
+    agent.pending_hostname = hb.hostname
+    agent.pending_seen_at = observed
+    agent.pending_features = features
+    agent.pending_labels = hb.labels
+
+
+async def upsert_agent_heartbeat(
+    session: AsyncSession, hb: HeartbeatRequest, cred: AgentCredential
+) -> str:
     observed = to_utc(hb.observed_at)
-    stmt = pg_insert(Agent).values(
-        agent_id=hb.agent_id,
-        hostname=hb.hostname,
-        status="alive",
-        last_seen_at=observed,
-        features=hb.features.model_dump(),
-        labels=hb.labels,
+    await reject_if_blocked(session, cred, hb.agent_id, hb.hostname, observed)
+
+    features = hb.features.model_dump()
+    res = await session.execute(select(Agent).where(Agent.accepted_key_hash == cred.key_hash))
+    accepted = res.scalar_one_or_none()
+    if accepted is not None:
+        await _reject_hostname_collision(session, accepted.agent_id, hb.hostname)
+        accepted.hostname = hb.hostname
+        accepted.status = "alive"
+        accepted.last_seen_at = observed
+        accepted.features = features
+        accepted.labels = hb.labels
+        metrics.heartbeats_total.labels(result="ok").inc()
+        return "accepted"
+
+    res = await session.execute(select(Agent).where(Agent.pending_key_hash == cred.key_hash))
+    pending = res.scalar_one_or_none()
+    if pending is not None:
+        await _reject_hostname_collision(session, pending.agent_id, hb.hostname)
+        _set_pending(pending, hb, cred, observed, features)
+        if pending.accepted_key_hash is None:
+            pending.hostname = hb.hostname
+            pending.status = "alive"
+            pending.last_seen_at = observed
+            pending.features = features
+            pending.labels = hb.labels
+        metrics.heartbeats_total.labels(result="pending").inc()
+        return "pending"
+
+    hostname_row = (
+        await session.execute(select(Agent).where(Agent.hostname == hb.hostname))
+    ).scalar_one_or_none()
+    if hostname_row is not None:
+        if hostname_row.pending_key_hash is not None:
+            raise HTTPException(
+                status_code=409,
+                detail="agent already has a different pending key",
+            )
+        await _ensure_pending_capacity(session)
+        _set_pending(hostname_row, hb, cred, observed, features)
+        metrics.heartbeats_total.labels(result="pending").inc()
+        return "pending"
+
+    agent_id_row = (
+        await session.execute(select(Agent).where(Agent.agent_id == hb.agent_id))
+    ).scalar_one_or_none()
+    if agent_id_row is not None:
+        raise HTTPException(status_code=409, detail="agent_id already exists")
+
+    await _ensure_pending_capacity(session)
+    session.add(
+        Agent(
+            agent_id=hb.agent_id,
+            hostname=hb.hostname,
+            status="alive",
+            last_seen_at=observed,
+            features=features,
+            labels=hb.labels,
+            pending_key_hash=cred.key_hash,
+            pending_key_fingerprint=cred.key_fingerprint,
+            pending_hostname=hb.hostname,
+            pending_seen_at=observed,
+            pending_features=features,
+            pending_labels=hb.labels,
+        )
     )
-    stmt = stmt.on_conflict_do_update(
-        index_elements=[Agent.agent_id],
-        set_={
-            "hostname": stmt.excluded.hostname,
-            "status": "alive",
-            "last_seen_at": stmt.excluded.last_seen_at,
-            "features": stmt.excluded.features,
-            "labels": stmt.excluded.labels,
-        },
-    )
-    await session.execute(stmt)
-    metrics.heartbeats_total.labels(result="ok").inc()
+    metrics.heartbeats_total.labels(result="pending").inc()
+    return "pending"
+
+
+async def accepted_agent_for_key(session: AsyncSession, cred: AgentCredential) -> Agent | None:
+    if await _blacklist_row(session, cred) is not None:
+        raise HTTPException(status_code=403, detail="agent blocked")
+    res = await session.execute(select(Agent).where(Agent.accepted_key_hash == cred.key_hash))
+    return res.scalar_one_or_none()
 
 
 async def _apply_incident(

server/monlet_server/services/notifier_worker.py

@@ -25,6 +25,43 @@ def _backoff(attempts: int) -> timedelta:
     return timedelta(seconds=_BACKOFF_SCHEDULE_SEC[idx])
 
 
+def _apply_output_policy(payload: dict) -> dict:
+    """PH-020: enforce a single notifier output policy.
+
+    Per-notifier code keeps its own format-specific shortening (e.g. Telegram
+    300 chars), but a hard cap and include-flag apply consistently before any
+    delivery. Truncation uses a clear marker so receivers can see it happened.
+    """
+    settings = get_settings()
+    out = dict(payload)
+    text_value = out.get("output")
+    if text_value is None:
+        return out
+    if not settings.notifier_include_output:
+        out["output"] = None
+        out["output_truncated"] = True
+        return out
+    max_bytes = max(settings.notifier_max_output_bytes, 0)
+    encoded = text_value.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return out
+    # Reserve room for the marker so the final string never exceeds max_bytes.
+    # If max_bytes is smaller than even the marker, drop the body entirely —
+    # this keeps the absolute cap a hard guarantee under any setting.
+    dropped = len(encoded) - max_bytes
+    marker = f"...[truncated {dropped} bytes]"
+    marker_bytes = len(marker.encode("utf-8"))
+    if marker_bytes >= max_bytes:
+        out["output"] = ""
+        out["output_truncated"] = True
+        return out
+    prefix_budget = max_bytes - marker_bytes
+    truncated = encoded[:prefix_budget].decode("utf-8", errors="ignore")
+    out["output"] = f"{truncated}{marker}"
+    out["output_truncated"] = True
+    return out
+
+
 async def _claim_batch(session, limit: int, lease_sec: int) -> list[dict]:
     res = await session.execute(
         text(
@@ -107,8 +144,10 @@ async def _process_row(
             metrics.notifications_total.labels(notifier=row["notifier"], result="discarded").inc()
             return
 
+        # PH-020: apply central output policy before handing to any notifier.
+        delivery_payload = _apply_output_policy(row["payload"])
         try:
-            result = await notifier.deliver(row["event_type"], row["payload"])
+            result = await notifier.deliver(row["event_type"], delivery_payload)
         except Exception as exc:  # defensive: notifier should not raise
             result_ok = False
             result_perm = False

server/monlet_server/settings.py

@@ -13,9 +13,9 @@ class Settings(BaseSettings):
     body_limit_bytes: int = 1_048_576
     output_max_bytes: int = 8192
     max_batch_events: int = 200
-    stale_after_sec: int = 90
-    dead_after_sec: int = 300
-    detector_tick_sec: int = 15
+    stale_after_sec: int = 20
+    dead_after_sec: int = 30
+    detector_tick_sec: int = 5
     host: str = "0.0.0.0"
     port: int = 8000
     enable_detector: bool = True
@@ -24,8 +24,18 @@ class Settings(BaseSettings):
     events_future_partitions: int = 3
     event_dedup_retention_days: int = 30
     event_dedup_prune_batch_size: int = 5000
+    max_pending_agents: int = 10_000
+    pending_agent_ttl_days: int = 7
+    pending_agent_prune_batch_size: int = 1000
     partition_maintenance_interval_sec: int = 3600
     outbox_retention_max_rows: int = 50_000
+    # PH-012: bounded prune per maintenance tick to avoid one huge DELETE.
+    outbox_prune_batch_size: int = 5000
+    # PH-016: explicit DB pool sizing for production tuning.
+    db_pool_size: int = 10
+    db_max_overflow: int = 10
+    db_pool_timeout_sec: float = 30.0
+    db_pool_recycle_sec: int = 1800
     notifier_tick_sec: int = 5
     notifier_batch_size: int = 20
     notifier_max_attempts: int = 8
@@ -33,6 +43,11 @@ class Settings(BaseSettings):
     notifier_http_timeout_sec: float = 10.0
     notification_time_zone: str = "UTC"
 
+    # PH-020: notifier output policy. include controls whether output text is
+    # sent at all; max_bytes truncates it before delivery (after redaction).
+    notifier_include_output: bool = True
+    notifier_max_output_bytes: int = 1024
+
     notifier_debug_enabled: bool = True
     notifier_telegram_enabled: bool = False
     notifier_telegram_token: str = ""
@@ -43,12 +58,26 @@ class Settings(BaseSettings):
     notifier_alertmanager_enabled: bool = False
     notifier_alertmanager_url: str = ""
 
+    @staticmethod
+    def _validate_token_list(value: str, env_name: str) -> str:
+        if not value or value == "changeme":
+            raise ValueError(f"{env_name} must be set to a non-default value")
+        tokens = [t for t in value.replace(",", " ").split() if t]
+        if not tokens:
+            raise ValueError(f"{env_name} must contain at least one token")
+        if any(t == "changeme" for t in tokens):
+            raise ValueError(f"{env_name} must be set to a non-default value")
+        return value
+
     @field_validator("auth_token")
     @classmethod
     def _validate_auth_token(cls, value: str) -> str:
-        if not value or value == "changeme":
-            raise ValueError("MONLET_AUTH_TOKEN must be set to a non-default value")
-        return value
+        return cls._validate_token_list(value, "MONLET_AUTH_TOKEN")
+
+    @property
+    def auth_tokens(self) -> list[str]:
+        """Active Bearer tokens (>=1). Multiple tokens enable overlap rotation."""
+        return [t for t in self.auth_token.replace(",", " ").split() if t]
 
     @field_validator("notification_time_zone")
     @classmethod