Implement Monlet MVP stack and UI updates

server/monlet_server/services/detector.py

@@ -0,0 +1,90 @@
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Event, Incident, NotificationOutbox
+from ..settings import get_settings
+
+log = get_logger("monlet.detector")
+
+OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
+
+
+async def _prune_history(session) -> None:
+    s = get_settings()
+    if s.events_retention_max_rows > 0:
+        keep_events = (
+            select(Event.event_id)
+            .order_by(Event.received_at.desc(), Event.event_id.desc())
+            .limit(s.events_retention_max_rows)
+        )
+        await session.execute(delete(Event).where(Event.event_id.not_in(keep_events)))
+
+    if s.outbox_retention_max_rows > 0:
+        keep_outbox = (
+            select(NotificationOutbox.id)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .order_by(NotificationOutbox.created_at.desc(), NotificationOutbox.id.desc())
+            .limit(s.outbox_retention_max_rows)
+        )
+        await session.execute(
+            delete(NotificationOutbox)
+            .where(NotificationOutbox.state.in_(OUTBOX_TERMINAL_STATES))
+            .where(NotificationOutbox.id.not_in(keep_outbox))
+        )
+
+
+async def _tick(sm: async_sessionmaker) -> None:
+    s = get_settings()
+    now = datetime.now(UTC)
+    stale_cut = now - timedelta(seconds=s.stale_after_sec)
+    dead_cut = now - timedelta(seconds=s.dead_after_sec)
+    async with sm() as session:
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= dead_cut)
+            .where(Agent.status != "dead")
+            .values(status="dead")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at <= stale_cut)
+            .where(Agent.last_seen_at > dead_cut)
+            .where(Agent.status != "stale")
+            .values(status="stale")
+        )
+        await session.execute(
+            update(Agent)
+            .where(Agent.last_seen_at > stale_cut)
+            .where(Agent.status != "alive")
+            .values(status="alive")
+        )
+        await _prune_history(session)
+        await session.commit()
+
+        for status in ("alive", "stale", "dead"):
+            r = await session.execute(
+                select(func.count()).select_from(Agent).where(Agent.status == status)
+            )
+            metrics.agents_gauge.labels(status=status).set(r.scalar_one())
+        r = await session.execute(
+            select(func.count()).select_from(Incident).where(Incident.state == "open")
+        )
+        metrics.open_incidents_gauge.set(r.scalar_one())
+
+
+async def run_detector(sm: async_sessionmaker, stop_event: asyncio.Event) -> None:
+    tick = get_settings().detector_tick_sec
+    while not stop_event.is_set():
+        try:
+            await _tick(sm)
+        except Exception as exc:
+            log.warning("detector_tick_failed", error=str(exc))
+        try:
+            await asyncio.wait_for(stop_event.wait(), timeout=tick)
+        except TimeoutError:
+            continue

server/monlet_server/services/ingestion.py

@@ -0,0 +1,237 @@
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+from fastapi import HTTPException
+from sqlalchemy import select, text
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..models import Agent, Check, Event, Incident, NotificationOutbox
+from ..redaction import redact
+from ..schemas import CheckResultEvent, HeartbeatRequest
+from ..settings import get_settings
+
+INCIDENT_KEY_MAX = 256
+
+log = get_logger("monlet.ingestion")
+
+_SEVERITY_RANK = {"unknown": 1, "warning": 2, "critical": 3}
+
+
+def _severity_for_status(status: str) -> str:
+    if status in ("warning", "critical", "unknown"):
+        return status
+    return "unknown"
+
+
+def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
+    key = ev.incident_key or f"{agent_id}:{ev.check_id}"
+    if len(key) > INCIDENT_KEY_MAX:
+        raise HTTPException(
+            status_code=400,
+            detail=f"incident_key exceeds {INCIDENT_KEY_MAX} chars",
+        )
+    return key
+
+
+async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
+    observed = (
+        hb.observed_at.astimezone(UTC)
+        if hb.observed_at.tzinfo
+        else hb.observed_at.replace(tzinfo=UTC)
+    )
+    stmt = pg_insert(Agent).values(
+        agent_id=hb.agent_id,
+        hostname=hb.hostname,
+        status="alive",
+        last_seen_at=observed,
+        features=hb.features.model_dump(),
+        labels=hb.labels,
+    )
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[Agent.agent_id],
+        set_={
+            "hostname": stmt.excluded.hostname,
+            "status": "alive",
+            "last_seen_at": stmt.excluded.last_seen_at,
+            "features": stmt.excluded.features,
+            "labels": stmt.excluded.labels,
+        },
+    )
+    await session.execute(stmt)
+    metrics.heartbeats_total.labels(result="ok").inc()
+
+
+async def _apply_incident(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+    incident_key: str,
+) -> tuple[bool, Incident | None]:
+    """Return (transition, incident). transition True if open->resolved or none->open."""
+    observed = ev.observed_at
+    if ev.status == "ok":
+        res = await session.execute(
+            select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+        )
+        inc = res.scalar_one_or_none()
+        if inc is None:
+            return False, None
+        inc.state = "resolved"
+        inc.resolved_at = observed
+        inc.last_event_id = UUID(ev.event_id)
+        metrics.incidents_resolved_total.inc()
+        return True, inc
+
+    severity = _severity_for_status(ev.status)
+    new_id = uuid4()
+    summary = (redact(ev.output) or "")[:200] if ev.output else None
+    stmt = pg_insert(Incident).values(
+        id=new_id,
+        incident_key=incident_key,
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        state="open",
+        severity=severity,
+        opened_at=observed,
+        last_event_id=UUID(ev.event_id),
+        summary=summary,
+    )
+    stmt = stmt.on_conflict_do_nothing(
+        index_elements=[Incident.incident_key],
+        index_where=text("state = 'open'"),
+    ).returning(Incident.id)
+    ins = await session.execute(stmt)
+    inserted_id = ins.scalar_one_or_none()
+    if inserted_id is not None:
+        await session.flush()
+        res = await session.execute(select(Incident).where(Incident.id == inserted_id))
+        metrics.incidents_opened_total.inc()
+        return True, res.scalar_one()
+
+    res = await session.execute(
+        select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
+    )
+    inc = res.scalar_one()
+    if inc.agent_id != agent_id or inc.check_id != ev.check_id:
+        raise HTTPException(
+            status_code=400,
+            detail="incident_key already bound to another (agent_id, check_id)",
+        )
+    cur = _SEVERITY_RANK.get(inc.severity, 0)
+    new = _SEVERITY_RANK.get(severity, 0)
+    if new > cur:
+        inc.severity = severity
+    inc.last_event_id = UUID(ev.event_id)
+    return False, inc
+
+
+async def _enqueue_outbox(
+    session: AsyncSession,
+    incident: Incident,
+    event_type: str,
+    payload: dict,
+) -> None:
+    notifiers = get_settings().enabled_notifiers
+    if not notifiers:
+        return
+    now = datetime.now(UTC)
+    for name in notifiers:
+        session.add(
+            NotificationOutbox(
+                id=uuid4(),
+                incident_id=incident.id,
+                notifier=name,
+                event_type=event_type,
+                state="pending",
+                attempts=0,
+                next_attempt_at=now,
+                payload=payload,
+            )
+        )
+
+
+async def ingest_event(
+    session: AsyncSession,
+    agent_id: str,
+    ev: CheckResultEvent,
+) -> bool:
+    """Return True if accepted, False if deduplicated."""
+    observed = ev.observed_at
+    output = redact(ev.output)
+    incident_key = _incident_key(agent_id, ev)
+
+    stmt = pg_insert(Event).values(
+        event_id=UUID(ev.event_id),
+        agent_id=agent_id,
+        check_id=ev.check_id,
+        observed_at=observed,
+        status=ev.status,
+        exit_code=ev.exit_code,
+        duration_ms=ev.duration_ms,
+        output=output,
+        output_truncated=ev.output_truncated,
+        notifications_enabled=ev.notifications_enabled,
+        incident_key=incident_key,
+    )
+    stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
+    result = await session.execute(stmt)
+    inserted = result.scalar_one_or_none()
+    if inserted is None:
+        metrics.events_total.labels(result="deduplicated").inc()
+        return False
+
+    metrics.events_total.labels(result="accepted").inc()
+
+    chk_res = await session.execute(
+        select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
+    )
+    cur = chk_res.scalar_one_or_none()
+    is_late = cur is not None and cur.last_observed_at > observed
+    if is_late:
+        return True
+
+    if cur is None:
+        session.add(
+            Check(
+                agent_id=agent_id,
+                check_id=ev.check_id,
+                status=ev.status,
+                exit_code=ev.exit_code,
+                last_observed_at=observed,
+                last_event_id=UUID(ev.event_id),
+                incident_key=incident_key,
+            )
+        )
+    else:
+        cur.status = ev.status
+        cur.exit_code = ev.exit_code
+        cur.last_observed_at = observed
+        cur.last_event_id = UUID(ev.event_id)
+        cur.incident_key = incident_key
+
+    transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
+    if transition and inc is not None and ev.notifications_enabled:
+        event_type = "resolved" if ev.status == "ok" else "firing"
+        await _enqueue_outbox(
+            session,
+            inc,
+            event_type,
+            {
+                "incident_id": str(inc.id),
+                "agent_id": agent_id,
+                "check_id": ev.check_id,
+                "status": ev.status,
+                "severity": inc.severity,
+                "incident_key": incident_key,
+                "observed_at": observed.isoformat(),
+                "opened_at": inc.opened_at.isoformat() if inc.opened_at else None,
+                "resolved_at": inc.resolved_at.isoformat() if inc.resolved_at else None,
+                "summary": inc.summary,
+                "output": output,
+                "event_type": event_type,
+            },
+        )
+    return True

server/monlet_server/services/notifier_worker.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from .. import metrics
+from ..logging_config import get_logger
+from ..redaction import redact
+from ..settings import Settings, get_settings
+from .notifiers import Notifier, build_registry
+
+log = get_logger("monlet.notifier_worker")
+
+
+# Per ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
+_BACKOFF_SCHEDULE_SEC = (5, 15, 60, 300, 1800, 3600, 7200, 14400)
+
+
+def _backoff(attempts: int) -> timedelta:
+    idx = max(0, min(attempts - 1, len(_BACKOFF_SCHEDULE_SEC) - 1))
+    return timedelta(seconds=_BACKOFF_SCHEDULE_SEC[idx])
+
+
+async def _claim_batch(session, limit: int, lease_sec: int) -> list[dict]:
+    res = await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = 'sending', updated_at = now()
+             WHERE id IN (
+                SELECT id FROM notification_outbox
+                 WHERE (
+                        state IN ('pending','retry')
+                        AND (next_attempt_at IS NULL OR next_attempt_at <= now())
+                       )
+                    OR (
+                        state = 'sending'
+                        AND updated_at < now() - make_interval(secs => :lease)
+                       )
+                 ORDER BY next_attempt_at NULLS FIRST
+                 LIMIT :lim
+                 FOR UPDATE SKIP LOCKED
+             )
+         RETURNING id, notifier, event_type, attempts, payload
+            """
+        ),
+        {"lim": limit, "lease": lease_sec},
+    )
+    rows = res.mappings().all()
+    return [dict(r) for r in rows]
+
+
+async def _finalize(
+    session,
+    row_id,
+    *,
+    state: str,
+    attempts: int,
+    next_attempt_at: datetime | None,
+    last_error: str | None,
+) -> None:
+    await session.execute(
+        text(
+            """
+            UPDATE notification_outbox
+               SET state = :state,
+                   attempts = :attempts,
+                   next_attempt_at = :next_attempt_at,
+                   last_error = :last_error,
+                   updated_at = now()
+             WHERE id = :id
+            """
+        ),
+        {
+            "state": state,
+            "attempts": attempts,
+            "next_attempt_at": next_attempt_at,
+            "last_error": last_error,
+            "id": row_id,
+        },
+    )
+
+
+async def _process_row(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    row: dict,
+    max_attempts: int,
+) -> None:
+    notifier = registry.get(row["notifier"])
+    attempts = int(row["attempts"]) + 1
+    async with sm() as session:
+        if notifier is None:
+            await _finalize(
+                session,
+                row["id"],
+                state="discarded",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=f"unknown_notifier:{row['notifier']}",
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=row["notifier"], result="discarded").inc()
+            return
+
+        try:
+            result = await notifier.deliver(row["event_type"], row["payload"])
+        except Exception as exc:  # defensive: notifier should not raise
+            result_ok = False
+            result_perm = False
+            result_err = f"exception:{type(exc).__name__}"
+        else:
+            result_ok = result.ok
+            result_perm = result.permanent
+            result_err = redact(result.error) if result.error else None
+
+        if result_ok:
+            await _finalize(
+                session,
+                row["id"],
+                state="sent",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=None,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="sent").inc()
+            return
+
+        if result_perm or attempts >= max_attempts:
+            await _finalize(
+                session,
+                row["id"],
+                state="failed",
+                attempts=attempts,
+                next_attempt_at=None,
+                last_error=result_err,
+            )
+            await session.commit()
+            metrics.notifications_total.labels(notifier=notifier.name, result="failed").inc()
+            return
+
+        next_at = datetime.now(UTC) + _backoff(attempts)
+        await _finalize(
+            session,
+            row["id"],
+            state="retry",
+            attempts=attempts,
+            next_attempt_at=next_at,
+            last_error=result_err,
+        )
+        await session.commit()
+        metrics.notifications_total.labels(notifier=notifier.name, result="retry").inc()
+
+
+async def tick(
+    sm: async_sessionmaker,
+    registry: dict[str, Notifier],
+    settings: Settings,
+) -> int:
+    async with sm() as session:
+        rows = await _claim_batch(
+            session, settings.notifier_batch_size, settings.notifier_lease_sec
+        )
+        await session.commit()
+    if not rows:
+        return 0
+    await asyncio.gather(
+        *(_process_row(sm, registry, r, settings.notifier_max_attempts) for r in rows)
+    )
+    return len(rows)
+
+
+async def run_notifier_worker(
+    sm: async_sessionmaker,
+    stop_event: asyncio.Event,
+    client: httpx.AsyncClient | None = None,
+) -> None:
+    settings = get_settings()
+    own_client = False
+    if client is None:
+        client = httpx.AsyncClient(timeout=settings.notifier_http_timeout_sec)
+        own_client = True
+    registry = build_registry(settings, client)
+    if not registry:
+        log.info("notifier_worker_disabled_no_notifiers")
+        if own_client:
+            await client.aclose()
+        return
+    log.info("notifier_worker_started", notifiers=list(registry.keys()))
+    try:
+        while not stop_event.is_set():
+            try:
+                await tick(sm, registry, settings)
+            except Exception as exc:
+                log.warning("notifier_tick_failed", error=str(exc))
+            try:
+                await asyncio.wait_for(stop_event.wait(), timeout=settings.notifier_tick_sec)
+            except TimeoutError:
+                continue
+    finally:
+        if own_client:
+            await client.aclose()

server/monlet_server/services/notifiers/__init__.py

@@ -0,0 +1,4 @@
+from .base import DeliveryResult, Notifier
+from .registry import build_registry
+
+__all__ = ["DeliveryResult", "Notifier", "build_registry"]

server/monlet_server/services/notifiers/alertmanager.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class AlertmanagerNotifier:
+    name = "alertmanager"
+
+    def __init__(self, client: httpx.AsyncClient, url: str) -> None:
+        self._client = client
+        self._url = url.rstrip("/")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        labels = {
+            "alertname": "monlet_incident",
+            "agent_id": str(payload.get("agent_id", "")),
+            "check_id": str(payload.get("check_id", "")),
+            "incident_key": str(payload.get("incident_key", "")),
+        }
+        annotations = {
+            "severity": str(payload.get("severity", "unknown")),
+            "status": str(payload.get("status", "")),
+            "summary": redact(payload.get("summary") or "") or "",
+        }
+        if payload.get("output"):
+            annotations["output"] = (redact(payload["output"]) or "")[:1024]
+        alert = {
+            "labels": labels,
+            "annotations": annotations,
+            "startsAt": payload.get("opened_at") or payload.get("observed_at"),
+        }
+        if event_type == "resolved" and payload.get("resolved_at"):
+            alert["endsAt"] = payload["resolved_at"]
+        try:
+            resp = await self._client.post(f"{self._url}/api/v2/alerts", json=[alert])
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/base.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+    ok: bool
+    permanent: bool = False
+    error: str | None = None
+
+
+class Notifier(Protocol):
+    name: str
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult: ...

server/monlet_server/services/notifiers/debug.py

@@ -0,0 +1,14 @@
+from ...logging_config import get_logger
+from ...redaction import redact_dict
+from .base import DeliveryResult
+
+
+class DebugNotifier:
+    name = "debug"
+
+    def __init__(self) -> None:
+        self._log = get_logger("monlet.notifier.debug")
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        self._log.info("notify", event_type=event_type, payload=redact_dict(payload))
+        return DeliveryResult(ok=True)

server/monlet_server/services/notifiers/http.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import httpx
+
+
+def classify_response(status: int) -> tuple[bool, bool]:
+    """Return (ok, permanent_failure)."""
+    if 200 <= status < 300:
+        return True, False
+    if 400 <= status < 500 and status not in (408, 425, 429):
+        return False, True
+    return False, False
+
+
+def classify_exception(exc: Exception) -> tuple[bool, bool]:
+    if isinstance(exc, httpx.TimeoutException | httpx.NetworkError):
+        return False, False
+    return False, False

server/monlet_server/services/notifiers/registry.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import httpx
+
+from ...settings import Settings
+from .alertmanager import AlertmanagerNotifier
+from .base import Notifier
+from .debug import DebugNotifier
+from .telegram import TelegramNotifier
+from .webhook import WebhookNotifier
+
+
+def build_registry(settings: Settings, client: httpx.AsyncClient) -> dict[str, Notifier]:
+    reg: dict[str, Notifier] = {}
+    if settings.notifier_debug_enabled:
+        reg["debug"] = DebugNotifier()
+    if (
+        settings.notifier_telegram_enabled
+        and settings.notifier_telegram_token
+        and settings.notifier_telegram_chat_id
+    ):
+        reg["telegram"] = TelegramNotifier(
+            client, settings.notifier_telegram_token, settings.notifier_telegram_chat_id
+        )
+    if settings.notifier_webhook_enabled and settings.notifier_webhook_url:
+        reg["webhook"] = WebhookNotifier(
+            client, settings.notifier_webhook_url, settings.notifier_webhook_token
+        )
+    if settings.notifier_alertmanager_enabled and settings.notifier_alertmanager_url:
+        reg["alertmanager"] = AlertmanagerNotifier(client, settings.notifier_alertmanager_url)
+    return reg

server/monlet_server/services/notifiers/telegram.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact
+from .base import DeliveryResult
+from .http import classify_response
+
+
+def _format_message(event_type: str, payload: dict) -> str:
+    icon = "[FIRING]" if event_type == "firing" else "[RESOLVED]"
+    sev = payload.get("severity", "unknown")
+    agent = payload.get("agent_id", "?")
+    check = payload.get("check_id", "?")
+    status = payload.get("status", "?")
+    incident_key = payload.get("incident_key", "?")
+    output = payload.get("output") or ""
+    lines = [
+        f"{icon} {sev} {agent}/{check}",
+        f"status: {status}",
+        f"incident_key: {incident_key}",
+        f"observed_at: {payload.get('observed_at', '?')}",
+    ]
+    if output:
+        lines.append(f"output: {output[:300]}")
+    return redact("\n".join(lines)) or ""
+
+
+class TelegramNotifier:
+    name = "telegram"
+
+    def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str) -> None:
+        self._client = client
+        self._token = token
+        self._chat_id = chat_id
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        url = f"https://api.telegram.org/bot{self._token}/sendMessage"
+        body = {"chat_id": self._chat_id, "text": _format_message(event_type, payload)}
+        try:
+            resp = await self._client.post(url, json=body)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")

server/monlet_server/services/notifiers/webhook.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import httpx
+
+from ...redaction import redact_dict
+from .base import DeliveryResult
+from .http import classify_response
+
+
+class WebhookNotifier:
+    name = "webhook"
+
+    def __init__(self, client: httpx.AsyncClient, url: str, token: str = "") -> None:
+        self._client = client
+        self._url = url
+        self._token = token
+
+    async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
+        headers = {"Content-Type": "application/json"}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+        body = {"event_type": event_type, "incident": redact_dict(payload)}
+        try:
+            resp = await self._client.post(self._url, json=body, headers=headers)
+        except httpx.HTTPError as exc:
+            return DeliveryResult(ok=False, permanent=False, error=type(exc).__name__)
+        ok, perm = classify_response(resp.status_code)
+        if ok:
+            return DeliveryResult(ok=True)
+        return DeliveryResult(ok=False, permanent=perm, error=f"http_{resp.status_code}")