Fix notifier verification coverage

deploy/e2e-showcase.sh

@@ -288,8 +288,9 @@ log "web pages render with expected content"
 curl -sfL "$WEB_BASE/"         | grep -q "Agents"  || fail "web / did not redirect to Agents"
 curl -sf "$WEB_BASE/agents"    | grep -q "mixed-status" || fail "web /agents missing mixed-status label"
 curl -sf "$WEB_BASE/checks"    | grep -q "critical_check" || fail "web /checks missing critical_check"
-curl -sf "$WEB_BASE/incidents?state=open&limit=100&sort=status&direction=desc" \
+INCIDENTS_HTML="$(curl -sfL "$WEB_BASE/incidents?state=open&sort=status&direction=desc")"
-  | grep -q "agent-08" || fail "web /incidents missing agent-08"
+echo "$INCIDENTS_HTML" | grep -q "agent-07" || fail "web /incidents missing agent-07"
+echo "$INCIDENTS_HTML" | grep -q "agent-08" || fail "web /incidents missing agent-08"
 curl -sf "$WEB_BASE/outbox"    | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names"
 curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200"
 

server/pyproject.toml

@@ -48,4 +48,4 @@ ignore = ["E501", "B008"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
-testpaths = ["tests"]
+testpaths = ["unit_tests", "tests"]

server/tests/test_detector.py

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox
 from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
-from monlet_server.settings import reset_settings_cache
+from monlet_server.settings import get_settings, reset_settings_cache
 
 from ._helpers import make_event, register_agent
 
@@ -19,15 +19,20 @@ async def test_detector_transitions(app_client, ui_auth_headers, engine, session
     await register_agent(app_client, ui_auth_headers, "agent-dead")
 
     now = datetime.now(UTC)
+    settings = get_settings()
+    assert settings.stale_after_sec < settings.dead_after_sec
+    stale_age = settings.stale_after_sec + (
+        (settings.dead_after_sec - settings.stale_after_sec) / 2
+    )
     await session.execute(
         update(Agent)
         .where(Agent.agent_id == "agent-stale")
-        .values(last_seen_at=now - timedelta(seconds=120))
+        .values(last_seen_at=now - timedelta(seconds=stale_age))
     )
     await session.execute(
         update(Agent)
         .where(Agent.agent_id == "agent-dead")
-        .values(last_seen_at=now - timedelta(minutes=10))
+        .values(last_seen_at=now - timedelta(seconds=settings.dead_after_sec + 10))
     )
     await session.commit()
 

server/tests/test_notifiers.py

@@ -1,22 +1,16 @@
 from __future__ import annotations
 
-import json
 import os
 from datetime import UTC, datetime, timedelta
 from uuid import uuid4
 
-import httpx
 import pytest
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import async_sessionmaker
 
 from monlet_server.models import Incident, NotificationOutbox
-from monlet_server.services.notifier_worker import _backoff, tick
+from monlet_server.services.notifier_worker import tick
-from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
 from monlet_server.services.notifiers.base import DeliveryResult
-from monlet_server.services.notifiers.debug import DebugNotifier
-from monlet_server.services.notifiers.telegram import TelegramNotifier
-from monlet_server.services.notifiers.webhook import WebhookNotifier
 from monlet_server.settings import Settings
 
 
@@ -35,135 +29,6 @@ def _payload() -> dict:
     }
 
 
-class _StubTransport(httpx.AsyncBaseTransport):
-    def __init__(self, handler):
-        self._handler = handler
-        self.requests: list[httpx.Request] = []
-
-    async def handle_async_request(self, request):
-        self.requests.append(request)
-        return self._handler(request)
-
-
-def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
-    t = _StubTransport(handler)
-    return httpx.AsyncClient(transport=t, base_url="http://x"), t
-
-
-@pytest.mark.asyncio
-async def test_debug_notifier_always_ok():
-    res = await DebugNotifier().deliver("firing", _payload())
-    assert res.ok and not res.permanent
-
-
-@pytest.mark.asyncio
-async def test_telegram_success_and_redaction():
-    def handler(req):
-        body = req.content.decode()
-        assert "token=abc123" not in body
-        assert "2026-05-27 09:00:00 +04" in body
-        assert "***" in body
-        return httpx.Response(200, json={"ok": True})
-
-    client, t = _client(handler)
-    try:
-        n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi")
-        res = await n.deliver("firing", _payload())
-        assert res.ok
-        assert "/botTOK/sendMessage" in str(t.requests[0].url)
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_telegram_4xx_permanent():
-    client, _ = _client(lambda r: httpx.Response(400, json={}))
-    try:
-        n = TelegramNotifier(client, "TOK", "123", "UTC")
-        res = await n.deliver("firing", _payload())
-        assert not res.ok and res.permanent
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_telegram_5xx_transient():
-    client, _ = _client(lambda r: httpx.Response(503, json={}))
-    try:
-        n = TelegramNotifier(client, "TOK", "123", "UTC")
-        res = await n.deliver("firing", _payload())
-        assert not res.ok and not res.permanent
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_telegram_network_error_transient():
-    def handler(req):
-        raise httpx.ConnectError("nope")
-
-    client, _ = _client(handler)
-    try:
-        n = TelegramNotifier(client, "TOK", "123", "UTC")
-        res = await n.deliver("firing", _payload())
-        assert not res.ok and not res.permanent
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_webhook_sends_bearer_and_redacts_payload():
-    captured: dict = {}
-
-    def handler(req):
-        captured["auth"] = req.headers.get("authorization")
-        captured["body"] = req.content.decode()
-        return httpx.Response(202)
-
-    client, _ = _client(handler)
-    try:
-        n = WebhookNotifier(client, "http://x/hook", token="WTOK")
-        res = await n.deliver("firing", _payload())
-        assert res.ok
-        assert captured["auth"] == "Bearer WTOK"
-        assert "token=abc123" not in captured["body"]
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_alertmanager_payload_shape():
-    captured: dict = {}
-
-    def handler(req):
-        captured["url"] = str(req.url)
-        captured["body"] = req.content.decode()
-        return httpx.Response(200)
-
-    client, _ = _client(handler)
-    try:
-        n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi")
-        res = await n.deliver("firing", _payload())
-        assert res.ok
-        assert captured["url"].endswith("/api/v2/alerts")
-        body = json.loads(captured["body"])
-        alert = body[0]
-        assert alert["labels"]["alertname"] == "monlet_incident"
-        assert alert["startsAt"] == "2026-05-27T05:00:00+00:00"
-        assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04"
-    finally:
-        await client.aclose()
-
-
-def test_backoff_matches_adr_0005():
-    # ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
-    expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
-    for i, sec in enumerate(expected, start=1):
-        assert _backoff(i).total_seconds() == sec
-    # Caps at last entry.
-    assert _backoff(99).total_seconds() == 14400
-
-
 # ---- Worker integration with real PG ----
 
 
@@ -344,46 +209,6 @@ async def test_worker_recovers_stuck_sending(engine, settings_full):
         assert fresh.state == "sent"
 
 
-@pytest.mark.asyncio
-async def test_alertmanager_severity_not_in_labels():
-    captured: dict = {}
-
-    def handler(req):
-        captured["body"] = req.content.decode()
-        return httpx.Response(200)
-
-    client, _ = _client(handler)
-    try:
-        n = AlertmanagerNotifier(client, "http://am", "UTC")
-        await n.deliver("firing", _payload())
-        import json as _json
-
-        alerts = _json.loads(captured["body"])
-        assert "severity" not in alerts[0]["labels"]
-        assert alerts[0]["annotations"]["severity"] == "critical"
-    finally:
-        await client.aclose()
-
-
-@pytest.mark.asyncio
-async def test_telegram_redacts_secret_in_incident_key():
-    captured: dict = {}
-
-    def handler(req):
-        captured["body"] = req.content.decode()
-        return httpx.Response(200)
-
-    client, _ = _client(handler)
-    try:
-        p = _payload()
-        p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
-        n = TelegramNotifier(client, "TOK", "1", "UTC")
-        await n.deliver("firing", p)
-        assert "leakedsecret" not in captured["body"]
-    finally:
-        await client.aclose()
-
-
 @pytest.mark.asyncio
 async def test_fanout_creates_row_per_enabled_notifier(
     app_client, ui_auth_headers, session, monkeypatch

server/unit_tests/test_notifier_live_optional.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import os
+from uuid import uuid4
+
+import httpx
+import pytest
+
+from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
+from monlet_server.services.notifiers.telegram import TelegramNotifier
+from monlet_server.services.notifiers.webhook import WebhookNotifier
+
+pytestmark = pytest.mark.asyncio
+
+
+def _live_enabled() -> bool:
+    return os.environ.get("MONLET_LIVE_NOTIFIER_TESTS") == "1"
+
+
+def _payload() -> dict:
+    return {
+        "incident_id": str(uuid4()),
+        "agent_id": "release-smoke",
+        "check_id": "notifier",
+        "status": "critical",
+        "severity": "critical",
+        "incident_key": f"release-smoke:notifier:{uuid4()}",
+        "observed_at": "2026-06-23T00:00:00+00:00",
+        "opened_at": "2026-06-23T00:00:00+00:00",
+        "summary": "Monlet release notifier smoke",
+        "output": "release notifier smoke",
+    }
+
+
+def _require_live_env(*names: str) -> None:
+    missing = [name for name in names if not os.environ.get(name)]
+    if not _live_enabled() or missing:
+        pytest.skip(
+            "set MONLET_LIVE_NOTIFIER_TESTS=1 and required notifier env to run live smoke"
+        )
+
+
+async def test_live_webhook_notifier_smoke():
+    _require_live_env("MONLET_LIVE_WEBHOOK_URL")
+    async with httpx.AsyncClient(timeout=10) as client:
+        notifier = WebhookNotifier(
+            client,
+            os.environ["MONLET_LIVE_WEBHOOK_URL"],
+            os.environ.get("MONLET_LIVE_WEBHOOK_TOKEN", ""),
+            "UTC",
+        )
+        result = await notifier.deliver("firing", _payload())
+    assert result.ok, result.error
+
+
+async def test_live_telegram_notifier_smoke():
+    _require_live_env("MONLET_LIVE_TELEGRAM_TOKEN", "MONLET_LIVE_TELEGRAM_CHAT_ID")
+    async with httpx.AsyncClient(timeout=10) as client:
+        notifier = TelegramNotifier(
+            client,
+            os.environ["MONLET_LIVE_TELEGRAM_TOKEN"],
+            os.environ["MONLET_LIVE_TELEGRAM_CHAT_ID"],
+            "UTC",
+        )
+        result = await notifier.deliver("firing", _payload())
+    assert result.ok, result.error
+
+
+async def test_live_alertmanager_notifier_smoke():
+    _require_live_env("MONLET_LIVE_ALERTMANAGER_URL")
+    async with httpx.AsyncClient(timeout=10) as client:
+        notifier = AlertmanagerNotifier(
+            client,
+            os.environ["MONLET_LIVE_ALERTMANAGER_URL"],
+            "UTC",
+        )
+        result = await notifier.deliver("firing", _payload())
+    assert result.ok, result.error

server/unit_tests/test_notifiers.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import json
+from uuid import uuid4
+
+import httpx
+import pytest
+
+from monlet_server.services.notifier_worker import _backoff
+from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
+from monlet_server.services.notifiers.debug import DebugNotifier
+from monlet_server.services.notifiers.telegram import TelegramNotifier
+from monlet_server.services.notifiers.webhook import WebhookNotifier
+
+
+def _payload() -> dict:
+    return {
+        "incident_id": str(uuid4()),
+        "agent_id": "agent-1",
+        "check_id": "chk-1",
+        "status": "critical",
+        "severity": "critical",
+        "incident_key": "agent-1:chk-1",
+        "observed_at": "2026-05-27T05:00:00+00:00",
+        "opened_at": "2026-05-27T05:00:00+00:00",
+        "summary": "boom",
+        "output": "stderr: boom token=abc123",
+    }
+
+
+class _StubTransport(httpx.AsyncBaseTransport):
+    def __init__(self, handler):
+        self._handler = handler
+        self.requests: list[httpx.Request] = []
+
+    async def handle_async_request(self, request):
+        self.requests.append(request)
+        return self._handler(request)
+
+
+def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
+    transport = _StubTransport(handler)
+    return httpx.AsyncClient(transport=transport, base_url="http://x"), transport
+
+
+@pytest.mark.asyncio
+async def test_debug_notifier_always_ok():
+    res = await DebugNotifier().deliver("firing", _payload())
+    assert res.ok and not res.permanent
+
+
+@pytest.mark.asyncio
+async def test_telegram_success_and_redaction():
+    def handler(req):
+        body = req.content.decode()
+        assert "token=abc123" not in body
+        assert "2026-05-27 09:00:00 +04" in body
+        assert "***" in body
+        return httpx.Response(200, json={"ok": True})
+
+    client, transport = _client(handler)
+    try:
+        notifier = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi")
+        res = await notifier.deliver("firing", _payload())
+        assert res.ok
+        assert "/botTOK/sendMessage" in str(transport.requests[0].url)
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_4xx_permanent():
+    client, _ = _client(lambda r: httpx.Response(400, json={}))
+    try:
+        notifier = TelegramNotifier(client, "TOK", "123", "UTC")
+        res = await notifier.deliver("firing", _payload())
+        assert not res.ok and res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_5xx_transient():
+    client, _ = _client(lambda r: httpx.Response(503, json={}))
+    try:
+        notifier = TelegramNotifier(client, "TOK", "123", "UTC")
+        res = await notifier.deliver("firing", _payload())
+        assert not res.ok and not res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_network_error_transient():
+    def handler(req):
+        raise httpx.ConnectError("nope")
+
+    client, _ = _client(handler)
+    try:
+        notifier = TelegramNotifier(client, "TOK", "123", "UTC")
+        res = await notifier.deliver("firing", _payload())
+        assert not res.ok and not res.permanent
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_webhook_sends_bearer_and_redacts_payload():
+    captured: dict = {}
+
+    def handler(req):
+        captured["auth"] = req.headers.get("authorization")
+        captured["body"] = req.content.decode()
+        return httpx.Response(202)
+
+    client, _ = _client(handler)
+    try:
+        notifier = WebhookNotifier(client, "http://x/hook", token="WTOK")
+        res = await notifier.deliver("firing", _payload())
+        assert res.ok
+        assert captured["auth"] == "Bearer WTOK"
+        assert "token=abc123" not in captured["body"]
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_alertmanager_payload_shape():
+    captured: dict = {}
+
+    def handler(req):
+        captured["url"] = str(req.url)
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        notifier = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi")
+        res = await notifier.deliver("firing", _payload())
+        assert res.ok
+        assert captured["url"].endswith("/api/v2/alerts")
+        body = json.loads(captured["body"])
+        alert = body[0]
+        assert alert["labels"]["alertname"] == "monlet_incident"
+        assert alert["startsAt"] == "2026-05-27T05:00:00+00:00"
+        assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04"
+    finally:
+        await client.aclose()
+
+
+def test_backoff_schedule():
+    expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
+    for i, sec in enumerate(expected, start=1):
+        assert _backoff(i).total_seconds() == sec
+    assert _backoff(99).total_seconds() == 14400
+
+
+@pytest.mark.asyncio
+async def test_alertmanager_severity_not_in_labels():
+    captured: dict = {}
+
+    def handler(req):
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        notifier = AlertmanagerNotifier(client, "http://am", "UTC")
+        await notifier.deliver("firing", _payload())
+        alerts = json.loads(captured["body"])
+        assert "severity" not in alerts[0]["labels"]
+        assert alerts[0]["annotations"]["severity"] == "critical"
+    finally:
+        await client.aclose()
+
+
+@pytest.mark.asyncio
+async def test_telegram_redacts_secret_in_incident_key():
+    captured: dict = {}
+
+    def handler(req):
+        captured["body"] = req.content.decode()
+        return httpx.Response(200)
+
+    client, _ = _client(handler)
+    try:
+        payload = _payload()
+        payload["incident_key"] = "agent-1:chk-1:token=leakedsecret"
+        notifier = TelegramNotifier(client, "TOK", "1", "UTC")
+        await notifier.deliver("firing", payload)
+        assert "leakedsecret" not in captured["body"]
+    finally:
+        await client.aclose()