From 6dbae537259dfce80c5931feeede8ec058990be8 Mon Sep 17 00:00:00 2001 From: Stanislav Rossovskii Date: Tue, 23 Jun 2026 16:06:58 +0400 Subject: [PATCH] Fix notifier verification coverage --- deploy/e2e-showcase.sh | 5 +- server/pyproject.toml | 2 +- server/tests/test_detector.py | 11 +- server/tests/test_notifiers.py | 177 +--------------- .../unit_tests/test_notifier_live_optional.py | 78 +++++++ server/unit_tests/test_notifiers.py | 193 ++++++++++++++++++ 6 files changed, 284 insertions(+), 182 deletions(-) create mode 100644 server/unit_tests/test_notifier_live_optional.py create mode 100644 server/unit_tests/test_notifiers.py diff --git a/deploy/e2e-showcase.sh b/deploy/e2e-showcase.sh index d75fe6c..69e671b 100755 --- a/deploy/e2e-showcase.sh +++ b/deploy/e2e-showcase.sh @@ -288,8 +288,9 @@ log "web pages render with expected content" curl -sfL "$WEB_BASE/" | grep -q "Agents" || fail "web / did not redirect to Agents" curl -sf "$WEB_BASE/agents" | grep -q "mixed-status" || fail "web /agents missing mixed-status label" curl -sf "$WEB_BASE/checks" | grep -q "critical_check" || fail "web /checks missing critical_check" -curl -sf "$WEB_BASE/incidents?state=open&limit=100&sort=status&direction=desc" \ - | grep -q "agent-08" || fail "web /incidents missing agent-08" +INCIDENTS_HTML="$(curl -sfL "$WEB_BASE/incidents?state=open&sort=status&direction=desc")" +echo "$INCIDENTS_HTML" | grep -q "agent-07" || fail "web /incidents missing agent-07" +echo "$INCIDENTS_HTML" | grep -q "agent-08" || fail "web /incidents missing agent-08" curl -sf "$WEB_BASE/outbox" | grep -q -E 'webhook|debug' || fail "web /outbox missing notifier names" curl -sf -o /dev/null "$WEB_BASE/events" || fail "web /events returned non-200" diff --git a/server/pyproject.toml b/server/pyproject.toml index a75ec21..61f26ac 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -48,4 +48,4 @@ ignore = ["E501", "B008"] [tool.pytest.ini_options] asyncio_mode = "auto" -testpaths = ["tests"] +testpaths = ["unit_tests", "tests"] diff --git a/server/tests/test_detector.py b/server/tests/test_detector.py index fe30c7d..4a412cb 100644 --- a/server/tests/test_detector.py +++ b/server/tests/test_detector.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick -from monlet_server.settings import reset_settings_cache +from monlet_server.settings import get_settings, reset_settings_cache from ._helpers import make_event, register_agent @@ -19,15 +19,20 @@ async def test_detector_transitions(app_client, ui_auth_headers, engine, session await register_agent(app_client, ui_auth_headers, "agent-dead") now = datetime.now(UTC) + settings = get_settings() + assert settings.stale_after_sec < settings.dead_after_sec + stale_age = settings.stale_after_sec + ( + (settings.dead_after_sec - settings.stale_after_sec) / 2 + ) await session.execute( update(Agent) .where(Agent.agent_id == "agent-stale") - .values(last_seen_at=now - timedelta(seconds=120)) + .values(last_seen_at=now - timedelta(seconds=stale_age)) ) await session.execute( update(Agent) .where(Agent.agent_id == "agent-dead") - .values(last_seen_at=now - timedelta(minutes=10)) + .values(last_seen_at=now - timedelta(seconds=settings.dead_after_sec + 10)) ) await session.commit() diff --git a/server/tests/test_notifiers.py b/server/tests/test_notifiers.py index 5917dee..7b485a1 100644 --- a/server/tests/test_notifiers.py +++ b/server/tests/test_notifiers.py @@ -1,22 +1,16 @@ from __future__ import annotations -import json import os from datetime import UTC, datetime, timedelta from uuid import uuid4 -import httpx import pytest from sqlalchemy import select from sqlalchemy.ext.asyncio import async_sessionmaker from monlet_server.models import Incident, NotificationOutbox -from monlet_server.services.notifier_worker import _backoff, tick -from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier +from monlet_server.services.notifier_worker import tick from monlet_server.services.notifiers.base import DeliveryResult -from monlet_server.services.notifiers.debug import DebugNotifier -from monlet_server.services.notifiers.telegram import TelegramNotifier -from monlet_server.services.notifiers.webhook import WebhookNotifier from monlet_server.settings import Settings @@ -35,135 +29,6 @@ def _payload() -> dict: } -class _StubTransport(httpx.AsyncBaseTransport): - def __init__(self, handler): - self._handler = handler - self.requests: list[httpx.Request] = [] - - async def handle_async_request(self, request): - self.requests.append(request) - return self._handler(request) - - -def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]: - t = _StubTransport(handler) - return httpx.AsyncClient(transport=t, base_url="http://x"), t - - -@pytest.mark.asyncio -async def test_debug_notifier_always_ok(): - res = await DebugNotifier().deliver("firing", _payload()) - assert res.ok and not res.permanent - - -@pytest.mark.asyncio -async def test_telegram_success_and_redaction(): - def handler(req): - body = req.content.decode() - assert "token=abc123" not in body - assert "2026-05-27 09:00:00 +04" in body - assert "***" in body - return httpx.Response(200, json={"ok": True}) - - client, t = _client(handler) - try: - n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi") - res = await n.deliver("firing", _payload()) - assert res.ok - assert "/botTOK/sendMessage" in str(t.requests[0].url) - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_telegram_4xx_permanent(): - client, _ = _client(lambda r: httpx.Response(400, json={})) - try: - n = TelegramNotifier(client, "TOK", "123", "UTC") - res = await n.deliver("firing", _payload()) - assert not res.ok and res.permanent - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_telegram_5xx_transient(): - client, _ = _client(lambda r: httpx.Response(503, json={})) - try: - n = TelegramNotifier(client, "TOK", "123", "UTC") - res = await n.deliver("firing", _payload()) - assert not res.ok and not res.permanent - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_telegram_network_error_transient(): - def handler(req): - raise httpx.ConnectError("nope") - - client, _ = _client(handler) - try: - n = TelegramNotifier(client, "TOK", "123", "UTC") - res = await n.deliver("firing", _payload()) - assert not res.ok and not res.permanent - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_webhook_sends_bearer_and_redacts_payload(): - captured: dict = {} - - def handler(req): - captured["auth"] = req.headers.get("authorization") - captured["body"] = req.content.decode() - return httpx.Response(202) - - client, _ = _client(handler) - try: - n = WebhookNotifier(client, "http://x/hook", token="WTOK") - res = await n.deliver("firing", _payload()) - assert res.ok - assert captured["auth"] == "Bearer WTOK" - assert "token=abc123" not in captured["body"] - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_alertmanager_payload_shape(): - captured: dict = {} - - def handler(req): - captured["url"] = str(req.url) - captured["body"] = req.content.decode() - return httpx.Response(200) - - client, _ = _client(handler) - try: - n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi") - res = await n.deliver("firing", _payload()) - assert res.ok - assert captured["url"].endswith("/api/v2/alerts") - body = json.loads(captured["body"]) - alert = body[0] - assert alert["labels"]["alertname"] == "monlet_incident" - assert alert["startsAt"] == "2026-05-27T05:00:00+00:00" - assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04" - finally: - await client.aclose() - - -def test_backoff_matches_adr_0005(): - # ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h. - expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400] - for i, sec in enumerate(expected, start=1): - assert _backoff(i).total_seconds() == sec - # Caps at last entry. - assert _backoff(99).total_seconds() == 14400 - - # ---- Worker integration with real PG ---- @@ -344,46 +209,6 @@ async def test_worker_recovers_stuck_sending(engine, settings_full): assert fresh.state == "sent" -@pytest.mark.asyncio -async def test_alertmanager_severity_not_in_labels(): - captured: dict = {} - - def handler(req): - captured["body"] = req.content.decode() - return httpx.Response(200) - - client, _ = _client(handler) - try: - n = AlertmanagerNotifier(client, "http://am", "UTC") - await n.deliver("firing", _payload()) - import json as _json - - alerts = _json.loads(captured["body"]) - assert "severity" not in alerts[0]["labels"] - assert alerts[0]["annotations"]["severity"] == "critical" - finally: - await client.aclose() - - -@pytest.mark.asyncio -async def test_telegram_redacts_secret_in_incident_key(): - captured: dict = {} - - def handler(req): - captured["body"] = req.content.decode() - return httpx.Response(200) - - client, _ = _client(handler) - try: - p = _payload() - p["incident_key"] = "agent-1:chk-1:token=leakedsecret" - n = TelegramNotifier(client, "TOK", "1", "UTC") - await n.deliver("firing", p) - assert "leakedsecret" not in captured["body"] - finally: - await client.aclose() - - @pytest.mark.asyncio async def test_fanout_creates_row_per_enabled_notifier( app_client, ui_auth_headers, session, monkeypatch diff --git a/server/unit_tests/test_notifier_live_optional.py b/server/unit_tests/test_notifier_live_optional.py new file mode 100644 index 0000000..63f96f1 --- /dev/null +++ b/server/unit_tests/test_notifier_live_optional.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import os +from uuid import uuid4 + +import httpx +import pytest + +from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier +from monlet_server.services.notifiers.telegram import TelegramNotifier +from monlet_server.services.notifiers.webhook import WebhookNotifier + +pytestmark = pytest.mark.asyncio + + +def _live_enabled() -> bool: + return os.environ.get("MONLET_LIVE_NOTIFIER_TESTS") == "1" + + +def _payload() -> dict: + return { + "incident_id": str(uuid4()), + "agent_id": "release-smoke", + "check_id": "notifier", + "status": "critical", + "severity": "critical", + "incident_key": f"release-smoke:notifier:{uuid4()}", + "observed_at": "2026-06-23T00:00:00+00:00", + "opened_at": "2026-06-23T00:00:00+00:00", + "summary": "Monlet release notifier smoke", + "output": "release notifier smoke", + } + + +def _require_live_env(*names: str) -> None: + missing = [name for name in names if not os.environ.get(name)] + if not _live_enabled() or missing: + pytest.skip( + "set MONLET_LIVE_NOTIFIER_TESTS=1 and required notifier env to run live smoke" + ) + + +async def test_live_webhook_notifier_smoke(): + _require_live_env("MONLET_LIVE_WEBHOOK_URL") + async with httpx.AsyncClient(timeout=10) as client: + notifier = WebhookNotifier( + client, + os.environ["MONLET_LIVE_WEBHOOK_URL"], + os.environ.get("MONLET_LIVE_WEBHOOK_TOKEN", ""), + "UTC", + ) + result = await notifier.deliver("firing", _payload()) + assert result.ok, result.error + + +async def test_live_telegram_notifier_smoke(): + _require_live_env("MONLET_LIVE_TELEGRAM_TOKEN", "MONLET_LIVE_TELEGRAM_CHAT_ID") + async with httpx.AsyncClient(timeout=10) as client: + notifier = TelegramNotifier( + client, + os.environ["MONLET_LIVE_TELEGRAM_TOKEN"], + os.environ["MONLET_LIVE_TELEGRAM_CHAT_ID"], + "UTC", + ) + result = await notifier.deliver("firing", _payload()) + assert result.ok, result.error + + +async def test_live_alertmanager_notifier_smoke(): + _require_live_env("MONLET_LIVE_ALERTMANAGER_URL") + async with httpx.AsyncClient(timeout=10) as client: + notifier = AlertmanagerNotifier( + client, + os.environ["MONLET_LIVE_ALERTMANAGER_URL"], + "UTC", + ) + result = await notifier.deliver("firing", _payload()) + assert result.ok, result.error diff --git a/server/unit_tests/test_notifiers.py b/server/unit_tests/test_notifiers.py new file mode 100644 index 0000000..4f2d23e --- /dev/null +++ b/server/unit_tests/test_notifiers.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json +from uuid import uuid4 + +import httpx +import pytest + +from monlet_server.services.notifier_worker import _backoff +from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier +from monlet_server.services.notifiers.debug import DebugNotifier +from monlet_server.services.notifiers.telegram import TelegramNotifier +from monlet_server.services.notifiers.webhook import WebhookNotifier + + +def _payload() -> dict: + return { + "incident_id": str(uuid4()), + "agent_id": "agent-1", + "check_id": "chk-1", + "status": "critical", + "severity": "critical", + "incident_key": "agent-1:chk-1", + "observed_at": "2026-05-27T05:00:00+00:00", + "opened_at": "2026-05-27T05:00:00+00:00", + "summary": "boom", + "output": "stderr: boom token=abc123", + } + + +class _StubTransport(httpx.AsyncBaseTransport): + def __init__(self, handler): + self._handler = handler + self.requests: list[httpx.Request] = [] + + async def handle_async_request(self, request): + self.requests.append(request) + return self._handler(request) + + +def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]: + transport = _StubTransport(handler) + return httpx.AsyncClient(transport=transport, base_url="http://x"), transport + + +@pytest.mark.asyncio +async def test_debug_notifier_always_ok(): + res = await DebugNotifier().deliver("firing", _payload()) + assert res.ok and not res.permanent + + +@pytest.mark.asyncio +async def test_telegram_success_and_redaction(): + def handler(req): + body = req.content.decode() + assert "token=abc123" not in body + assert "2026-05-27 09:00:00 +04" in body + assert "***" in body + return httpx.Response(200, json={"ok": True}) + + client, transport = _client(handler) + try: + notifier = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi") + res = await notifier.deliver("firing", _payload()) + assert res.ok + assert "/botTOK/sendMessage" in str(transport.requests[0].url) + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_telegram_4xx_permanent(): + client, _ = _client(lambda r: httpx.Response(400, json={})) + try: + notifier = TelegramNotifier(client, "TOK", "123", "UTC") + res = await notifier.deliver("firing", _payload()) + assert not res.ok and res.permanent + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_telegram_5xx_transient(): + client, _ = _client(lambda r: httpx.Response(503, json={})) + try: + notifier = TelegramNotifier(client, "TOK", "123", "UTC") + res = await notifier.deliver("firing", _payload()) + assert not res.ok and not res.permanent + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_telegram_network_error_transient(): + def handler(req): + raise httpx.ConnectError("nope") + + client, _ = _client(handler) + try: + notifier = TelegramNotifier(client, "TOK", "123", "UTC") + res = await notifier.deliver("firing", _payload()) + assert not res.ok and not res.permanent + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_webhook_sends_bearer_and_redacts_payload(): + captured: dict = {} + + def handler(req): + captured["auth"] = req.headers.get("authorization") + captured["body"] = req.content.decode() + return httpx.Response(202) + + client, _ = _client(handler) + try: + notifier = WebhookNotifier(client, "http://x/hook", token="WTOK") + res = await notifier.deliver("firing", _payload()) + assert res.ok + assert captured["auth"] == "Bearer WTOK" + assert "token=abc123" not in captured["body"] + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_alertmanager_payload_shape(): + captured: dict = {} + + def handler(req): + captured["url"] = str(req.url) + captured["body"] = req.content.decode() + return httpx.Response(200) + + client, _ = _client(handler) + try: + notifier = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi") + res = await notifier.deliver("firing", _payload()) + assert res.ok + assert captured["url"].endswith("/api/v2/alerts") + body = json.loads(captured["body"]) + alert = body[0] + assert alert["labels"]["alertname"] == "monlet_incident" + assert alert["startsAt"] == "2026-05-27T05:00:00+00:00" + assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04" + finally: + await client.aclose() + + +def test_backoff_schedule(): + expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400] + for i, sec in enumerate(expected, start=1): + assert _backoff(i).total_seconds() == sec + assert _backoff(99).total_seconds() == 14400 + + +@pytest.mark.asyncio +async def test_alertmanager_severity_not_in_labels(): + captured: dict = {} + + def handler(req): + captured["body"] = req.content.decode() + return httpx.Response(200) + + client, _ = _client(handler) + try: + notifier = AlertmanagerNotifier(client, "http://am", "UTC") + await notifier.deliver("firing", _payload()) + alerts = json.loads(captured["body"]) + assert "severity" not in alerts[0]["labels"] + assert alerts[0]["annotations"]["severity"] == "critical" + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_telegram_redacts_secret_in_incident_key(): + captured: dict = {} + + def handler(req): + captured["body"] = req.content.decode() + return httpx.Response(200) + + client, _ = _client(handler) + try: + payload = _payload() + payload["incident_key"] = "agent-1:chk-1:token=leakedsecret" + notifier = TelegramNotifier(client, "TOK", "1", "UTC") + await notifier.deliver("firing", payload) + assert "leakedsecret" not in captured["body"] + finally: + await client.aclose()