from __future__ import annotations import json import os from datetime import UTC, datetime, timedelta from uuid import uuid4 import httpx import pytest from sqlalchemy import select from sqlalchemy.ext.asyncio import async_sessionmaker from monlet_server.models import Incident, NotificationOutbox from monlet_server.services.notifier_worker import _backoff, tick from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier from monlet_server.services.notifiers.base import DeliveryResult from monlet_server.services.notifiers.debug import DebugNotifier from monlet_server.services.notifiers.telegram import TelegramNotifier from monlet_server.services.notifiers.webhook import WebhookNotifier from monlet_server.settings import Settings def _payload() -> dict: return { "incident_id": str(uuid4()), "agent_id": "agent-1", "check_id": "chk-1", "status": "critical", "severity": "critical", "incident_key": "agent-1:chk-1", "observed_at": "2026-05-27T05:00:00+00:00", "opened_at": "2026-05-27T05:00:00+00:00", "summary": "boom", "output": "stderr: boom token=abc123", } class _StubTransport(httpx.AsyncBaseTransport): def __init__(self, handler): self._handler = handler self.requests: list[httpx.Request] = [] async def handle_async_request(self, request): self.requests.append(request) return self._handler(request) def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]: t = _StubTransport(handler) return httpx.AsyncClient(transport=t, base_url="http://x"), t @pytest.mark.asyncio async def test_debug_notifier_always_ok(): res = await DebugNotifier().deliver("firing", _payload()) assert res.ok and not res.permanent @pytest.mark.asyncio async def test_telegram_success_and_redaction(): def handler(req): body = req.content.decode() assert "token=abc123" not in body assert "2026-05-27 09:00:00 +04" in body assert "***" in body return httpx.Response(200, json={"ok": True}) client, t = _client(handler) try: n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi") res = await n.deliver("firing", _payload()) assert res.ok assert "/botTOK/sendMessage" in str(t.requests[0].url) finally: await client.aclose() @pytest.mark.asyncio async def test_telegram_4xx_permanent(): client, _ = _client(lambda r: httpx.Response(400, json={})) try: n = TelegramNotifier(client, "TOK", "123", "UTC") res = await n.deliver("firing", _payload()) assert not res.ok and res.permanent finally: await client.aclose() @pytest.mark.asyncio async def test_telegram_5xx_transient(): client, _ = _client(lambda r: httpx.Response(503, json={})) try: n = TelegramNotifier(client, "TOK", "123", "UTC") res = await n.deliver("firing", _payload()) assert not res.ok and not res.permanent finally: await client.aclose() @pytest.mark.asyncio async def test_telegram_network_error_transient(): def handler(req): raise httpx.ConnectError("nope") client, _ = _client(handler) try: n = TelegramNotifier(client, "TOK", "123", "UTC") res = await n.deliver("firing", _payload()) assert not res.ok and not res.permanent finally: await client.aclose() @pytest.mark.asyncio async def test_webhook_sends_bearer_and_redacts_payload(): captured: dict = {} def handler(req): captured["auth"] = req.headers.get("authorization") captured["body"] = req.content.decode() return httpx.Response(202) client, _ = _client(handler) try: n = WebhookNotifier(client, "http://x/hook", token="WTOK") res = await n.deliver("firing", _payload()) assert res.ok assert captured["auth"] == "Bearer WTOK" assert "token=abc123" not in captured["body"] finally: await client.aclose() @pytest.mark.asyncio async def test_alertmanager_payload_shape(): captured: dict = {} def handler(req): captured["url"] = str(req.url) captured["body"] = req.content.decode() return httpx.Response(200) client, _ = _client(handler) try: n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi") res = await n.deliver("firing", _payload()) assert res.ok assert captured["url"].endswith("/api/v2/alerts") body = json.loads(captured["body"]) alert = body[0] assert alert["labels"]["alertname"] == "monlet_incident" assert alert["startsAt"] == "2026-05-27T05:00:00+00:00" assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04" finally: await client.aclose() def test_backoff_matches_adr_0005(): # ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h. expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400] for i, sec in enumerate(expected, start=1): assert _backoff(i).total_seconds() == sec # Caps at last entry. assert _backoff(99).total_seconds() == 14400 # ---- Worker integration with real PG ---- @pytest.fixture def settings_full() -> Settings: s = Settings() s.notifier_max_attempts = 3 s.notifier_batch_size = 50 return s async def _seed_incident_and_outbox( sm: async_sessionmaker, notifier: str ) -> tuple[NotificationOutbox, Incident]: async with sm() as session: # Need agent for FK on incidents.agent_id? No FK on incidents.agent_id per schema. inc = Incident( id=uuid4(), incident_key=f"k-{uuid4()}", agent_id="agent-1", check_id="chk-1", state="open", severity="critical", opened_at=datetime.now(UTC), last_event_id=uuid4(), summary="boom", ) session.add(inc) await session.flush() row = NotificationOutbox( id=uuid4(), incident_id=inc.id, notifier=notifier, event_type="firing", state="pending", attempts=0, next_attempt_at=datetime.now(UTC) - timedelta(seconds=1), payload=_payload(), ) session.add(row) await session.commit() return row, inc class _StubNotifier: def __init__(self, name: str, results: list[DeliveryResult]) -> None: self.name = name self._results = list(results) self.calls = 0 async def deliver(self, event_type: str, payload: dict) -> DeliveryResult: self.calls += 1 if self._results: return self._results.pop(0) return DeliveryResult(ok=True) @pytest.mark.asyncio async def test_worker_marks_sent(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "stub") registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])} n = await tick(sm, registry, settings_full) assert n == 1 async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "sent" assert fresh.attempts == 1 assert fresh.last_error is None @pytest.mark.asyncio async def test_worker_retries_then_succeeds(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "stub") notifier = _StubNotifier( "stub", [DeliveryResult(ok=False, permanent=False, error="net"), DeliveryResult(ok=True)], ) registry = {"stub": notifier} await tick(sm, registry, settings_full) async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "retry" assert fresh.attempts == 1 assert fresh.next_attempt_at is not None # Force eligibility for second tick. fresh.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1) await session.commit() await tick(sm, registry, settings_full) async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "sent" assert fresh.attempts == 2 @pytest.mark.asyncio async def test_worker_permanent_failure_marked_failed(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "stub") registry = { "stub": _StubNotifier("stub", [DeliveryResult(ok=False, permanent=True, error="http_400")]) } await tick(sm, registry, settings_full) async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "failed" assert fresh.last_error == "http_400" @pytest.mark.asyncio async def test_worker_unknown_notifier_discarded(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "ghost") await tick(sm, {}, settings_full) async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "discarded" @pytest.mark.asyncio async def test_worker_exhausts_attempts(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "stub") notifier = _StubNotifier( "stub", [DeliveryResult(ok=False, permanent=False, error="boom")] * 10, ) registry = {"stub": notifier} for _ in range(settings_full.notifier_max_attempts): async with sm() as session: await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) # Force ready. await session.execute( NotificationOutbox.__table__.update() .where(NotificationOutbox.id == row.id) .values(next_attempt_at=datetime.now(UTC) - timedelta(seconds=1)) ) await session.commit() await tick(sm, registry, settings_full) async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "failed" assert fresh.attempts == settings_full.notifier_max_attempts @pytest.mark.asyncio async def test_worker_recovers_stuck_sending(engine, settings_full): sm = async_sessionmaker(engine, expire_on_commit=False) row, _ = await _seed_incident_and_outbox(sm, "stub") async with sm() as session: await session.execute( NotificationOutbox.__table__.update() .where(NotificationOutbox.id == row.id) .values(state="sending", updated_at=datetime.now(UTC) - timedelta(hours=1)) ) await session.commit() settings_full.notifier_lease_sec = 60 registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])} n = await tick(sm, registry, settings_full) assert n == 1 async with sm() as session: fresh = ( await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id)) ).scalar_one() assert fresh.state == "sent" @pytest.mark.asyncio async def test_alertmanager_severity_not_in_labels(): captured: dict = {} def handler(req): captured["body"] = req.content.decode() return httpx.Response(200) client, _ = _client(handler) try: n = AlertmanagerNotifier(client, "http://am", "UTC") await n.deliver("firing", _payload()) import json as _json alerts = _json.loads(captured["body"]) assert "severity" not in alerts[0]["labels"] assert alerts[0]["annotations"]["severity"] == "critical" finally: await client.aclose() @pytest.mark.asyncio async def test_telegram_redacts_secret_in_incident_key(): captured: dict = {} def handler(req): captured["body"] = req.content.decode() return httpx.Response(200) client, _ = _client(handler) try: p = _payload() p["incident_key"] = "agent-1:chk-1:token=leakedsecret" n = TelegramNotifier(client, "TOK", "1", "UTC") await n.deliver("firing", p) assert "leakedsecret" not in captured["body"] finally: await client.aclose() @pytest.mark.asyncio async def test_fanout_creates_row_per_enabled_notifier( app_client, ui_auth_headers, session, monkeypatch ): # Enable telegram (token+chat_id satisfied) and webhook in addition to debug. os.environ["MONLET_NOTIFIER_TELEGRAM_ENABLED"] = "true" os.environ["MONLET_NOTIFIER_TELEGRAM_TOKEN"] = "tok" os.environ["MONLET_NOTIFIER_TELEGRAM_CHAT_ID"] = "1" os.environ["MONLET_NOTIFIER_WEBHOOK_ENABLED"] = "true" os.environ["MONLET_NOTIFIER_WEBHOOK_URL"] = "http://x/hook" from monlet_server.settings import reset_settings_cache reset_settings_cache() try: from tests._helpers import make_event, register_agent auth_headers = await register_agent(app_client, ui_auth_headers) ev = make_event(status="critical", exit_code=2) r = await app_client.post( "/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers, ) assert r.status_code == 202 rows = (await session.execute(select(NotificationOutbox))).scalars().all() names = sorted(r.notifier for r in rows) assert names == ["debug", "telegram", "webhook"] finally: for k in ( "MONLET_NOTIFIER_TELEGRAM_ENABLED", "MONLET_NOTIFIER_TELEGRAM_TOKEN", "MONLET_NOTIFIER_TELEGRAM_CHAT_ID", "MONLET_NOTIFIER_WEBHOOK_ENABLED", "MONLET_NOTIFIER_WEBHOOK_URL", ): os.environ.pop(k, None) reset_settings_cache()