Fix notifier verification coverage
This commit is contained in:
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox
|
||||
from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
from monlet_server.settings import get_settings, reset_settings_cache
|
||||
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
@@ -19,15 +19,20 @@ async def test_detector_transitions(app_client, ui_auth_headers, engine, session
|
||||
await register_agent(app_client, ui_auth_headers, "agent-dead")
|
||||
|
||||
now = datetime.now(UTC)
|
||||
settings = get_settings()
|
||||
assert settings.stale_after_sec < settings.dead_after_sec
|
||||
stale_age = settings.stale_after_sec + (
|
||||
(settings.dead_after_sec - settings.stale_after_sec) / 2
|
||||
)
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.agent_id == "agent-stale")
|
||||
.values(last_seen_at=now - timedelta(seconds=120))
|
||||
.values(last_seen_at=now - timedelta(seconds=stale_age))
|
||||
)
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.agent_id == "agent-dead")
|
||||
.values(last_seen_at=now - timedelta(minutes=10))
|
||||
.values(last_seen_at=now - timedelta(seconds=settings.dead_after_sec + 10))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
@@ -1,22 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from monlet_server.models import Incident, NotificationOutbox
|
||||
from monlet_server.services.notifier_worker import _backoff, tick
|
||||
from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
|
||||
from monlet_server.services.notifier_worker import tick
|
||||
from monlet_server.services.notifiers.base import DeliveryResult
|
||||
from monlet_server.services.notifiers.debug import DebugNotifier
|
||||
from monlet_server.services.notifiers.telegram import TelegramNotifier
|
||||
from monlet_server.services.notifiers.webhook import WebhookNotifier
|
||||
from monlet_server.settings import Settings
|
||||
|
||||
|
||||
@@ -35,135 +29,6 @@ def _payload() -> dict:
|
||||
}
|
||||
|
||||
|
||||
class _StubTransport(httpx.AsyncBaseTransport):
|
||||
def __init__(self, handler):
|
||||
self._handler = handler
|
||||
self.requests: list[httpx.Request] = []
|
||||
|
||||
async def handle_async_request(self, request):
|
||||
self.requests.append(request)
|
||||
return self._handler(request)
|
||||
|
||||
|
||||
def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
|
||||
t = _StubTransport(handler)
|
||||
return httpx.AsyncClient(transport=t, base_url="http://x"), t
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_debug_notifier_always_ok():
|
||||
res = await DebugNotifier().deliver("firing", _payload())
|
||||
assert res.ok and not res.permanent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_success_and_redaction():
|
||||
def handler(req):
|
||||
body = req.content.decode()
|
||||
assert "token=abc123" not in body
|
||||
assert "2026-05-27 09:00:00 +04" in body
|
||||
assert "***" in body
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
client, t = _client(handler)
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert "/botTOK/sendMessage" in str(t.requests[0].url)
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_4xx_permanent():
|
||||
client, _ = _client(lambda r: httpx.Response(400, json={}))
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123", "UTC")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_5xx_transient():
|
||||
client, _ = _client(lambda r: httpx.Response(503, json={}))
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123", "UTC")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and not res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_network_error_transient():
|
||||
def handler(req):
|
||||
raise httpx.ConnectError("nope")
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123", "UTC")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and not res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_webhook_sends_bearer_and_redacts_payload():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["auth"] = req.headers.get("authorization")
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(202)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = WebhookNotifier(client, "http://x/hook", token="WTOK")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert captured["auth"] == "Bearer WTOK"
|
||||
assert "token=abc123" not in captured["body"]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alertmanager_payload_shape():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["url"] = str(req.url)
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert captured["url"].endswith("/api/v2/alerts")
|
||||
body = json.loads(captured["body"])
|
||||
alert = body[0]
|
||||
assert alert["labels"]["alertname"] == "monlet_incident"
|
||||
assert alert["startsAt"] == "2026-05-27T05:00:00+00:00"
|
||||
assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04"
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def test_backoff_matches_adr_0005():
|
||||
# ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
|
||||
expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
|
||||
for i, sec in enumerate(expected, start=1):
|
||||
assert _backoff(i).total_seconds() == sec
|
||||
# Caps at last entry.
|
||||
assert _backoff(99).total_seconds() == 14400
|
||||
|
||||
|
||||
# ---- Worker integration with real PG ----
|
||||
|
||||
|
||||
@@ -344,46 +209,6 @@ async def test_worker_recovers_stuck_sending(engine, settings_full):
|
||||
assert fresh.state == "sent"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alertmanager_severity_not_in_labels():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = AlertmanagerNotifier(client, "http://am", "UTC")
|
||||
await n.deliver("firing", _payload())
|
||||
import json as _json
|
||||
|
||||
alerts = _json.loads(captured["body"])
|
||||
assert "severity" not in alerts[0]["labels"]
|
||||
assert alerts[0]["annotations"]["severity"] == "critical"
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_redacts_secret_in_incident_key():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
p = _payload()
|
||||
p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
|
||||
n = TelegramNotifier(client, "TOK", "1", "UTC")
|
||||
await n.deliver("firing", p)
|
||||
assert "leakedsecret" not in captured["body"]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fanout_creates_row_per_enabled_notifier(
|
||||
app_client, ui_auth_headers, session, monkeypatch
|
||||
|
||||
Reference in New Issue
Block a user