Fix notifier verification coverage
Some checks failed
ci / openapi (push) Failing after 43s
ci / agent (push) Failing after 24s
ci / server (push) Failing after 42s
ci / stack-smoke (push) Has been skipped
ci / web (push) Failing after 7s

This commit is contained in:
Stanislav Rossovskii
2026-06-23 16:06:58 +04:00
parent 03bf0edddb
commit 6dbae53725
6 changed files with 284 additions and 182 deletions

View File

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from monlet_server.models import Agent, Check, Event, Incident, NotificationOutbox
from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
from monlet_server.settings import reset_settings_cache
from monlet_server.settings import get_settings, reset_settings_cache
from ._helpers import make_event, register_agent
@@ -19,15 +19,20 @@ async def test_detector_transitions(app_client, ui_auth_headers, engine, session
await register_agent(app_client, ui_auth_headers, "agent-dead")
now = datetime.now(UTC)
settings = get_settings()
assert settings.stale_after_sec < settings.dead_after_sec
stale_age = settings.stale_after_sec + (
(settings.dead_after_sec - settings.stale_after_sec) / 2
)
await session.execute(
update(Agent)
.where(Agent.agent_id == "agent-stale")
.values(last_seen_at=now - timedelta(seconds=120))
.values(last_seen_at=now - timedelta(seconds=stale_age))
)
await session.execute(
update(Agent)
.where(Agent.agent_id == "agent-dead")
.values(last_seen_at=now - timedelta(minutes=10))
.values(last_seen_at=now - timedelta(seconds=settings.dead_after_sec + 10))
)
await session.commit()

View File

@@ -1,22 +1,16 @@
from __future__ import annotations
import json
import os
from datetime import UTC, datetime, timedelta
from uuid import uuid4
import httpx
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from monlet_server.models import Incident, NotificationOutbox
from monlet_server.services.notifier_worker import _backoff, tick
from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
from monlet_server.services.notifier_worker import tick
from monlet_server.services.notifiers.base import DeliveryResult
from monlet_server.services.notifiers.debug import DebugNotifier
from monlet_server.services.notifiers.telegram import TelegramNotifier
from monlet_server.services.notifiers.webhook import WebhookNotifier
from monlet_server.settings import Settings
@@ -35,135 +29,6 @@ def _payload() -> dict:
}
class _StubTransport(httpx.AsyncBaseTransport):
def __init__(self, handler):
self._handler = handler
self.requests: list[httpx.Request] = []
async def handle_async_request(self, request):
self.requests.append(request)
return self._handler(request)
def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
t = _StubTransport(handler)
return httpx.AsyncClient(transport=t, base_url="http://x"), t
@pytest.mark.asyncio
async def test_debug_notifier_always_ok():
res = await DebugNotifier().deliver("firing", _payload())
assert res.ok and not res.permanent
@pytest.mark.asyncio
async def test_telegram_success_and_redaction():
def handler(req):
body = req.content.decode()
assert "token=abc123" not in body
assert "2026-05-27 09:00:00 +04" in body
assert "***" in body
return httpx.Response(200, json={"ok": True})
client, t = _client(handler)
try:
n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi")
res = await n.deliver("firing", _payload())
assert res.ok
assert "/botTOK/sendMessage" in str(t.requests[0].url)
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_telegram_4xx_permanent():
client, _ = _client(lambda r: httpx.Response(400, json={}))
try:
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and res.permanent
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_telegram_5xx_transient():
client, _ = _client(lambda r: httpx.Response(503, json={}))
try:
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and not res.permanent
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_telegram_network_error_transient():
def handler(req):
raise httpx.ConnectError("nope")
client, _ = _client(handler)
try:
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and not res.permanent
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_webhook_sends_bearer_and_redacts_payload():
captured: dict = {}
def handler(req):
captured["auth"] = req.headers.get("authorization")
captured["body"] = req.content.decode()
return httpx.Response(202)
client, _ = _client(handler)
try:
n = WebhookNotifier(client, "http://x/hook", token="WTOK")
res = await n.deliver("firing", _payload())
assert res.ok
assert captured["auth"] == "Bearer WTOK"
assert "token=abc123" not in captured["body"]
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_alertmanager_payload_shape():
captured: dict = {}
def handler(req):
captured["url"] = str(req.url)
captured["body"] = req.content.decode()
return httpx.Response(200)
client, _ = _client(handler)
try:
n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi")
res = await n.deliver("firing", _payload())
assert res.ok
assert captured["url"].endswith("/api/v2/alerts")
body = json.loads(captured["body"])
alert = body[0]
assert alert["labels"]["alertname"] == "monlet_incident"
assert alert["startsAt"] == "2026-05-27T05:00:00+00:00"
assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04"
finally:
await client.aclose()
def test_backoff_matches_adr_0005():
# ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
for i, sec in enumerate(expected, start=1):
assert _backoff(i).total_seconds() == sec
# Caps at last entry.
assert _backoff(99).total_seconds() == 14400
# ---- Worker integration with real PG ----
@@ -344,46 +209,6 @@ async def test_worker_recovers_stuck_sending(engine, settings_full):
assert fresh.state == "sent"
@pytest.mark.asyncio
async def test_alertmanager_severity_not_in_labels():
captured: dict = {}
def handler(req):
captured["body"] = req.content.decode()
return httpx.Response(200)
client, _ = _client(handler)
try:
n = AlertmanagerNotifier(client, "http://am", "UTC")
await n.deliver("firing", _payload())
import json as _json
alerts = _json.loads(captured["body"])
assert "severity" not in alerts[0]["labels"]
assert alerts[0]["annotations"]["severity"] == "critical"
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_telegram_redacts_secret_in_incident_key():
captured: dict = {}
def handler(req):
captured["body"] = req.content.decode()
return httpx.Response(200)
client, _ = _client(handler)
try:
p = _payload()
p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
n = TelegramNotifier(client, "TOK", "1", "UTC")
await n.deliver("firing", p)
assert "leakedsecret" not in captured["body"]
finally:
await client.aclose()
@pytest.mark.asyncio
async def test_fanout_creates_row_per_enabled_notifier(
app_client, ui_auth_headers, session, monkeypatch