Implement Monlet MVP stack and UI updates
This commit is contained in:
Stanislav Rossovskii
417
server/tests/test_notifiers.py
Normal file
417
server/tests/test_notifiers.py
Normal file
@@ -0,0 +1,417 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from monlet_server.models import Incident, NotificationOutbox
|
||||
from monlet_server.services.notifier_worker import _backoff, tick
|
||||
from monlet_server.services.notifiers.alertmanager import AlertmanagerNotifier
|
||||
from monlet_server.services.notifiers.base import DeliveryResult
|
||||
from monlet_server.services.notifiers.debug import DebugNotifier
|
||||
from monlet_server.services.notifiers.telegram import TelegramNotifier
|
||||
from monlet_server.services.notifiers.webhook import WebhookNotifier
|
||||
from monlet_server.settings import Settings
|
||||
|
||||
|
||||
def _payload() -> dict:
|
||||
return {
|
||||
"incident_id": str(uuid4()),
|
||||
"agent_id": "agent-1",
|
||||
"check_id": "chk-1",
|
||||
"status": "critical",
|
||||
"severity": "critical",
|
||||
"incident_key": "agent-1:chk-1",
|
||||
"observed_at": datetime.now(UTC).isoformat(),
|
||||
"opened_at": datetime.now(UTC).isoformat(),
|
||||
"summary": "boom",
|
||||
"output": "stderr: boom token=abc123",
|
||||
}
|
||||
|
||||
|
||||
class _StubTransport(httpx.AsyncBaseTransport):
|
||||
def __init__(self, handler):
|
||||
self._handler = handler
|
||||
self.requests: list[httpx.Request] = []
|
||||
|
||||
async def handle_async_request(self, request):
|
||||
self.requests.append(request)
|
||||
return self._handler(request)
|
||||
|
||||
|
||||
def _client(handler) -> tuple[httpx.AsyncClient, _StubTransport]:
|
||||
t = _StubTransport(handler)
|
||||
return httpx.AsyncClient(transport=t, base_url="http://x"), t
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_debug_notifier_always_ok():
|
||||
res = await DebugNotifier().deliver("firing", _payload())
|
||||
assert res.ok and not res.permanent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_success_and_redaction():
|
||||
def handler(req):
|
||||
body = req.content.decode()
|
||||
assert "token=abc123" not in body
|
||||
assert "***" in body
|
||||
return httpx.Response(200, json={"ok": True})
|
||||
|
||||
client, t = _client(handler)
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert "/botTOK/sendMessage" in str(t.requests[0].url)
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_4xx_permanent():
|
||||
client, _ = _client(lambda r: httpx.Response(400, json={}))
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_5xx_transient():
|
||||
client, _ = _client(lambda r: httpx.Response(503, json={}))
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and not res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_network_error_transient():
|
||||
def handler(req):
|
||||
raise httpx.ConnectError("nope")
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = TelegramNotifier(client, "TOK", "123")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert not res.ok and not res.permanent
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_webhook_sends_bearer_and_redacts_payload():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["auth"] = req.headers.get("authorization")
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(202)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = WebhookNotifier(client, "http://x/hook", token="WTOK")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert captured["auth"] == "Bearer WTOK"
|
||||
assert "token=abc123" not in captured["body"]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alertmanager_payload_shape():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["url"] = str(req.url)
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = AlertmanagerNotifier(client, "http://am/")
|
||||
res = await n.deliver("firing", _payload())
|
||||
assert res.ok
|
||||
assert captured["url"].endswith("/api/v2/alerts")
|
||||
assert '"alertname":"monlet_incident"' in captured["body"]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def test_backoff_matches_adr_0005():
|
||||
# ADR-0005: 5s, 15s, 60s, 5m, 30m, 1h, 2h, 4h.
|
||||
expected = [5, 15, 60, 300, 1800, 3600, 7200, 14400]
|
||||
for i, sec in enumerate(expected, start=1):
|
||||
assert _backoff(i).total_seconds() == sec
|
||||
# Caps at last entry.
|
||||
assert _backoff(99).total_seconds() == 14400
|
||||
|
||||
|
||||
# ---- Worker integration with real PG ----
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def settings_full() -> Settings:
|
||||
s = Settings()
|
||||
s.notifier_max_attempts = 3
|
||||
s.notifier_batch_size = 50
|
||||
return s
|
||||
|
||||
|
||||
async def _seed_incident_and_outbox(
|
||||
sm: async_sessionmaker, notifier: str
|
||||
) -> tuple[NotificationOutbox, Incident]:
|
||||
async with sm() as session:
|
||||
# Need agent for FK on incidents.agent_id? No FK on incidents.agent_id per schema.
|
||||
inc = Incident(
|
||||
id=uuid4(),
|
||||
incident_key=f"k-{uuid4()}",
|
||||
agent_id="agent-1",
|
||||
check_id="chk-1",
|
||||
state="open",
|
||||
severity="critical",
|
||||
opened_at=datetime.now(UTC),
|
||||
last_event_id=uuid4(),
|
||||
summary="boom",
|
||||
)
|
||||
session.add(inc)
|
||||
await session.flush()
|
||||
row = NotificationOutbox(
|
||||
id=uuid4(),
|
||||
incident_id=inc.id,
|
||||
notifier=notifier,
|
||||
event_type="firing",
|
||||
state="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
payload=_payload(),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return row, inc
|
||||
|
||||
|
||||
class _StubNotifier:
|
||||
def __init__(self, name: str, results: list[DeliveryResult]) -> None:
|
||||
self.name = name
|
||||
self._results = list(results)
|
||||
self.calls = 0
|
||||
|
||||
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
|
||||
self.calls += 1
|
||||
if self._results:
|
||||
return self._results.pop(0)
|
||||
return DeliveryResult(ok=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_marks_sent(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "stub")
|
||||
registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])}
|
||||
n = await tick(sm, registry, settings_full)
|
||||
assert n == 1
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "sent"
|
||||
assert fresh.attempts == 1
|
||||
assert fresh.last_error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_retries_then_succeeds(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "stub")
|
||||
notifier = _StubNotifier(
|
||||
"stub",
|
||||
[DeliveryResult(ok=False, permanent=False, error="net"), DeliveryResult(ok=True)],
|
||||
)
|
||||
registry = {"stub": notifier}
|
||||
await tick(sm, registry, settings_full)
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "retry"
|
||||
assert fresh.attempts == 1
|
||||
assert fresh.next_attempt_at is not None
|
||||
# Force eligibility for second tick.
|
||||
fresh.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await session.commit()
|
||||
await tick(sm, registry, settings_full)
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "sent"
|
||||
assert fresh.attempts == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_permanent_failure_marked_failed(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "stub")
|
||||
registry = {
|
||||
"stub": _StubNotifier("stub", [DeliveryResult(ok=False, permanent=True, error="http_400")])
|
||||
}
|
||||
await tick(sm, registry, settings_full)
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "failed"
|
||||
assert fresh.last_error == "http_400"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_unknown_notifier_discarded(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "ghost")
|
||||
await tick(sm, {}, settings_full)
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "discarded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_exhausts_attempts(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "stub")
|
||||
notifier = _StubNotifier(
|
||||
"stub",
|
||||
[DeliveryResult(ok=False, permanent=False, error="boom")] * 10,
|
||||
)
|
||||
registry = {"stub": notifier}
|
||||
for _ in range(settings_full.notifier_max_attempts):
|
||||
async with sm() as session:
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
# Force ready.
|
||||
await session.execute(
|
||||
NotificationOutbox.__table__.update()
|
||||
.where(NotificationOutbox.id == row.id)
|
||||
.values(next_attempt_at=datetime.now(UTC) - timedelta(seconds=1))
|
||||
)
|
||||
await session.commit()
|
||||
await tick(sm, registry, settings_full)
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "failed"
|
||||
assert fresh.attempts == settings_full.notifier_max_attempts
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_recovers_stuck_sending(engine, settings_full):
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
row, _ = await _seed_incident_and_outbox(sm, "stub")
|
||||
async with sm() as session:
|
||||
await session.execute(
|
||||
NotificationOutbox.__table__.update()
|
||||
.where(NotificationOutbox.id == row.id)
|
||||
.values(state="sending", updated_at=datetime.now(UTC) - timedelta(hours=1))
|
||||
)
|
||||
await session.commit()
|
||||
settings_full.notifier_lease_sec = 60
|
||||
registry = {"stub": _StubNotifier("stub", [DeliveryResult(ok=True)])}
|
||||
n = await tick(sm, registry, settings_full)
|
||||
assert n == 1
|
||||
async with sm() as session:
|
||||
fresh = (
|
||||
await session.execute(select(NotificationOutbox).where(NotificationOutbox.id == row.id))
|
||||
).scalar_one()
|
||||
assert fresh.state == "sent"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alertmanager_severity_not_in_labels():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
n = AlertmanagerNotifier(client, "http://am")
|
||||
await n.deliver("firing", _payload())
|
||||
import json as _json
|
||||
|
||||
alerts = _json.loads(captured["body"])
|
||||
assert "severity" not in alerts[0]["labels"]
|
||||
assert alerts[0]["annotations"]["severity"] == "critical"
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_redacts_secret_in_incident_key():
|
||||
captured: dict = {}
|
||||
|
||||
def handler(req):
|
||||
captured["body"] = req.content.decode()
|
||||
return httpx.Response(200)
|
||||
|
||||
client, _ = _client(handler)
|
||||
try:
|
||||
p = _payload()
|
||||
p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
|
||||
n = TelegramNotifier(client, "TOK", "1")
|
||||
await n.deliver("firing", p)
|
||||
assert "leakedsecret" not in captured["body"]
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fanout_creates_row_per_enabled_notifier(
|
||||
app_client, auth_headers, session, monkeypatch
|
||||
):
|
||||
# Enable telegram (token+chat_id satisfied) and webhook in addition to debug.
|
||||
os.environ["MONLET_NOTIFIER_TELEGRAM_ENABLED"] = "true"
|
||||
os.environ["MONLET_NOTIFIER_TELEGRAM_TOKEN"] = "tok"
|
||||
os.environ["MONLET_NOTIFIER_TELEGRAM_CHAT_ID"] = "1"
|
||||
os.environ["MONLET_NOTIFIER_WEBHOOK_ENABLED"] = "true"
|
||||
os.environ["MONLET_NOTIFIER_WEBHOOK_URL"] = "http://x/hook"
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
|
||||
reset_settings_cache()
|
||||
try:
|
||||
from tests._helpers import make_event, make_heartbeat
|
||||
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [ev]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
assert r.status_code == 202
|
||||
rows = (await session.execute(select(NotificationOutbox))).scalars().all()
|
||||
names = sorted(r.notifier for r in rows)
|
||||
assert names == ["debug", "telegram", "webhook"]
|
||||
finally:
|
||||
for k in (
|
||||
"MONLET_NOTIFIER_TELEGRAM_ENABLED",
|
||||
"MONLET_NOTIFIER_TELEGRAM_TOKEN",
|
||||
"MONLET_NOTIFIER_TELEGRAM_CHAT_ID",
|
||||
"MONLET_NOTIFIER_WEBHOOK_ENABLED",
|
||||
"MONLET_NOTIFIER_WEBHOOK_URL",
|
||||
):
|
||||
os.environ.pop(k, None)
|
||||
reset_settings_cache()
|
||||
Reference in New Issue
Block a user