Harden production workflows and agent admission
This commit is contained in:
@@ -49,7 +49,28 @@ def make_heartbeat(agent_id: str = "agent-1") -> dict:
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"observed_at": now_iso(),
|
||||
"hostname": "host-1",
|
||||
"hostname": agent_id,
|
||||
"features": {"push": True, "metrics": True},
|
||||
"labels": {},
|
||||
}
|
||||
|
||||
|
||||
def agent_token(agent_id: str = "agent-1") -> str:
|
||||
safe = "".join(ch if ch.isalnum() else "0" for ch in agent_id)
|
||||
return f"test-agent-key-{safe}-00000000000000000000000000000000"
|
||||
|
||||
|
||||
def agent_headers(agent_id: str = "agent-1") -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {agent_token(agent_id)}"}
|
||||
|
||||
|
||||
async def register_agent(
|
||||
client, ui_headers: dict[str, str], agent_id: str = "agent-1"
|
||||
) -> dict[str, str]:
|
||||
headers = agent_headers(agent_id)
|
||||
r = await client.post("/api/v1/heartbeat", json=make_heartbeat(agent_id), headers=headers)
|
||||
assert r.status_code == 202
|
||||
r = await client.post(f"/api/v1/agents/{agent_id}/accept", headers=ui_headers)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 1
|
||||
return headers
|
||||
|
||||
@@ -13,7 +13,7 @@ from testcontainers.postgres import PostgresContainer
|
||||
|
||||
from alembic import command
|
||||
|
||||
os.environ.setdefault("MONLET_AUTH_TOKEN", "test-token")
|
||||
os.environ.setdefault("MONLET_AUTH_TOKEN", "test-ui-token")
|
||||
os.environ.setdefault("MONLET_ENABLE_DETECTOR", "false")
|
||||
os.environ.setdefault("MONLET_ENABLE_NOTIFIER_WORKER", "false")
|
||||
|
||||
@@ -97,7 +97,7 @@ async def _truncate_tables(engine) -> AsyncIterator[None]:
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
"TRUNCATE TABLE notification_outbox, incidents, events, event_ingest_dedup, checks, agents RESTART IDENTITY CASCADE"
|
||||
"TRUNCATE TABLE notification_outbox, incidents, events, event_ingest_dedup, checks, agent_blacklist, agents RESTART IDENTITY CASCADE"
|
||||
)
|
||||
)
|
||||
|
||||
@@ -116,4 +116,11 @@ async def app_client(database_url: str) -> AsyncIterator[AsyncClient]:
|
||||
|
||||
@pytest.fixture
|
||||
def auth_headers() -> dict[str, str]:
|
||||
return {"Authorization": "Bearer test-token"}
|
||||
from tests._helpers import agent_headers
|
||||
|
||||
return agent_headers()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ui_auth_headers() -> dict[str, str]:
|
||||
return {"Authorization": "Bearer test-ui-token"}
|
||||
|
||||
354
server/tests/test_agent_admission.py
Normal file
354
server/tests/test_agent_admission.py
Normal file
@@ -0,0 +1,354 @@
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from monlet_server.models import Agent, AgentBlacklist, Check
|
||||
from monlet_server.services.detector import _prune_history
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
|
||||
from ._helpers import agent_headers, make_event, make_heartbeat, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_heartbeat_creates_pending_agent(app_client, auth_headers, session):
|
||||
r = await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-pending"), headers=auth_headers
|
||||
)
|
||||
assert r.status_code == 202
|
||||
agent = (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "agent-pending"))
|
||||
).scalar_one()
|
||||
assert agent.pending_key_hash
|
||||
assert agent.accepted_key_hash is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_events_are_dropped_until_accept(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
assert r.status_code == 202
|
||||
assert r.json() == {"accepted": 0, "deduplicated": 0}
|
||||
n = (await session.execute(select(func.count()).select_from(Check))).scalar_one()
|
||||
assert n == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accept_enables_events(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
assert r.json() == {"accepted": 1, "deduplicated": 0}
|
||||
n = (await session.execute(select(func.count()).select_from(Check))).scalar_one()
|
||||
assert n == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_block_pending_key(app_client, auth_headers, ui_auth_headers, session):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-block"), headers=auth_headers
|
||||
)
|
||||
r = await app_client.post("/api/v1/agents/agent-block/block", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 1
|
||||
assert (await session.execute(select(func.count()).select_from(Agent))).scalar_one() == 0
|
||||
assert (
|
||||
await session.execute(select(func.count()).select_from(AgentBlacklist))
|
||||
).scalar_one() == 1
|
||||
|
||||
r = await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-block"), headers=auth_headers
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert r.json()["error"]["code"] == "agent_blocked"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blacklist_list_delete_and_clear(app_client, auth_headers, ui_auth_headers):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-block-a"), headers=auth_headers
|
||||
)
|
||||
await app_client.post("/api/v1/agents/agent-block-a/block", headers=ui_auth_headers)
|
||||
|
||||
other_headers = agent_headers("agent-block-b")
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat",
|
||||
json=make_heartbeat("agent-block-b"),
|
||||
headers=other_headers,
|
||||
)
|
||||
await app_client.post("/api/v1/agents/agent-block-b/block", headers=ui_auth_headers)
|
||||
|
||||
r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
items = r.json()["items"]
|
||||
assert len(items) == 2
|
||||
|
||||
blocked_id = items[0]["id"]
|
||||
r = await app_client.delete(
|
||||
f"/api/v1/agents/admission/blacklist/{blocked_id}",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
assert r.json()["count"] == 1
|
||||
|
||||
r = await app_client.delete("/api/v1/agents/admission/blacklist", headers=ui_auth_headers)
|
||||
assert r.json()["count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_accepted_agent_reappears_pending(app_client, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers, "agent-remove")
|
||||
r = await app_client.delete("/api/v1/agents/agent-remove", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
|
||||
r = await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-remove"), headers=auth_headers
|
||||
)
|
||||
assert r.status_code == 202
|
||||
r = await app_client.get("/api/v1/agents/agent-remove", headers=ui_auth_headers)
|
||||
assert r.json()["admission_state"] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_key_same_hostname_is_pending_replacement(app_client, ui_auth_headers, session):
|
||||
old_headers = await register_agent(app_client, ui_auth_headers, "old-id")
|
||||
hb = make_heartbeat("new-id")
|
||||
hb["hostname"] = "old-id"
|
||||
new_headers = agent_headers("new-id")
|
||||
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=new_headers)
|
||||
assert r.status_code == 202
|
||||
r = await app_client.get("/api/v1/agents/old-id", headers=ui_auth_headers)
|
||||
assert r.json()["admission_state"] == "pending"
|
||||
assert r.json()["rotation_pending"] is True
|
||||
|
||||
ev = make_event(status="ok")
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "old-id", "events": [ev]}, headers=old_headers
|
||||
)
|
||||
assert r.json()["accepted"] == 1
|
||||
|
||||
r = await app_client.post("/api/v1/agents/old-id/block", headers=ui_auth_headers)
|
||||
assert r.json()["count"] == 1
|
||||
agent = (await session.execute(select(Agent).where(Agent.agent_id == "old-id"))).scalar_one()
|
||||
assert agent.accepted_key_hash
|
||||
assert agent.pending_key_hash is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accept_all_skips_rotations_without_explicit_flag(
|
||||
app_client, ui_auth_headers, session
|
||||
):
|
||||
await register_agent(app_client, ui_auth_headers, "stable-host")
|
||||
hb = make_heartbeat("replacement-id")
|
||||
hb["hostname"] = "stable-host"
|
||||
await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("replacement"))
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("new-host"), headers=agent_headers("new-host")
|
||||
)
|
||||
|
||||
r = await app_client.post("/api/v1/agents/admission/accept-all", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 1
|
||||
|
||||
stable = (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "stable-host"))
|
||||
).scalar_one()
|
||||
assert stable.pending_key_hash is not None
|
||||
new_host = (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "new-host"))
|
||||
).scalar_one()
|
||||
assert new_host.pending_key_hash is None
|
||||
assert new_host.accepted_key_hash is not None
|
||||
|
||||
r = await app_client.post(
|
||||
"/api/v1/agents/admission/accept-all?include_rotation=true",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["count"] == 1
|
||||
await session.refresh(stable)
|
||||
assert stable.pending_key_hash is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_key_cannot_hijack_existing_pending_hostname(app_client, auth_headers):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("pending-host"), headers=auth_headers
|
||||
)
|
||||
r = await app_client.post(
|
||||
"/api/v1/heartbeat",
|
||||
json=make_heartbeat("pending-host"),
|
||||
headers=agent_headers("other-key"),
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert r.json()["error"]["code"] == "conflict"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_key_agent_id_collision_is_rejected(app_client, ui_auth_headers):
|
||||
await register_agent(app_client, ui_auth_headers, "agent-a")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-b")
|
||||
|
||||
hb = make_heartbeat("agent-a")
|
||||
hb["hostname"] = "new-host"
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("new-key"))
|
||||
assert r.status_code == 409
|
||||
assert r.json()["error"]["code"] == "conflict"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accepted_key_cannot_take_another_hostname(app_client, ui_auth_headers):
|
||||
headers = await register_agent(app_client, ui_auth_headers, "agent-a")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-b")
|
||||
|
||||
hb = make_heartbeat("agent-a")
|
||||
hb["hostname"] = "agent-b"
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=headers)
|
||||
assert r.status_code == 409
|
||||
assert r.json()["error"]["code"] == "conflict"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hostname_replacement_still_wins_over_body_agent_id(app_client, ui_auth_headers):
|
||||
await register_agent(app_client, ui_auth_headers, "agent-a")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-b")
|
||||
|
||||
hb = make_heartbeat("agent-a")
|
||||
hb["hostname"] = "agent-b"
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("new-key"))
|
||||
assert r.status_code == 202
|
||||
|
||||
assert (await app_client.get("/api/v1/agents/agent-b", headers=ui_auth_headers)).json()[
|
||||
"admission_state"
|
||||
] == "pending"
|
||||
assert (await app_client.get("/api/v1/agents/agent-a", headers=ui_auth_headers)).json()[
|
||||
"admission_state"
|
||||
] == "accepted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_pending_replacement_keeps_accepted_agent(
|
||||
app_client, ui_auth_headers, session
|
||||
):
|
||||
old_headers = await register_agent(app_client, ui_auth_headers, "stable-host")
|
||||
hb = make_heartbeat("replacement-id")
|
||||
hb["hostname"] = "stable-host"
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("replacement"))
|
||||
assert r.status_code == 202
|
||||
|
||||
r = await app_client.delete("/api/v1/agents/stable-host", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
|
||||
agent = (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "stable-host"))
|
||||
).scalar_one()
|
||||
assert agent.accepted_key_hash
|
||||
assert agent.pending_key_hash is None
|
||||
|
||||
ev = make_event(status="ok")
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "stable-host", "events": [ev]}, headers=old_headers
|
||||
)
|
||||
assert r.json()["accepted"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_agent_cap_rejects_new_pending(app_client, auth_headers, monkeypatch):
|
||||
monkeypatch.setenv("MONLET_MAX_PENDING_AGENTS", "1")
|
||||
reset_settings_cache()
|
||||
r = await app_client.post("/api/v1/heartbeat", json=make_heartbeat("one"), headers=auth_headers)
|
||||
assert r.status_code == 202
|
||||
r = await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("two"), headers=agent_headers("two")
|
||||
)
|
||||
assert r.status_code == 429
|
||||
assert r.json()["error"]["code"] == "rate_limited"
|
||||
monkeypatch.delenv("MONLET_MAX_PENDING_AGENTS", raising=False)
|
||||
reset_settings_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_prune_deletes_pending_only_and_clears_replacement(
|
||||
app_client, ui_auth_headers, session, monkeypatch
|
||||
):
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("pending-only"), headers=agent_headers("p1")
|
||||
)
|
||||
await register_agent(app_client, ui_auth_headers, "accepted")
|
||||
hb = make_heartbeat("replacement")
|
||||
hb["hostname"] = "accepted"
|
||||
await app_client.post("/api/v1/heartbeat", json=hb, headers=agent_headers("p2"))
|
||||
|
||||
old = datetime.now(UTC) - timedelta(days=2)
|
||||
rows = (
|
||||
(await session.execute(select(Agent).where(Agent.pending_key_hash.is_not(None))))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for row in rows:
|
||||
row.pending_seen_at = old
|
||||
await session.commit()
|
||||
|
||||
monkeypatch.setenv("MONLET_PENDING_AGENT_TTL_DAYS", "1")
|
||||
reset_settings_cache()
|
||||
await _prune_history(session)
|
||||
await session.commit()
|
||||
|
||||
assert (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "pending-only"))
|
||||
).scalar_one_or_none() is None
|
||||
accepted = (
|
||||
await session.execute(select(Agent).where(Agent.agent_id == "accepted"))
|
||||
).scalar_one()
|
||||
assert accepted.accepted_key_hash
|
||||
assert accepted.pending_key_hash is None
|
||||
monkeypatch.delenv("MONLET_PENDING_AGENT_TTL_DAYS", raising=False)
|
||||
reset_settings_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blacklist_delete_uses_unique_row_id(app_client, ui_auth_headers, session):
|
||||
from datetime import UTC, datetime
|
||||
|
||||
now = datetime.now(UTC)
|
||||
same_fingerprint = "0123456789abcdef"
|
||||
first_hash = "a" * 64
|
||||
second_hash = "b" * 64
|
||||
session.add_all(
|
||||
[
|
||||
AgentBlacklist(
|
||||
key_hash=first_hash,
|
||||
key_fingerprint=same_fingerprint,
|
||||
agent_id="agent-a",
|
||||
hostname="host-a",
|
||||
last_seen_at=now,
|
||||
),
|
||||
AgentBlacklist(
|
||||
key_hash=second_hash,
|
||||
key_fingerprint=same_fingerprint,
|
||||
agent_id="agent-b",
|
||||
hostname="host-b",
|
||||
last_seen_at=now,
|
||||
),
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
items = r.json()["items"]
|
||||
assert [i["id"] for i in items] == [first_hash, second_hash]
|
||||
|
||||
r = await app_client.delete(
|
||||
f"/api/v1/agents/admission/blacklist/{first_hash}",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
assert r.json()["count"] == 1
|
||||
|
||||
r = await app_client.get("/api/v1/agents/admission/blacklist", headers=ui_auth_headers)
|
||||
assert [i["id"] for i in r.json()["items"]] == [second_hash]
|
||||
@@ -30,6 +30,69 @@ async def test_wrong_token_401(app_client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_valid_token_200(app_client, auth_headers):
|
||||
r = await app_client.get("/api/v1/agents", headers=auth_headers)
|
||||
async def test_valid_token_200(app_client, ui_auth_headers):
|
||||
r = await app_client.get("/api/v1/agents", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_token_is_separate_from_agent_self_key(monkeypatch, database_url):
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from monlet_server import db as db_module
|
||||
from monlet_server.main import create_app
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
from tests._helpers import agent_headers, make_heartbeat
|
||||
|
||||
monkeypatch.setenv("MONLET_AUTH_TOKEN", "ui-token")
|
||||
reset_settings_cache()
|
||||
await db_module.dispose_engine()
|
||||
app = create_app()
|
||||
async with app.router.lifespan_context(app):
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
ui = {"Authorization": "Bearer ui-token"}
|
||||
agent_a = agent_headers("agent-a")
|
||||
agent_b = agent_headers("agent-b")
|
||||
assert (await client.get("/api/v1/agents", headers=ui)).status_code == 200
|
||||
assert (await client.get("/api/v1/agents", headers=agent_a)).status_code == 401
|
||||
|
||||
r = await client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-a"), headers=agent_a
|
||||
)
|
||||
assert r.status_code == 202
|
||||
r = await client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-b"), headers=agent_b
|
||||
)
|
||||
assert r.status_code == 202
|
||||
r = await client.post("/api/v1/heartbeat", json=make_heartbeat("agent-c"), headers=ui)
|
||||
assert r.status_code == 401
|
||||
await db_module.dispose_engine()
|
||||
monkeypatch.setenv("MONLET_AUTH_TOKEN", "test-ui-token")
|
||||
reset_settings_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_token_overlap(monkeypatch, database_url):
|
||||
"""Two active tokens both authenticate; rotation can overlap without 401s."""
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from monlet_server import db as db_module
|
||||
from monlet_server.main import create_app
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
|
||||
monkeypatch.setenv("MONLET_AUTH_TOKEN", "old-token new-token")
|
||||
reset_settings_cache()
|
||||
await db_module.dispose_engine()
|
||||
app = create_app()
|
||||
async with app.router.lifespan_context(app):
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
for tok in ("old-token", "new-token"):
|
||||
r = await client.get("/api/v1/agents", headers={"Authorization": f"Bearer {tok}"})
|
||||
assert r.status_code == 200, tok
|
||||
r = await client.get("/api/v1/agents", headers={"Authorization": "Bearer revoked"})
|
||||
assert r.status_code == 401
|
||||
await db_module.dispose_engine()
|
||||
monkeypatch.setenv("MONLET_AUTH_TOKEN", "test-ui-token")
|
||||
reset_settings_cache()
|
||||
|
||||
@@ -9,20 +9,14 @@ from monlet_server.models import Agent, Check, Event, Incident, NotificationOutb
|
||||
from monlet_server.services.detector import LIVENESS_CHECK_ID, _tick
|
||||
from monlet_server.settings import reset_settings_cache
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detector_transitions(app_client, auth_headers, engine, session):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
|
||||
)
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
|
||||
)
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
|
||||
)
|
||||
async def test_detector_transitions(app_client, ui_auth_headers, engine, session):
|
||||
await register_agent(app_client, ui_auth_headers, "agent-alive")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-stale")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-dead")
|
||||
|
||||
now = datetime.now(UTC)
|
||||
await session.execute(
|
||||
@@ -82,10 +76,57 @@ async def test_detector_transitions(app_client, auth_headers, engine, session):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detector_owns_liveness_check_and_events(app_client, auth_headers, engine, session):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-flap"), headers=auth_headers
|
||||
async def test_detector_noop_tick_does_not_rewrite_liveness(
|
||||
app_client, ui_auth_headers, engine, session
|
||||
):
|
||||
"""PH-009: when no agent's liveness status changes, the detector must not
|
||||
rewrite the liveness Check rows. We verify by tagging last_observed_at to a
|
||||
sentinel before the tick and checking the row was untouched."""
|
||||
await register_agent(app_client, ui_auth_headers, "agent-still")
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
# Force a transition first so a liveness Check row exists.
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.agent_id == "agent-still")
|
||||
.values(last_seen_at=datetime.now(UTC) - timedelta(minutes=10))
|
||||
)
|
||||
await session.commit()
|
||||
await _tick(sm)
|
||||
session.expire_all()
|
||||
row_before = (
|
||||
await session.execute(
|
||||
select(Check).where(
|
||||
Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
sentinel = datetime(2030, 1, 1, tzinfo=UTC)
|
||||
await session.execute(
|
||||
update(Check)
|
||||
.where(Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID)
|
||||
.values(last_observed_at=sentinel)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# Second tick with no liveness change: row must NOT be rewritten.
|
||||
await _tick(sm)
|
||||
session.expire_all()
|
||||
row_after = (
|
||||
await session.execute(
|
||||
select(Check).where(
|
||||
Check.agent_id == "agent-still", Check.check_id == LIVENESS_CHECK_ID
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
assert row_after.last_observed_at == sentinel
|
||||
assert row_after.last_event_id == row_before.last_event_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detector_owns_liveness_check_and_events(
|
||||
app_client, ui_auth_headers, engine, session
|
||||
):
|
||||
await register_agent(app_client, ui_auth_headers, "agent-flap")
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
# Tick 1: alive → Check exists with status=ok, one event.
|
||||
@@ -144,13 +185,11 @@ async def test_detector_owns_liveness_check_and_events(app_client, auth_headers,
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detector_prunes_outbox(app_client, auth_headers, engine, session, monkeypatch):
|
||||
async def test_detector_prunes_outbox(app_client, ui_auth_headers, engine, session, monkeypatch):
|
||||
monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
|
||||
reset_settings_cache()
|
||||
try:
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
|
||||
)
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune")
|
||||
critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
@@ -192,3 +231,73 @@ async def test_detector_prunes_outbox(app_client, auth_headers, engine, session,
|
||||
assert terminal_outbox == 2
|
||||
finally:
|
||||
reset_settings_cache()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detector_prune_preserves_active_outbox(
|
||||
app_client, ui_auth_headers, engine, session, monkeypatch
|
||||
):
|
||||
"""PH-012: prune must never delete active rows (pending/sending/retry),
|
||||
even when retention is exceeded. The keep-N-newest list applies only to
|
||||
terminal states; active rows are filtered out of the eligible set."""
|
||||
monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "1")
|
||||
reset_settings_cache()
|
||||
try:
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers, "agent-prune-active")
|
||||
critical = make_event(check_id="active_prune", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-prune-active", "events": [critical]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
inc = (
|
||||
await session.execute(select(Incident).where(Incident.check_id == "active_prune"))
|
||||
).scalar_one()
|
||||
now = datetime.now(UTC)
|
||||
# Mix of active and terminal states.
|
||||
for state, count in (("sent", 5), ("pending", 3), ("retry", 2), ("sending", 1)):
|
||||
for _ in range(count):
|
||||
session.add(
|
||||
NotificationOutbox(
|
||||
id=uuid4(),
|
||||
incident_id=inc.id,
|
||||
notifier="debug",
|
||||
event_type="firing",
|
||||
state=state,
|
||||
attempts=1,
|
||||
next_attempt_at=None,
|
||||
payload={},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
await _tick(sm)
|
||||
|
||||
# Active rows must be untouched. Ingestion may add extra pending rows
|
||||
# for the debug notifier; we only assert ours are still present and
|
||||
# that no active rows were deleted by the prune.
|
||||
for state, injected in (("pending", 3), ("retry", 2), ("sending", 1)):
|
||||
n = (
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(NotificationOutbox)
|
||||
.where(NotificationOutbox.state == state)
|
||||
)
|
||||
).scalar_one()
|
||||
assert n >= injected, (state, n, injected)
|
||||
# Terminal rows must be capped to retention=1 (or 2 after notifier worker
|
||||
# produces extras from the critical event we ingested; but worker is
|
||||
# disabled in tests so we just check terminal pruning).
|
||||
sent = (
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(NotificationOutbox)
|
||||
.where(NotificationOutbox.state == "sent")
|
||||
)
|
||||
).scalar_one()
|
||||
assert sent <= 2 # 1 retained + at most 1 created by detector enqueue
|
||||
finally:
|
||||
reset_settings_cache()
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from monlet_server.models import Check, Event
|
||||
from monlet_server.models import Agent, Check, Event
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_event_accepted_and_state(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_event_accepted_and_state(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="ok", output="disk ok")
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
@@ -23,8 +23,20 @@ async def test_event_accepted_and_state(app_client, auth_headers, session):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_event_deduplicated(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_event_before_heartbeat_is_dropped(app_client, auth_headers, session):
|
||||
ev = make_event(status="warning", exit_code=1)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-replay", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
assert r.status_code == 202
|
||||
assert r.json() == {"accepted": 0, "deduplicated": 0}
|
||||
n = (await session.execute(select(func.count()).select_from(Agent))).scalar_one()
|
||||
assert n == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_event_deduplicated(app_client, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
r1 = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
@@ -46,8 +58,8 @@ async def test_empty_batch_400(app_client, auth_headers):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("observed_at", ["2026-05-27T09:00:00", "2026-05-27T09:00:00+04:00"])
|
||||
async def test_event_requires_utc_timestamp(app_client, auth_headers, observed_at):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_event_requires_utc_timestamp(app_client, ui_auth_headers, observed_at):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(observed_at=observed_at)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
@@ -66,8 +78,8 @@ async def test_oversize_batch_400(app_client, auth_headers):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_same_status_not_stored(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_repeated_same_status_not_stored(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
evs = [make_event(status="ok") for _ in range(5)]
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
|
||||
@@ -79,8 +91,8 @@ async def test_repeated_same_status_not_stored(app_client, auth_headers, session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_transitions_stored(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_status_transitions_stored(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
seq = [
|
||||
make_event(status="ok"),
|
||||
make_event(status="warning", exit_code=1),
|
||||
@@ -98,8 +110,8 @@ async def test_status_transitions_stored(app_client, auth_headers, session):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_status_different_exit_code_stored(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_same_status_different_exit_code_stored(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
seq = [
|
||||
make_event(status="warning", exit_code=1),
|
||||
make_event(status="warning", exit_code=3),
|
||||
@@ -113,10 +125,10 @@ async def test_same_status_different_exit_code_stored(app_client, auth_headers,
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_late_event_does_not_move_state(app_client, auth_headers, session):
|
||||
async def test_late_event_does_not_move_state(app_client, ui_auth_headers, session):
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
newer = datetime.now(UTC).replace(microsecond=0)
|
||||
older = (newer - timedelta(minutes=5)).isoformat()
|
||||
ev_now = make_event(
|
||||
|
||||
@@ -12,8 +12,10 @@ async def test_heartbeat_creates_agent(app_client, auth_headers, session):
|
||||
assert r.status_code == 202
|
||||
res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
|
||||
a = res.scalar_one()
|
||||
assert a.hostname == "host-1"
|
||||
assert a.hostname == "agent-1"
|
||||
assert a.status == "alive"
|
||||
assert a.pending_key_hash
|
||||
assert a.accepted_key_hash is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -24,7 +26,7 @@ async def test_heartbeat_upsert(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
|
||||
res = await session.execute(select(Agent).where(Agent.agent_id == "agent-1"))
|
||||
a = res.scalar_one()
|
||||
assert a.features == {"push": True, "metrics": False}
|
||||
assert a.pending_features == {"push": True, "metrics": False}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -1,30 +1,30 @@
|
||||
import pytest
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incident_key_ownership_rejected(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-1"), headers=auth_headers)
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat("agent-2"), headers=auth_headers)
|
||||
async def test_incident_key_ownership_rejected(app_client, ui_auth_headers):
|
||||
agent_1_headers = await register_agent(app_client, ui_auth_headers, "agent-1")
|
||||
agent_2_headers = await register_agent(app_client, ui_auth_headers, "agent-2")
|
||||
|
||||
ev1 = make_event(status="critical", exit_code=2, incident_key="shared-key")
|
||||
r1 = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=auth_headers
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev1]}, headers=agent_1_headers
|
||||
)
|
||||
assert r1.status_code == 202
|
||||
|
||||
ev2 = make_event(status="critical", exit_code=2, incident_key="shared-key")
|
||||
r2 = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=auth_headers
|
||||
"/api/v1/events", json={"agent_id": "agent-2", "events": [ev2]}, headers=agent_2_headers
|
||||
)
|
||||
assert r2.status_code == 400
|
||||
assert r2.json()["error"]["code"] == "validation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incident_key_too_long(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_incident_key_too_long(app_client, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2, incident_key="x" * 257)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
@@ -33,20 +33,20 @@ async def test_incident_key_too_long(app_client, auth_headers):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_query_cursor_pagination(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_events_query_cursor_pagination(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
evs = [make_event(check_id=f"chk-{i}") for i in range(5)]
|
||||
await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": evs}, headers=auth_headers
|
||||
)
|
||||
r = await app_client.get("/api/v1/events/query?limit=2", headers=auth_headers)
|
||||
r = await app_client.get("/api/v1/events/query?limit=2", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
page1 = r.json()
|
||||
assert len(page1["items"]) == 2
|
||||
assert page1["next_cursor"]
|
||||
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=auth_headers
|
||||
f"/api/v1/events/query?limit=2&cursor={page1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
page2 = r2.json()
|
||||
assert len(page2["items"]) == 2
|
||||
|
||||
@@ -3,12 +3,12 @@ from sqlalchemy import select
|
||||
|
||||
from monlet_server.models import Incident, NotificationOutbox
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_escalate_resolve_reopen(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_open_escalate_resolve_reopen(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
a = "agent-1"
|
||||
|
||||
# warning -> open warning
|
||||
@@ -52,8 +52,8 @@ async def test_open_escalate_resolve_reopen(app_client, auth_headers, session):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_skipped_when_notifications_disabled(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_outbox_skipped_when_notifications_disabled(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2, notifications_enabled=False)
|
||||
await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
@@ -63,8 +63,8 @@ async def test_outbox_skipped_when_notifications_disabled(app_client, auth_heade
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_firing_and_resolved(app_client, auth_headers, session):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_outbox_firing_and_resolved(app_client, ui_auth_headers, session):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
a = "agent-1"
|
||||
ev_open = make_event(status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
@@ -77,3 +77,42 @@ async def test_outbox_firing_and_resolved(app_client, auth_headers, session):
|
||||
rows = (await session.execute(select(NotificationOutbox))).scalars().all()
|
||||
types = sorted(r.event_type for r in rows)
|
||||
assert types == ["firing", "resolved"]
|
||||
|
||||
|
||||
def test_status_rank_map_covers_all_pairs():
|
||||
"""PH-review: the SQL-side rank is computed by a STORED generated column
|
||||
(`incidents.status_rank`). The Python `_STATUS_RANK` map mirrors that CASE
|
||||
for cursor encoding only. This test pins the map so the two cannot drift —
|
||||
if a new (state, severity) pair is added, both must update."""
|
||||
from monlet_server.api.incidents import _STATUS_RANK
|
||||
|
||||
assert _STATUS_RANK == {
|
||||
("open", "critical"): 5,
|
||||
("open", "warning"): 4,
|
||||
("open", "unknown"): 3,
|
||||
("resolved", "critical"): 2,
|
||||
("resolved", "warning"): 1,
|
||||
("resolved", "unknown"): 0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_rank_generated_column_matches_python_map(
|
||||
app_client, ui_auth_headers, session
|
||||
):
|
||||
"""End-to-end check: insert one event per (state, severity) pair and
|
||||
confirm the DB-side generated column equals the Python map value."""
|
||||
from monlet_server.api.incidents import _STATUS_RANK
|
||||
|
||||
# Open critical incident comes from a single critical event.
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
crit = make_event(check_id="crit", status="critical", exit_code=2)
|
||||
warn = make_event(check_id="warn", status="warning", exit_code=1)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [crit, warn]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
rows = (await session.execute(select(Incident))).scalars().all()
|
||||
for r in rows:
|
||||
assert r.status_rank == _STATUS_RANK[(r.state, r.severity)]
|
||||
|
||||
@@ -10,7 +10,7 @@ EXAMPLES = os.path.join(os.path.dirname(__file__), "..", "..", "examples")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_examples_payload(app_client, auth_headers, session):
|
||||
async def test_examples_payload(app_client, auth_headers, ui_auth_headers, session):
|
||||
with open(os.path.join(EXAMPLES, "heartbeat.json")) as f:
|
||||
hb = json.load(f)
|
||||
with open(os.path.join(EXAMPLES, "event-batch.json")) as f:
|
||||
@@ -18,6 +18,8 @@ async def test_examples_payload(app_client, auth_headers, session):
|
||||
|
||||
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
|
||||
assert r.status_code == 202
|
||||
r = await app_client.post(f"/api/v1/agents/{hb['agent_id']}/accept", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
|
||||
r = await app_client.post("/api/v1/events", json=batch, headers=auth_headers)
|
||||
assert r.status_code == 202
|
||||
|
||||
@@ -200,7 +200,7 @@ async def _seed_incident_and_outbox(
|
||||
event_type="firing",
|
||||
state="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
next_attempt_at=datetime.now(UTC) - timedelta(seconds=1),
|
||||
payload=_payload(),
|
||||
)
|
||||
session.add(row)
|
||||
@@ -386,7 +386,7 @@ async def test_telegram_redacts_secret_in_incident_key():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fanout_creates_row_per_enabled_notifier(
|
||||
app_client, auth_headers, session, monkeypatch
|
||||
app_client, ui_auth_headers, session, monkeypatch
|
||||
):
|
||||
# Enable telegram (token+chat_id satisfied) and webhook in addition to debug.
|
||||
os.environ["MONLET_NOTIFIER_TELEGRAM_ENABLED"] = "true"
|
||||
@@ -398,9 +398,9 @@ async def test_fanout_creates_row_per_enabled_notifier(
|
||||
|
||||
reset_settings_cache()
|
||||
try:
|
||||
from tests._helpers import make_event, make_heartbeat
|
||||
from tests._helpers import make_event, register_agent
|
||||
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
r = await app_client.post(
|
||||
"/api/v1/events",
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
import pytest
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import agent_headers, make_event, make_heartbeat, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agents_cursor(app_client, auth_headers):
|
||||
async def test_agents_cursor(app_client, ui_auth_headers):
|
||||
for i in range(5):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat", json=make_heartbeat(f"a-{i}"), headers=auth_headers
|
||||
)
|
||||
r1 = await app_client.get("/api/v1/agents?limit=2", headers=auth_headers)
|
||||
await register_agent(app_client, ui_auth_headers, f"a-{i}")
|
||||
r1 = await app_client.get("/api/v1/agents?limit=2", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 2
|
||||
assert p1["next_cursor"]
|
||||
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/agents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
|
||||
f"/api/v1/agents?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
p2 = r2.json()
|
||||
assert len(p2["items"]) == 2
|
||||
@@ -25,76 +23,204 @@ async def test_agents_cursor(app_client, auth_headers):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checks_cursor(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_agents_cursor_sorts_pending_before_accepted(app_client, ui_auth_headers):
|
||||
for i in range(3):
|
||||
await register_agent(app_client, ui_auth_headers, f"accepted-{i}")
|
||||
for i in range(2):
|
||||
await app_client.post(
|
||||
"/api/v1/heartbeat",
|
||||
json=make_heartbeat(f"pending-{i}"),
|
||||
headers=agent_headers(f"pending-{i}"),
|
||||
)
|
||||
|
||||
r1 = await app_client.get("/api/v1/agents?limit=3", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert [i["admission_state"] for i in p1["items"][:2]] == ["pending", "pending"]
|
||||
assert p1["next_cursor"]
|
||||
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/agents?limit=3&cursor={p1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
assert {i["agent_id"] for i in r2.json()["items"]}.isdisjoint(
|
||||
{i["agent_id"] for i in p1["items"]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checks_cursor(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
for i in range(4):
|
||||
ev = make_event(check_id=f"chk-{i}")
|
||||
await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
r1 = await app_client.get("/api/v1/checks?limit=2", headers=auth_headers)
|
||||
r1 = await app_client.get("/api/v1/checks?limit=2", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 2
|
||||
assert p1["next_cursor"]
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/checks?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
|
||||
f"/api/v1/checks?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
p2 = r2.json()
|
||||
assert len(p2["items"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checks_rejects_invalid_agent_id_query(app_client, auth_headers):
|
||||
r = await app_client.get("/api/v1/checks?agent_id=bad%20id", headers=auth_headers)
|
||||
async def test_checks_rejects_invalid_agent_id_query(app_client, ui_auth_headers):
|
||||
r = await app_client.get("/api/v1/checks?agent_id=bad%20id", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
assert r.json()["error"]["code"] == "validation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_query_rejects_invalid_id_queries(app_client, auth_headers):
|
||||
r = await app_client.get("/api/v1/events/query?agent_id=bad%20id", headers=auth_headers)
|
||||
async def test_events_query_rejects_invalid_id_queries(app_client, ui_auth_headers):
|
||||
r = await app_client.get("/api/v1/events/query?agent_id=bad%20id", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
assert r.json()["error"]["code"] == "validation"
|
||||
|
||||
r = await app_client.get("/api/v1/events/query?check_id=bad%20id", headers=auth_headers)
|
||||
r = await app_client.get("/api/v1/events/query?check_id=bad%20id", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
assert r.json()["error"]["code"] == "validation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incidents_cursor(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_incidents_cursor(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
for i in range(3):
|
||||
ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
r1 = await app_client.get("/api/v1/incidents?limit=2", headers=auth_headers)
|
||||
r1 = await app_client.get("/api/v1/incidents?limit=2", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 2
|
||||
assert p1["next_cursor"]
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/incidents?limit=2&cursor={p1['next_cursor']}", headers=auth_headers
|
||||
f"/api/v1/incidents?limit=2&cursor={p1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
p2 = r2.json()
|
||||
assert len(p2["items"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_cursor(app_client, auth_headers):
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
async def test_incidents_default_sort_is_status(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
warning = make_event(check_id="warn-check", status="warning", exit_code=1)
|
||||
critical = make_event(check_id="crit-check", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [warning, critical]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
|
||||
r = await app_client.get("/api/v1/incidents?limit=10", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
items = r.json()["items"]
|
||||
assert [i["severity"] for i in items[:2]] == ["critical", "warning"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_cursor(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
for i in range(3):
|
||||
ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
|
||||
)
|
||||
r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=auth_headers)
|
||||
r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 2
|
||||
assert p1["next_cursor"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_cursor_400(app_client, auth_headers):
|
||||
r = await app_client.get("/api/v1/agents?cursor=$$$bad", headers=auth_headers)
|
||||
async def test_invalid_cursor_400(app_client, ui_auth_headers):
|
||||
r = await app_client.get("/api/v1/agents?cursor=$$$bad", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_query_invalid_cursor_uuid_400(app_client, ui_auth_headers):
|
||||
"""PH-010: cursor encodes (received_at, event_id) and event_id must parse
|
||||
as a UUID before reaching the SQL layer."""
|
||||
import base64
|
||||
|
||||
bad = base64.urlsafe_b64encode(b"2026-01-01T00:00:00+00:00|not-a-uuid").decode().rstrip("=")
|
||||
r = await app_client.get(f"/api/v1/events/query?cursor={bad}", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incidents_bidirectional_cursor_round_trip(app_client, auth_headers, ui_auth_headers):
|
||||
"""PH-review: page1 → next → prev returns page1 with the same items and
|
||||
order. prev_cursor must be null on the first page."""
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
for i in range(6):
|
||||
ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [ev]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
|
||||
r1 = await app_client.get("/api/v1/incidents?limit=3", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 3
|
||||
assert p1["next_cursor"]
|
||||
assert p1.get("prev_cursor") is None # first page
|
||||
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/incidents?limit=3&cursor={p1['next_cursor']}", headers=ui_auth_headers
|
||||
)
|
||||
p2 = r2.json()
|
||||
assert len(p2["items"]) == 3
|
||||
assert p2["prev_cursor"]
|
||||
ids1 = [i["id"] for i in p1["items"]]
|
||||
ids2 = [i["id"] for i in p2["items"]]
|
||||
assert set(ids1).isdisjoint(set(ids2))
|
||||
|
||||
r_back = await app_client.get(
|
||||
f"/api/v1/incidents?limit=3&cursor={p2['prev_cursor']}&back=true",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
p_back = r_back.json()
|
||||
assert [i["id"] for i in p_back["items"]] == ids1
|
||||
# back to first page => prev_cursor must be null again
|
||||
assert p_back.get("prev_cursor") is None
|
||||
assert p_back["next_cursor"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_bidirectional_cursor_round_trip(app_client, auth_headers, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
for i in range(5):
|
||||
ev = make_event(check_id=f"chk-{i}", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [ev]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
|
||||
r1 = await app_client.get("/api/v1/notifiers/outbox?limit=2", headers=ui_auth_headers)
|
||||
p1 = r1.json()
|
||||
assert len(p1["items"]) == 2
|
||||
assert p1["next_cursor"]
|
||||
assert p1.get("prev_cursor") is None
|
||||
|
||||
r2 = await app_client.get(
|
||||
f"/api/v1/notifiers/outbox?limit=2&cursor={p1['next_cursor']}",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
p2 = r2.json()
|
||||
assert p2["prev_cursor"]
|
||||
ids1 = [i["id"] for i in p1["items"]]
|
||||
ids2 = [i["id"] for i in p2["items"]]
|
||||
assert set(ids1).isdisjoint(set(ids2))
|
||||
|
||||
r_back = await app_client.get(
|
||||
f"/api/v1/notifiers/outbox?limit=2&cursor={p2['prev_cursor']}&back=true",
|
||||
headers=ui_auth_headers,
|
||||
)
|
||||
p_back = r_back.json()
|
||||
assert [i["id"] for i in p_back["items"]] == ids1
|
||||
assert p_back.get("prev_cursor") is None
|
||||
|
||||
@@ -67,14 +67,14 @@ async def test_drop_old_partitions_uses_retention(engine, monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_prevents_replay_via_ingest(app_client, auth_headers, session):
|
||||
async def test_dedup_prevents_replay_via_ingest(app_client, ui_auth_headers, session):
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from monlet_server.models import Event
|
||||
|
||||
from ._helpers import make_event, make_heartbeat
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers)
|
||||
ev = make_event(status="critical", exit_code=2)
|
||||
|
||||
r1 = await app_client.post(
|
||||
|
||||
35
server/tests/test_system_summary.py
Normal file
35
server/tests/test_system_summary.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import pytest
|
||||
|
||||
from ._helpers import make_event, register_agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_summary_aggregate(app_client, ui_auth_headers):
|
||||
auth_headers = await register_agent(app_client, ui_auth_headers, "agent-1")
|
||||
await register_agent(app_client, ui_auth_headers, "agent-2")
|
||||
ev = make_event(check_id="c1", status="critical", exit_code=2)
|
||||
await app_client.post(
|
||||
"/api/v1/events",
|
||||
json={"agent_id": "agent-1", "events": [ev]},
|
||||
headers=auth_headers,
|
||||
)
|
||||
|
||||
r = await app_client.get("/api/v1/system-summary", headers=ui_auth_headers)
|
||||
assert r.status_code == 200
|
||||
body = r.json()
|
||||
assert body["agents_online"] == 2
|
||||
assert body["agents_offline"] == 0
|
||||
assert body["checks_critical"] >= 1
|
||||
assert "generated_at" in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_summary_requires_auth(app_client):
|
||||
r = await app_client.get("/api/v1/system-summary")
|
||||
assert r.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outbox_invalid_state_400(app_client, ui_auth_headers):
|
||||
r = await app_client.get("/api/v1/notifiers/outbox?state=bogus", headers=ui_auth_headers)
|
||||
assert r.status_code == 400
|
||||
Reference in New Issue
Block a user