110 lines
3.6 KiB
Python
110 lines
3.6 KiB
Python
from datetime import UTC, datetime, timedelta
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select, update
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
from monlet_server.models import Agent, Event, Incident, NotificationOutbox
|
|
from monlet_server.services.detector import _tick
|
|
from monlet_server.settings import reset_settings_cache
|
|
|
|
from ._helpers import make_event, make_heartbeat
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_detector_transitions(app_client, auth_headers, engine, session):
|
|
await app_client.post(
|
|
"/api/v1/heartbeat", json=make_heartbeat("agent-alive"), headers=auth_headers
|
|
)
|
|
await app_client.post(
|
|
"/api/v1/heartbeat", json=make_heartbeat("agent-stale"), headers=auth_headers
|
|
)
|
|
await app_client.post(
|
|
"/api/v1/heartbeat", json=make_heartbeat("agent-dead"), headers=auth_headers
|
|
)
|
|
|
|
now = datetime.now(UTC)
|
|
await session.execute(
|
|
update(Agent)
|
|
.where(Agent.agent_id == "agent-stale")
|
|
.values(last_seen_at=now - timedelta(seconds=120))
|
|
)
|
|
await session.execute(
|
|
update(Agent)
|
|
.where(Agent.agent_id == "agent-dead")
|
|
.values(last_seen_at=now - timedelta(minutes=10))
|
|
)
|
|
await session.commit()
|
|
|
|
sm = async_sessionmaker(engine, expire_on_commit=False)
|
|
await _tick(sm)
|
|
|
|
statuses = {
|
|
r.agent_id: r.status for r in (await session.execute(select(Agent))).scalars().all()
|
|
}
|
|
assert statuses["agent-alive"] == "alive"
|
|
assert statuses["agent-stale"] == "stale"
|
|
assert statuses["agent-dead"] == "dead"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_detector_prunes_bounded_history(
|
|
app_client, auth_headers, engine, session, monkeypatch
|
|
):
|
|
monkeypatch.setenv("MONLET_EVENTS_RETENTION_MAX_ROWS", "2")
|
|
monkeypatch.setenv("MONLET_OUTBOX_RETENTION_MAX_ROWS", "2")
|
|
reset_settings_cache()
|
|
try:
|
|
await app_client.post(
|
|
"/api/v1/heartbeat", json=make_heartbeat("agent-prune"), headers=auth_headers
|
|
)
|
|
events = [make_event(check_id="prune_check") for _ in range(5)]
|
|
await app_client.post(
|
|
"/api/v1/events",
|
|
json={"agent_id": "agent-prune", "events": events},
|
|
headers=auth_headers,
|
|
)
|
|
critical = make_event(check_id="critical_prune", status="critical", exit_code=2)
|
|
await app_client.post(
|
|
"/api/v1/events",
|
|
json={"agent_id": "agent-prune", "events": [critical]},
|
|
headers=auth_headers,
|
|
)
|
|
|
|
inc = (
|
|
await session.execute(select(Incident).where(Incident.check_id == "critical_prune"))
|
|
).scalar_one()
|
|
now = datetime.now(UTC)
|
|
for _ in range(5):
|
|
session.add(
|
|
NotificationOutbox(
|
|
id=uuid4(),
|
|
incident_id=inc.id,
|
|
notifier="debug",
|
|
event_type="firing",
|
|
state="sent",
|
|
attempts=1,
|
|
next_attempt_at=None,
|
|
payload={},
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
sm = async_sessionmaker(engine, expire_on_commit=False)
|
|
await _tick(sm)
|
|
|
|
assert (await session.execute(select(func.count()).select_from(Event))).scalar_one() == 2
|
|
terminal_outbox = (
|
|
await session.execute(
|
|
select(func.count())
|
|
.select_from(NotificationOutbox)
|
|
.where(NotificationOutbox.state == "sent")
|
|
)
|
|
).scalar_one()
|
|
assert terminal_outbox == 2
|
|
finally:
|
|
reset_settings_cache()
|