store only status-change events; UI client-side polling refactor
This commit is contained in:
Stanislav Rossovskii
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from .. import metrics
|
||||
@@ -12,6 +14,138 @@ from ..settings import get_settings
|
||||
log = get_logger("monlet.detector")
|
||||
|
||||
OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
|
||||
LIVENESS_CHECK_ID = "agent_liveness"
|
||||
|
||||
|
||||
def _liveness_key(agent_id: str) -> str:
|
||||
return f"agent:{agent_id}:liveness"
|
||||
|
||||
|
||||
def _liveness_severity(status: str) -> str:
|
||||
return "critical" if status == "dead" else "warning"
|
||||
|
||||
|
||||
async def _enqueue_liveness_outbox(
|
||||
session,
|
||||
incident: Incident,
|
||||
event_type: str,
|
||||
agent: Agent,
|
||||
status: str,
|
||||
now: datetime,
|
||||
summary: str,
|
||||
severity: str,
|
||||
) -> None:
|
||||
notifiers = get_settings().enabled_notifiers
|
||||
if not notifiers:
|
||||
return
|
||||
for name in notifiers:
|
||||
session.add(
|
||||
NotificationOutbox(
|
||||
id=uuid4(),
|
||||
incident_id=incident.id,
|
||||
notifier=name,
|
||||
event_type=event_type,
|
||||
state="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=now,
|
||||
payload={
|
||||
"incident_id": str(incident.id),
|
||||
"agent_id": agent.agent_id,
|
||||
"check_id": LIVENESS_CHECK_ID,
|
||||
"status": status,
|
||||
"severity": severity,
|
||||
"incident_key": incident.incident_key,
|
||||
"observed_at": now.isoformat(),
|
||||
"opened_at": incident.opened_at.isoformat() if incident.opened_at else None,
|
||||
"resolved_at": incident.resolved_at.isoformat()
|
||||
if incident.resolved_at
|
||||
else None,
|
||||
"summary": summary,
|
||||
"output": summary,
|
||||
"event_type": event_type,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
|
||||
severity = _liveness_severity(status)
|
||||
summary = f"agent is {status}"
|
||||
incident_key = _liveness_key(agent.agent_id)
|
||||
new_id = uuid4()
|
||||
stmt = (
|
||||
pg_insert(Incident)
|
||||
.values(
|
||||
id=new_id,
|
||||
incident_key=incident_key,
|
||||
agent_id=agent.agent_id,
|
||||
check_id=LIVENESS_CHECK_ID,
|
||||
state="open",
|
||||
severity=severity,
|
||||
opened_at=now,
|
||||
summary=summary,
|
||||
last_event_id=uuid4(),
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=[Incident.incident_key],
|
||||
index_where=text("state = 'open'"),
|
||||
)
|
||||
.returning(Incident.id)
|
||||
)
|
||||
inserted_id = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if inserted_id is not None:
|
||||
await session.flush()
|
||||
inc = (
|
||||
await session.execute(select(Incident).where(Incident.id == inserted_id))
|
||||
).scalar_one()
|
||||
metrics.incidents_opened_total.inc()
|
||||
await _enqueue_liveness_outbox(
|
||||
session, inc, "firing", agent, status, now, summary=summary, severity=severity
|
||||
)
|
||||
return
|
||||
|
||||
inc = (
|
||||
await session.execute(
|
||||
select(Incident).where(
|
||||
Incident.incident_key == incident_key,
|
||||
Incident.state == "open",
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
if status == "dead" and inc.severity != "critical":
|
||||
inc.severity = "critical"
|
||||
inc.summary = summary
|
||||
inc.last_event_id = uuid4()
|
||||
await _enqueue_liveness_outbox(
|
||||
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
|
||||
inc = (
|
||||
await session.execute(
|
||||
select(Incident).where(
|
||||
Incident.incident_key == _liveness_key(agent.agent_id),
|
||||
Incident.state == "open",
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if inc is None:
|
||||
return
|
||||
inc.state = "resolved"
|
||||
inc.resolved_at = now
|
||||
inc.last_event_id = uuid4()
|
||||
metrics.incidents_resolved_total.inc()
|
||||
await _enqueue_liveness_outbox(
|
||||
session,
|
||||
inc,
|
||||
"resolved",
|
||||
agent,
|
||||
"ok",
|
||||
now,
|
||||
summary="agent is alive",
|
||||
severity="ok",
|
||||
)
|
||||
|
||||
|
||||
async def _prune_history(session) -> None:
|
||||
@@ -44,25 +178,20 @@ async def _tick(sm: async_sessionmaker) -> None:
|
||||
stale_cut = now - timedelta(seconds=s.stale_after_sec)
|
||||
dead_cut = now - timedelta(seconds=s.dead_after_sec)
|
||||
async with sm() as session:
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.last_seen_at <= dead_cut)
|
||||
.where(Agent.status != "dead")
|
||||
.values(status="dead")
|
||||
)
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.last_seen_at <= stale_cut)
|
||||
.where(Agent.last_seen_at > dead_cut)
|
||||
.where(Agent.status != "stale")
|
||||
.values(status="stale")
|
||||
)
|
||||
await session.execute(
|
||||
update(Agent)
|
||||
.where(Agent.last_seen_at > stale_cut)
|
||||
.where(Agent.status != "alive")
|
||||
.values(status="alive")
|
||||
)
|
||||
agents = (await session.execute(select(Agent))).scalars().all()
|
||||
for agent in agents:
|
||||
if agent.last_seen_at <= dead_cut:
|
||||
next_status = "dead"
|
||||
elif agent.last_seen_at <= stale_cut:
|
||||
next_status = "stale"
|
||||
else:
|
||||
next_status = "alive"
|
||||
if agent.status != next_status:
|
||||
agent.status = next_status
|
||||
if next_status in ("stale", "dead"):
|
||||
await _open_liveness_incident(session, agent, next_status, now)
|
||||
else:
|
||||
await _resolve_liveness_incident(session, agent, now)
|
||||
await _prune_history(session)
|
||||
await session.commit()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user