creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

store only status-change events; UI client-side polling refactor

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 14:58:46 +04:00
parent 71f0035b0b
commit d5f312f200
48 changed files with 2247 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
server/monlet_server/services/detector.py
View File

@@ -1,7 +1,9 @@
import asyncio
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from sqlalchemy import delete, func, select, update
from sqlalchemy import delete, func, select, text
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import async_sessionmaker
from .. import metrics
@@ -12,6 +14,138 @@ from ..settings import get_settings
log = get_logger("monlet.detector")
OUTBOX_TERMINAL_STATES = ("sent", "failed", "discarded")
LIVENESS_CHECK_ID = "agent_liveness"
def _liveness_key(agent_id: str) -> str:
return f"agent:{agent_id}:liveness"
def _liveness_severity(status: str) -> str:
return "critical" if status == "dead" else "warning"
async def _enqueue_liveness_outbox(
session,
incident: Incident,
event_type: str,
agent: Agent,
status: str,
now: datetime,
summary: str,
severity: str,
) -> None:
notifiers = get_settings().enabled_notifiers
if not notifiers:
return
for name in notifiers:
session.add(
NotificationOutbox(
id=uuid4(),
incident_id=incident.id,
notifier=name,
event_type=event_type,
state="pending",
attempts=0,
next_attempt_at=now,
payload={
"incident_id": str(incident.id),
"agent_id": agent.agent_id,
"check_id": LIVENESS_CHECK_ID,
"status": status,
"severity": severity,
"incident_key": incident.incident_key,
"observed_at": now.isoformat(),
"opened_at": incident.opened_at.isoformat() if incident.opened_at else None,
"resolved_at": incident.resolved_at.isoformat()
if incident.resolved_at
else None,
"summary": summary,
"output": summary,
"event_type": event_type,
},
)
)
async def _open_liveness_incident(session, agent: Agent, status: str, now: datetime) -> None:
severity = _liveness_severity(status)
summary = f"agent is {status}"
incident_key = _liveness_key(agent.agent_id)
new_id = uuid4()
stmt = (
pg_insert(Incident)
.values(
id=new_id,
incident_key=incident_key,
agent_id=agent.agent_id,
check_id=LIVENESS_CHECK_ID,
state="open",
severity=severity,
opened_at=now,
summary=summary,
last_event_id=uuid4(),
)
.on_conflict_do_nothing(
index_elements=[Incident.incident_key],
index_where=text("state = 'open'"),
)
.returning(Incident.id)
)
inserted_id = (await session.execute(stmt)).scalar_one_or_none()
if inserted_id is not None:
await session.flush()
inc = (
await session.execute(select(Incident).where(Incident.id == inserted_id))
).scalar_one()
metrics.incidents_opened_total.inc()
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity=severity
)
return
inc = (
await session.execute(
select(Incident).where(
Incident.incident_key == incident_key,
Incident.state == "open",
)
)
).scalar_one()
if status == "dead" and inc.severity != "critical":
inc.severity = "critical"
inc.summary = summary
inc.last_event_id = uuid4()
await _enqueue_liveness_outbox(
session, inc, "firing", agent, status, now, summary=summary, severity="critical"
)
async def _resolve_liveness_incident(session, agent: Agent, now: datetime) -> None:
inc = (
await session.execute(
select(Incident).where(
Incident.incident_key == _liveness_key(agent.agent_id),
Incident.state == "open",
)
)
).scalar_one_or_none()
if inc is None:
return
inc.state = "resolved"
inc.resolved_at = now
inc.last_event_id = uuid4()
metrics.incidents_resolved_total.inc()
await _enqueue_liveness_outbox(
session,
inc,
"resolved",
agent,
"ok",
now,
summary="agent is alive",
severity="ok",
)
async def _prune_history(session) -> None:
@@ -44,25 +178,20 @@ async def _tick(sm: async_sessionmaker) -> None:
stale_cut = now - timedelta(seconds=s.stale_after_sec)
dead_cut = now - timedelta(seconds=s.dead_after_sec)
async with sm() as session:
await session.execute(
update(Agent)
.where(Agent.last_seen_at <= dead_cut)
.where(Agent.status != "dead")
.values(status="dead")
)
await session.execute(
update(Agent)
.where(Agent.last_seen_at <= stale_cut)
.where(Agent.last_seen_at > dead_cut)
.where(Agent.status != "stale")
.values(status="stale")
)
await session.execute(
update(Agent)
.where(Agent.last_seen_at > stale_cut)
.where(Agent.status != "alive")
.values(status="alive")
)
agents = (await session.execute(select(Agent))).scalars().all()
for agent in agents:
if agent.last_seen_at <= dead_cut:
next_status = "dead"
elif agent.last_seen_at <= stale_cut:
next_status = "stale"
else:
next_status = "alive"
if agent.status != next_status:
agent.status = next_status
if next_status in ("stale", "dead"):
await _open_liveness_incident(session, agent, next_status, now)
else:
await _resolve_liveness_incident(session, agent, now)
await _prune_history(session)
await session.commit()

102
server/monlet_server/services/ingestion.py
View File

@@ -155,59 +155,79 @@ async def ingest_event(
agent_id: str,
ev: CheckResultEvent,
) -> bool:
"""Return True if accepted, False if deduplicated."""
"""Return True if a new change row was written, False if not (duplicate/unchanged/late)."""
observed = to_utc(ev.observed_at)
output = redact(ev.output)
check_summary = (output or "")[:200] if output else None
incident_key = _incident_key(agent_id, ev)
stmt = pg_insert(Event).values(
event_id=UUID(ev.event_id),
agent_id=agent_id,
check_id=ev.check_id,
observed_at=observed,
status=ev.status,
exit_code=ev.exit_code,
duration_ms=ev.duration_ms,
output=output,
output_truncated=ev.output_truncated,
notifications_enabled=ev.notifications_enabled,
incident_key=incident_key,
)
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
result = await session.execute(stmt)
inserted = result.scalar_one_or_none()
if inserted is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
metrics.events_total.labels(result="accepted").inc()
chk_res = await session.execute(
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
)
cur = chk_res.scalar_one_or_none()
is_late = cur is not None and cur.last_observed_at > observed
if is_late:
return True
changed = cur is None or cur.status != ev.status or cur.exit_code != ev.exit_code
if cur is None:
session.add(
Check(
agent_id=agent_id,
check_id=ev.check_id,
status=ev.status,
exit_code=ev.exit_code,
last_observed_at=observed,
last_event_id=UUID(ev.event_id),
incident_key=incident_key,
)
if is_late:
metrics.events_total.labels(result="late").inc()
return False
if changed:
stmt = pg_insert(Event).values(
event_id=UUID(ev.event_id),
agent_id=agent_id,
check_id=ev.check_id,
observed_at=observed,
status=ev.status,
exit_code=ev.exit_code,
duration_ms=ev.duration_ms,
output=output,
output_truncated=ev.output_truncated,
notifications_enabled=ev.notifications_enabled,
incident_key=incident_key,
)
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(
Event.event_id
)
result = await session.execute(stmt)
inserted = result.scalar_one_or_none()
if inserted is None:
metrics.events_total.labels(result="deduplicated").inc()
return False
metrics.events_total.labels(result="accepted").inc()
else:
cur.status = ev.status
cur.exit_code = ev.exit_code
cur.last_observed_at = observed
cur.last_event_id = UUID(ev.event_id)
cur.incident_key = incident_key
metrics.events_total.labels(result="unchanged").inc()
# Upsert Check to be safe under concurrent first-write race for the same (agent_id, check_id).
# When changed=False keep the existing last_event_id and incident_key — incident_key changes
# are only meaningful at transitions and would otherwise desync from any open Incident.
check_values = {
"agent_id": agent_id,
"check_id": ev.check_id,
"status": ev.status,
"exit_code": ev.exit_code,
"last_observed_at": observed,
"last_event_id": UUID(ev.event_id),
"incident_key": incident_key,
"summary": check_summary,
}
update_set = {
"status": ev.status,
"exit_code": ev.exit_code,
"last_observed_at": observed,
"summary": check_summary,
}
if changed:
update_set["last_event_id"] = UUID(ev.event_id)
update_set["incident_key"] = incident_key
chk_stmt = pg_insert(Check).values(**check_values)
chk_stmt = chk_stmt.on_conflict_do_update(
index_elements=[Check.agent_id, Check.check_id], set_=update_set
)
await session.execute(chk_stmt)
if not changed:
return False
transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
if transition and inc is not None and ev.notifications_enabled: