Implement Monlet MVP stack and UI updates
This commit is contained in:
Stanislav Rossovskii
237
server/monlet_server/services/ingestion.py
Normal file
237
server/monlet_server/services/ingestion.py
Normal file
@@ -0,0 +1,237 @@
|
||||
from datetime import UTC, datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from .. import metrics
|
||||
from ..logging_config import get_logger
|
||||
from ..models import Agent, Check, Event, Incident, NotificationOutbox
|
||||
from ..redaction import redact
|
||||
from ..schemas import CheckResultEvent, HeartbeatRequest
|
||||
from ..settings import get_settings
|
||||
|
||||
INCIDENT_KEY_MAX = 256
|
||||
|
||||
log = get_logger("monlet.ingestion")
|
||||
|
||||
_SEVERITY_RANK = {"unknown": 1, "warning": 2, "critical": 3}
|
||||
|
||||
|
||||
def _severity_for_status(status: str) -> str:
|
||||
if status in ("warning", "critical", "unknown"):
|
||||
return status
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
|
||||
key = ev.incident_key or f"{agent_id}:{ev.check_id}"
|
||||
if len(key) > INCIDENT_KEY_MAX:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"incident_key exceeds {INCIDENT_KEY_MAX} chars",
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
|
||||
observed = (
|
||||
hb.observed_at.astimezone(UTC)
|
||||
if hb.observed_at.tzinfo
|
||||
else hb.observed_at.replace(tzinfo=UTC)
|
||||
)
|
||||
stmt = pg_insert(Agent).values(
|
||||
agent_id=hb.agent_id,
|
||||
hostname=hb.hostname,
|
||||
status="alive",
|
||||
last_seen_at=observed,
|
||||
features=hb.features.model_dump(),
|
||||
labels=hb.labels,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=[Agent.agent_id],
|
||||
set_={
|
||||
"hostname": stmt.excluded.hostname,
|
||||
"status": "alive",
|
||||
"last_seen_at": stmt.excluded.last_seen_at,
|
||||
"features": stmt.excluded.features,
|
||||
"labels": stmt.excluded.labels,
|
||||
},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
metrics.heartbeats_total.labels(result="ok").inc()
|
||||
|
||||
|
||||
async def _apply_incident(
|
||||
session: AsyncSession,
|
||||
agent_id: str,
|
||||
ev: CheckResultEvent,
|
||||
incident_key: str,
|
||||
) -> tuple[bool, Incident | None]:
|
||||
"""Return (transition, incident). transition True if open->resolved or none->open."""
|
||||
observed = ev.observed_at
|
||||
if ev.status == "ok":
|
||||
res = await session.execute(
|
||||
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
|
||||
)
|
||||
inc = res.scalar_one_or_none()
|
||||
if inc is None:
|
||||
return False, None
|
||||
inc.state = "resolved"
|
||||
inc.resolved_at = observed
|
||||
inc.last_event_id = UUID(ev.event_id)
|
||||
metrics.incidents_resolved_total.inc()
|
||||
return True, inc
|
||||
|
||||
severity = _severity_for_status(ev.status)
|
||||
new_id = uuid4()
|
||||
summary = (redact(ev.output) or "")[:200] if ev.output else None
|
||||
stmt = pg_insert(Incident).values(
|
||||
id=new_id,
|
||||
incident_key=incident_key,
|
||||
agent_id=agent_id,
|
||||
check_id=ev.check_id,
|
||||
state="open",
|
||||
severity=severity,
|
||||
opened_at=observed,
|
||||
last_event_id=UUID(ev.event_id),
|
||||
summary=summary,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_nothing(
|
||||
index_elements=[Incident.incident_key],
|
||||
index_where=text("state = 'open'"),
|
||||
).returning(Incident.id)
|
||||
ins = await session.execute(stmt)
|
||||
inserted_id = ins.scalar_one_or_none()
|
||||
if inserted_id is not None:
|
||||
await session.flush()
|
||||
res = await session.execute(select(Incident).where(Incident.id == inserted_id))
|
||||
metrics.incidents_opened_total.inc()
|
||||
return True, res.scalar_one()
|
||||
|
||||
res = await session.execute(
|
||||
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
|
||||
)
|
||||
inc = res.scalar_one()
|
||||
if inc.agent_id != agent_id or inc.check_id != ev.check_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="incident_key already bound to another (agent_id, check_id)",
|
||||
)
|
||||
cur = _SEVERITY_RANK.get(inc.severity, 0)
|
||||
new = _SEVERITY_RANK.get(severity, 0)
|
||||
if new > cur:
|
||||
inc.severity = severity
|
||||
inc.last_event_id = UUID(ev.event_id)
|
||||
return False, inc
|
||||
|
||||
|
||||
async def _enqueue_outbox(
|
||||
session: AsyncSession,
|
||||
incident: Incident,
|
||||
event_type: str,
|
||||
payload: dict,
|
||||
) -> None:
|
||||
notifiers = get_settings().enabled_notifiers
|
||||
if not notifiers:
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
for name in notifiers:
|
||||
session.add(
|
||||
NotificationOutbox(
|
||||
id=uuid4(),
|
||||
incident_id=incident.id,
|
||||
notifier=name,
|
||||
event_type=event_type,
|
||||
state="pending",
|
||||
attempts=0,
|
||||
next_attempt_at=now,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def ingest_event(
|
||||
session: AsyncSession,
|
||||
agent_id: str,
|
||||
ev: CheckResultEvent,
|
||||
) -> bool:
|
||||
"""Return True if accepted, False if deduplicated."""
|
||||
observed = ev.observed_at
|
||||
output = redact(ev.output)
|
||||
incident_key = _incident_key(agent_id, ev)
|
||||
|
||||
stmt = pg_insert(Event).values(
|
||||
event_id=UUID(ev.event_id),
|
||||
agent_id=agent_id,
|
||||
check_id=ev.check_id,
|
||||
observed_at=observed,
|
||||
status=ev.status,
|
||||
exit_code=ev.exit_code,
|
||||
duration_ms=ev.duration_ms,
|
||||
output=output,
|
||||
output_truncated=ev.output_truncated,
|
||||
notifications_enabled=ev.notifications_enabled,
|
||||
incident_key=incident_key,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
|
||||
result = await session.execute(stmt)
|
||||
inserted = result.scalar_one_or_none()
|
||||
if inserted is None:
|
||||
metrics.events_total.labels(result="deduplicated").inc()
|
||||
return False
|
||||
|
||||
metrics.events_total.labels(result="accepted").inc()
|
||||
|
||||
chk_res = await session.execute(
|
||||
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
|
||||
)
|
||||
cur = chk_res.scalar_one_or_none()
|
||||
is_late = cur is not None and cur.last_observed_at > observed
|
||||
if is_late:
|
||||
return True
|
||||
|
||||
if cur is None:
|
||||
session.add(
|
||||
Check(
|
||||
agent_id=agent_id,
|
||||
check_id=ev.check_id,
|
||||
status=ev.status,
|
||||
exit_code=ev.exit_code,
|
||||
last_observed_at=observed,
|
||||
last_event_id=UUID(ev.event_id),
|
||||
incident_key=incident_key,
|
||||
)
|
||||
)
|
||||
else:
|
||||
cur.status = ev.status
|
||||
cur.exit_code = ev.exit_code
|
||||
cur.last_observed_at = observed
|
||||
cur.last_event_id = UUID(ev.event_id)
|
||||
cur.incident_key = incident_key
|
||||
|
||||
transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
|
||||
if transition and inc is not None and ev.notifications_enabled:
|
||||
event_type = "resolved" if ev.status == "ok" else "firing"
|
||||
await _enqueue_outbox(
|
||||
session,
|
||||
inc,
|
||||
event_type,
|
||||
{
|
||||
"incident_id": str(inc.id),
|
||||
"agent_id": agent_id,
|
||||
"check_id": ev.check_id,
|
||||
"status": ev.status,
|
||||
"severity": inc.severity,
|
||||
"incident_key": incident_key,
|
||||
"observed_at": observed.isoformat(),
|
||||
"opened_at": inc.opened_at.isoformat() if inc.opened_at else None,
|
||||
"resolved_at": inc.resolved_at.isoformat() if inc.resolved_at else None,
|
||||
"summary": inc.summary,
|
||||
"output": output,
|
||||
"event_type": event_type,
|
||||
},
|
||||
)
|
||||
return True
|
||||
Reference in New Issue
Block a user