235 lines
7.4 KiB
Python
235 lines
7.4 KiB
Python
from datetime import UTC, datetime
|
|
from uuid import UUID, uuid4
|
|
|
|
from fastapi import HTTPException
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from .. import metrics
|
|
from ..logging_config import get_logger
|
|
from ..models import Agent, Check, Event, Incident, NotificationOutbox
|
|
from ..redaction import redact
|
|
from ..schemas import CheckResultEvent, HeartbeatRequest
|
|
from ..settings import get_settings
|
|
from ..timeutil import to_utc
|
|
|
|
INCIDENT_KEY_MAX = 256
|
|
|
|
log = get_logger("monlet.ingestion")
|
|
|
|
_SEVERITY_RANK = {"unknown": 1, "warning": 2, "critical": 3}
|
|
|
|
|
|
def _severity_for_status(status: str) -> str:
|
|
if status in ("warning", "critical", "unknown"):
|
|
return status
|
|
return "unknown"
|
|
|
|
|
|
def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
|
|
key = ev.incident_key or f"{agent_id}:{ev.check_id}"
|
|
if len(key) > INCIDENT_KEY_MAX:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"incident_key exceeds {INCIDENT_KEY_MAX} chars",
|
|
)
|
|
return key
|
|
|
|
|
|
async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
|
|
observed = to_utc(hb.observed_at)
|
|
stmt = pg_insert(Agent).values(
|
|
agent_id=hb.agent_id,
|
|
hostname=hb.hostname,
|
|
status="alive",
|
|
last_seen_at=observed,
|
|
features=hb.features.model_dump(),
|
|
labels=hb.labels,
|
|
)
|
|
stmt = stmt.on_conflict_do_update(
|
|
index_elements=[Agent.agent_id],
|
|
set_={
|
|
"hostname": stmt.excluded.hostname,
|
|
"status": "alive",
|
|
"last_seen_at": stmt.excluded.last_seen_at,
|
|
"features": stmt.excluded.features,
|
|
"labels": stmt.excluded.labels,
|
|
},
|
|
)
|
|
await session.execute(stmt)
|
|
metrics.heartbeats_total.labels(result="ok").inc()
|
|
|
|
|
|
async def _apply_incident(
|
|
session: AsyncSession,
|
|
agent_id: str,
|
|
ev: CheckResultEvent,
|
|
incident_key: str,
|
|
) -> tuple[bool, Incident | None]:
|
|
"""Return (transition, incident). transition True if open->resolved or none->open."""
|
|
observed = to_utc(ev.observed_at)
|
|
if ev.status == "ok":
|
|
res = await session.execute(
|
|
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
|
|
)
|
|
inc = res.scalar_one_or_none()
|
|
if inc is None:
|
|
return False, None
|
|
inc.state = "resolved"
|
|
inc.resolved_at = observed
|
|
inc.last_event_id = UUID(ev.event_id)
|
|
metrics.incidents_resolved_total.inc()
|
|
return True, inc
|
|
|
|
severity = _severity_for_status(ev.status)
|
|
new_id = uuid4()
|
|
summary = (redact(ev.output) or "")[:200] if ev.output else None
|
|
stmt = pg_insert(Incident).values(
|
|
id=new_id,
|
|
incident_key=incident_key,
|
|
agent_id=agent_id,
|
|
check_id=ev.check_id,
|
|
state="open",
|
|
severity=severity,
|
|
opened_at=observed,
|
|
last_event_id=UUID(ev.event_id),
|
|
summary=summary,
|
|
)
|
|
stmt = stmt.on_conflict_do_nothing(
|
|
index_elements=[Incident.incident_key],
|
|
index_where=text("state = 'open'"),
|
|
).returning(Incident.id)
|
|
ins = await session.execute(stmt)
|
|
inserted_id = ins.scalar_one_or_none()
|
|
if inserted_id is not None:
|
|
await session.flush()
|
|
res = await session.execute(select(Incident).where(Incident.id == inserted_id))
|
|
metrics.incidents_opened_total.inc()
|
|
return True, res.scalar_one()
|
|
|
|
res = await session.execute(
|
|
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
|
|
)
|
|
inc = res.scalar_one()
|
|
if inc.agent_id != agent_id or inc.check_id != ev.check_id:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="incident_key already bound to another (agent_id, check_id)",
|
|
)
|
|
cur = _SEVERITY_RANK.get(inc.severity, 0)
|
|
new = _SEVERITY_RANK.get(severity, 0)
|
|
if new > cur:
|
|
inc.severity = severity
|
|
inc.last_event_id = UUID(ev.event_id)
|
|
return False, inc
|
|
|
|
|
|
async def _enqueue_outbox(
|
|
session: AsyncSession,
|
|
incident: Incident,
|
|
event_type: str,
|
|
payload: dict,
|
|
) -> None:
|
|
notifiers = get_settings().enabled_notifiers
|
|
if not notifiers:
|
|
return
|
|
now = datetime.now(UTC)
|
|
for name in notifiers:
|
|
session.add(
|
|
NotificationOutbox(
|
|
id=uuid4(),
|
|
incident_id=incident.id,
|
|
notifier=name,
|
|
event_type=event_type,
|
|
state="pending",
|
|
attempts=0,
|
|
next_attempt_at=now,
|
|
payload=payload,
|
|
)
|
|
)
|
|
|
|
|
|
async def ingest_event(
|
|
session: AsyncSession,
|
|
agent_id: str,
|
|
ev: CheckResultEvent,
|
|
) -> bool:
|
|
"""Return True if accepted, False if deduplicated."""
|
|
observed = to_utc(ev.observed_at)
|
|
output = redact(ev.output)
|
|
incident_key = _incident_key(agent_id, ev)
|
|
|
|
stmt = pg_insert(Event).values(
|
|
event_id=UUID(ev.event_id),
|
|
agent_id=agent_id,
|
|
check_id=ev.check_id,
|
|
observed_at=observed,
|
|
status=ev.status,
|
|
exit_code=ev.exit_code,
|
|
duration_ms=ev.duration_ms,
|
|
output=output,
|
|
output_truncated=ev.output_truncated,
|
|
notifications_enabled=ev.notifications_enabled,
|
|
incident_key=incident_key,
|
|
)
|
|
stmt = stmt.on_conflict_do_nothing(index_elements=[Event.event_id]).returning(Event.event_id)
|
|
result = await session.execute(stmt)
|
|
inserted = result.scalar_one_or_none()
|
|
if inserted is None:
|
|
metrics.events_total.labels(result="deduplicated").inc()
|
|
return False
|
|
|
|
metrics.events_total.labels(result="accepted").inc()
|
|
|
|
chk_res = await session.execute(
|
|
select(Check).where(Check.agent_id == agent_id, Check.check_id == ev.check_id)
|
|
)
|
|
cur = chk_res.scalar_one_or_none()
|
|
is_late = cur is not None and cur.last_observed_at > observed
|
|
if is_late:
|
|
return True
|
|
|
|
if cur is None:
|
|
session.add(
|
|
Check(
|
|
agent_id=agent_id,
|
|
check_id=ev.check_id,
|
|
status=ev.status,
|
|
exit_code=ev.exit_code,
|
|
last_observed_at=observed,
|
|
last_event_id=UUID(ev.event_id),
|
|
incident_key=incident_key,
|
|
)
|
|
)
|
|
else:
|
|
cur.status = ev.status
|
|
cur.exit_code = ev.exit_code
|
|
cur.last_observed_at = observed
|
|
cur.last_event_id = UUID(ev.event_id)
|
|
cur.incident_key = incident_key
|
|
|
|
transition, inc = await _apply_incident(session, agent_id, ev, incident_key)
|
|
if transition and inc is not None and ev.notifications_enabled:
|
|
event_type = "resolved" if ev.status == "ok" else "firing"
|
|
await _enqueue_outbox(
|
|
session,
|
|
inc,
|
|
event_type,
|
|
{
|
|
"incident_id": str(inc.id),
|
|
"agent_id": agent_id,
|
|
"check_id": ev.check_id,
|
|
"status": ev.status,
|
|
"severity": inc.severity,
|
|
"incident_key": incident_key,
|
|
"observed_at": observed.isoformat(),
|
|
"opened_at": inc.opened_at.isoformat() if inc.opened_at else None,
|
|
"resolved_at": inc.resolved_at.isoformat() if inc.resolved_at else None,
|
|
"summary": inc.summary,
|
|
"output": output,
|
|
"event_type": event_type,
|
|
},
|
|
)
|
|
return True
|