creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve timestamp handling and event navigation

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 11:01:27 +04:00
parent edc51e9c59
commit 71f0035b0b
37 changed files with 485 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
server/monlet_server/services/ingestion.py
View File

@@ -12,6 +12,7 @@ from ..models import Agent, Check, Event, Incident, NotificationOutbox
from ..redaction import redact
from ..schemas import CheckResultEvent, HeartbeatRequest
from ..settings import get_settings
from ..timeutil import to_utc
INCIDENT_KEY_MAX = 256
@@ -37,11 +38,7 @@ def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
observed = (
hb.observed_at.astimezone(UTC)
if hb.observed_at.tzinfo
else hb.observed_at.replace(tzinfo=UTC)
)
observed = to_utc(hb.observed_at)
stmt = pg_insert(Agent).values(
agent_id=hb.agent_id,
hostname=hb.hostname,
@@ -71,7 +68,7 @@ async def _apply_incident(
incident_key: str,
) -> tuple[bool, Incident | None]:
"""Return (transition, incident). transition True if open->resolved or none->open."""
observed = ev.observed_at
observed = to_utc(ev.observed_at)
if ev.status == "ok":
res = await session.execute(
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
@@ -159,7 +156,7 @@ async def ingest_event(
ev: CheckResultEvent,
) -> bool:
"""Return True if accepted, False if deduplicated."""
observed = ev.observed_at
observed = to_utc(ev.observed_at)
output = redact(ev.output)
incident_key = _incident_key(agent_id, ev)

9
server/monlet_server/services/notifiers/alertmanager.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import httpx
from ...redaction import redact
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
@@ -10,11 +11,13 @@ from .http import classify_response
class AlertmanagerNotifier:
name = "alertmanager"
def __init__(self, client: httpx.AsyncClient, url: str) -> None:
def __init__(self, client: httpx.AsyncClient, url: str, time_zone: str) -> None:
self._client = client
self._url = url.rstrip("/")
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
display_payload = payload_with_display_times(payload, self._time_zone)
labels = {
"alertname": "monlet_incident",
"agent_id": str(payload.get("agent_id", "")),
@@ -25,7 +28,11 @@ class AlertmanagerNotifier:
"severity": str(payload.get("severity", "unknown")),
"status": str(payload.get("status", "")),
"summary": redact(payload.get("summary") or "") or "",
"time_zone": display_payload["time_zone"],
}
for key in ("observed_at_display", "opened_at_display", "resolved_at_display"):
if display_payload.get(key):
annotations[key.removesuffix("_display")] = display_payload[key]
if payload.get("output"):
annotations["output"] = (redact(payload["output"]) or "")[:1024]
alert = {

10
server/monlet_server/services/notifiers/debug.py
View File

@@ -1,14 +1,20 @@
from ...logging_config import get_logger
from ...redaction import redact_dict
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
class DebugNotifier:
name = "debug"
def __init__(self) -> None:
def __init__(self, time_zone: str = "UTC") -> None:
self._log = get_logger("monlet.notifier.debug")
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
self._log.info("notify", event_type=event_type, payload=redact_dict(payload))
self._log.info(
"notify",
event_type=event_type,
payload=redact_dict(payload_with_display_times(payload, self._time_zone)),
)
return DeliveryResult(ok=True)

16
server/monlet_server/services/notifiers/registry.py
View File

@@ -13,19 +13,27 @@ from .webhook import WebhookNotifier
def build_registry(settings: Settings, client: httpx.AsyncClient) -> dict[str, Notifier]:
reg: dict[str, Notifier] = {}
if settings.notifier_debug_enabled:
reg["debug"] = DebugNotifier()
reg["debug"] = DebugNotifier(settings.notification_time_zone)
if (
settings.notifier_telegram_enabled
and settings.notifier_telegram_token
and settings.notifier_telegram_chat_id
):
reg["telegram"] = TelegramNotifier(
client, settings.notifier_telegram_token, settings.notifier_telegram_chat_id
client,
settings.notifier_telegram_token,
settings.notifier_telegram_chat_id,
settings.notification_time_zone,
)
if settings.notifier_webhook_enabled and settings.notifier_webhook_url:
reg["webhook"] = WebhookNotifier(
client, settings.notifier_webhook_url, settings.notifier_webhook_token
client,
settings.notifier_webhook_url,
settings.notifier_webhook_token,
settings.notification_time_zone,
)
if settings.notifier_alertmanager_enabled and settings.notifier_alertmanager_url:
reg["alertmanager"] = AlertmanagerNotifier(client, settings.notifier_alertmanager_url)
reg["alertmanager"] = AlertmanagerNotifier(
client, settings.notifier_alertmanager_url, settings.notification_time_zone
)
return reg

14
server/monlet_server/services/notifiers/telegram.py
View File

@@ -3,11 +3,13 @@ from __future__ import annotations
import httpx
from ...redaction import redact
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
def _format_message(event_type: str, payload: dict) -> str:
def _format_message(event_type: str, payload: dict, time_zone: str) -> str:
payload = payload_with_display_times(payload, time_zone)
icon = "[FIRING]" if event_type == "firing" else "[RESOLVED]"
sev = payload.get("severity", "unknown")
agent = payload.get("agent_id", "?")
@@ -19,7 +21,7 @@ def _format_message(event_type: str, payload: dict) -> str:
f"{icon} {sev} {agent}/{check}",
f"status: {status}",
f"incident_key: {incident_key}",
f"observed_at: {payload.get('observed_at', '?')}",
f"observed_at: {payload.get('observed_at_display') or payload.get('observed_at', '?')}",
]
if output:
lines.append(f"output: {output[:300]}")
@@ -29,14 +31,18 @@ def _format_message(event_type: str, payload: dict) -> str:
class TelegramNotifier:
name = "telegram"
def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str) -> None:
def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str, time_zone: str) -> None:
self._client = client
self._token = token
self._chat_id = chat_id
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
url = f"https://api.telegram.org/bot{self._token}/sendMessage"
body = {"chat_id": self._chat_id, "text": _format_message(event_type, payload)}
body = {
"chat_id": self._chat_id,
"text": _format_message(event_type, payload, self._time_zone),
}
try:
resp = await self._client.post(url, json=body)
except httpx.HTTPError as exc:

11
server/monlet_server/services/notifiers/webhook.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import httpx
from ...redaction import redact_dict
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
@@ -10,16 +11,22 @@ from .http import classify_response
class WebhookNotifier:
name = "webhook"
def __init__(self, client: httpx.AsyncClient, url: str, token: str = "") -> None:
def __init__(
self, client: httpx.AsyncClient, url: str, token: str = "", time_zone: str = "UTC"
) -> None:
self._client = client
self._url = url
self._token = token
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
headers = {"Content-Type": "application/json"}
if self._token:
headers["Authorization"] = f"Bearer {self._token}"
body = {"event_type": event_type, "incident": redact_dict(payload)}
body = {
"event_type": event_type,
"incident": redact_dict(payload_with_display_times(payload, self._time_zone)),
}
try:
resp = await self._client.post(self._url, json=body, headers=headers)
except httpx.HTTPError as exc: