creator/monlet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve timestamp handling and event navigation

Browse Source
This commit is contained in:
Stanislav Rossovskii
2026-05-27 11:01:27 +04:00
parent edc51e9c59
commit 71f0035b0b
37 changed files with 485 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/.env.example
View File

@@ -16,6 +16,7 @@ MONLET_NOTIFIER_BATCH_SIZE=20
MONLET_NOTIFIER_MAX_ATTEMPTS=8
MONLET_NOTIFIER_LEASE_SEC=300
MONLET_NOTIFIER_HTTP_TIMEOUT_SEC=10
MONLET_NOTIFICATION_TIME_ZONE=UTC
MONLET_NOTIFIER_DEBUG_ENABLED=true
MONLET_NOTIFIER_TELEGRAM_ENABLED=false
MONLET_NOTIFIER_TELEGRAM_TOKEN=

1
server/README.md
View File

@@ -47,6 +47,7 @@ UV_PROJECT_ENVIRONMENT=.venv UV_CACHE_DIR=../.cache/uv uv run fastapi dev monlet
| `MONLET_NOTIFIER_MAX_ATTEMPTS` | `8` | Terminal-failure threshold (ADR-0005) |
| `MONLET_NOTIFIER_LEASE_SEC` | `300` | Recovery lease for stuck `sending` rows |
| `MONLET_NOTIFIER_HTTP_TIMEOUT_SEC` | `10` | Outbound HTTP timeout |
| `MONLET_NOTIFICATION_TIME_ZONE` | `UTC` | IANA timezone for human-readable notification timestamps |
| `MONLET_NOTIFIER_DEBUG_ENABLED` | `true` | Log-only notifier |
| `MONLET_NOTIFIER_TELEGRAM_ENABLED` | `false` | Telegram notifier on/off |
| `MONLET_NOTIFIER_TELEGRAM_TOKEN` | `` | Telegram bot token |

3
server/monlet_server/api/agents.py
View File

@@ -6,6 +6,7 @@ from ..cursors import decode, encode
from ..db import get_session
from ..models import Agent as AgentModel
from ..schemas import Agent, AgentsPage
from ..timeutil import to_utc
router = APIRouter()
@@ -15,7 +16,7 @@ def _to_schema(a: AgentModel) -> Agent:
agent_id=a.agent_id,
hostname=a.hostname,
status=a.status,
last_seen_at=a.last_seen_at,
last_seen_at=to_utc(a.last_seen_at),
features=a.features or {"push": False, "metrics": False},
labels=a.labels or {},
)

3
server/monlet_server/api/checks.py
View File

@@ -6,6 +6,7 @@ from ..cursors import decode, encode
from ..db import get_session
from ..models import Check
from ..schemas import ID_PATTERN, ChecksPage, CheckState
from ..timeutil import to_utc
router = APIRouter()
@@ -38,7 +39,7 @@ async def list_checks(
agent_id=r.agent_id,
check_id=r.check_id,
status=r.status,
last_observed_at=r.last_observed_at,
last_observed_at=to_utc(r.last_observed_at),
exit_code=r.exit_code,
incident_key=r.incident_key,
)

9
server/monlet_server/api/events.py
View File

@@ -16,12 +16,13 @@ from ..schemas import (
StoredCheckResultEvent,
)
from ..services.ingestion import ingest_event
from ..timeutil import to_utc
router = APIRouter()
def _encode_cursor(received_at: datetime, event_id) -> str:
raw = f"{received_at.isoformat()}|{event_id}".encode()
raw = f"{to_utc(received_at).isoformat()}|{event_id}".encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
@@ -30,7 +31,7 @@ def _decode_cursor(cursor: str) -> tuple[datetime, str]:
try:
raw = base64.urlsafe_b64decode(cursor + pad).decode()
ts_str, eid = raw.split("|", 1)
return datetime.fromisoformat(ts_str), eid
return to_utc(datetime.fromisoformat(ts_str)), eid
except (binascii.Error, ValueError) as exc:
raise HTTPException(status_code=400, detail="invalid cursor") from exc
@@ -87,8 +88,8 @@ async def query_events(
event_id=str(r.event_id),
agent_id=r.agent_id,
check_id=r.check_id,
observed_at=r.observed_at,
received_at=r.received_at,
observed_at=to_utc(r.observed_at),
received_at=to_utc(r.received_at),
status=r.status,
exit_code=r.exit_code,
duration_ms=r.duration_ms,

7
server/monlet_server/api/incidents.py
View File

@@ -6,6 +6,7 @@ from ..cursors import decode, decode_datetime, encode
from ..db import get_session
from ..models import Incident as IncidentModel
from ..schemas import Incident, IncidentsPage
from ..timeutil import to_utc
router = APIRouter()
@@ -18,8 +19,8 @@ def _to_schema(i: IncidentModel) -> Incident:
severity=i.severity,
agent_id=i.agent_id,
check_id=i.check_id,
opened_at=i.opened_at,
resolved_at=i.resolved_at,
opened_at=to_utc(i.opened_at),
resolved_at=to_utc(i.resolved_at) if i.resolved_at else None,
summary=i.summary,
)
@@ -49,5 +50,5 @@ async def list_incidents(
next_cursor = None
if len(rows) > limit:
rows = rows[:limit]
next_cursor = encode(rows[-1].opened_at.isoformat(), rows[-1].id)
next_cursor = encode(to_utc(rows[-1].opened_at).isoformat(), rows[-1].id)
return IncidentsPage(items=[_to_schema(r) for r in rows], next_cursor=next_cursor)

7
server/monlet_server/api/outbox.py
View File

@@ -6,6 +6,7 @@ from ..cursors import decode, decode_datetime, encode
from ..db import get_session
from ..models import NotificationOutbox
from ..schemas import NotificationOutboxItem, OutboxPage
from ..timeutil import to_utc
router = APIRouter()
@@ -38,7 +39,7 @@ async def list_outbox(
next_cursor = None
if len(rows) > limit:
rows = rows[:limit]
next_cursor = encode(rows[-1].created_at.isoformat(), rows[-1].id)
next_cursor = encode(to_utc(rows[-1].created_at).isoformat(), rows[-1].id)
return OutboxPage(
items=[
NotificationOutboxItem(
@@ -48,9 +49,9 @@ async def list_outbox(
incident_id=str(r.incident_id),
event_type=r.event_type,
attempts=r.attempts,
next_attempt_at=r.next_attempt_at,
next_attempt_at=to_utc(r.next_attempt_at) if r.next_attempt_at else None,
last_error=r.last_error,
updated_at=r.updated_at,
updated_at=to_utc(r.updated_at),
)
for r in rows
],

12
server/monlet_server/schemas.py
View File

@@ -5,6 +5,8 @@ from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator
from .timeutil import require_utc_datetime
ID_PATTERN = r"^[A-Za-z0-9._:-]+$"
UUIDV7_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
ID_RE = re.compile(ID_PATTERN)
@@ -80,6 +82,11 @@ class HeartbeatRequest(BaseModel):
features: AgentFeatures
labels: dict[str, str] = Field(default_factory=dict)
@field_validator("observed_at")
@classmethod
def _observed_at_must_be_utc(cls, v: datetime) -> datetime:
return require_utc_datetime(v)
@field_validator("labels")
@classmethod
def _check_labels(cls, v: dict[str, str]) -> dict[str, str]:
@@ -109,6 +116,11 @@ class CheckResultEvent(BaseModel):
notifications_enabled: bool = True
incident_key: str | None = Field(default=None, max_length=256)
@field_validator("observed_at")
@classmethod
def _observed_at_must_be_utc(cls, v: datetime) -> datetime:
return require_utc_datetime(v)
@field_validator("output")
@classmethod
def _check_output_bytes(cls, v: str | None) -> str | None:

11
server/monlet_server/services/ingestion.py
View File

@@ -12,6 +12,7 @@ from ..models import Agent, Check, Event, Incident, NotificationOutbox
from ..redaction import redact
from ..schemas import CheckResultEvent, HeartbeatRequest
from ..settings import get_settings
from ..timeutil import to_utc
INCIDENT_KEY_MAX = 256
@@ -37,11 +38,7 @@ def _incident_key(agent_id: str, ev: CheckResultEvent) -> str:
async def upsert_agent_heartbeat(session: AsyncSession, hb: HeartbeatRequest) -> None:
observed = (
hb.observed_at.astimezone(UTC)
if hb.observed_at.tzinfo
else hb.observed_at.replace(tzinfo=UTC)
)
observed = to_utc(hb.observed_at)
stmt = pg_insert(Agent).values(
agent_id=hb.agent_id,
hostname=hb.hostname,
@@ -71,7 +68,7 @@ async def _apply_incident(
incident_key: str,
) -> tuple[bool, Incident | None]:
"""Return (transition, incident). transition True if open->resolved or none->open."""
observed = ev.observed_at
observed = to_utc(ev.observed_at)
if ev.status == "ok":
res = await session.execute(
select(Incident).where(Incident.incident_key == incident_key, Incident.state == "open")
@@ -159,7 +156,7 @@ async def ingest_event(
ev: CheckResultEvent,
) -> bool:
"""Return True if accepted, False if deduplicated."""
observed = ev.observed_at
observed = to_utc(ev.observed_at)
output = redact(ev.output)
incident_key = _incident_key(agent_id, ev)

9
server/monlet_server/services/notifiers/alertmanager.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import httpx
from ...redaction import redact
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
@@ -10,11 +11,13 @@ from .http import classify_response
class AlertmanagerNotifier:
name = "alertmanager"
def __init__(self, client: httpx.AsyncClient, url: str) -> None:
def __init__(self, client: httpx.AsyncClient, url: str, time_zone: str) -> None:
self._client = client
self._url = url.rstrip("/")
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
display_payload = payload_with_display_times(payload, self._time_zone)
labels = {
"alertname": "monlet_incident",
"agent_id": str(payload.get("agent_id", "")),
@@ -25,7 +28,11 @@ class AlertmanagerNotifier:
"severity": str(payload.get("severity", "unknown")),
"status": str(payload.get("status", "")),
"summary": redact(payload.get("summary") or "") or "",
"time_zone": display_payload["time_zone"],
}
for key in ("observed_at_display", "opened_at_display", "resolved_at_display"):
if display_payload.get(key):
annotations[key.removesuffix("_display")] = display_payload[key]
if payload.get("output"):
annotations["output"] = (redact(payload["output"]) or "")[:1024]
alert = {

10
server/monlet_server/services/notifiers/debug.py
View File

@@ -1,14 +1,20 @@
from ...logging_config import get_logger
from ...redaction import redact_dict
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
class DebugNotifier:
name = "debug"
def __init__(self) -> None:
def __init__(self, time_zone: str = "UTC") -> None:
self._log = get_logger("monlet.notifier.debug")
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
self._log.info("notify", event_type=event_type, payload=redact_dict(payload))
self._log.info(
"notify",
event_type=event_type,
payload=redact_dict(payload_with_display_times(payload, self._time_zone)),
)
return DeliveryResult(ok=True)

16
server/monlet_server/services/notifiers/registry.py
View File

@@ -13,19 +13,27 @@ from .webhook import WebhookNotifier
def build_registry(settings: Settings, client: httpx.AsyncClient) -> dict[str, Notifier]:
reg: dict[str, Notifier] = {}
if settings.notifier_debug_enabled:
reg["debug"] = DebugNotifier()
reg["debug"] = DebugNotifier(settings.notification_time_zone)
if (
settings.notifier_telegram_enabled
and settings.notifier_telegram_token
and settings.notifier_telegram_chat_id
):
reg["telegram"] = TelegramNotifier(
client, settings.notifier_telegram_token, settings.notifier_telegram_chat_id
client,
settings.notifier_telegram_token,
settings.notifier_telegram_chat_id,
settings.notification_time_zone,
)
if settings.notifier_webhook_enabled and settings.notifier_webhook_url:
reg["webhook"] = WebhookNotifier(
client, settings.notifier_webhook_url, settings.notifier_webhook_token
client,
settings.notifier_webhook_url,
settings.notifier_webhook_token,
settings.notification_time_zone,
)
if settings.notifier_alertmanager_enabled and settings.notifier_alertmanager_url:
reg["alertmanager"] = AlertmanagerNotifier(client, settings.notifier_alertmanager_url)
reg["alertmanager"] = AlertmanagerNotifier(
client, settings.notifier_alertmanager_url, settings.notification_time_zone
)
return reg

14
server/monlet_server/services/notifiers/telegram.py
View File

@@ -3,11 +3,13 @@ from __future__ import annotations
import httpx
from ...redaction import redact
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
def _format_message(event_type: str, payload: dict) -> str:
def _format_message(event_type: str, payload: dict, time_zone: str) -> str:
payload = payload_with_display_times(payload, time_zone)
icon = "[FIRING]" if event_type == "firing" else "[RESOLVED]"
sev = payload.get("severity", "unknown")
agent = payload.get("agent_id", "?")
@@ -19,7 +21,7 @@ def _format_message(event_type: str, payload: dict) -> str:
f"{icon} {sev} {agent}/{check}",
f"status: {status}",
f"incident_key: {incident_key}",
f"observed_at: {payload.get('observed_at', '?')}",
f"observed_at: {payload.get('observed_at_display') or payload.get('observed_at', '?')}",
]
if output:
lines.append(f"output: {output[:300]}")
@@ -29,14 +31,18 @@ def _format_message(event_type: str, payload: dict) -> str:
class TelegramNotifier:
name = "telegram"
def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str) -> None:
def __init__(self, client: httpx.AsyncClient, token: str, chat_id: str, time_zone: str) -> None:
self._client = client
self._token = token
self._chat_id = chat_id
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
url = f"https://api.telegram.org/bot{self._token}/sendMessage"
body = {"chat_id": self._chat_id, "text": _format_message(event_type, payload)}
body = {
"chat_id": self._chat_id,
"text": _format_message(event_type, payload, self._time_zone),
}
try:
resp = await self._client.post(url, json=body)
except httpx.HTTPError as exc:

11
server/monlet_server/services/notifiers/webhook.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import httpx
from ...redaction import redact_dict
from ...timeutil import payload_with_display_times
from .base import DeliveryResult
from .http import classify_response
@@ -10,16 +11,22 @@ from .http import classify_response
class WebhookNotifier:
name = "webhook"
def __init__(self, client: httpx.AsyncClient, url: str, token: str = "") -> None:
def __init__(
self, client: httpx.AsyncClient, url: str, token: str = "", time_zone: str = "UTC"
) -> None:
self._client = client
self._url = url
self._token = token
self._time_zone = time_zone
async def deliver(self, event_type: str, payload: dict) -> DeliveryResult:
headers = {"Content-Type": "application/json"}
if self._token:
headers["Authorization"] = f"Bearer {self._token}"
body = {"event_type": event_type, "incident": redact_dict(payload)}
body = {
"event_type": event_type,
"incident": redact_dict(payload_with_display_times(payload, self._time_zone)),
}
try:
resp = await self._client.post(self._url, json=body, headers=headers)
except httpx.HTTPError as exc:

8
server/monlet_server/settings.py
View File

@@ -1,6 +1,8 @@
from pydantic import field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from .timeutil import validate_time_zone
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="MONLET_", env_file=".env", extra="ignore")
@@ -25,6 +27,7 @@ class Settings(BaseSettings):
notifier_max_attempts: int = 8
notifier_lease_sec: int = 300
notifier_http_timeout_sec: float = 10.0
notification_time_zone: str = "UTC"
notifier_debug_enabled: bool = True
notifier_telegram_enabled: bool = False
@@ -43,6 +46,11 @@ class Settings(BaseSettings):
raise ValueError("MONLET_AUTH_TOKEN must be set to a non-default value")
return value
@field_validator("notification_time_zone")
@classmethod
def _validate_notification_time_zone(cls, value: str) -> str:
return validate_time_zone(value)
@property
def enabled_notifiers(self) -> list[str]:
out: list[str] = []

53
server/monlet_server/timeutil.py Normal file
View File

@@ -0,0 +1,53 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
DEFAULT_TIME_ZONE = "UTC"
_TIMESTAMP_KEYS = ("observed_at", "opened_at", "resolved_at")
def validate_time_zone(value: str) -> str:
name = value.strip() or DEFAULT_TIME_ZONE
try:
ZoneInfo(name)
except ZoneInfoNotFoundError as exc:
raise ValueError(f"invalid timezone: {name}") from exc
return name
def require_utc_datetime(value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("timestamp must include UTC timezone")
if value.utcoffset() != timedelta(0):
raise ValueError("timestamp must be UTC")
return value.astimezone(UTC)
def to_utc(value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def format_display_datetime(value: str | datetime | None, time_zone: str) -> str:
if not value:
return ""
if isinstance(value, datetime):
dt = value
else:
dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
return (
to_utc(dt)
.astimezone(ZoneInfo(validate_time_zone(time_zone)))
.strftime("%Y-%m-%d %H:%M:%S %Z")
)
def payload_with_display_times(payload: dict, time_zone: str) -> dict:
out = dict(payload)
out["time_zone"] = validate_time_zone(time_zone)
for key in _TIMESTAMP_KEYS:
if payload.get(key):
out[f"{key}_display"] = format_display_datetime(payload[key], out["time_zone"])
return out

12
server/tests/test_events_ingestion.py
View File

@@ -43,6 +43,18 @@ async def test_empty_batch_400(app_client, auth_headers):
assert r.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize("observed_at", ["2026-05-27T09:00:00", "2026-05-27T09:00:00+04:00"])
async def test_event_requires_utc_timestamp(app_client, auth_headers, observed_at):
await app_client.post("/api/v1/heartbeat", json=make_heartbeat(), headers=auth_headers)
ev = make_event(observed_at=observed_at)
r = await app_client.post(
"/api/v1/events", json={"agent_id": "agent-1", "events": [ev]}, headers=auth_headers
)
assert r.status_code == 400
assert r.json()["error"]["code"] == "validation"
@pytest.mark.asyncio
async def test_oversize_batch_400(app_client, auth_headers):
evs = [make_event() for _ in range(201)]

10
server/tests/test_heartbeat.py
View File

@@ -43,6 +43,16 @@ async def test_heartbeat_validation_400(app_client, auth_headers):
assert r.json()["error"]["code"] == "validation"
@pytest.mark.asyncio
@pytest.mark.parametrize("observed_at", ["2026-05-27T09:00:00", "2026-05-27T09:00:00+04:00"])
async def test_heartbeat_requires_utc_timestamp(app_client, auth_headers, observed_at):
hb = make_heartbeat()
hb["observed_at"] = observed_at
r = await app_client.post("/api/v1/heartbeat", json=hb, headers=auth_headers)
assert r.status_code == 400
assert r.json()["error"]["code"] == "validation"
@pytest.mark.asyncio
async def test_heartbeat_rejects_extra_fields(app_client, auth_headers):
hb = make_heartbeat()

26
server/tests/test_notifiers.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
import os
from datetime import UTC, datetime, timedelta
from uuid import uuid4
@@ -27,8 +28,8 @@ def _payload() -> dict:
"status": "critical",
"severity": "critical",
"incident_key": "agent-1:chk-1",
"observed_at": datetime.now(UTC).isoformat(),
"opened_at": datetime.now(UTC).isoformat(),
"observed_at": "2026-05-27T05:00:00+00:00",
"opened_at": "2026-05-27T05:00:00+00:00",
"summary": "boom",
"output": "stderr: boom token=abc123",
}
@@ -60,12 +61,13 @@ async def test_telegram_success_and_redaction():
def handler(req):
body = req.content.decode()
assert "token=abc123" not in body
assert "2026-05-27 09:00:00 +04" in body
assert "***" in body
return httpx.Response(200, json={"ok": True})
client, t = _client(handler)
try:
n = TelegramNotifier(client, "TOK", "123")
n = TelegramNotifier(client, "TOK", "123", "Asia/Tbilisi")
res = await n.deliver("firing", _payload())
assert res.ok
assert "/botTOK/sendMessage" in str(t.requests[0].url)
@@ -77,7 +79,7 @@ async def test_telegram_success_and_redaction():
async def test_telegram_4xx_permanent():
client, _ = _client(lambda r: httpx.Response(400, json={}))
try:
n = TelegramNotifier(client, "TOK", "123")
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and res.permanent
finally:
@@ -88,7 +90,7 @@ async def test_telegram_4xx_permanent():
async def test_telegram_5xx_transient():
client, _ = _client(lambda r: httpx.Response(503, json={}))
try:
n = TelegramNotifier(client, "TOK", "123")
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and not res.permanent
finally:
@@ -102,7 +104,7 @@ async def test_telegram_network_error_transient():
client, _ = _client(handler)
try:
n = TelegramNotifier(client, "TOK", "123")
n = TelegramNotifier(client, "TOK", "123", "UTC")
res = await n.deliver("firing", _payload())
assert not res.ok and not res.permanent
finally:
@@ -140,11 +142,15 @@ async def test_alertmanager_payload_shape():
client, _ = _client(handler)
try:
n = AlertmanagerNotifier(client, "http://am/")
n = AlertmanagerNotifier(client, "http://am/", "Asia/Tbilisi")
res = await n.deliver("firing", _payload())
assert res.ok
assert captured["url"].endswith("/api/v2/alerts")
assert '"alertname":"monlet_incident"' in captured["body"]
body = json.loads(captured["body"])
alert = body[0]
assert alert["labels"]["alertname"] == "monlet_incident"
assert alert["startsAt"] == "2026-05-27T05:00:00+00:00"
assert alert["annotations"]["observed_at"] == "2026-05-27 09:00:00 +04"
finally:
await client.aclose()
@@ -348,7 +354,7 @@ async def test_alertmanager_severity_not_in_labels():
client, _ = _client(handler)
try:
n = AlertmanagerNotifier(client, "http://am")
n = AlertmanagerNotifier(client, "http://am", "UTC")
await n.deliver("firing", _payload())
import json as _json
@@ -371,7 +377,7 @@ async def test_telegram_redacts_secret_in_incident_key():
try:
p = _payload()
p["incident_key"] = "agent-1:chk-1:token=leakedsecret"
n = TelegramNotifier(client, "TOK", "1")
n = TelegramNotifier(client, "TOK", "1", "UTC")
await n.deliver("firing", p)
assert "leakedsecret" not in captured["body"]
finally: