Harden production workflows and agent admission

deploy/showcase/agents/dead.toml

@@ -1,4 +1,4 @@
-hostname = "agent-dead"
+agent_id = "agent-08"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 

deploy/showcase/agents/flap.toml

@@ -1,4 +1,4 @@
-hostname = "agent-flap"
+agent_id = "agent-03"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 
@@ -22,4 +21,4 @@ name = "Flapping check (oscillates ok/critical)"
 command = "/opt/monlet/checks/flap.sh"
 interval = "10s"
 timeout = "3s"
-dedupe_key = "agent-flap:flap"
+dedupe_key = "agent-03:flap"

deploy/showcase/agents/mixed.toml

@@ -1,4 +1,4 @@
-hostname = "agent-mixed"
+agent_id = "agent-01"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 
@@ -36,7 +35,7 @@ name = "Showcase critical (permanent webhook fail)"
 command = "/opt/monlet/checks/exit_2.sh"
 interval = "5s"
 timeout = "3s"
-dedupe_key = "agent-mixed:critical:fail"
+dedupe_key = "agent-01:critical:fail"
 
 [[checks]]
 id = "unknown_check"

deploy/showcase/agents/prometheus_owned.toml

@@ -1,4 +1,4 @@
-hostname = "agent-prometheus-owned"
+agent_id = "agent-05"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 

deploy/showcase/agents/resolve.toml

@@ -1,4 +1,4 @@
-hostname = "agent-resolve"
+agent_id = "agent-02"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 
@@ -22,4 +21,4 @@ name = "Flapping check (critical then ok)"
 command = "/opt/monlet/checks/flapping.sh"
 interval = "5s"
 timeout = "3s"
-dedupe_key = "agent-resolve:flapping:retry"
+dedupe_key = "agent-02:flapping:retry"

deploy/showcase/agents/spool_replay.toml

@@ -1,4 +1,4 @@
-hostname = "agent-spool-replay"
+agent_id = "agent-06"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 

deploy/showcase/agents/stale.toml

@@ -1,4 +1,4 @@
-hostname = "agent-stale"
+agent_id = "agent-07"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 

deploy/showcase/agents/timeout.toml

@@ -1,4 +1,4 @@
-hostname = "agent-timeout"
+agent_id = "agent-04"
 state_dir = "/var/lib/monlet-agent"
 
 [labels]
@@ -8,7 +8,6 @@ role = "showcase"
 [server]
 enabled = true
 url = "http://server:8000"
-token = "monlet-dev-token"
 heartbeat_interval = "5s"
 batch_interval = "3s"
 
@@ -22,7 +21,7 @@ name = "Healthy probe"
 command = "/opt/monlet/checks/exit_0.sh"
 interval = "10s"
 timeout = "3s"
-dedupe_key = "agent-timeout:ok_probe"
+dedupe_key = "agent-04:ok_probe"
 
 [[checks]]
 id = "slow_probe"
@@ -30,4 +29,4 @@ name = "Probe that exceeds the timeout"
 command = "/opt/monlet/checks/sleep_long.sh"
 interval = "10s"
 timeout = "2s"
-dedupe_key = "agent-timeout:slow_probe"
+dedupe_key = "agent-04:slow_probe"

deploy/showcase/checks/replay_check.sh

@@ -1,3 +1,15 @@
 #!/bin/sh
-echo "ok: replay tick"
+# PH-review: spool-replay scenario needs many DISTINCT events because the
+# server only persists an event row when the check status TRANSITIONS
+# (server/monlet_server/services/ingestion.py). Alternate ok/warning on each
+# run so every tick produces a fresh event for the spool to replay.
+STATE_FILE=/var/lib/monlet-agent/replay_tick
+COUNT=$(cat "$STATE_FILE" 2>/dev/null || echo 0)
+COUNT=$((COUNT + 1))
+echo "$COUNT" > "$STATE_FILE"
+if [ $((COUNT % 2)) -eq 0 ]; then
+  echo "warning: replay tick $COUNT"
+  exit 1
+fi
+echo "ok: replay tick $COUNT"
 exit 0

deploy/showcase/seed_discarded.sql

@@ -1,16 +0,0 @@
--- Seed one outbox row with an unknown notifier name so the worker marks it 'discarded'.
--- Attached to any existing incident; if none exists yet the script is a no-op.
-INSERT INTO notification_outbox
-    (id, incident_id, notifier, event_type, state, attempts, next_attempt_at, payload)
-SELECT
-    gen_random_uuid(),
-    i.id,
-    'showcase-unknown',
-    'firing',
-    'pending',
-    0,
-    now(),
-    '{"showcase":"discarded"}'::jsonb
-FROM incidents i
-ORDER BY i.opened_at DESC
-LIMIT 1;