monlet/agent/internal/scheduler/scheduler_test.go

package scheduler

import (
	"context"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/monlet/agent/internal/config"
)

func runScheduler(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) {
	m := NewManager(ctx, agentID, out, h)
	m.Update(checks)
	<-ctx.Done()
	m.Stop()
	m.Wait()
}

func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig {
	return config.CheckConfig{
		ID:       id,
		Command:  cmd,
		Interval: config.Duration{Duration: interval},
		Timeout:  config.Duration{Duration: timeout},
	}
}

func TestEmitsEvents(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
	defer cancel()
	out := make(chan Event, 10)
	go runScheduler(ctx, "a1", []config.CheckConfig{mkCheck("c1", "echo hi", 200*time.Millisecond, 100*time.Millisecond)}, out, Hooks{})
	select {
	case ev := <-out:
		if ev.CheckID != "c1" || ev.Status != "ok" {
			t.Fatalf("unexpected event: %+v", ev)
		}
		if ev.IncidentKey != "a1:c1" {
			t.Fatalf("incident_key: %q", ev.IncidentKey)
		}
	case <-ctx.Done():
		t.Fatal("no event")
	}
}

func TestNoOverlapSkips(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 600*time.Millisecond)
	defer cancel()
	out := make(chan Event, 10)
	var skipped int32
	hooks := Hooks{OnSkipped: func(string) { atomic.AddInt32(&skipped, 1) }}
	c := mkCheck("slow", "sleep 1", 100*time.Millisecond, 900*time.Millisecond)
	go runScheduler(ctx, "a1", []config.CheckConfig{c}, out, hooks)
	<-ctx.Done()
	if atomic.LoadInt32(&skipped) == 0 {
		t.Fatal("expected at least one skipped tick")
	}
}

func TestReportsFullEventChannel(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	out := make(chan Event, 1)
	out <- Event{EventID: "occupied"}
	var full int32
	m := NewManager(ctx, "a1", out, Hooks{
		OnEventChannelFull: func() { atomic.AddInt32(&full, 1) },
	})
	m.Update([]config.CheckConfig{mkCheck("c1", "echo hi", time.Hour, time.Second)})

	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if atomic.LoadInt32(&full) > 0 {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	if atomic.LoadInt32(&full) == 0 {
		t.Fatal("expected full channel hook")
	}
	<-out
	select {
	case ev := <-out:
		if ev.CheckID != "c1" {
			t.Fatalf("unexpected event: %+v", ev)
		}
	case <-ctx.Done():
		t.Fatal("blocked event was not delivered")
	}
	m.Stop()
	m.Wait()
}

func TestManagerReloadChangedCheckWaitsForRunning(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	out := make(chan Event, 10)
	m := NewManager(ctx, "a1", out, Hooks{})
	m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.3; echo old", time.Second, time.Second)})
	time.Sleep(50 * time.Millisecond)
	m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)})

	var first, second Event
	select {
	case first = <-out:
	case <-ctx.Done():
		t.Fatal("no first event")
	}
	select {
	case second = <-out:
	case <-ctx.Done():
		t.Fatal("no second event")
	}
	m.Stop()
	m.Wait()

	if !strings.Contains(first.Output, "old") {
		t.Fatalf("first output: %q", first.Output)
	}
	if !strings.Contains(second.Output, "new") {
		t.Fatalf("second output: %q", second.Output)
	}
}

func TestManagerReloadRemovedCheckLetsRunningFinish(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	out := make(chan Event, 10)
	stopped := make(chan string, 1)
	m := NewManager(ctx, "a1", out, Hooks{OnStopped: func(id string) { stopped <- id }})
	m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.2; echo old", time.Second, time.Second)})
	time.Sleep(50 * time.Millisecond)
	m.Update(nil)

	select {
	case ev := <-out:
		if !strings.Contains(ev.Output, "old") {
			t.Fatalf("output: %q", ev.Output)
		}
	case <-ctx.Done():
		t.Fatal("removed running check did not finish")
	}
	select {
	case id := <-stopped:
		if id != "c1" {
			t.Fatalf("stopped id: %q", id)
		}
	case <-ctx.Done():
		t.Fatal("removed check did not stop")
	}
	m.Stop()
	m.Wait()
}

func TestManagerRemoveThenReaddSameCheckDoesNotOverlap(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancel()
	out := make(chan Event, 10)
	m := NewManager(ctx, "a1", out, Hooks{})
	m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)})
	time.Sleep(50 * time.Millisecond)
	m.Update(nil)
	m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)})

	var first, second Event
	select {
	case first = <-out:
	case <-ctx.Done():
		t.Fatal("no first event")
	}
	select {
	case second = <-out:
	case <-ctx.Done():
		t.Fatal("no second event")
	}
	m.Stop()
	m.Wait()

	if !strings.Contains(first.Output, "old") {
		t.Fatalf("first output: %q", first.Output)
	}
	if !strings.Contains(second.Output, "new") {
		t.Fatalf("second output: %q", second.Output)
	}
}

func TestManagerUsesLatestPendingReloadAfterRunningFinishes(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancel()
	out := make(chan Event, 10)
	m := NewManager(ctx, "a1", out, Hooks{})
	m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)})
	time.Sleep(50 * time.Millisecond)
	m.Update([]config.CheckConfig{mkCheck("c1", "echo intermediate", time.Second, time.Second)})
	m.Update([]config.CheckConfig{mkCheck("c1", "echo latest", time.Second, time.Second)})

	var first, second Event
	select {
	case first = <-out:
	case <-ctx.Done():
		t.Fatal("no first event")
	}
	select {
	case second = <-out:
	case <-ctx.Done():
		t.Fatal("no second event")
	}
	m.Stop()
	m.Wait()

	if !strings.Contains(first.Output, "old") {
		t.Fatalf("first output: %q", first.Output)
	}
	if strings.Contains(second.Output, "intermediate") || !strings.Contains(second.Output, "latest") {
		t.Fatalf("second output: %q", second.Output)
	}
}