299 lines
8.0 KiB
Go
299 lines
8.0 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/monlet/agent/internal/config"
|
|
"github.com/robfig/cron/v3"
|
|
)
|
|
|
|
func runScheduler(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) {
|
|
m := NewManager(ctx, agentID, out, h)
|
|
m.Update(checks)
|
|
<-ctx.Done()
|
|
m.Stop()
|
|
m.Wait()
|
|
}
|
|
|
|
func mkCheck(id, cmd string, interval, timeout time.Duration) config.CheckConfig {
|
|
return config.CheckConfig{
|
|
ID: id,
|
|
Command: cmd,
|
|
Interval: config.Duration{Duration: interval},
|
|
Timeout: config.Duration{Duration: timeout},
|
|
}
|
|
}
|
|
|
|
type fakeCronSchedule struct {
|
|
delay time.Duration
|
|
}
|
|
|
|
func (s fakeCronSchedule) Next(t time.Time) time.Time {
|
|
return t.Add(s.delay)
|
|
}
|
|
|
|
func withFakeCron(t *testing.T, delay time.Duration) {
|
|
t.Helper()
|
|
old := parseCronSchedule
|
|
parseCronSchedule = func(string) (cron.Schedule, error) {
|
|
return fakeCronSchedule{delay: delay}, nil
|
|
}
|
|
t.Cleanup(func() { parseCronSchedule = old })
|
|
}
|
|
|
|
func mkCronCheck(id, cmd string, timeout time.Duration) config.CheckConfig {
|
|
return config.CheckConfig{
|
|
ID: id,
|
|
Command: cmd,
|
|
Cron: "0 * * * *",
|
|
Timeout: config.Duration{Duration: timeout},
|
|
}
|
|
}
|
|
|
|
func TestEmitsEvents(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
go runScheduler(ctx, "a1", []config.CheckConfig{mkCheck("c1", "echo hi", 200*time.Millisecond, 100*time.Millisecond)}, out, Hooks{})
|
|
select {
|
|
case ev := <-out:
|
|
if ev.CheckID != "c1" || ev.Status != "ok" {
|
|
t.Fatalf("unexpected event: %+v", ev)
|
|
}
|
|
if ev.IncidentKey != "a1:c1" {
|
|
t.Fatalf("incident_key: %q", ev.IncidentKey)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("no event")
|
|
}
|
|
}
|
|
|
|
func TestCronDoesNotRunImmediately(t *testing.T) {
|
|
withFakeCron(t, 200*time.Millisecond)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
m := NewManager(ctx, "a1", out, Hooks{})
|
|
m.Update([]config.CheckConfig{mkCronCheck("cron", "echo cron", 100*time.Millisecond)})
|
|
select {
|
|
case ev := <-out:
|
|
t.Fatalf("cron check ran before next slot: %+v", ev)
|
|
case <-ctx.Done():
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
}
|
|
|
|
func TestCronEmitsOnDueSlot(t *testing.T) {
|
|
withFakeCron(t, 30*time.Millisecond)
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
m := NewManager(ctx, "a1", out, Hooks{})
|
|
m.Update([]config.CheckConfig{mkCronCheck("cron", "echo cron", 100*time.Millisecond)})
|
|
select {
|
|
case ev := <-out:
|
|
if ev.CheckID != "cron" || ev.Status != "ok" {
|
|
t.Fatalf("unexpected event: %+v", ev)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("no cron event")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
}
|
|
|
|
func TestNoOverlapSkips(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 600*time.Millisecond)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
var skipped int32
|
|
hooks := Hooks{OnSkipped: func(string) { atomic.AddInt32(&skipped, 1) }}
|
|
c := mkCheck("slow", "sleep 1", 100*time.Millisecond, 900*time.Millisecond)
|
|
go runScheduler(ctx, "a1", []config.CheckConfig{c}, out, hooks)
|
|
<-ctx.Done()
|
|
if atomic.LoadInt32(&skipped) == 0 {
|
|
t.Fatal("expected at least one skipped tick")
|
|
}
|
|
}
|
|
|
|
func TestCronNoOverlapSkips(t *testing.T) {
|
|
withFakeCron(t, 50*time.Millisecond)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
var skipped int32
|
|
hooks := Hooks{OnSkipped: func(string) { atomic.AddInt32(&skipped, 1) }}
|
|
m := NewManager(ctx, "a1", out, hooks)
|
|
m.Update([]config.CheckConfig{mkCronCheck("cron", "sleep 0.2", 500*time.Millisecond)})
|
|
<-ctx.Done()
|
|
m.Stop()
|
|
m.Wait()
|
|
if atomic.LoadInt32(&skipped) == 0 {
|
|
t.Fatal("expected at least one skipped cron slot")
|
|
}
|
|
}
|
|
|
|
func TestReportsFullEventChannel(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 1)
|
|
out <- Event{EventID: "occupied"}
|
|
var full int32
|
|
m := NewManager(ctx, "a1", out, Hooks{
|
|
OnEventChannelFull: func() { atomic.AddInt32(&full, 1) },
|
|
})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "echo hi", time.Hour, time.Second)})
|
|
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if atomic.LoadInt32(&full) > 0 {
|
|
break
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if atomic.LoadInt32(&full) == 0 {
|
|
t.Fatal("expected full channel hook")
|
|
}
|
|
<-out
|
|
select {
|
|
case ev := <-out:
|
|
if ev.CheckID != "c1" {
|
|
t.Fatalf("unexpected event: %+v", ev)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("blocked event was not delivered")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
}
|
|
|
|
func TestManagerReloadChangedCheckWaitsForRunning(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
m := NewManager(ctx, "a1", out, Hooks{})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.3; echo old", time.Second, time.Second)})
|
|
time.Sleep(50 * time.Millisecond)
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)})
|
|
|
|
var first, second Event
|
|
select {
|
|
case first = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no first event")
|
|
}
|
|
select {
|
|
case second = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no second event")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
|
|
if !strings.Contains(first.Output, "old") {
|
|
t.Fatalf("first output: %q", first.Output)
|
|
}
|
|
if !strings.Contains(second.Output, "new") {
|
|
t.Fatalf("second output: %q", second.Output)
|
|
}
|
|
}
|
|
|
|
func TestManagerReloadRemovedCheckLetsRunningFinish(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
stopped := make(chan string, 1)
|
|
m := NewManager(ctx, "a1", out, Hooks{OnStopped: func(id string) { stopped <- id }})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.2; echo old", time.Second, time.Second)})
|
|
time.Sleep(50 * time.Millisecond)
|
|
m.Update(nil)
|
|
|
|
select {
|
|
case ev := <-out:
|
|
if !strings.Contains(ev.Output, "old") {
|
|
t.Fatalf("output: %q", ev.Output)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("removed running check did not finish")
|
|
}
|
|
select {
|
|
case id := <-stopped:
|
|
if id != "c1" {
|
|
t.Fatalf("stopped id: %q", id)
|
|
}
|
|
case <-ctx.Done():
|
|
t.Fatal("removed check did not stop")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
}
|
|
|
|
func TestManagerRemoveThenReaddSameCheckDoesNotOverlap(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
m := NewManager(ctx, "a1", out, Hooks{})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)})
|
|
time.Sleep(50 * time.Millisecond)
|
|
m.Update(nil)
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "echo new", time.Second, time.Second)})
|
|
|
|
var first, second Event
|
|
select {
|
|
case first = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no first event")
|
|
}
|
|
select {
|
|
case second = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no second event")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
|
|
if !strings.Contains(first.Output, "old") {
|
|
t.Fatalf("first output: %q", first.Output)
|
|
}
|
|
if !strings.Contains(second.Output, "new") {
|
|
t.Fatalf("second output: %q", second.Output)
|
|
}
|
|
}
|
|
|
|
func TestManagerUsesLatestPendingReloadAfterRunningFinishes(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
out := make(chan Event, 10)
|
|
m := NewManager(ctx, "a1", out, Hooks{})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "sleep 0.25; echo old", time.Second, time.Second)})
|
|
time.Sleep(50 * time.Millisecond)
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "echo intermediate", time.Second, time.Second)})
|
|
m.Update([]config.CheckConfig{mkCheck("c1", "echo latest", time.Second, time.Second)})
|
|
|
|
var first, second Event
|
|
select {
|
|
case first = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no first event")
|
|
}
|
|
select {
|
|
case second = <-out:
|
|
case <-ctx.Done():
|
|
t.Fatal("no second event")
|
|
}
|
|
m.Stop()
|
|
m.Wait()
|
|
|
|
if !strings.Contains(first.Output, "old") {
|
|
t.Fatalf("first output: %q", first.Output)
|
|
}
|
|
if strings.Contains(second.Output, "intermediate") || !strings.Contains(second.Output, "latest") {
|
|
t.Fatalf("second output: %q", second.Output)
|
|
}
|
|
}
|