114 lines
2.6 KiB
Go
114 lines
2.6 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/monlet/agent/internal/config"
|
|
"github.com/monlet/agent/internal/eventid"
|
|
"github.com/monlet/agent/internal/runner"
|
|
)
|
|
|
|
// Event is a contract-shaped check result emitted by the scheduler.
|
|
type Event struct {
|
|
EventID string `json:"event_id"`
|
|
CheckID string `json:"check_id"`
|
|
ObservedAt time.Time `json:"observed_at"`
|
|
Status string `json:"status"`
|
|
ExitCode int `json:"exit_code"`
|
|
DurationMs int64 `json:"duration_ms"`
|
|
Output string `json:"output,omitempty"`
|
|
OutputTruncated bool `json:"output_truncated,omitempty"`
|
|
NotificationOwner string `json:"notification_owner,omitempty"`
|
|
IncidentKey string `json:"incident_key,omitempty"`
|
|
}
|
|
|
|
type Hooks struct {
|
|
OnSkipped func(checkID string)
|
|
OnResult func(ev Event)
|
|
}
|
|
|
|
// Run starts one goroutine per check. Emits events to out. Blocks until ctx is done.
|
|
func Run(ctx context.Context, agentID string, checks []config.CheckConfig, out chan<- Event, h Hooks) {
|
|
var wg sync.WaitGroup
|
|
for i := range checks {
|
|
c := checks[i]
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
runCheck(ctx, agentID, c, out, h)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func runCheck(ctx context.Context, agentID string, c config.CheckConfig, out chan<- Event, h Hooks) {
|
|
t := time.NewTicker(c.Interval.Duration)
|
|
defer t.Stop()
|
|
var running bool
|
|
var mu sync.Mutex
|
|
|
|
exec := func() {
|
|
mu.Lock()
|
|
if running {
|
|
mu.Unlock()
|
|
if h.OnSkipped != nil {
|
|
h.OnSkipped(c.ID)
|
|
}
|
|
return
|
|
}
|
|
running = true
|
|
mu.Unlock()
|
|
defer func() {
|
|
mu.Lock()
|
|
running = false
|
|
mu.Unlock()
|
|
}()
|
|
|
|
res := runner.Run(ctx, c.Command, c.Timeout.Duration)
|
|
id, err := eventid.New()
|
|
if err != nil {
|
|
return
|
|
}
|
|
ev := Event{
|
|
EventID: id,
|
|
CheckID: c.ID,
|
|
ObservedAt: time.Now().UTC(),
|
|
Status: string(res.Status),
|
|
ExitCode: res.ExitCode,
|
|
DurationMs: res.DurationMs,
|
|
Output: res.Output,
|
|
OutputTruncated: res.OutputTruncated,
|
|
NotificationOwner: c.NotificationOwner,
|
|
IncidentKey: incidentKey(agentID, c),
|
|
}
|
|
if h.OnResult != nil {
|
|
h.OnResult(ev)
|
|
}
|
|
select {
|
|
case out <- ev:
|
|
case <-ctx.Done():
|
|
}
|
|
}
|
|
|
|
// Fire first run immediately so smoke tests don't wait a full interval.
|
|
go exec()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
go exec()
|
|
}
|
|
}
|
|
}
|
|
|
|
func incidentKey(agentID string, c config.CheckConfig) string {
|
|
if c.DedupeKey != "" {
|
|
return c.DedupeKey
|
|
}
|
|
return agentID + ":" + c.ID
|
|
}
|