Git designed for AI agents.
package loadmonitor
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/feder-cr/git-ai/internal/cgroups"
"github.com/feder-cr/git-ai/internal/helper"
"github.com/feder-cr/git-ai/internal/log"
"github.com/prometheus/client_golang/prometheus"
)
const (
// defaultPollInterval is the default poll interval to poll resource usage stats
defaultPollInterval = time.Second * 10
// defaultConditionTimeout is the timeout given to a Condition
defaultConditionTimeout = time.Second * 5
)
// ErrAlreadyStarted is returned when `Start` is called when the manager
// has already been started.
var ErrAlreadyStarted = errors.New("monitor already started")
// ErrNotRunning is returned when an operation on a non-running Monitor is performed.
var ErrNotRunning = errors.New("monitor not running")
// conditionFn is the type for a Condition function
type conditionFn func(ctx context.Context, previous, current Stats, interval time.Duration) (emitEvent bool, name string)
// Condition is a type that represent a condition to be evaluated by the LoadMonitor
// on every poll event.
type Condition struct {
// Name is the name of the Condition. This is used to identify a Condition.
// Ideally names should be unique across the application.
Name string
// Fn is a function type that takes the previous and current system Stats struct as arguments. A user
// can define a function to compare system stats between two polling event and make a decision an event
// should be sent or not.
// If a particular Event occurred, as defined in the Condition, Fn must return true, along with the name
// of the event. Else it must return false, and the name can be any string. an empty string is fine.
// The duration between the two polled stats is also provided.
// Implementation of this type must honor the context cancellation.
Fn conditionFn
}
// Monitor is the base interface for the load monitor.
type Monitor interface {
// Start starts the Monitor. When `ctx` is canceled, the Monitor will terminate
// all polling and notifying activities.
Start(ctx context.Context) error
// Stop will stop the Monitor and wait for all shutdown operations to complete
// before returning. It is an alternative to cancelling the `ctx` passed
// to `Start`. It is safe to both cancel the context and call `Stop`.
Stop()
// NotifyOn takes a list of custom Condition as input and returns an Event channel.
// The Monitor evaluates the conditions at a periodic interval. When a condition is
// true, an event is sent down the Event channel. The returned channel should not be
// closed by the receiver. It must be closed by the sender (Monitor).
NotifyOn(conditions ...Condition) (<-chan Event, error)
}
// Event is the struct that is sent in an Event channel when a condition evaluates to `true`.
// It contains information about the condition that was violated.
type Event struct {
// ConditionName is the name of the condition that triggered this event
ConditionName string
// Description is the string value returned from a Condition.
// It contains the reason why this event was fired.
Description string
// CurrentStats are the current statistics that were evaluated by the Condition
// prior to firing this event.
CurrentStats Stats
// PreviousStats are the previous statistics that were evaluated by the Condition
// prior to firing this event.
PreviousStats Stats
}
// consumer is a struct representing a consumer of the Monitor.
type eventConsumer struct {
out chan Event
conditions []Condition
}
// Config holds the various configurations to configure the behavior
// of the Monitor. It does not hold its dependencies.
type Config struct {
// PollInterval defines at which interval to poll for resource stats
PollInterval time.Duration
// NotifyTimeout is the timeout before the Monitor abort evaluating conditions
// after a poll. This is to prevent blocked or long-running Conditions from
// blocking the main loop.
NotifyTimeout time.Duration
}
type monitorState struct {
// previousStats are the stats of the previous polling event
previousStats Stats
// currentStats are the current stats
currentStats Stats
// consumers is a list of eventConsumer to notify when their
// condition is reached
consumers []eventConsumer
// lastPoll is the time of the last poll
lastPoll time.Time
// lastPollInterval is the interval between the two last poll.
// Value is 0 when 1 or 0 poll occurred.
lastPollInterval time.Duration
}
// DefaultMonitor is the default implementation of a LoadMonitor
type DefaultMonitor struct {
// cfg is the configuration for a manager's instance
cfg Config
// metrics holds the various metrics this monitor emits
metrics *monitorMetrics
// logger the logger used in the monitor
logger log.Logger
// cgroupManager is the cgroup manager from which cgroup
// stats are polled.
cgroupManager cgroups.Manager
// state holds the current state of the Monitor
state monitorState
// stateLock synchronize access to the state
stateLock sync.Mutex
// pollTicker is the ticker used to synchronize polling
// to get new (up-to-date) resource usage data
pollTicker helper.Ticker
// started is used to determine if the Monitor is started or not
// Value is `false` by default.
started atomic.Bool
// stopOnce is used to make sure Stop is executed only once
stopOnce sync.Once
// shutdownOnce is used to make sure the shutdown logic is executed only once
// This is needed in addition to `stopOnce` because the Monitor can be stopped
// in two ways:
// * Calling Stop()
// * Cancelling the context passed in `Start()`
shutdownOnce sync.Once
// isShuttingDown is set to true as soon as the Monitor is shutting down
isShuttingDown bool
// shutdownMutex synchronize access to `isShuttingDown` variable.
// This ensures no new operations are performed while the Monitor is
// shutting down.
shutdownMutex sync.Mutex
// stopCh is used by Stop to signal to the main loop
// that is has to stop.
stopCh chan struct{}
// doneCh is used by the main loop to signal to `Stop` that
// the Monitor is now fully and properly stopped.
doneCh chan struct{}
}
var _ Monitor = (*DefaultMonitor)(nil)
// NewLoadMonitor creates a new Monitor using the defaultMonitor implementation.
func NewLoadMonitor(cfg Config, logger log.Logger, cgroupManager cgroups.Manager) *DefaultMonitor {
if cfg.PollInterval == 0 {
cfg.PollInterval = defaultPollInterval
}
if cfg.NotifyTimeout == 0 {
cfg.NotifyTimeout = defaultConditionTimeout
}
m := &DefaultMonitor{
cfg: cfg,
logger: logger,
metrics: newMonitorMetrics(),
cgroupManager: cgroupManager,
state: monitorState{
consumers: make([]eventConsumer, 0),
},
pollTicker: helper.NewTimerTicker(cfg.PollInterval),
stopCh: make(chan struct{}, 1),
doneCh: make(chan struct{}),
}
return m
}
// Start starts the monitor main loop.
// It polls the stats provider at every `PollInterval` and then calls
// `notify()` to evaluate all Conditions against the newly polled stats.
func (m *DefaultMonitor) Start(ctx context.Context) error {
if !m.markAsRunning() {
return ErrAlreadyStarted
}
go func() {
defer close(m.doneCh)
for {
// Reset the ticker before each loop to account
// for the poll/notify/collect phase.
m.pollTicker.Reset()
select {
// This case stops the Monitor when m.Stop() is called
case <-m.stopCh:
m.shutdown()
return
// This case stops the Monitor when the context is canceled
case <-ctx.Done():
m.shutdown()
return
case <-m.pollTicker.C():
// We do not notify consumers when polled failed because we do not
// have the latest stats to validate the Conditions
err := m.poll()
if err != nil {
m.logger.WithError(err).Error("poll failed")
continue
}
m.notify(ctx)
m.setMetrics()
}
}
}()
return nil
}
// Stop will stop the Monitor and wait for a graceful shutdown to
// occur before returning.
func (m *DefaultMonitor) Stop() {
// Return immediately if the Monitor is not running yet
if !m.isRunning() {
return
}
m.stopOnce.Do(func() {
// m.stopCh should be available for sending a value
// unless the context has been canceled, in which case
// `Start` has already returned and `stopCh` has no more
// listener.
select {
// If the monitor has already been closed by means of context cancellation
// then we exit here. nothing else to do.
case <-m.doneCh:
return
case m.stopCh <- struct{}{}:
<-m.doneCh
}
})
}
// NotifyOn registers a list of Condition on the Monitor.
// The Conditions are evaluated at every PollInterval.
// The Monitor will notify the returned Event channel
// only when a condition is met.
func (m *DefaultMonitor) NotifyOn(conditions ...Condition) (<-chan Event, error) {
// Make sure we do not create a consumer while the Monitor is shutting down
// as this could lead to this channel not being closed on shutdown.
m.shutdownMutex.Lock()
defer m.shutdownMutex.Unlock()
if m.isShuttingDown || !m.isRunning() {
return nil, ErrNotRunning
}
// This is a bit arbitrary, but using a buffer of 2 to allow
// for the previous value and the current one.
out := make(chan Event, 2)
// NotifyOn can be called concurrently from many places.
// Hence, we must synchronize the writes here.
m.stateLock.Lock()
defer m.stateLock.Unlock()
m.state.consumers = append(m.state.consumers, eventConsumer{
out: out,
conditions: conditions,
})
return out, nil
}
// Describe is used to generate description information for each emitted Prometheus metrics
func (m *DefaultMonitor) Describe(ch chan<- *prometheus.Desc) {
m.metrics.Describe(ch)
}
// Collect is used to collect the current values of each emitted Prometheus metrics
func (m *DefaultMonitor) Collect(ch chan<- prometheus.Metric) {
m.metrics.Collect(ch)
}
// poll Polls the StatsProvider to get resource usage information.
// It then sets the previous and current Stats on the Monitor, which
// will be used to evaluate the consumer's Condition.
// This method is not thread safe
func (m *DefaultMonitor) poll() error {
// Record the time right before polling
now := time.Now()
// Poll the cgroups stats
cgroupStats, err := m.cgroupManager.Stats()
if err != nil {
return fmt.Errorf("polling cgroup stats failed: %w", err)
}
// There is no need to lock access to the state here because poll is called
// synchronously before notify
if m.state.lastPoll.IsZero() {
// This is the first poll, set the previous state to the current state.
m.state.previousStats.CGroup = cgroupStats
} else {
m.state.previousStats = m.state.currentStats
m.state.lastPollInterval = now.Sub(m.state.lastPoll)
}
m.state.currentStats.CGroup = cgroupStats
m.state.currentStats.pollTime = now
m.state.lastPoll = now
return nil
}
// notify calls each consumer in a loop and evaluate their
// condition. If the condition evaluates to `true`, an Event
// is sent down the consumer channel. To avoid the case where a
// consumer does not read properly its channel, which would block
// the current function, we discard the event if the channel is
// blocked for writing. It is the consumer's responsibility to
// read the channel properly.
func (m *DefaultMonitor) notify(ctx context.Context) (timeoutExpired bool) {
// Consumers can be added via `NotifyOn()` concurrently
// to this method. Let's make a copy of the consumers and
// work with the copy. That way we don't hold the lock for
// too long.
m.stateLock.Lock()
consumers := make([]eventConsumer, len(m.state.consumers))
copy(consumers, m.state.consumers)
m.stateLock.Unlock()
// This is fine without lock, as `notify` is called synchronously after poll.
prev, curr := m.state.previousStats, m.state.currentStats
lastPoll := m.state.lastPollInterval
notifyCtx, cancelNotify := context.WithTimeout(ctx, m.cfg.NotifyTimeout)
// We don't use the `cancel` function. This call is only to release context's resources.
// We let the timeout set on the context to expire, which also cancels the context.
defer cancelNotify()
wg := sync.WaitGroup{}
for _, cs := range consumers {
for _, cd := range cs.conditions {
wg.Add(1)
go func(consumer eventConsumer, condition Condition) {
defer wg.Done()
// This is for monitoring purposes. We want to log each Condition
// that has not returned when the NotifyTimeout was reached.
// We only log when the deadline was actually exceeded, not when
// notifyCtx was cancelled by the deferred cancelNotify() in the
// parent — otherwise this races with `doneCh` and emits spurious
// warnings on fast condition completions.
doneCh := make(chan struct{}, 1)
go func() {
select {
case <-doneCh:
case <-notifyCtx.Done():
if !errors.Is(notifyCtx.Err(), context.DeadlineExceeded) {
return
}
t := m.cfg.NotifyTimeout.String()
d := condition.Name
msg := fmt.Sprintf("LoadMonitor notify timeout (%q) reached! Condition %q did not complete in time.", t, d)
m.logger.Warn(msg)
}
}()
if ok, desc := condition.Fn(notifyCtx, prev, curr, lastPoll); ok {
// Record the event emission before attempting delivery so
// it is counted even when the consumer's channel is full or
// has no reader and the event is discarded below.
m.metrics.incEventEmitted(condition.Name)
// The shutdown logic closes all consumers channel.
// To avoid sending on a closed channel, we hold the
// shutdown lock while sending an Event. Shutdown cannot
// occur while we hold this lock.
m.shutdownMutex.Lock()
if !m.isShuttingDown && notifyCtx.Err() == nil {
select {
case consumer.out <- Event{
ConditionName: condition.Name,
Description: desc,
CurrentStats: curr,
PreviousStats: prev,
}:
default:
// Discard the Event if the channel is full
// or does not have a reader.
}
}
m.shutdownMutex.Unlock()
}
doneCh <- struct{}{}
}(cs, cd)
}
}
wgDoneCh := make(chan struct{})
go func() {
wg.Wait()
close(wgDoneCh)
}()
select {
case <-wgDoneCh:
return false
case <-notifyCtx.Done():
t := m.cfg.NotifyTimeout.String()
msg := fmt.Sprintf("LoadMonitor notify timeout (%q) reached: some conditions might not have finished in time.", t)
m.logger.Warn(msg)
return true
}
}
// collect collects Prometheus metrics from the polled stats
func (m *DefaultMonitor) setMetrics() {
m.metrics.setStats(m.state.currentStats)
}
// shutdown closes all consumer's channel to notify
// them that the Manager has been closed.
func (m *DefaultMonitor) shutdown() {
m.shutdownOnce.Do(func() {
m.started.Store(false)
m.shutdownMutex.Lock()
m.isShuttingDown = true
m.stateLock.Lock()
for _, c := range m.state.consumers {
close(c.out)
}
m.stateLock.Unlock()
m.shutdownMutex.Unlock()
})
}
// markAsRunning returns true if the Monitor can be started, false otherwise.
// Currently, it only makes sure Start is not called twice using an atomic value.
func (m *DefaultMonitor) markAsRunning() bool {
return m.started.CompareAndSwap(false, true)
}
// isRunning returns true if the Monitor is already running
func (m *DefaultMonitor) isRunning() bool {
return m.started.Load()
}