Skip to content

Git designed for AI agents.

5.3 KB
Raw Blame
package tracker

import (
	"context"
	"errors"
	"sync"
	"time"

	"github.com/feder-cr/git-ai/internal/praefect/config"
)

// ErrorTracker allows tracking how many read/write errors have occurred, and whether or not it has
// exceeded a configured threshold in a configured time window
type ErrorTracker interface {
	// IncrReadErr increases read errors by 1
	IncrReadErr(nodeStorage string)
	// IncrWriteErr increases write errors by 1
	IncrWriteErr(nodeStorage string)
	// ReadThresholdReached returns whether or not the read threshold was reached
	ReadThresholdReached(nodeStorage string) bool
	// WriteThresholdReached returns whether or not the read threshold was reached
	WriteThresholdReached(nodeStorage string) bool
}

// ErrorWindowFunction is a function that is passed to `NewErrors()`. This function shall return
// `true` if the time when a specific error occurred should be considered part of the error window.
type ErrorWindowFunction func(now time.Time, errorTime time.Time) bool

// NewErrorWindowFunction derives an error window from the given configuration.
func NewErrorWindowFunction(cfg config.Failover) (ErrorWindowFunction, error) {
	if cfg.ErrorThresholdWindow == 0 {
		return nil, errors.New("errorWindow must be non zero")
	}

	errorWindow := cfg.ErrorThresholdWindow.Duration()
	return func(now time.Time, errorTime time.Time) bool {
		return errorTime.After(now.Add(-errorWindow))
	}, nil
}

type errorTracker struct {
	isInErrorWindow               ErrorWindowFunction
	m                             sync.RWMutex
	writeThreshold, readThreshold int
	readErrors, writeErrors       map[string][]time.Time
	ctx                           context.Context
}

func newErrors(ctx context.Context, isInErrorWindow ErrorWindowFunction, readThreshold, writeThreshold uint32) (*errorTracker, error) {
	if readThreshold == 0 {
		return nil, errors.New("readThreshold must be non zero")
	}

	if writeThreshold == 0 {
		return nil, errors.New("writeThreshold must be non zero")
	}

	e := &errorTracker{
		isInErrorWindow: isInErrorWindow,
		readErrors:      make(map[string][]time.Time),
		writeErrors:     make(map[string][]time.Time),
		readThreshold:   int(readThreshold),
		writeThreshold:  int(writeThreshold),
		ctx:             ctx,
	}
	go e.periodicallyClear()

	return e, nil
}

// NewErrors creates a new Error instance given a time window in seconds, and read and write thresholds
func NewErrors(ctx context.Context, isInErrorWindow ErrorWindowFunction, readThreshold, writeThreshold uint32) (ErrorTracker, error) {
	return newErrors(ctx, isInErrorWindow, readThreshold, writeThreshold)
}

// IncrReadErr increases the read errors for a node by 1
func (e *errorTracker) IncrReadErr(node string) {
	e.incrReadErrTime(node, time.Now())
}

func (e *errorTracker) incrReadErrTime(node string, t time.Time) {
	select {
	case <-e.ctx.Done():
		return
	default:
		e.m.Lock()
		defer e.m.Unlock()

		e.readErrors[node] = append(e.readErrors[node], t)

		if len(e.readErrors[node]) > e.readThreshold {
			e.readErrors[node] = e.readErrors[node][1:]
		}
	}
}

// IncrWriteErr increases the read errors for a node by 1
func (e *errorTracker) IncrWriteErr(node string) {
	e.incrWriteErrTime(node, time.Now())
}

func (e *errorTracker) incrWriteErrTime(node string, t time.Time) {
	select {
	case <-e.ctx.Done():
		return
	default:
		e.m.Lock()
		defer e.m.Unlock()

		e.writeErrors[node] = append(e.writeErrors[node], t)

		if len(e.writeErrors[node]) > e.writeThreshold {
			e.writeErrors[node] = e.writeErrors[node][1:]
		}
	}
}

// ReadThresholdReached indicates whether or not the read threshold has been reached within the time window
func (e *errorTracker) ReadThresholdReached(node string) bool {
	select {
	case <-e.ctx.Done():
		break
	default:
		e.m.RLock()
		defer e.m.RUnlock()

		now := time.Now()

		for i, errTime := range e.readErrors[node] {
			if e.isInErrorWindow(now, errTime) {
				if i == 0 {
					return len(e.readErrors[node]) >= e.readThreshold
				}
				return len(e.readErrors[node][i-1:]) >= e.readThreshold
			}
		}
	}

	return false
}

// WriteThresholdReached indicates whether or not the write threshold has been reached within the time window
func (e *errorTracker) WriteThresholdReached(node string) bool {
	select {
	case <-e.ctx.Done():
		break
	default:
		e.m.RLock()
		defer e.m.RUnlock()

		now := time.Now()

		for i, errTime := range e.writeErrors[node] {
			if e.isInErrorWindow(now, errTime) {
				if i == 0 {
					return len(e.writeErrors[node]) >= e.writeThreshold
				}
				return len(e.writeErrors[node][i-1:]) >= e.writeThreshold
			}
		}
	}

	return false
}

// periodicallyClear runs in an infinite loop clearing out old error entries
func (e *errorTracker) periodicallyClear() {
	ticker := time.NewTicker(1 * time.Second)
	for {
		select {
		case <-ticker.C:
			e.clear()
		case <-e.ctx.Done():
			e.m.Lock()
			defer e.m.Unlock()
			e.readErrors = nil
			e.writeErrors = nil
			return
		}
	}
}

func (e *errorTracker) clear() {
	e.m.Lock()
	defer e.m.Unlock()

	now := time.Now()

	e.clearErrors(e.writeErrors, now)
	e.clearErrors(e.readErrors, now)
}

func (e *errorTracker) clearErrors(errs map[string][]time.Time, now time.Time) {
	for node, errors := range errs {
		for i, errTime := range errors {
			if e.isInErrorWindow(now, errTime) {
				errs[node] = errors[i:]
				break
			}

			if i == len(errors)-1 {
				errs[node] = errs[node][:0]
			}
		}
	}
}