The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package main
import (
"context"
"math/rand"
"net/http"
"sync"
"sync/atomic"
"time"
"golang.org/x/time/rate"
)
type ingester struct {
cfg config
client *http.Client
stats *latencyTracker
sender signalSender
limiter *rate.Limiter
batchSize atomic.Int64
requestRate atomic.Int64 // requests/sec, scaled by 1e3 so we keep fractional resolution
attemptedItems atomic.Int64
rejectedItems atomic.Int64
okItems atomic.Int64 // items in 2xx responses minus PartialSuccess rejects — rows that actually landed
// rng pool: each worker grabs one to avoid contending on the global source
rngPool sync.Pool
// Two contexts let the ramp stop accepting new requests at step boundaries
// without killing in-flight HTTP. acceptCancel ends the limiter.Wait loop;
// the request ctx (parent of acceptCtx) remains alive so HTTP responses
// can complete and be counted. Stop() forces a hard cancel by deriving the
// hard-stop ctx from the parent.
workerWg *sync.WaitGroup
acceptCancel context.CancelFunc
}
func newIngester(cfg config, client *http.Client, stats *latencyTracker) (*ingester, error) {
sender, err := pickSender(cfg.signal)
if err != nil {
return nil, err
}
return &ingester{
cfg: cfg,
client: client,
stats: stats,
sender: sender,
limiter: rate.NewLimiter(rate.Limit(1), 1),
rngPool: sync.Pool{New: func() any { return rand.New(rand.NewSource(rand.Int63())) }},
}, nil
}
func pickSender(signal string) (signalSender, error) {
switch signal {
case "spans":
return spansSender{}, nil
case "metrics":
return metricsSender{}, nil
case "logs":
return logsSender{}, nil
default:
return nil, errUnknownSignal(signal)
}
}
type errUnknownSignal string
func (e errUnknownSignal) Error() string {
return "unknown signal: " + string(e) + " (expected spans|metrics|logs)"
}
func (i *ingester) Name() string {
return i.sender.Name()
}
func (i *ingester) SetBatchSize(b int) {
if b < 1 {
b = 1
}
i.batchSize.Store(int64(b))
}
func (i *ingester) SetRequestRate(rps float64) {
if rps < 0.001 {
rps = 0.001
}
i.limiter.SetLimit(rate.Limit(rps))
burst := int(rps) + 1
if burst < 1 {
burst = 1
}
i.limiter.SetBurst(burst)
i.requestRate.Store(int64(rps * 1000))
}
func (i *ingester) RequestRate() float64 {
return float64(i.requestRate.Load()) / 1000.0
}
// SnapshotAndResetItems returns attempted/rejected since the last call and
// resets them. Called at step boundaries alongside latencyTracker.
func (i *ingester) SnapshotAndResetItems() (attempted, rejected int64) {
attempted = i.attemptedItems.Swap(0)
rejected = i.rejectedItems.Swap(0)
return
}
// SnapshotAndResetOkItems returns items confirmed by 2xx responses (minus
// OTLP PartialSuccess rejects) since the last call. The read-probe fill
// tracks this instead of attempted items so admission-gate 503s and other
// HTTP failures don't count phantom rows toward a fill target.
func (i *ingester) SnapshotAndResetOkItems() int64 {
return i.okItems.Swap(0)
}
// Start launches the worker pool sized for the current request rate. The
// passed ctx is used directly for HTTP requests so in-flight requests survive
// across step boundaries; the limiter loop uses a derived acceptCtx that
// StopAccepting can cancel independently.
func (i *ingester) Start(ctx context.Context) {
acceptCtx, cancelAccept := context.WithCancel(ctx)
wg := &sync.WaitGroup{}
i.acceptCancel = cancelAccept
i.workerWg = wg
workers := workerCountFor(i.RequestRate())
wg.Add(workers)
for w := 0; w < workers; w++ {
go func() {
defer wg.Done()
rng := i.rngPool.Get().(*rand.Rand)
defer i.rngPool.Put(rng)
for {
if err := i.limiter.Wait(acceptCtx); err != nil {
return
}
batchSize := int(i.batchSize.Load())
// HTTP uses the parent ctx (not acceptCtx) so requests in
// flight when StopAccepting fires aren't canceled mid-flight.
sendOneOTLP(ctx, i.client, i.cfg, i.sender, rng, batchSize, i.stats, &i.attemptedItems, &i.rejectedItems, &i.okItems)
}
}()
}
}
// StopAccepting tells the worker pool to stop pulling from the rate limiter.
// In-flight HTTP requests continue under the parent ctx until they finish or
// the parent ctx is canceled. Pair with WaitForDrain to give them a grace
// window before the next step starts.
func (i *ingester) StopAccepting() {
if i.acceptCancel != nil {
i.acceptCancel()
}
}
// WaitForDrain blocks up to d for the worker pool to finish any in-flight
// HTTP requests. Returns earlier when all workers exit.
func (i *ingester) WaitForDrain(d time.Duration) {
if i.workerWg == nil {
return
}
done := make(chan struct{})
go func() { i.workerWg.Wait(); close(done) }()
if d <= 0 {
<-done
return
}
select {
case <-done:
case <-time.After(d):
}
}
// Stop ends acceptance and waits forever for the pool to drain. The parent
// ctx still gates HTTP cancellation, so this won't hang past the overall run
// deadline.
func (i *ingester) Stop() {
i.StopAccepting()
i.WaitForDrain(0)
i.acceptCancel = nil
i.workerWg = nil
}
// workerCountFor scales worker count with request rate. The headroom is
// generous because high-latency steps (multi-second P50) can otherwise starve
// the rate limiter: at rps=5 with P50=4s, 10 workers only deliver 2.5 req/sec.
// 30 seconds of in-flight headroom guarantees we hit target rate unless the
// SUT is truly past its cliff.
func workerCountFor(rps float64) int {
n := int(rps * 30)
if n < 32 {
n = 32
}
if n > 512 {
n = 512
}
return n
}