The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package main
import (
"context"
"fmt"
"net/http"
"sync/atomic"
"time"
)
// Cross-phase state for detecting CH restarts. Single-run CLI; package-level
// state is fine. lastCHUptime stores the most recent step's CH uptime (0
// before the first reachable snapshot); chRestarted is set sticky-true once a
// restart is observed so main() can mark the run failed and exit non-zero.
var (
lastCHUptime atomic.Int64
chRestarted atomic.Bool
sutDied atomic.Bool
)
type stepResult struct {
Step int `json:"step"`
BatchSize int `json:"batchSize"`
RequestRate float64 `json:"requestRate"`
AttemptedItemsPerSec float64 `json:"attemptedItemsPerSec"`
ActualItemsPerSec float64 `json:"actualItemsPerSec"`
Rejected int64 `json:"rejected"`
Ingest latencySnapshot `json:"ingest"`
CH chSnapshot `json:"ch"`
// SutIngest is the end-of-step write-path snapshot from /health/deep
// (cumulative counters plus DuckDB engine gauges); DroppedItems is the
// per-step delta of silently dropped rows, which fails the step when
// nonzero.
SutIngest *ingestStatsSnapshot `json:"sutIngest,omitempty"`
DroppedItems int64 `json:"droppedItems,omitempty"`
Passed bool `json:"passed"`
FailReason string `json:"failReason,omitempty"`
}
type phaseResult struct {
Kind string `json:"kind"`
FixedRequestRate float64 `json:"fixedRequestRate,omitempty"`
FixedBatchSize int `json:"fixedBatchSize,omitempty"`
Steps []stepResult `json:"steps"`
MaxBatchSize int `json:"maxBatchSize,omitempty"`
MaxRequestRate float64 `json:"maxRequestRate,omitempty"`
}
type finalReport struct {
Tier string `json:"tier"`
Mode string `json:"mode"`
Signal string `json:"signal"`
Scenario string `json:"scenario"`
StartedAt string `json:"startedAt"`
EndedAt string `json:"endedAt"`
Phase1 *phaseResult `json:"phase1,omitempty"`
Phase2 *phaseResult `json:"phase2,omitempty"`
Phase3 *phaseResult `json:"phase3,omitempty"`
ReadProbe *readProbeResult `json:"readProbe,omitempty"`
MaxSustainableItemsPerSec float64 `json:"maxSustainableItemsPerSec,omitempty"`
MaxFillLevelPassed int64 `json:"maxFillLevelPassed,omitempty"`
ChRestarted bool `json:"chRestarted,omitempty"`
// SutDied: passing steps stay valid but the headline is only a lower
// bound — the cliff was never bisected.
SutDied bool `json:"sutDied,omitempty"`
}
func (r *finalReport) computeHeadline() {
// Take the max ActualItemsPerSec across passing steps from all three
// throughput phases. Different phases probe different shapes (collector
// fat batches, SDK small-batch fleet), and the headline is the SUT's
// best sustained number regardless of shape. Per-phase numbers live in
// the JSON for shape-specific analysis.
maxFromPhase := func(p *phaseResult) float64 {
if p == nil {
return 0
}
var best float64
for _, s := range p.Steps {
if s.Passed && s.ActualItemsPerSec > best {
best = s.ActualItemsPerSec
}
}
return best
}
for _, candidate := range []float64{
maxFromPhase(r.Phase2),
maxFromPhase(r.Phase3),
maxFromPhase(r.Phase1), // fallback when phases 2/3 produce nothing
} {
if candidate > r.MaxSustainableItemsPerSec {
r.MaxSustainableItemsPerSec = candidate
}
}
if r.ReadProbe != nil {
r.MaxFillLevelPassed = r.ReadProbe.MaxFillLevelPassed
}
}
// phaseCheckpoint is invoked after every step in the throughput phases so the
// caller can persist a partial report — we never want to lose progress when
// the process dies mid-run.
type phaseCheckpoint func(phaseResult)
// runBatchSizeRamp holds requestRate fixed (phase1FixedRate) and grows batch
// size step by step. Stops at the first failing step. Returns a phaseResult
// whose MaxBatchSize is the largest batch that passed. Calls `checkpoint`
// after each step (after both pass and fail) so partial state is durable.
func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
Kind: "batch-size-ramp",
FixedRequestRate: cfg.phase1FixedRate,
}
ing.SetRequestRate(cfg.phase1FixedRate)
for idx, batch := range cfg.phase1BatchSizes {
if ctx.Err() != nil {
break
}
ing.SetBatchSize(batch)
s := runOneStep(ctx, cfg, ing, ingest, client, idx+1, batch, cfg.phase1FixedRate)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "phase1 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
if s.Passed {
res.MaxBatchSize = batch
}
if checkpoint != nil {
checkpoint(res)
}
if !s.Passed {
break
}
}
return res
}
// runRequestRateRamp holds batchSize fixed at min(phase1.MaxBatchSize, cfg.phase2BatchCap)
// and grows request rate step by step. Phase 2's purpose: at the largest
// single-request payload the SUT can absorb (collector-shape), find the
// req/sec ceiling.
func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {
batch := phase1.MaxBatchSize
if batch <= 0 {
batch = cfg.phase2BatchCap
}
if batch > cfg.phase2BatchCap {
batch = cfg.phase2BatchCap
}
return runRateRamp(ctx, cfg, ing, ingest, client, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)
}
// runSmallBatchRateRamp holds batchSize fixed at cfg.phase3BatchSize (default
// 100 — typical SDK batch shape rather than collector-default 8192) and
// ramps request rate. Phase 3's purpose: measure the SUT's behaviour under
// many small requests, as opposed to Phase 2's few-fat-requests shape.
// SDK fleets in the wild send hundreds-to-thousands of small batches per
// second — language-SDK BatchSpanProcessor defaults are 512 per batch on a
// 5 s rotation, often much less in practice; this phase covers that regime.
func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {
return runRateRamp(ctx, cfg, ing, ingest, client, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)
}
// runRateRamp is the shared engine for Phase 2 and Phase 3: hold batch
// fixed, ramp request rate, bisect after the first failing step. logPrefix
// is just for stderr messages ("phase2" / "phase3").
func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
Kind: kind,
FixedBatchSize: batch,
}
if batch <= 0 || len(rates) == 0 {
return res
}
ing.SetBatchSize(batch)
var lastPassRate, firstFailRate float64
stepNo := 0
for _, rate := range rates {
if ctx.Err() != nil {
break
}
stepNo++
ing.SetRequestRate(rate)
s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, rate)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "%s step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
if s.Passed {
res.MaxRequestRate = rate
lastPassRate = rate
}
if checkpoint != nil {
checkpoint(res)
}
if !s.Passed {
firstFailRate = rate
if !ensureSutRecovered(ctx, cfg, logPrefix) {
return res
}
break
}
}
// Bisection refinement. After the coarse ramp finds an adjacent
// (passing, failing) pair, halve the gap up to phase2BisectMaxSteps
// times to pin the real cliff. Phase 3 reuses the same tunables since
// the math is identical regardless of batch size.
if cfg.phase2BisectMaxSteps > 0 && lastPassRate > 0 && firstFailRate > lastPassRate {
for b := 0; b < cfg.phase2BisectMaxSteps; b++ {
if ctx.Err() != nil {
break
}
gap := (firstFailRate - lastPassRate) / lastPassRate
if gap <= cfg.phase2BisectTolerance {
break
}
mid := (lastPassRate + firstFailRate) / 2
stepNo++
ing.SetRequestRate(mid)
s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, mid)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "%s bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
if s.Passed {
lastPassRate = mid
res.MaxRequestRate = mid
} else {
firstFailRate = mid
}
if checkpoint != nil {
checkpoint(res)
}
if !s.Passed && !ensureSutRecovered(ctx, cfg, logPrefix) {
break
}
}
}
return res
}
// ensureSutRecovered gates only failed steps — a crash can hide there, while
// a passing step proves liveness. Without it, one fatal step leaves the rest
// of the bisect firing at a dead server and converging to the last pass
// instead of the real cliff.
func ensureSutRecovered(ctx context.Context, cfg config, logPrefix string) bool {
if err := waitForSutHealthy(ctx, cfg.target, cfg.sutHealthTimeoutSeconds); err != nil {
// A canceled ctx is the run deadline, not a verdict on the SUT.
if ctx.Err() == nil {
sutDied.Store(true)
fmt.Fprintf(stderrPrefix(), "%s: SUT did not recover after failed step — ending phase, results are lower bounds: %v\n", logPrefix, err)
}
return false
}
return true
}
// runOneStep resizes the worker pool for the new rate, holds the step for
// stepDuration, drains in-flight HTTP for stepDrainSeconds, then snapshots
// latency + item counters. The drain window matters: without it, every step
// boundary cancels ~workerCount in-flight requests mid-flight, inflating the
// recorded error count and depressing the OK count.
func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, stepNo, batchSize int, requestRate float64) stepResult {
ingest.SnapshotAndReset()
ing.SnapshotAndResetItems()
_, ingestStatsBase := fetchDeepHealth(ctx, cfg, client)
ing.Start(ctx)
stepCtx, cancel := context.WithTimeout(ctx, cfg.stepDuration)
start := time.Now()
<-stepCtx.Done()
cancel()
// Stop accepting new requests; let in-flight ones complete. elapsed
// includes the drain window so the rate divisor reflects the full window
// during which items could have arrived.
ing.StopAccepting()
ing.WaitForDrain(cfg.stepDrainSeconds)
elapsed := time.Since(start)
ing.Stop()
snap := ingest.SnapshotAndReset()
attempted, rejected := ing.SnapshotAndResetItems()
ch, ingestStatsEnd := fetchDeepHealth(ctx, cfg, client)
var droppedItems int64
if ingestStatsBase.Reachable && ingestStatsEnd.Reachable {
droppedItems = ingestStatsEnd.DroppedRowsTotal - ingestStatsBase.DroppedRowsTotal
if droppedItems < 0 {
droppedItems = 0
}
}
var sutIngest *ingestStatsSnapshot
if ingestStatsEnd.Reachable {
sutIngest = &ingestStatsEnd
}
// Restart detection: if ClickHouse's uptime is materially less than what
// it was at the previous step's snapshot, the CH process restarted during
// this step. The compose restart policy may have brought it back, but the
// numbers from this step (and the preceding one) are no longer
// interpretable, so the run is marked failed and remaining phases skipped.
prev := lastCHUptime.Load()
restartedThisStep := false
if ch.Reachable && ch.UptimeSec > 0 {
if prev > 0 && ch.UptimeSec < prev+int64(elapsed.Seconds()) {
restartedThisStep = true
chRestarted.Store(true)
}
lastCHUptime.Store(ch.UptimeSec)
}
var attemptedIps, actualIps float64
if elapsed > 0 {
attemptedIps = float64(attempted) / elapsed.Seconds()
actualItems := attempted - rejected
// Discount failed HTTP requests too — their items never made it in.
if attempted > 0 {
httpFailItems := int64(float64(snap.Errors) / float64(snap.OK+snap.Errors) * float64(attempted))
actualItems -= httpFailItems
}
if actualItems < 0 {
actualItems = 0
}
actualIps = float64(actualItems) / elapsed.Seconds()
}
passed, reason := evaluateStep(cfg, snap, attempted, rejected, requestRate, elapsed, droppedItems)
if restartedThisStep {
passed = false
reason = fmt.Sprintf("CH restarted mid-step (uptime %ds < previous %ds + step %ds)", ch.UptimeSec, prev, int64(elapsed.Seconds()))
}
return stepResult{
Step: stepNo,
BatchSize: batchSize,
RequestRate: requestRate,
AttemptedItemsPerSec: attemptedIps,
ActualItemsPerSec: actualIps,
Rejected: rejected,
Ingest: snap,
CH: ch,
SutIngest: sutIngest,
DroppedItems: droppedItems,
Passed: passed,
FailReason: reason,
}
}
// evaluateStep combines four failure criteria:
// 1. HTTP-level error rate + OTLP partial-success rejections > threshold
// (combined item-error budget).
// 2. Silent drops: the SUT acked requests but its telemetry write path
// discarded rows mid-step (DuckDB drops rows the appender rejects).
// Throughput from such a step is not real ingested throughput, so any
// drop fails the step.
// 3. Soft cliff: achieved request rate is far below target. This catches
// the "workers saturated but SUT not yet erroring" state, where latency
// has cliffed to multiple seconds and only error-rate-based detection
// would let the ramp keep climbing one more step before noticing.
func evaluateStep(cfg config, snap latencySnapshot, attempted, rejected int64, targetRate float64, elapsed time.Duration, droppedItems int64) (bool, string) {
totalReq := snap.OK + snap.Errors
if totalReq == 0 {
return false, "no requests completed"
}
httpErrRate := float64(snap.Errors) / float64(totalReq)
var rejectRate float64
if attempted > 0 {
rejectRate = float64(rejected) / float64(attempted)
}
combined := httpErrRate + rejectRate
if combined > cfg.ingestErrThreshold {
return false, fmt.Sprintf("combined error rate %.2f%% (http %.2f%% + rejected %.2f%%) > %.2f%% threshold",
combined*100, httpErrRate*100, rejectRate*100, cfg.ingestErrThreshold*100)
}
if droppedItems > 0 {
return false, fmt.Sprintf("SUT silently dropped %d rows mid-step (acked but not stored)", droppedItems)
}
if targetRate > 0 && elapsed > 0 && cfg.softCliffRatio > 0 {
achievedRate := float64(snap.OK) / elapsed.Seconds()
if achievedRate < targetRate*cfg.softCliffRatio {
return false, fmt.Sprintf("achieved %.2f req/sec is below %.0f%% of target %.2f req/sec (workers saturated, SUT past cliff)",
achievedRate, cfg.softCliffRatio*100, targetRate)
}
}
return true, ""
}