The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
cooldown
ddux66d2e0614c35
diff --git a/benchmarks/README.md b/benchmarks/README.mdindex c934ca0c..900d01bd 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -55,8 +55,15 @@ Per matrix entry (one tier × one mode × one signal):- **Phase 1 — batch-size ramp.** Single client at a fixed 5 req/sec.Batch sizes step through `256,1024,4096,8192,16384`. Each step holds for`--step-duration` (default 2 min). Stops at the first failing step.+ - **Inter-phase cooldown.** Phase 1's last step typically runs the SUT+ near saturation; the loadgen sleeps `--inter-phase-cooldown-seconds`+ (default 30 s) before starting Phase 2 so the SUT can drain its+ queues (CH merges, PG WAL, HTTP handler pool). Without this pause,+ pgch runs reliably produced "0 OK / 0 errors" Phase 2 because new+ requests sat on the SUT-side TCP backlog while the SUT was still+ digesting Phase 1's final wave.- **Phase 2 — request-rate ramp.** Batch size fixed at- `min(Phase 1 winner, --phase2-batch-cap=8192)`. Request rates step+ `min(Phase 1 winner, --phase2-batch-cap=16384)`. Request rates stepthrough `1,5,25,100,400`. When the coarse ramp finds the first failingstep, up to `--phase2-bisect-max-steps` (default 3) bisection steps runbetween the last passing rate and the failing rate to pin the cliff@@ -73,7 +80,10 @@ Per matrix entry (one tier × one mode × one signal):The headline `maxSustainableItemsPerSec` is the highest `actualItemsPerSec`recorded across passing Phase 2 steps — measured from real OK responses,not the formula `batchSize × targetRate` (which over-reports when workers- saturate before the limiter does).+ saturate before the limiter does). If Phase 2 produces zero passing steps+ (rare, but happens when Phase 1 leaves the SUT in an unrecoverable state+ even after the cooldown), the headline falls back to the best Phase 1+ `actualItemsPerSec` so the run still reports a meaningful number.7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.After all matrix entries finish, `chart.py` renders three bar chartsdiff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.goindex a7e607d0..a0879b44 100644--- a/benchmarks/loadgen/main.go+++ b/benchmarks/loadgen/main.go@@ -27,11 +27,12 @@ type config struct {phase2RequestRates []float64phase1FixedRate float64phase2BatchCap int- ingestErrThreshold float64- softCliffRatio float64- stepDrainSeconds time.Duration- phase2BisectMaxSteps int- phase2BisectTolerance float64+ ingestErrThreshold float64+ softCliffRatio float64+ stepDrainSeconds time.Duration+ interPhaseCooldownSeconds time.Duration+ phase2BisectMaxSteps int+ phase2BisectTolerance float64fillLevels []int64readThresholdMs intsettleSeconds time.Duration@@ -61,10 +62,11 @@ func main() {flag.StringVar(&phase1BatchesStr, "phase1-batch-sizes", "256,1024,4096,8192,16384", "Comma-separated batch sizes for Phase 1 (throughput scenario)")flag.StringVar(&phase2RatesStr, "phase2-request-rates", "1,5,25,100,400", "Comma-separated request rates for Phase 2 (throughput scenario)")flag.Float64Var(&cfg.phase1FixedRate, "phase1-fixed-rate", 5, "Fixed request rate during Phase 1 (req/sec)")- flag.IntVar(&cfg.phase2BatchCap, "phase2-batch-cap", 8192, "Cap on Phase 2 batch size; Phase 2 uses min(this, Phase 1 winner)")+ flag.IntVar(&cfg.phase2BatchCap, "phase2-batch-cap", 16384, "Cap on Phase 2 batch size; Phase 2 uses min(this, Phase 1 winner). Bumped from the OTel collector default of 8192 because pgch SUTs can usefully exceed it.")flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if combined (HTTP error + OTLP rejected) item rate exceeds this")flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")+ flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 30*time.Second, "Pause between Phase 1 and Phase 2 to let the SUT drain queues and recover from Phase 1's final-step load. 0 disables.")flag.IntVar(&cfg.phase2BisectMaxSteps, "phase2-bisect-max-steps", 3, "After Phase 2 finds the cliff, run up to this many bisection steps between the last passing and first failing rate to narrow the cliff. 0 disables.")flag.Float64Var(&cfg.phase2BisectTolerance, "phase2-bisect-tolerance", 0.20, "Bisection stops when (firstFailRate-lastPassRate)/lastPassRate falls below this fraction (e.g. 0.20 = stop when the cliff is pinned to within 20% of the last passing rate).")flag.StringVar(&fillLevelsStr, "fill-levels", "100000,1000000,10000000,100000000", "Comma-separated row counts to fill before probing a read (read-probe scenario)")@@ -155,6 +157,19 @@ func main() {switch cfg.scenario {case "throughput":phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats)+ // Cool-down between phases: Phase 1's last step often runs the SUT at+ // 70-99% of capacity, leaving its internal queues (CH merges, PG WAL,+ // HTTP handler pool) saturated. Jumping straight into Phase 2 with the+ // SUT still digesting that wave produces "0 OK / 0 errors" garbage+ // because new requests sit on the SUT-side TCP backlog without ever+ // reaching a handler or timing out.+ if cfg.interPhaseCooldownSeconds > 0 {+ fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)+ select {+ case <-time.After(cfg.interPhaseCooldownSeconds):+ case <-ctx.Done():+ }+ }phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1)out.Phase1 = &phase1out.Phase2 = &phase2diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.goindex fcfc4fb7..cf8a1dd1 100644--- a/benchmarks/loadgen/ramp.go+++ b/benchmarks/loadgen/ramp.go@@ -56,6 +56,17 @@ func (r *finalReport) computeHeadline() {}r.MaxSustainableItemsPerSec = best}+ // Fall back to Phase 1 when Phase 2 produced no passing steps. This+ // happens when Phase 1's heavy final step poisons the SUT and Phase 2+ // can't recover within its drain window — without this fallback the+ // headline would be 0 even though Phase 1 has perfectly valid data.+ if r.MaxSustainableItemsPerSec == 0 && r.Phase1 != nil {+ for _, s := range r.Phase1.Steps {+ if s.Passed && s.ActualItemsPerSec > r.MaxSustainableItemsPerSec {+ r.MaxSustainableItemsPerSec = s.ActualItemsPerSec+ }+ }+ }if r.ReadProbe != nil {r.MaxFillLevelPassed = r.ReadProbe.MaxFillLevelPassed}