The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
added 3rd phase
ddux8ee8ed9e21bd
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.ymlindex 945db4ef..bdd3f561 100644--- a/.github/workflows/benchmark-hardware.yml+++ b/.github/workflows/benchmark-hardware.yml@@ -165,15 +165,35 @@ jobs:git config user.name "traceway-bot"git config user.email "[email protected]"+ staged="${{ steps.stage.outputs.dst }}"+# The benchmark run takes 2-3 hours, so by the time we get here our# checkout is almost always behind main. Sync to whatever main looks- # like *right now* before committing. The result files are untracked,- # so a hard reset to origin/main leaves them in place.+ # like *right now* before committing.+ #+ # CRITICAL: previously this was just `git reset --hard origin/main`,+ # which assumed the staged dir contained only untracked new files.+ # That assumption breaks when an earlier dispatch today already+ # committed `benchmarks/results/<date>/` — our re-rendered charts ++ # summary.md become tracked-but-modified, and `reset --hard` reverts+ # them to the old (incomplete) versions. Only the genuinely-new+ # JSON survives, producing a misleading partial commit.+ #+ # Fix: copy the staged dir aside, reset to main, then restore. Our+ # versions always win on conflict, which is what we want — they+ # reflect the union of all matrix entries that have run today.+ tmp_staged="$(mktemp -d)/staged"+ cp -a "${staged}" "${tmp_staged}"+echo "syncing checkout to latest origin/main before committing"git fetch origin maingit reset --hard origin/main- git add "${{ steps.stage.outputs.dst }}"+ mkdir -p "${staged}"+ cp -a "${tmp_staged}/." "${staged}/"+ rm -rf "${tmp_staged%/staged}"++ git add "${staged}"if git diff --cached --quiet; thenecho "nothing to commit"exit 0diff --git a/benchmarks/README.md b/benchmarks/README.mdindex 900d01bd..625fd248 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -62,7 +62,7 @@ Per matrix entry (one tier × one mode × one signal):pgch runs reliably produced "0 OK / 0 errors" Phase 2 because newrequests sat on the SUT-side TCP backlog while the SUT was stilldigesting Phase 1's final wave.- - **Phase 2 — request-rate ramp.** Batch size fixed at+ - **Phase 2 — request-rate ramp (collector shape).** Batch size fixed at`min(Phase 1 winner, --phase2-batch-cap=16384)`. Request rates stepthrough `1,5,25,100,400`. When the coarse ramp finds the first failingstep, up to `--phase2-bisect-max-steps` (default 3) bisection steps run@@ -70,6 +70,16 @@ Per matrix entry (one tier × one mode × one signal):within `--phase2-bisect-tolerance` (default 20%). So `5→25` (a 5× jump)gets refined into something like `5, 15, 10, 12` until the gap is<20% of the last passing rate.+ - **Inter-phase cooldown** before Phase 3 too.+ - **Phase 3 — request-rate ramp (SDK-fleet shape).** Batch size fixed at+ `--phase3-batch-size` (default **100**, matching typical language-SDK+ `BatchSpanProcessor` output rather than the collector's 8192). Request+ rates step through `--phase3-request-rates` (default+ `10,100,1000,5000,10000` — much higher than Phase 2 because each+ request is much cheaper at batch=100). Same bisection logic as Phase 2.+ Measures the SUT under "thousands of small batches/sec from a real+ SDK fleet" load, which stresses the HTTP/auth/decode/queue path more+ than the raw insert path Phase 2 measures.6. A step "fails" when **either**:- combined error rate (HTTP failures + OTLP `PartialSuccess` rejected items)exceeds `--ingest-err-threshold` (default 5%), **or**@@ -78,12 +88,12 @@ Per matrix entry (one tier × one mode × one signal):SUT has cliffed on latency, and we'd be erroring out one step later anyway.The headline `maxSustainableItemsPerSec` is the highest `actualItemsPerSec`- recorded across passing Phase 2 steps — measured from real OK responses,- not the formula `batchSize × targetRate` (which over-reports when workers- saturate before the limiter does). If Phase 2 produces zero passing steps- (rare, but happens when Phase 1 leaves the SUT in an unrecoverable state- even after the cooldown), the headline falls back to the best Phase 1- `actualItemsPerSec` so the run still reports a meaningful number.+ recorded across passing steps from **any** of Phase 1, Phase 2, or Phase 3+ — measured from real OK responses, not the formula `batchSize × targetRate`+ (which over-reports when workers saturate before the limiter does).+ Different phases probe different shapes (collector fat-batch vs SDK+ small-batch fleet) and the SUT's best-shape ceiling is what the headline+ reports; per-phase numbers stay in the JSON for shape-specific analysis.7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.After all matrix entries finish, `chart.py` renders three bar chartsdiff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.goindex 973f7ad3..7152dee8 100644--- a/benchmarks/loadgen/main.go+++ b/benchmarks/loadgen/main.go@@ -25,8 +25,10 @@ type config struct {stepDuration time.Durationphase1BatchSizes []intphase2RequestRates []float64+ phase3RequestRates []float64phase1FixedRate float64phase2BatchCap int+ phase3BatchSize intingestErrThreshold float64softCliffRatio float64stepDrainSeconds time.Duration@@ -48,6 +50,7 @@ func main() {cfg configphase1BatchesStr stringphase2RatesStr string+ phase3RatesStr stringfillLevelsStr string)@@ -63,6 +66,8 @@ func main() {flag.StringVar(&phase2RatesStr, "phase2-request-rates", "1,5,25,100,400", "Comma-separated request rates for Phase 2 (throughput scenario)")flag.Float64Var(&cfg.phase1FixedRate, "phase1-fixed-rate", 5, "Fixed request rate during Phase 1 (req/sec)")flag.IntVar(&cfg.phase2BatchCap, "phase2-batch-cap", 16384, "Cap on Phase 2 batch size; Phase 2 uses min(this, Phase 1 winner). Bumped from the OTel collector default of 8192 because pgch SUTs can usefully exceed it.")+ flag.IntVar(&cfg.phase3BatchSize, "phase3-batch-size", 100, "Fixed batch size for Phase 3 (SDK-fleet shape — many small batches at high rate). Default 100 reflects typical language-SDK BatchSpanProcessor output, not the collector's 8192.")+ flag.StringVar(&phase3RatesStr, "phase3-request-rates", "10,100,1000,5000,10000", "Comma-separated request rates for Phase 3 (small-batch rate ramp). Goes higher than Phase 2 because each request is much cheaper at batch=100.")flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if combined (HTTP error + OTLP rejected) item rate exceeds this")flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")@@ -95,8 +100,14 @@ func main() {fmt.Fprintf(os.Stderr, "invalid --phase2-request-rates: %v\n", err)os.Exit(2)}+ phase3Rates, err := parseFloats(phase3RatesStr)+ if err != nil {+ fmt.Fprintf(os.Stderr, "invalid --phase3-request-rates: %v\n", err)+ os.Exit(2)+ }cfg.phase1BatchSizes = batchescfg.phase2RequestRates = rates+ cfg.phase3RequestRates = phase3RatesfillLevels, err := parseInt64s(fillLevelsStr)if err != nil {@@ -191,6 +202,22 @@ func main() {writeCheckpoint()})out.Phase2 = &phase2++ // Phase 3 — small-batch high-rate (SDK-fleet shape). Independent of+ // Phase 1/2 results, so it runs even if Phase 2 produced no passing+ // steps. Same cooldown before it as between 1 and 2.+ if cfg.interPhaseCooldownSeconds > 0 && ctx.Err() == nil {+ fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)+ select {+ case <-time.After(cfg.interPhaseCooldownSeconds):+ case <-ctx.Done():+ }+ }+ phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {+ out.Phase3 = &p+ writeCheckpoint()+ })+ out.Phase3 = &phase3case "read-probe":probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient, func(p readProbeResult) {out.ReadProbe = &pdiff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.goindex 67212984..50b97350 100644--- a/benchmarks/loadgen/ramp.go+++ b/benchmarks/loadgen/ramp.go@@ -36,35 +36,37 @@ type finalReport struct {EndedAt string `json:"endedAt"`Phase1 *phaseResult `json:"phase1,omitempty"`Phase2 *phaseResult `json:"phase2,omitempty"`+ Phase3 *phaseResult `json:"phase3,omitempty"`ReadProbe *readProbeResult `json:"readProbe,omitempty"`MaxSustainableItemsPerSec float64 `json:"maxSustainableItemsPerSec,omitempty"`MaxFillLevelPassed int64 `json:"maxFillLevelPassed,omitempty"`}func (r *finalReport) computeHeadline() {- if r.Phase2 != nil {- // Use the achieved throughput of the last passing step. The- // FixedBatchSize × MaxRequestRate formula is unreliable: at the cliff,- // workers saturate on HTTP latency and never deliver the target rate,- // so the formula overstates capacity. ActualItemsPerSec is measured- // from real OK responses and tells the truth.+ // Take the max ActualItemsPerSec across passing steps from all three+ // throughput phases. Different phases probe different shapes (collector+ // fat batches, SDK small-batch fleet), and the headline is the SUT's+ // best sustained number regardless of shape. Per-phase numbers live in+ // the JSON for shape-specific analysis.+ maxFromPhase := func(p *phaseResult) float64 {+ if p == nil {+ return 0+ }var best float64- for _, s := range r.Phase2.Steps {+ for _, s := range p.Steps {if s.Passed && s.ActualItemsPerSec > best {best = s.ActualItemsPerSec}}- r.MaxSustainableItemsPerSec = best+ return best}- // Fall back to Phase 1 when Phase 2 produced no passing steps. This- // happens when Phase 1's heavy final step poisons the SUT and Phase 2- // can't recover within its drain window — without this fallback the- // headline would be 0 even though Phase 1 has perfectly valid data.- if r.MaxSustainableItemsPerSec == 0 && r.Phase1 != nil {- for _, s := range r.Phase1.Steps {- if s.Passed && s.ActualItemsPerSec > r.MaxSustainableItemsPerSec {- r.MaxSustainableItemsPerSec = s.ActualItemsPerSec- }+ for _, candidate := range []float64{+ maxFromPhase(r.Phase2),+ maxFromPhase(r.Phase3),+ maxFromPhase(r.Phase1), // fallback when phases 2/3 produce nothing+ } {+ if candidate > r.MaxSustainableItemsPerSec {+ r.MaxSustainableItemsPerSec = candidate}}if r.ReadProbe != nil {@@ -113,8 +115,9 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la}// runRequestRateRamp holds batchSize fixed at min(phase1.MaxBatchSize, cfg.phase2BatchCap)-// and grows request rate step by step. Calls `checkpoint` after each coarse-// step and after each bisection step so partial state is durable.+// and grows request rate step by step. Phase 2's purpose: at the largest+// single-request payload the SUT can absorb (collector-shape), find the+// req/sec ceiling.func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {batch := phase1.MaxBatchSizeif batch <= 0 {@@ -123,12 +126,29 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *if batch > cfg.phase2BatchCap {batch = cfg.phase2BatchCap}+ return runRateRamp(ctx, cfg, ing, ingest, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)+}+// runSmallBatchRateRamp holds batchSize fixed at cfg.phase3BatchSize (default+// 100 — typical SDK batch shape rather than collector-default 8192) and+// ramps request rate. Phase 3's purpose: measure the SUT's behaviour under+// many small requests, as opposed to Phase 2's few-fat-requests shape.+// SDK fleets in the wild send hundreds-to-thousands of small batches per+// second — language-SDK BatchSpanProcessor defaults are 512 per batch on a+// 5 s rotation, often much less in practice; this phase covers that regime.+func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {+ return runRateRamp(ctx, cfg, ing, ingest, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)+}++// runRateRamp is the shared engine for Phase 2 and Phase 3: hold batch+// fixed, ramp request rate, bisect after the first failing step. logPrefix+// is just for stderr messages ("phase2" / "phase3").+func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {res := phaseResult{- Kind: "request-rate-ramp",+ Kind: kind,FixedBatchSize: batch,}- if batch <= 0 {+ if batch <= 0 || len(rates) == 0 {return res}@@ -137,7 +157,7 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *var lastPassRate, firstFailRate float64stepNo := 0- for _, rate := range cfg.phase2RequestRates {+ for _, rate := range rates {if ctx.Err() != nil {break}@@ -145,8 +165,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *ing.SetRequestRate(rate)s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, rate)res.Steps = append(res.Steps, s)- fmt.Fprintf(stderrPrefix(), "phase2 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",- s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)+ fmt.Fprintf(stderrPrefix(), "%s step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",+ logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)if s.Passed {res.MaxRequestRate = ratelastPassRate = rate@@ -162,9 +182,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *// Bisection refinement. After the coarse ramp finds an adjacent// (passing, failing) pair, halve the gap up to phase2BisectMaxSteps- // times to pin the real cliff. Skipped when nothing failed (no cliff- // in the configured range) or nothing passed (cliff below the- // smallest configured rate — bisecting below it isn't informative).+ // times to pin the real cliff. Phase 3 reuses the same tunables since+ // the math is identical regardless of batch size.if cfg.phase2BisectMaxSteps > 0 && lastPassRate > 0 && firstFailRate > lastPassRate {for b := 0; b < cfg.phase2BisectMaxSteps; b++ {if ctx.Err() != nil {@@ -179,8 +198,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *ing.SetRequestRate(mid)s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, mid)res.Steps = append(res.Steps, s)- fmt.Fprintf(stderrPrefix(), "phase2 bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",- s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)+ fmt.Fprintf(stderrPrefix(), "%s bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",+ logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)if s.Passed {lastPassRate = midres.MaxRequestRate = middiff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.pyindex c88e3efe..b141827a 100755--- a/benchmarks/scripts/chart.py+++ b/benchmarks/scripts/chart.py@@ -203,19 +203,20 @@ def render_summary(runs: list[dict], out: Path) -> None:continuelines.append(f"### {signal.capitalize()}")lines.append("")- lines.append(f"| Tier | Mode | Max {ITEM_LABEL[signal]} | Phase 1 max batch | Phase 2 max req/sec | Ingest P99 @ max (ms) |")- lines.append("|------|------|---------|-------------------|---------------------|------------------------|")+ lines.append(f"| Tier | Mode | Max {ITEM_LABEL[signal]} | P1 max batch | P2 max req/sec @ batch | P3 max req/sec @ batch=100 |")+ lines.append("|------|------|---------|--------------|-------------------------|-------------------------------|")for run in sorted(sig_runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):max_items = int(run.get("maxSustainableItemsPerSec", 0))max_batch = (run.get("phase1") or {}).get("maxBatchSize", 0)- max_rate = (run.get("phase2") or {}).get("maxRequestRate", 0)- last_pass = None- for s in reversed((run.get("phase2") or {}).get("steps") or []):- if s.get("passed"):- last_pass = s- break- ingest_p99 = int(last_pass["ingest"]["p99"]) if last_pass else 0- lines.append(f"| {run['tier']} | {run['mode']} | {max_items:,} | {max_batch:,} | {max_rate:g} | {ingest_p99} |")+ p2 = run.get("phase2") or {}+ p2_rate = p2.get("maxRequestRate", 0)+ p2_batch = p2.get("fixedBatchSize", 0)+ p2_cell = f"{p2_rate:g} @ {p2_batch:,}" if p2_rate else "—"+ p3 = run.get("phase3") or {}+ p3_rate = p3.get("maxRequestRate", 0)+ p3_batch = p3.get("fixedBatchSize", 0)+ p3_cell = f"{p3_rate:g} @ {p3_batch:,}" if p3_rate else "—"+ lines.append(f"| {run['tier']} | {run['mode']} | {max_items:,} | {max_batch:,} | {p2_cell} | {p3_cell} |")lines.append("")readprobe_runs = [r for r in runs if r["scenario"] == "read-probe"]diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.shindex afd0f05a..0f25a09b 100755--- a/benchmarks/scripts/run-matrix-entry.sh+++ b/benchmarks/scripts/run-matrix-entry.sh@@ -74,7 +74,7 @@ if [[ "${SMOKE}" == "smoke" ]]; thenif [[ "${SCENARIO}" == "read-probe" ]]; thenextra_args+=( --fill-levels 100000,1000000 --settle-seconds 5s )else- extra_args+=( --phase1-batch-sizes 256,1024 --phase2-request-rates 1,5 --step-duration 15s )+ extra_args+=( --phase1-batch-sizes 256,1024 --phase2-request-rates 1,5 --phase3-request-rates 10,100 --step-duration 15s )fifi