Skip to content

The only tool you need to know what is happening and how to fix it.

added 3rd phase

ddux8ee8ed9e21bd
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.yml
index 945db4ef..bdd3f561 100644
--- a/.github/workflows/benchmark-hardware.yml
+++ b/.github/workflows/benchmark-hardware.yml
@@ -165,15 +165,35 @@ jobs:
git config user.name "traceway-bot"
git config user.email "[email protected]"
+ staged="${{ steps.stage.outputs.dst }}"
+
# The benchmark run takes 2-3 hours, so by the time we get here our
# checkout is almost always behind main. Sync to whatever main looks
- # like *right now* before committing. The result files are untracked,
- # so a hard reset to origin/main leaves them in place.
+ # like *right now* before committing.
+ #
+ # CRITICAL: previously this was just `git reset --hard origin/main`,
+ # which assumed the staged dir contained only untracked new files.
+ # That assumption breaks when an earlier dispatch today already
+ # committed `benchmarks/results/<date>/` — our re-rendered charts +
+ # summary.md become tracked-but-modified, and `reset --hard` reverts
+ # them to the old (incomplete) versions. Only the genuinely-new
+ # JSON survives, producing a misleading partial commit.
+ #
+ # Fix: copy the staged dir aside, reset to main, then restore. Our
+ # versions always win on conflict, which is what we want — they
+ # reflect the union of all matrix entries that have run today.
+ tmp_staged="$(mktemp -d)/staged"
+ cp -a "${staged}" "${tmp_staged}"
+
echo "syncing checkout to latest origin/main before committing"
git fetch origin main
git reset --hard origin/main
- git add "${{ steps.stage.outputs.dst }}"
+ mkdir -p "${staged}"
+ cp -a "${tmp_staged}/." "${staged}/"
+ rm -rf "${tmp_staged%/staged}"
+
+ git add "${staged}"
if git diff --cached --quiet; then
echo "nothing to commit"
exit 0
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 900d01bd..625fd248 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -62,7 +62,7 @@ Per matrix entry (one tier × one mode × one signal):
pgch runs reliably produced "0 OK / 0 errors" Phase 2 because new
requests sat on the SUT-side TCP backlog while the SUT was still
digesting Phase 1's final wave.
- - **Phase 2 — request-rate ramp.** Batch size fixed at
+ - **Phase 2 — request-rate ramp (collector shape).** Batch size fixed at
`min(Phase 1 winner, --phase2-batch-cap=16384)`. Request rates step
through `1,5,25,100,400`. When the coarse ramp finds the first failing
step, up to `--phase2-bisect-max-steps` (default 3) bisection steps run
@@ -70,6 +70,16 @@ Per matrix entry (one tier × one mode × one signal):
within `--phase2-bisect-tolerance` (default 20%). So `5→25` (a 5× jump)
gets refined into something like `5, 15, 10, 12` until the gap is
<20% of the last passing rate.
+ - **Inter-phase cooldown** before Phase 3 too.
+ - **Phase 3 — request-rate ramp (SDK-fleet shape).** Batch size fixed at
+ `--phase3-batch-size` (default **100**, matching typical language-SDK
+ `BatchSpanProcessor` output rather than the collector's 8192). Request
+ rates step through `--phase3-request-rates` (default
+ `10,100,1000,5000,10000` — much higher than Phase 2 because each
+ request is much cheaper at batch=100). Same bisection logic as Phase 2.
+ Measures the SUT under "thousands of small batches/sec from a real
+ SDK fleet" load, which stresses the HTTP/auth/decode/queue path more
+ than the raw insert path Phase 2 measures.
6. A step "fails" when **either**:
- combined error rate (HTTP failures + OTLP `PartialSuccess` rejected items)
exceeds `--ingest-err-threshold` (default 5%), **or**
@@ -78,12 +88,12 @@ Per matrix entry (one tier × one mode × one signal):
SUT has cliffed on latency, and we'd be erroring out one step later anyway.
The headline `maxSustainableItemsPerSec` is the highest `actualItemsPerSec`
- recorded across passing Phase 2 steps — measured from real OK responses,
- not the formula `batchSize × targetRate` (which over-reports when workers
- saturate before the limiter does). If Phase 2 produces zero passing steps
- (rare, but happens when Phase 1 leaves the SUT in an unrecoverable state
- even after the cooldown), the headline falls back to the best Phase 1
- `actualItemsPerSec` so the run still reports a meaningful number.
+ recorded across passing steps from **any** of Phase 1, Phase 2, or Phase 3
+ — measured from real OK responses, not the formula `batchSize × targetRate`
+ (which over-reports when workers saturate before the limiter does).
+ Different phases probe different shapes (collector fat-batch vs SDK
+ small-batch fleet) and the SUT's best-shape ceiling is what the headline
+ reports; per-phase numbers stay in the JSON for shape-specific analysis.
7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.
After all matrix entries finish, `chart.py` renders three bar charts
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
index 973f7ad3..7152dee8 100644
--- a/benchmarks/loadgen/main.go
+++ b/benchmarks/loadgen/main.go
@@ -25,8 +25,10 @@ type config struct {
stepDuration time.Duration
phase1BatchSizes []int
phase2RequestRates []float64
+ phase3RequestRates []float64
phase1FixedRate float64
phase2BatchCap int
+ phase3BatchSize int
ingestErrThreshold float64
softCliffRatio float64
stepDrainSeconds time.Duration
@@ -48,6 +50,7 @@ func main() {
cfg config
phase1BatchesStr string
phase2RatesStr string
+ phase3RatesStr string
fillLevelsStr string
)
@@ -63,6 +66,8 @@ func main() {
flag.StringVar(&phase2RatesStr, "phase2-request-rates", "1,5,25,100,400", "Comma-separated request rates for Phase 2 (throughput scenario)")
flag.Float64Var(&cfg.phase1FixedRate, "phase1-fixed-rate", 5, "Fixed request rate during Phase 1 (req/sec)")
flag.IntVar(&cfg.phase2BatchCap, "phase2-batch-cap", 16384, "Cap on Phase 2 batch size; Phase 2 uses min(this, Phase 1 winner). Bumped from the OTel collector default of 8192 because pgch SUTs can usefully exceed it.")
+ flag.IntVar(&cfg.phase3BatchSize, "phase3-batch-size", 100, "Fixed batch size for Phase 3 (SDK-fleet shape — many small batches at high rate). Default 100 reflects typical language-SDK BatchSpanProcessor output, not the collector's 8192.")
+ flag.StringVar(&phase3RatesStr, "phase3-request-rates", "10,100,1000,5000,10000", "Comma-separated request rates for Phase 3 (small-batch rate ramp). Goes higher than Phase 2 because each request is much cheaper at batch=100.")
flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if combined (HTTP error + OTLP rejected) item rate exceeds this")
flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")
flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")
@@ -95,8 +100,14 @@ func main() {
fmt.Fprintf(os.Stderr, "invalid --phase2-request-rates: %v\n", err)
os.Exit(2)
}
+ phase3Rates, err := parseFloats(phase3RatesStr)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "invalid --phase3-request-rates: %v\n", err)
+ os.Exit(2)
+ }
cfg.phase1BatchSizes = batches
cfg.phase2RequestRates = rates
+ cfg.phase3RequestRates = phase3Rates
fillLevels, err := parseInt64s(fillLevelsStr)
if err != nil {
@@ -191,6 +202,22 @@ func main() {
writeCheckpoint()
})
out.Phase2 = &phase2
+
+ // Phase 3 — small-batch high-rate (SDK-fleet shape). Independent of
+ // Phase 1/2 results, so it runs even if Phase 2 produced no passing
+ // steps. Same cooldown before it as between 1 and 2.
+ if cfg.interPhaseCooldownSeconds > 0 && ctx.Err() == nil {
+ fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)
+ select {
+ case <-time.After(cfg.interPhaseCooldownSeconds):
+ case <-ctx.Done():
+ }
+ }
+ phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {
+ out.Phase3 = &p
+ writeCheckpoint()
+ })
+ out.Phase3 = &phase3
case "read-probe":
probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient, func(p readProbeResult) {
out.ReadProbe = &p
diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.go
index 67212984..50b97350 100644
--- a/benchmarks/loadgen/ramp.go
+++ b/benchmarks/loadgen/ramp.go
@@ -36,35 +36,37 @@ type finalReport struct {
EndedAt string `json:"endedAt"`
Phase1 *phaseResult `json:"phase1,omitempty"`
Phase2 *phaseResult `json:"phase2,omitempty"`
+ Phase3 *phaseResult `json:"phase3,omitempty"`
ReadProbe *readProbeResult `json:"readProbe,omitempty"`
MaxSustainableItemsPerSec float64 `json:"maxSustainableItemsPerSec,omitempty"`
MaxFillLevelPassed int64 `json:"maxFillLevelPassed,omitempty"`
}
func (r *finalReport) computeHeadline() {
- if r.Phase2 != nil {
- // Use the achieved throughput of the last passing step. The
- // FixedBatchSize × MaxRequestRate formula is unreliable: at the cliff,
- // workers saturate on HTTP latency and never deliver the target rate,
- // so the formula overstates capacity. ActualItemsPerSec is measured
- // from real OK responses and tells the truth.
+ // Take the max ActualItemsPerSec across passing steps from all three
+ // throughput phases. Different phases probe different shapes (collector
+ // fat batches, SDK small-batch fleet), and the headline is the SUT's
+ // best sustained number regardless of shape. Per-phase numbers live in
+ // the JSON for shape-specific analysis.
+ maxFromPhase := func(p *phaseResult) float64 {
+ if p == nil {
+ return 0
+ }
var best float64
- for _, s := range r.Phase2.Steps {
+ for _, s := range p.Steps {
if s.Passed && s.ActualItemsPerSec > best {
best = s.ActualItemsPerSec
}
}
- r.MaxSustainableItemsPerSec = best
+ return best
}
- // Fall back to Phase 1 when Phase 2 produced no passing steps. This
- // happens when Phase 1's heavy final step poisons the SUT and Phase 2
- // can't recover within its drain window — without this fallback the
- // headline would be 0 even though Phase 1 has perfectly valid data.
- if r.MaxSustainableItemsPerSec == 0 && r.Phase1 != nil {
- for _, s := range r.Phase1.Steps {
- if s.Passed && s.ActualItemsPerSec > r.MaxSustainableItemsPerSec {
- r.MaxSustainableItemsPerSec = s.ActualItemsPerSec
- }
+ for _, candidate := range []float64{
+ maxFromPhase(r.Phase2),
+ maxFromPhase(r.Phase3),
+ maxFromPhase(r.Phase1), // fallback when phases 2/3 produce nothing
+ } {
+ if candidate > r.MaxSustainableItemsPerSec {
+ r.MaxSustainableItemsPerSec = candidate
}
}
if r.ReadProbe != nil {
@@ -113,8 +115,9 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la
}
// runRequestRateRamp holds batchSize fixed at min(phase1.MaxBatchSize, cfg.phase2BatchCap)
-// and grows request rate step by step. Calls `checkpoint` after each coarse
-// step and after each bisection step so partial state is durable.
+// and grows request rate step by step. Phase 2's purpose: at the largest
+// single-request payload the SUT can absorb (collector-shape), find the
+// req/sec ceiling.
func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {
batch := phase1.MaxBatchSize
if batch <= 0 {
@@ -123,12 +126,29 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
if batch > cfg.phase2BatchCap {
batch = cfg.phase2BatchCap
}
+ return runRateRamp(ctx, cfg, ing, ingest, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)
+}
+// runSmallBatchRateRamp holds batchSize fixed at cfg.phase3BatchSize (default
+// 100 — typical SDK batch shape rather than collector-default 8192) and
+// ramps request rate. Phase 3's purpose: measure the SUT's behaviour under
+// many small requests, as opposed to Phase 2's few-fat-requests shape.
+// SDK fleets in the wild send hundreds-to-thousands of small batches per
+// second — language-SDK BatchSpanProcessor defaults are 512 per batch on a
+// 5 s rotation, often much less in practice; this phase covers that regime.
+func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {
+ return runRateRamp(ctx, cfg, ing, ingest, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)
+}
+
+// runRateRamp is the shared engine for Phase 2 and Phase 3: hold batch
+// fixed, ramp request rate, bisect after the first failing step. logPrefix
+// is just for stderr messages ("phase2" / "phase3").
+func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
- Kind: "request-rate-ramp",
+ Kind: kind,
FixedBatchSize: batch,
}
- if batch <= 0 {
+ if batch <= 0 || len(rates) == 0 {
return res
}
@@ -137,7 +157,7 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
var lastPassRate, firstFailRate float64
stepNo := 0
- for _, rate := range cfg.phase2RequestRates {
+ for _, rate := range rates {
if ctx.Err() != nil {
break
}
@@ -145,8 +165,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
ing.SetRequestRate(rate)
s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, rate)
res.Steps = append(res.Steps, s)
- fmt.Fprintf(stderrPrefix(), "phase2 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
- s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
+ fmt.Fprintf(stderrPrefix(), "%s step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
+ logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
if s.Passed {
res.MaxRequestRate = rate
lastPassRate = rate
@@ -162,9 +182,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
// Bisection refinement. After the coarse ramp finds an adjacent
// (passing, failing) pair, halve the gap up to phase2BisectMaxSteps
- // times to pin the real cliff. Skipped when nothing failed (no cliff
- // in the configured range) or nothing passed (cliff below the
- // smallest configured rate — bisecting below it isn't informative).
+ // times to pin the real cliff. Phase 3 reuses the same tunables since
+ // the math is identical regardless of batch size.
if cfg.phase2BisectMaxSteps > 0 && lastPassRate > 0 && firstFailRate > lastPassRate {
for b := 0; b < cfg.phase2BisectMaxSteps; b++ {
if ctx.Err() != nil {
@@ -179,8 +198,8 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
ing.SetRequestRate(mid)
s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, mid)
res.Steps = append(res.Steps, s)
- fmt.Fprintf(stderrPrefix(), "phase2 bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
- s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
+ fmt.Fprintf(stderrPrefix(), "%s bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
+ logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
if s.Passed {
lastPassRate = mid
res.MaxRequestRate = mid
diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.py
index c88e3efe..b141827a 100755
--- a/benchmarks/scripts/chart.py
+++ b/benchmarks/scripts/chart.py
@@ -203,19 +203,20 @@ def render_summary(runs: list[dict], out: Path) -> None:
continue
lines.append(f"### {signal.capitalize()}")
lines.append("")
- lines.append(f"| Tier | Mode | Max {ITEM_LABEL[signal]} | Phase 1 max batch | Phase 2 max req/sec | Ingest P99 @ max (ms) |")
- lines.append("|------|------|---------|-------------------|---------------------|------------------------|")
+ lines.append(f"| Tier | Mode | Max {ITEM_LABEL[signal]} | P1 max batch | P2 max req/sec @ batch | P3 max req/sec @ batch=100 |")
+ lines.append("|------|------|---------|--------------|-------------------------|-------------------------------|")
for run in sorted(sig_runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):
max_items = int(run.get("maxSustainableItemsPerSec", 0))
max_batch = (run.get("phase1") or {}).get("maxBatchSize", 0)
- max_rate = (run.get("phase2") or {}).get("maxRequestRate", 0)
- last_pass = None
- for s in reversed((run.get("phase2") or {}).get("steps") or []):
- if s.get("passed"):
- last_pass = s
- break
- ingest_p99 = int(last_pass["ingest"]["p99"]) if last_pass else 0
- lines.append(f"| {run['tier']} | {run['mode']} | {max_items:,} | {max_batch:,} | {max_rate:g} | {ingest_p99} |")
+ p2 = run.get("phase2") or {}
+ p2_rate = p2.get("maxRequestRate", 0)
+ p2_batch = p2.get("fixedBatchSize", 0)
+ p2_cell = f"{p2_rate:g} @ {p2_batch:,}" if p2_rate else "—"
+ p3 = run.get("phase3") or {}
+ p3_rate = p3.get("maxRequestRate", 0)
+ p3_batch = p3.get("fixedBatchSize", 0)
+ p3_cell = f"{p3_rate:g} @ {p3_batch:,}" if p3_rate else "—"
+ lines.append(f"| {run['tier']} | {run['mode']} | {max_items:,} | {max_batch:,} | {p2_cell} | {p3_cell} |")
lines.append("")
readprobe_runs = [r for r in runs if r["scenario"] == "read-probe"]
diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.sh
index afd0f05a..0f25a09b 100755
--- a/benchmarks/scripts/run-matrix-entry.sh
+++ b/benchmarks/scripts/run-matrix-entry.sh
@@ -74,7 +74,7 @@ if [[ "${SMOKE}" == "smoke" ]]; then
if [[ "${SCENARIO}" == "read-probe" ]]; then
extra_args+=( --fill-levels 100000,1000000 --settle-seconds 5s )
else
- extra_args+=( --phase1-batch-sizes 256,1024 --phase2-request-rates 1,5 --step-duration 15s )
+ extra_args+=( --phase1-batch-sizes 256,1024 --phase2-request-rates 1,5 --phase3-request-rates 10,100 --step-duration 15s )
fi
fi