Skip to content

The only tool you need to know what is happening and how to fix it.

more reliable testing

dduxc431503e8041
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 625fd248..b304570a 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -96,6 +96,21 @@ Per matrix entry (one tier × one mode × one signal):
reports; per-phase numbers stay in the JSON for shape-specific analysis.
7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.
+### SUT crash recovery
+
+`pgch` and `managed-ch` containers declare `restart: on-failure:3` in their
+compose files. When ClickHouse OOMs under Phase 1's heaviest step (typical
+on `ccx33` + `pgch` once Phase 2 climbs past `batch=16384 × rate=100`), Docker
+auto-restarts the crashed service within seconds. After each inter-phase
+cooldown the loadgen then polls `GET <target>/health` for up to
+`--sut-health-timeout-seconds` (default 60 s) to confirm the SUT actually
+came back. If `/health` never returns 200, the loadgen skips the remaining
+phases and writes the partial JSON via the existing per-step checkpoint —
+no data already collected is lost. The bounded `on-failure:3` retry count
+prevents a permanently-broken SUT from restart-looping forever; if the
+third restart also crashes, the container stays down and the health-poll
+fires the clean-abort path.
+
After all matrix entries finish, `chart.py` renders three bar charts
(`chart-spans.png`, `chart-metrics.png`, `chart-logs.png`), three detail
charts (`chart-detail-<signal>.png`), and a combined `summary.md`.
diff --git a/benchmarks/compose/docker-compose.managed-ch.yml b/benchmarks/compose/docker-compose.managed-ch.yml
index 5ee27523..e1d52ce8 100644
--- a/benchmarks/compose/docker-compose.managed-ch.yml
+++ b/benchmarks/compose/docker-compose.managed-ch.yml
@@ -22,7 +22,7 @@ services:
depends_on:
postgres:
condition: service_healthy
- restart: "no"
+ restart: on-failure:3
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost/health"]
interval: 5s
@@ -44,7 +44,7 @@ services:
timeout: 3s
start_period: 10s
retries: 20
- restart: "no"
+ restart: on-failure:3
volumes:
postgres-bench-data:
diff --git a/benchmarks/compose/docker-compose.pgch.yml b/benchmarks/compose/docker-compose.pgch.yml
index 87e55be4..6ba6e55f 100644
--- a/benchmarks/compose/docker-compose.pgch.yml
+++ b/benchmarks/compose/docker-compose.pgch.yml
@@ -24,7 +24,7 @@ services:
condition: service_healthy
postgres:
condition: service_healthy
- restart: "no"
+ restart: on-failure:3
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost/health"]
interval: 5s
@@ -45,7 +45,7 @@ services:
timeout: 3s
start_period: 10s
retries: 20
- restart: "no"
+ restart: on-failure:3
ulimits:
nofile:
soft: 262144
@@ -65,7 +65,7 @@ services:
timeout: 3s
start_period: 10s
retries: 20
- restart: "no"
+ restart: on-failure:3
volumes:
clickhouse-bench-data:
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
index 7152dee8..e3fabead 100644
--- a/benchmarks/loadgen/main.go
+++ b/benchmarks/loadgen/main.go
@@ -33,6 +33,7 @@ type config struct {
softCliffRatio float64
stepDrainSeconds time.Duration
interPhaseCooldownSeconds time.Duration
+ sutHealthTimeoutSeconds time.Duration
phase2BisectMaxSteps int
phase2BisectTolerance float64
fillLevels []int64
@@ -72,6 +73,7 @@ func main() {
flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")
flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")
flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 30*time.Second, "Pause between Phase 1 and Phase 2 to let the SUT drain queues and recover from Phase 1's final-step load. 0 disables.")
+ flag.DurationVar(&cfg.sutHealthTimeoutSeconds, "sut-health-timeout-seconds", 60*time.Second, "After each inter-phase cooldown, poll GET <target>/health for up to this long. If the SUT doesn't return 200 in time, remaining phases are skipped and the partial JSON is written via the existing checkpoint. 0 disables the check.")
flag.IntVar(&cfg.phase2BisectMaxSteps, "phase2-bisect-max-steps", 3, "After Phase 2 finds the cliff, run up to this many bisection steps between the last passing and first failing rate to narrow the cliff. 0 disables.")
flag.Float64Var(&cfg.phase2BisectTolerance, "phase2-bisect-tolerance", 0.20, "Bisection stops when (firstFailRate-lastPassRate)/lastPassRate falls below this fraction (e.g. 0.20 = stop when the cliff is pinned to within 20% of the last passing rate).")
flag.StringVar(&fillLevelsStr, "fill-levels", "100000,1000000,10000000,100000000", "Comma-separated row counts to fill before probing a read (read-probe scenario)")
@@ -197,6 +199,14 @@ func main() {
case <-ctx.Done():
}
}
+ // Verify the SUT survived Phase 1 before launching Phase 2. If the
+ // compose restart policy didn't bring the SUT back, skip remaining
+ // phases — the final writeCheckpoint after the switch captures
+ // whatever data was collected up to this point.
+ if err := waitForSutHealthy(ctx, cfg.target, cfg.sutHealthTimeoutSeconds); err != nil {
+ fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 1 cooldown — skipping Phase 2/3: %v\n", err)
+ break
+ }
phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1, func(p phaseResult) {
out.Phase2 = &p
writeCheckpoint()
@@ -213,6 +223,10 @@ func main() {
case <-ctx.Done():
}
}
+ if err := waitForSutHealthy(ctx, cfg.target, cfg.sutHealthTimeoutSeconds); err != nil {
+ fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 2 cooldown — skipping Phase 3: %v\n", err)
+ break
+ }
phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {
out.Phase3 = &p
writeCheckpoint()
@@ -240,6 +254,51 @@ func main() {
}
}
+// waitForSutHealthy polls GET <target>/health until it returns 200 or the
+// timeout expires. Used between phases to detect SUT-side crashes: when
+// Phase 1's heavy final step OOM-kills ClickHouse, the compose restart
+// policy (on-failure:3) brings the container back within a few seconds.
+// This poll confirms recovery before we ask Phase 2 to fire requests at a
+// SUT that may still be coming up.
+//
+// /health returns 200 unconditionally when traceway is alive — it doesn't
+// validate downstream CH connectivity. That's intentional: the per-step
+// ingestErrThreshold + softCliffRatio already detect CH-down-but-traceway-up,
+// so this poll only catches the case where the traceway container itself
+// went away.
+func waitForSutHealthy(ctx context.Context, target string, timeout time.Duration) error {
+ if timeout <= 0 {
+ return nil
+ }
+ deadline := time.Now().Add(timeout)
+ client := &http.Client{Timeout: 5 * time.Second}
+ interval := 2 * time.Second
+ var lastErr error
+ for {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ resp, err := client.Get(target + "/health")
+ if err == nil {
+ resp.Body.Close()
+ if resp.StatusCode == 200 {
+ return nil
+ }
+ lastErr = fmt.Errorf("status %d", resp.StatusCode)
+ } else {
+ lastErr = err
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf("/health did not return 200 within %v (last err: %v)", timeout, lastErr)
+ }
+ select {
+ case <-time.After(interval):
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+}
+
// writeReportAtomic encodes the report into a sibling .tmp file and renames
// it over the destination, so a kill or disk-full mid-write can't leave a
// half-encoded file. Called after every step.