Skip to content

The only tool you need to know what is happening and how to fix it.

capturing results after each step

dduxa0f79c4840ca
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
index a0879b44..973f7ad3 100644
--- a/benchmarks/loadgen/main.go
+++ b/benchmarks/loadgen/main.go
@@ -154,9 +154,25 @@ func main() {
StartedAt: startedAt.Format(time.RFC3339),
}
+ // Persist a partial report after every step so a mid-run crash, OOM,
+ // or dropped SSH session still leaves usable data on disk. Atomic
+ // write via tempfile + rename means concurrent readers (and any partial
+ // kill of the writer process itself) never see a half-written file.
+ writeCheckpoint := func() {
+ out.EndedAt = time.Now().UTC().Format(time.RFC3339)
+ out.computeHeadline()
+ if err := writeReportAtomic(cfg.reportOut, &out); err != nil {
+ fmt.Fprintf(os.Stderr, "checkpoint write failed: %v\n", err)
+ }
+ }
+
switch cfg.scenario {
case "throughput":
- phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats)
+ phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {
+ out.Phase1 = &p
+ writeCheckpoint()
+ })
+ out.Phase1 = &phase1
// Cool-down between phases: Phase 1's last step often runs the SUT at
// 70-99% of capacity, leaving its internal queues (CH merges, PG WAL,
// HTTP handler pool) saturated. Jumping straight into Phase 2 with the
@@ -170,28 +186,23 @@ func main() {
case <-ctx.Done():
}
}
- phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1)
- out.Phase1 = &phase1
+ phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1, func(p phaseResult) {
+ out.Phase2 = &p
+ writeCheckpoint()
+ })
out.Phase2 = &phase2
case "read-probe":
- probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient)
+ probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient, func(p readProbeResult) {
+ out.ReadProbe = &p
+ writeCheckpoint()
+ })
out.ReadProbe = &probe
}
- out.EndedAt = time.Now().UTC().Format(time.RFC3339)
- out.computeHeadline()
- f, err := os.Create(cfg.reportOut)
- if err != nil {
- fmt.Fprintf(os.Stderr, "create %s: %v\n", cfg.reportOut, err)
- os.Exit(1)
- }
- defer f.Close()
- enc := json.NewEncoder(f)
- enc.SetIndent("", " ")
- if err := enc.Encode(&out); err != nil {
- fmt.Fprintf(os.Stderr, "write %s: %v\n", cfg.reportOut, err)
- os.Exit(1)
- }
+ // Final write — even if everything above ran cleanly, do one last
+ // atomic rewrite so the file reflects the EndedAt and headline.
+ writeCheckpoint()
+
switch cfg.scenario {
case "throughput":
fmt.Fprintf(os.Stderr, "wrote %s: signal=%s max sustainable %s/sec = %.0f\n",
@@ -202,6 +213,29 @@ func main() {
}
}
+// writeReportAtomic encodes the report into a sibling .tmp file and renames
+// it over the destination, so a kill or disk-full mid-write can't leave a
+// half-encoded file. Called after every step.
+func writeReportAtomic(path string, report *finalReport) error {
+ tmp := path + ".tmp"
+ f, err := os.Create(tmp)
+ if err != nil {
+ return err
+ }
+ enc := json.NewEncoder(f)
+ enc.SetIndent("", " ")
+ if err := enc.Encode(report); err != nil {
+ f.Close()
+ os.Remove(tmp)
+ return err
+ }
+ if err := f.Close(); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return os.Rename(tmp, path)
+}
+
func parseInt64s(s string) ([]int64, error) {
parts := strings.Split(s, ",")
out := make([]int64, 0, len(parts))
diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.go
index cf8a1dd1..67212984 100644
--- a/benchmarks/loadgen/ramp.go
+++ b/benchmarks/loadgen/ramp.go
@@ -72,10 +72,16 @@ func (r *finalReport) computeHeadline() {
}
}
+// phaseCheckpoint is invoked after every step in the throughput phases so the
+// caller can persist a partial report — we never want to lose progress when
+// the process dies mid-run.
+type phaseCheckpoint func(phaseResult)
+
// runBatchSizeRamp holds requestRate fixed (phase1FixedRate) and grows batch
// size step by step. Stops at the first failing step. Returns a phaseResult
-// whose MaxBatchSize is the largest batch that passed.
-func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker) phaseResult {
+// whose MaxBatchSize is the largest batch that passed. Calls `checkpoint`
+// after each step (after both pass and fail) so partial state is durable.
+func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
Kind: "batch-size-ramp",
FixedRequestRate: cfg.phase1FixedRate,
@@ -92,18 +98,24 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "phase1 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
+ if s.Passed {
+ res.MaxBatchSize = batch
+ }
+ if checkpoint != nil {
+ checkpoint(res)
+ }
if !s.Passed {
break
}
- res.MaxBatchSize = batch
}
return res
}
// runRequestRateRamp holds batchSize fixed at min(phase1.MaxBatchSize, cfg.phase2BatchCap)
-// and grows request rate step by step.
-func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult) phaseResult {
+// and grows request rate step by step. Calls `checkpoint` after each coarse
+// step and after each bisection step so partial state is durable.
+func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {
batch := phase1.MaxBatchSize
if batch <= 0 {
batch = cfg.phase2BatchCap
@@ -135,12 +147,17 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "phase2 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
+ if s.Passed {
+ res.MaxRequestRate = rate
+ lastPassRate = rate
+ }
+ if checkpoint != nil {
+ checkpoint(res)
+ }
if !s.Passed {
firstFailRate = rate
break
}
- res.MaxRequestRate = rate
- lastPassRate = rate
}
// Bisection refinement. After the coarse ramp finds an adjacent
@@ -170,6 +187,9 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
} else {
firstFailRate = mid
}
+ if checkpoint != nil {
+ checkpoint(res)
+ }
}
}
diff --git a/benchmarks/loadgen/read_probe.go b/benchmarks/loadgen/read_probe.go
index 1775ff01..5cdcd823 100644
--- a/benchmarks/loadgen/read_probe.go
+++ b/benchmarks/loadgen/read_probe.go
@@ -36,7 +36,12 @@ type readProbeResult struct {
// issues one read probe. If the probe times out, errors, or exceeds
// cfg.readThresholdMs, the step fails and the loop stops without ingesting
// further.
-func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client) readProbeResult {
+// readProbeCheckpoint is invoked after every fill-level step so the caller
+// can persist a partial report — we never want to lose progress when the
+// process dies mid-run.
+type readProbeCheckpoint func(readProbeResult)
+
+func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client, checkpoint readProbeCheckpoint) readProbeResult {
res := readProbeResult{
ReadPath: readPathForSignal(cfg.signal),
ReadThresholdMs: cfg.readThresholdMs,
@@ -84,6 +89,9 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l
}
if ctx.Err() != nil {
res.Steps = append(res.Steps, step)
+ if checkpoint != nil {
+ checkpoint(res)
+ }
break
}
@@ -107,6 +115,9 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l
fmt.Fprintf(stderrPrefix(), "read-probe target=%d read=%.0fms ok=%t passed=%t %s\n",
target, latencyMs, step.ReadOk, step.Passed, step.FailReason)
res.Steps = append(res.Steps, step)
+ if checkpoint != nil {
+ checkpoint(res)
+ }
if !step.Passed {
break
}
diff --git a/benchmarks/scripts/loadgen-bootstrap.sh b/benchmarks/scripts/loadgen-bootstrap.sh
index 67b68e8f..c4e3cdd3 100755
--- a/benchmarks/scripts/loadgen-bootstrap.sh
+++ b/benchmarks/scripts/loadgen-bootstrap.sh
@@ -56,6 +56,13 @@ bench_ssh "${LG_IP}" "chmod +x /root/loadgen/loadgen"
# so the read-probe scenario can hit dashboard endpoints; the throughput
# scenario ignores them.
echo "running loadgen on ${LG_IP} -> http://${SUT_PRIVATE_IP} (tier=${TIER} mode=${MODE} signal=${SIGNAL} duration=${DURATION})" >&2
+
+# Capture the loadgen's exit code without letting set -e abort the script.
+# The loadgen now writes its result.json incrementally after every step, so
+# even if it dies mid-run (OOM, SSH drop, panic) the file on the SUT has
+# everything up to the last completed step. We must scp regardless of the
+# loadgen's exit status to avoid losing that partial data.
+loadgen_rc=0
bench_ssh "${LG_IP}" /root/loadgen/loadgen \
--target "http://${SUT_PRIVATE_IP}" \
--token "${TOKEN}" \
@@ -66,8 +73,17 @@ bench_ssh "${LG_IP}" /root/loadgen/loadgen \
--tier "${TIER}" \
--mode "${MODE}" \
--report-out /root/loadgen/result.json \
- "$@"
+ "$@" || loadgen_rc=$?
+
+if [[ "${loadgen_rc}" -ne 0 ]]; then
+ echo "loadgen exited with status ${loadgen_rc} — attempting to fetch any partial result.json from the loadgen box" >&2
+fi
mkdir -p "$(dirname "${OUT_PATH}")"
-bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}"
-echo "wrote ${OUT_PATH}" >&2
+if bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}" 2>/dev/null; then
+ echo "wrote ${OUT_PATH}" >&2
+else
+ echo "no result.json on loadgen box (loadgen died before writing the first checkpoint)" >&2
+fi
+
+exit "${loadgen_rc}"