The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
capturing results after each step
dduxa0f79c4840ca
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.goindex a0879b44..973f7ad3 100644--- a/benchmarks/loadgen/main.go+++ b/benchmarks/loadgen/main.go@@ -154,9 +154,25 @@ func main() {StartedAt: startedAt.Format(time.RFC3339),}+ // Persist a partial report after every step so a mid-run crash, OOM,+ // or dropped SSH session still leaves usable data on disk. Atomic+ // write via tempfile + rename means concurrent readers (and any partial+ // kill of the writer process itself) never see a half-written file.+ writeCheckpoint := func() {+ out.EndedAt = time.Now().UTC().Format(time.RFC3339)+ out.computeHeadline()+ if err := writeReportAtomic(cfg.reportOut, &out); err != nil {+ fmt.Fprintf(os.Stderr, "checkpoint write failed: %v\n", err)+ }+ }+switch cfg.scenario {case "throughput":- phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats)+ phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {+ out.Phase1 = &p+ writeCheckpoint()+ })+ out.Phase1 = &phase1// Cool-down between phases: Phase 1's last step often runs the SUT at// 70-99% of capacity, leaving its internal queues (CH merges, PG WAL,// HTTP handler pool) saturated. Jumping straight into Phase 2 with the@@ -170,28 +186,23 @@ func main() {case <-ctx.Done():}}- phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1)- out.Phase1 = &phase1+ phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1, func(p phaseResult) {+ out.Phase2 = &p+ writeCheckpoint()+ })out.Phase2 = &phase2case "read-probe":- probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient)+ probe := runReadProbe(ctx, cfg, ing, ingestStats, httpClient, func(p readProbeResult) {+ out.ReadProbe = &p+ writeCheckpoint()+ })out.ReadProbe = &probe}- out.EndedAt = time.Now().UTC().Format(time.RFC3339)- out.computeHeadline()- f, err := os.Create(cfg.reportOut)- if err != nil {- fmt.Fprintf(os.Stderr, "create %s: %v\n", cfg.reportOut, err)- os.Exit(1)- }- defer f.Close()- enc := json.NewEncoder(f)- enc.SetIndent("", " ")- if err := enc.Encode(&out); err != nil {- fmt.Fprintf(os.Stderr, "write %s: %v\n", cfg.reportOut, err)- os.Exit(1)- }+ // Final write — even if everything above ran cleanly, do one last+ // atomic rewrite so the file reflects the EndedAt and headline.+ writeCheckpoint()+switch cfg.scenario {case "throughput":fmt.Fprintf(os.Stderr, "wrote %s: signal=%s max sustainable %s/sec = %.0f\n",@@ -202,6 +213,29 @@ func main() {}}+// writeReportAtomic encodes the report into a sibling .tmp file and renames+// it over the destination, so a kill or disk-full mid-write can't leave a+// half-encoded file. Called after every step.+func writeReportAtomic(path string, report *finalReport) error {+ tmp := path + ".tmp"+ f, err := os.Create(tmp)+ if err != nil {+ return err+ }+ enc := json.NewEncoder(f)+ enc.SetIndent("", " ")+ if err := enc.Encode(report); err != nil {+ f.Close()+ os.Remove(tmp)+ return err+ }+ if err := f.Close(); err != nil {+ os.Remove(tmp)+ return err+ }+ return os.Rename(tmp, path)+}+func parseInt64s(s string) ([]int64, error) {parts := strings.Split(s, ",")out := make([]int64, 0, len(parts))diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.goindex cf8a1dd1..67212984 100644--- a/benchmarks/loadgen/ramp.go+++ b/benchmarks/loadgen/ramp.go@@ -72,10 +72,16 @@ func (r *finalReport) computeHeadline() {}}+// phaseCheckpoint is invoked after every step in the throughput phases so the+// caller can persist a partial report — we never want to lose progress when+// the process dies mid-run.+type phaseCheckpoint func(phaseResult)+// runBatchSizeRamp holds requestRate fixed (phase1FixedRate) and grows batch// size step by step. Stops at the first failing step. Returns a phaseResult-// whose MaxBatchSize is the largest batch that passed.-func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker) phaseResult {+// whose MaxBatchSize is the largest batch that passed. Calls `checkpoint`+// after each step (after both pass and fail) so partial state is durable.+func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {res := phaseResult{Kind: "batch-size-ramp",FixedRequestRate: cfg.phase1FixedRate,@@ -92,18 +98,24 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *lares.Steps = append(res.Steps, s)fmt.Fprintf(stderrPrefix(), "phase1 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)+ if s.Passed {+ res.MaxBatchSize = batch+ }+ if checkpoint != nil {+ checkpoint(res)+ }if !s.Passed {break}- res.MaxBatchSize = batch}return res}// runRequestRateRamp holds batchSize fixed at min(phase1.MaxBatchSize, cfg.phase2BatchCap)-// and grows request rate step by step.-func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult) phaseResult {+// and grows request rate step by step. Calls `checkpoint` after each coarse+// step and after each bisection step so partial state is durable.+func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {batch := phase1.MaxBatchSizeif batch <= 0 {batch = cfg.phase2BatchCap@@ -135,12 +147,17 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *res.Steps = append(res.Steps, s)fmt.Fprintf(stderrPrefix(), "phase2 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)+ if s.Passed {+ res.MaxRequestRate = rate+ lastPassRate = rate+ }+ if checkpoint != nil {+ checkpoint(res)+ }if !s.Passed {firstFailRate = ratebreak}- res.MaxRequestRate = rate- lastPassRate = rate}// Bisection refinement. After the coarse ramp finds an adjacent@@ -170,6 +187,9 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *} else {firstFailRate = mid}+ if checkpoint != nil {+ checkpoint(res)+ }}}diff --git a/benchmarks/loadgen/read_probe.go b/benchmarks/loadgen/read_probe.goindex 1775ff01..5cdcd823 100644--- a/benchmarks/loadgen/read_probe.go+++ b/benchmarks/loadgen/read_probe.go@@ -36,7 +36,12 @@ type readProbeResult struct {// issues one read probe. If the probe times out, errors, or exceeds// cfg.readThresholdMs, the step fails and the loop stops without ingesting// further.-func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client) readProbeResult {+// readProbeCheckpoint is invoked after every fill-level step so the caller+// can persist a partial report — we never want to lose progress when the+// process dies mid-run.+type readProbeCheckpoint func(readProbeResult)++func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client, checkpoint readProbeCheckpoint) readProbeResult {res := readProbeResult{ReadPath: readPathForSignal(cfg.signal),ReadThresholdMs: cfg.readThresholdMs,@@ -84,6 +89,9 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l}if ctx.Err() != nil {res.Steps = append(res.Steps, step)+ if checkpoint != nil {+ checkpoint(res)+ }break}@@ -107,6 +115,9 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *lfmt.Fprintf(stderrPrefix(), "read-probe target=%d read=%.0fms ok=%t passed=%t %s\n",target, latencyMs, step.ReadOk, step.Passed, step.FailReason)res.Steps = append(res.Steps, step)+ if checkpoint != nil {+ checkpoint(res)+ }if !step.Passed {break}diff --git a/benchmarks/scripts/loadgen-bootstrap.sh b/benchmarks/scripts/loadgen-bootstrap.shindex 67b68e8f..c4e3cdd3 100755--- a/benchmarks/scripts/loadgen-bootstrap.sh+++ b/benchmarks/scripts/loadgen-bootstrap.sh@@ -56,6 +56,13 @@ bench_ssh "${LG_IP}" "chmod +x /root/loadgen/loadgen"# so the read-probe scenario can hit dashboard endpoints; the throughput# scenario ignores them.echo "running loadgen on ${LG_IP} -> http://${SUT_PRIVATE_IP} (tier=${TIER} mode=${MODE} signal=${SIGNAL} duration=${DURATION})" >&2++# Capture the loadgen's exit code without letting set -e abort the script.+# The loadgen now writes its result.json incrementally after every step, so+# even if it dies mid-run (OOM, SSH drop, panic) the file on the SUT has+# everything up to the last completed step. We must scp regardless of the+# loadgen's exit status to avoid losing that partial data.+loadgen_rc=0bench_ssh "${LG_IP}" /root/loadgen/loadgen \--target "http://${SUT_PRIVATE_IP}" \--token "${TOKEN}" \@@ -66,8 +73,17 @@ bench_ssh "${LG_IP}" /root/loadgen/loadgen \--tier "${TIER}" \--mode "${MODE}" \--report-out /root/loadgen/result.json \- "$@"+ "$@" || loadgen_rc=$?++if [[ "${loadgen_rc}" -ne 0 ]]; then+ echo "loadgen exited with status ${loadgen_rc} — attempting to fetch any partial result.json from the loadgen box" >&2+fimkdir -p "$(dirname "${OUT_PATH}")"-bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}"-echo "wrote ${OUT_PATH}" >&2+if bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}" 2>/dev/null; then+ echo "wrote ${OUT_PATH}" >&2+else+ echo "no result.json on loadgen box (loadgen died before writing the first checkpoint)" >&2+fi++exit "${loadgen_rc}"