Skip to content

The only tool you need to know what is happening and how to fix it.

bench: gate read-probes on WAL digestion, uncap probe timeout, add phase2_request_rates input

jstojiljkovic6ec79924a94a
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.yml
index 06a64f24..7ee1ef6f 100644
--- a/.github/workflows/benchmark-hardware.yml
+++ b/.github/workflows/benchmark-hardware.yml
@@ -54,6 +54,10 @@ on:
description: 'Read-probe fill ladder override, comma-separated row counts (default 100000,1000000,10000000,100000000). e.g. 10000000,100000000,1000000000 to probe 1B — raise duration to cover the fill, it is a hard cap on the whole run. Ignored unless scenario=read-probe.'
required: false
default: ''
+ phase2_request_rates:
+ description: 'Throughput Phase 2 request-rate ladder override, comma-separated req/sec (default 1,5,25,100,400). Use a finer ramp (e.g. 1,5,10,15,20,25) to pin a cliff the default 5-to-25 jump skips over — needed when the failing step kills the SUT so post-fail bisection cannot run. Ignored unless scenario=throughput.'
+ required: false
+ default: ''
read_threshold_ms:
description: 'Read-probe pass threshold override in ms (default 5000; probe timeout is threshold + 1s). Raise for diagnostic dispatches (e.g. 60000) to measure how long slow queries actually take instead of truncating at 6s. Leave empty for headline runs so the series stays comparable. Ignored unless scenario=read-probe.'
required: false
@@ -118,6 +122,7 @@ jobs:
BENCH_FILL_REQUEST_RATE: ${{ github.event.inputs.fill_request_rate }}
BENCH_FILL_LEVELS: ${{ github.event.inputs.fill_levels }}
BENCH_READ_THRESHOLD_MS: ${{ github.event.inputs.read_threshold_ms }}
+ BENCH_PHASE2_REQUEST_RATES: ${{ github.event.inputs.phase2_request_rates }}
# Managed-ClickHouse credentials. Used only when modes contains
# managed-ch; empty for sqlite/pgch entries (harmless — sut-bootstrap
# only validates them when MODE=managed-ch).
diff --git a/benchmarks/README.md b/benchmarks/README.md
index f01fde8f..13e38736 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -46,6 +46,21 @@ the `--scenario` flag.
5 s). Results land in `benchmarks/results-probe/`. Answers "how big
can the table grow before the dashboard read on this endpoint cliffs?"
+ Between the fill and the probes sits a **digestion gate**
+ (`--max-digest-wait`, default 10 m): the loadgen polls `/api/health/deep`
+ until the engine's db+WAL file sizes are stable across two consecutive
+ polls. DuckDB checkpoints its WAL asynchronously after ingest stops, and
+ on a small tier that checkpoint can outlast the settle window — probing
+ through it produces uniform timeouts on sub-millisecond queries
+ (a wedged engine, not query cost; observed as a nondeterministic
+ logs/spans "cliff" on ccx13 before the gate existed). The wait is
+ recorded per fill level as `digestSeconds` (with `digestTimedOut` when
+ the cap expired), so "how long until the box answers again after a
+ burst" is itself a published number. Backends without engine gauges
+ (SQLite, ClickHouse) pass the gate on the first poll, and the probes
+ use a client without the shared 30 s timeout so `--read-threshold-ms`
+ above 29 s actually takes effect.
+
The two scenarios write to sibling folders so one never overwrites the
other. Each folder is wiped on each dispatch of its scenario.
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
index 857f5efd..4fd4fdc9 100644
--- a/benchmarks/loadgen/main.go
+++ b/benchmarks/loadgen/main.go
@@ -40,6 +40,7 @@ type config struct {
fillLevels []int64
readThresholdMs int
settleSeconds time.Duration
+ maxDigestWait time.Duration
fillBatchSize int
fillRequestRate float64
reportOut string
@@ -81,6 +82,7 @@ func main() {
flag.StringVar(&fillLevelsStr, "fill-levels", "100000,1000000,10000000,100000000", "Comma-separated row counts to fill before probing a read (read-probe scenario)")
flag.IntVar(&cfg.readThresholdMs, "read-threshold-ms", 5000, "Read latency threshold in ms; step fails if a probe exceeds it (read-probe scenario)")
flag.DurationVar(&cfg.settleSeconds, "settle-seconds", 10*time.Second, "Wait between finishing ingest and probing the read (read-probe scenario)")
+ flag.DurationVar(&cfg.maxDigestWait, "max-digest-wait", 10*time.Minute, "After each read-probe fill, poll /api/health/deep until the engine's db+WAL file sizes are stable across two consecutive polls (DuckDB checkpoints the WAL after ingest stops; probing mid-checkpoint measures a wedged engine, not query cost). Cap on that wait; 0 disables. No-op on backends without engine gauges (SQLite, ClickHouse).")
flag.IntVar(&cfg.fillBatchSize, "fill-batch-size", 8192, "OTLP batch size used during the fill phase (read-probe scenario)")
flag.Float64Var(&cfg.fillRequestRate, "fill-request-rate", 100, "OTLP request rate (req/sec) during the fill phase (read-probe scenario)")
flag.StringVar(&cfg.reportOut, "report-out", "", "Path to write JSON results (required)")
diff --git a/benchmarks/loadgen/read_probe.go b/benchmarks/loadgen/read_probe.go
index aa2cde4f..8f85670d 100644
--- a/benchmarks/loadgen/read_probe.go
+++ b/benchmarks/loadgen/read_probe.go
@@ -27,7 +27,14 @@ type readProbeStep struct {
// DroppedRows is cumulative since scenario start, mirroring RowsIngested.
// Nonzero means the SUT silently discarded rows during fill, so the true
// table size is RowsIngested minus DroppedRows.
- DroppedRows int64 `json:"droppedRows,omitempty"`
+ DroppedRows int64 `json:"droppedRows,omitempty"`
+ // DigestSeconds is how long the SUT's engine needed after the fill before
+ // its db+WAL file sizes went stable (the WAL-checkpoint gate). Probes run
+ // after this wait, so latencies measure steady-state query cost, not a
+ // mid-checkpoint wedge. DigestTimedOut means the cap expired first and
+ // the probes ran against a possibly still-digesting engine.
+ DigestSeconds float64 `json:"digestSeconds,omitempty"`
+ DigestTimedOut bool `json:"digestTimedOut,omitempty"`
Probes []endpointProbeResult `json:"probes"`
MedianLatencyMs float64 `json:"medianLatencyMs"`
// Back-compat: equals MedianLatencyMs and AND-of-all-probe-Ok respectively.
@@ -82,6 +89,11 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l
res.ReadPath = endpoints[0].Path
}
+ // The shared client carries a 30s Timeout that would silently cap probes
+ // below a raised --read-threshold-ms; probes get their own client so the
+ // per-probe context (threshold + 1s) is the only deadline.
+ probeClient := &http.Client{Transport: client.Transport}
+
ing.SetBatchSize(cfg.fillBatchSize)
ing.SetRequestRate(cfg.fillRequestRate)
@@ -123,8 +135,13 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l
}
}
- fmt.Fprintf(stderrPrefix(), "read-probe fill=%d rows reached in %.1fs (signal=%s) — settling %ds\n",
- step.RowsIngested, step.IngestSecondsElapsed, cfg.signal, res.SettleSeconds)
+ step.DigestSeconds, step.DigestTimedOut = waitForIngestDigestion(ctx, cfg, client.Transport)
+ if step.DigestTimedOut {
+ fmt.Fprintf(stderrPrefix(), "read-probe: engine still digesting after %.0fs cap — probing anyway\n", step.DigestSeconds)
+ }
+
+ fmt.Fprintf(stderrPrefix(), "read-probe fill=%d rows reached in %.1fs (signal=%s) — digested in %.1fs, settling %ds\n",
+ step.RowsIngested, step.IngestSecondsElapsed, cfg.signal, step.DigestSeconds, res.SettleSeconds)
select {
case <-time.After(cfg.settleSeconds):
@@ -142,7 +159,7 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *l
allOk := true
latencies := make([]float64, 0, len(endpoints))
for _, ep := range endpoints {
- latency, err := probeReadEndpoint(ctx, client, cfg, ep)
+ latency, err := probeReadEndpoint(ctx, probeClient, cfg, ep)
pr := endpointProbeResult{
Name: ep.Name,
Path: ep.Path,
@@ -261,6 +278,53 @@ func endpointSetForSignal(signal string) []readProbeEndpoint {
return nil
}
+// waitForIngestDigestion blocks until the SUT's storage engine has absorbed
+// the fill burst, or cfg.maxDigestWait expires. DuckDB checkpoints the WAL
+// asynchronously after ingest stops; probing mid-checkpoint on a small box
+// measures a wedged engine, not query cost — the symptom is sub-millisecond
+// queries timing out uniformly. /api/health/deep is the canary: on DuckDB it
+// executes real engine queries, so it only answers promptly when a dashboard
+// query would too. "Digested" means two consecutive successful polls, 5s
+// apart, reporting identical db+WAL file sizes. Backends without engine
+// gauges (SQLite, ClickHouse) return on the first successful poll, keeping
+// their timing behavior identical to before this gate existed. Returns the
+// seconds waited and whether the cap expired first.
+func waitForIngestDigestion(ctx context.Context, cfg config, transport http.RoundTripper) (float64, bool) {
+ if cfg.maxDigestWait <= 0 {
+ return 0, false
+ }
+ pollClient := &http.Client{Timeout: 5 * time.Second, Transport: transport}
+ start := time.Now()
+ deadline := start.Add(cfg.maxDigestWait)
+ var prev *ingestStatsSnapshot
+ for time.Now().Before(deadline) {
+ if ctx.Err() != nil {
+ return time.Since(start).Seconds(), false
+ }
+ _, cur := fetchDeepHealth(ctx, cfg, pollClient)
+ if cur.Reachable {
+ if cur.DBSizeBytes == 0 && cur.WALSizeBytes == 0 {
+ return time.Since(start).Seconds(), false
+ }
+ if prev != nil && prev.DBSizeBytes == cur.DBSizeBytes && prev.WALSizeBytes == cur.WALSizeBytes {
+ return time.Since(start).Seconds(), false
+ }
+ snap := cur
+ prev = &snap
+ } else {
+ // A slow or failed poll is the engine (or the box) not answering —
+ // it breaks the stability streak rather than counting toward it.
+ prev = nil
+ }
+ select {
+ case <-time.After(5 * time.Second):
+ case <-ctx.Done():
+ return time.Since(start).Seconds(), false
+ }
+ }
+ return time.Since(start).Seconds(), true
+}
+
// probeReadEndpoint issues one HTTP request for the given endpoint and returns
// wall-clock latency in ms plus any error. Hard-capped at threshold + 1s so a
// hanging SUT can't deadlock the loop.
diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.sh
index 79323a70..067e58a7 100755
--- a/benchmarks/scripts/run-matrix-entry.sh
+++ b/benchmarks/scripts/run-matrix-entry.sh
@@ -116,6 +116,14 @@ if [[ "${SCENARIO}" == "read-probe" ]]; then
fi
fi
+# Throughput Phase 2 ladder override (e.g. 1,5,10,15,20,25). The default
+# 5-to-25 jump can skip straight from a passing step to one that kills the
+# SUT, and post-fail bisection needs a live SUT — a finer ramp pins the
+# cliff from the passing side instead. Loadgen ignores it outside throughput.
+if [[ "${SCENARIO}" == "throughput" && -n "${BENCH_PHASE2_REQUEST_RATES:-}" ]]; then
+ extra_args+=( --phase2-request-rates "${BENCH_PHASE2_REQUEST_RATES}" )
+fi
+
# SQLite and DuckDB have no merge-idle equivalent — /health/deep returns
# chReachable=false and waitForMergesIdle skips immediately. Compensate with a
# longer per-step drain and a fixed inter-phase cooldown so the SUT can finish