The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
benchmarks
dduxf224c69fd6f2
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.ymlindex 3b8d3f4c..bbb15f7d 100644--- a/.github/workflows/benchmark-hardware.yml+++ b/.github/workflows/benchmark-hardware.yml@@ -33,6 +33,11 @@ on:description: 'Hetzner datacenter (nbg1, fsn1, hel1, ash, hil)'required: truedefault: 'nbg1'+ async_insert:+ description: 'When true, sets CH_ASYNC_INSERT=1 on the SUT for this dispatch. Result files gain a -async suffix so paired bars can be rendered alongside the default (off) pass.'+ required: true+ type: boolean+ default: falseconcurrency:group: benchmark-hardware@@ -119,12 +124,17 @@ jobs:run: ./benchmarks/scripts/preflight.sh- name: Run matrix entry (${{ matrix.tier }} / ${{ matrix.mode }} / ${{ matrix.signal }})- run: ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ matrix.signal }}" "${{ github.event.inputs.duration }}" benchmarks/_artifact+ run: |+ async_arg=""+ if [[ "${{ github.event.inputs.async_insert }}" == "true" ]]; then+ async_arg="async"+ fi+ ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ matrix.signal }}" "${{ github.event.inputs.duration }}" benchmarks/_artifact "" "${async_arg}"- uses: actions/upload-artifact@v4if: always()with:- name: result-${{ matrix.tier }}-${{ matrix.mode }}-${{ matrix.signal }}+ name: result-${{ matrix.tier }}-${{ matrix.mode }}-${{ matrix.signal }}${{ github.event.inputs.async_insert == 'true' && '-async' || '' }}path: benchmarks/_artifact/*.jsonif-no-files-found: warndiff --git a/backend/app/chdb/batch_ctx.go b/backend/app/chdb/batch_ctx.gonew file mode 100644index 00000000..c5d58a5e--- /dev/null+++ b/backend/app/chdb/batch_ctx.go@@ -0,0 +1,19 @@+//go:build pgch++package chdb++import (+ "context"+ "os"++ "github.com/ClickHouse/clickhouse-go/v2"+)++var asyncInsertEnabled = os.Getenv("CH_ASYNC_INSERT") == "1"++// BatchCtx is the canonical context for clickhouse-go PrepareBatch calls.+// async_insert is off by default; benchmark dispatches set CH_ASYNC_INSERT=1+// to measure the realistic production ceiling.+func BatchCtx() context.Context {+ return clickhouse.Context(context.Background(), clickhouse.WithAsync(asyncInsertEnabled))+}diff --git a/backend/app/controllers/health_deep.go b/backend/app/controllers/health_deep.gonew file mode 100644index 00000000..efbb6d69--- /dev/null+++ b/backend/app/controllers/health_deep.go@@ -0,0 +1,43 @@+package controllers++import (+ "net/http"++ "github.com/gin-gonic/gin"+)++type healthDeepController struct{}++type TableParts struct {+ Table string `json:"table"`+ Parts int64 `json:"parts"`+}++type CHError struct {+ Name string `json:"name"`+ Value int64 `json:"value"`+ LastErrorTime string `json:"lastErrorTime,omitempty"`+}++type HealthDeepResponse struct {+ CHReachable bool `json:"chReachable"`+ CHUptimeSec int64 `json:"chUptimeSec"`+ PartsCount int64 `json:"partsCount"`+ PartsByTable []TableParts `json:"partsByTable,omitempty"`+ ActiveMerges int64 `json:"activeMerges"`+ LongestMergeSec float64 `json:"longestMergeSec"`+ ErrorsRecent []CHError `json:"errorsRecent,omitempty"`+ MemoryUsageBytes int64 `json:"memoryUsageBytes,omitempty"`+ MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`+}++func (h healthDeepController) Get(c *gin.Context) {+ resp := fetchCHHealth(c.Request.Context())+ if !resp.CHReachable {+ c.JSON(http.StatusServiceUnavailable, resp)+ return+ }+ c.JSON(http.StatusOK, resp)+}++var HealthDeepController = healthDeepController{}diff --git a/backend/app/controllers/health_deep_pgch.go b/backend/app/controllers/health_deep_pgch.gonew file mode 100644index 00000000..173e224b--- /dev/null+++ b/backend/app/controllers/health_deep_pgch.go@@ -0,0 +1,77 @@+//go:build pgch++package controllers++import (+ "context"+ "time"++ "github.com/tracewayapp/traceway/backend/app/chdb"+)++func fetchCHHealth(parent context.Context) HealthDeepResponse {+ ctx, cancel := context.WithTimeout(parent, 5*time.Second)+ defer cancel()++ resp := HealthDeepResponse{}++ if err := chdb.Conn.Exec(ctx, "SELECT 1"); err != nil {+ resp.CHReachable = false+ return resp+ }+ resp.CHReachable = true++ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(uptime())"); row != nil {+ var uptime int64+ if err := row.Scan(&uptime); err == nil {+ resp.CHUptimeSec = uptime+ }+ }++ if rows, err := chdb.Conn.Query(ctx, "SELECT table, sum(rows > 0) AS parts FROM system.parts WHERE active GROUP BY table ORDER BY parts DESC"); err == nil {+ for rows.Next() {+ var pt TableParts+ if err := rows.Scan(&pt.Table, &pt.Parts); err != nil {+ continue+ }+ resp.PartsByTable = append(resp.PartsByTable, pt)+ resp.PartsCount += pt.Parts+ }+ rows.Close()+ }++ if row := chdb.Conn.QueryRow(ctx, "SELECT count(), coalesce(max(elapsed), 0) FROM system.merges"); row != nil {+ var n int64+ var longest float64+ if err := row.Scan(&n, &longest); err == nil {+ resp.ActiveMerges = n+ resp.LongestMergeSec = longest+ }+ }++ if rows, err := chdb.Conn.Query(ctx, "SELECT name, value, toString(last_error_time) FROM system.errors WHERE value > 0 ORDER BY last_error_time DESC LIMIT 20"); err == nil {+ for rows.Next() {+ var e CHError+ if err := rows.Scan(&e.Name, &e.Value, &e.LastErrorTime); err != nil {+ continue+ }+ resp.ErrorsRecent = append(resp.ErrorsRecent, e)+ }+ rows.Close()+ }++ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(value) FROM system.asynchronous_metrics WHERE metric = 'MemoryResident'"); row != nil {+ var v int64+ if err := row.Scan(&v); err == nil {+ resp.MemoryUsageBytes = v+ }+ }+ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(value) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryTotal'"); row != nil {+ var v int64+ if err := row.Scan(&v); err == nil {+ resp.MemoryTotalBytes = v+ }+ }++ return resp+}diff --git a/backend/app/controllers/health_deep_sqlite.go b/backend/app/controllers/health_deep_sqlite.gonew file mode 100644index 00000000..ace2d896--- /dev/null+++ b/backend/app/controllers/health_deep_sqlite.go@@ -0,0 +1,9 @@+//go:build !pgch++package controllers++import "context"++func fetchCHHealth(_ context.Context) HealthDeepResponse {+ return HealthDeepResponse{CHReachable: false}+}diff --git a/backend/app/controllers/routes.go b/backend/app/controllers/routes.goindex 8d85eb83..f1ecef61 100644--- a/backend/app/controllers/routes.go+++ b/backend/app/controllers/routes.go@@ -44,6 +44,8 @@ func RegisterControllers(router *gin.RouterGroup) {router.PUT("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.UpdateProject)router.DELETE("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.DeleteProject)+ router.GET("/health/deep", middleware.UseAppAuth, HealthDeepController.Get)+// Dashboard endpoints (projectId in query param)router.POST("/stats", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricRecordController.FindHomepageStats)router.GET("/dashboard", middleware.UseAppAuth, middleware.RequireProjectAccess, DashboardController.GetDashboard)diff --git a/backend/app/repositories/ai_trace.repository.go b/backend/app/repositories/ai_trace.repository.goindex 031b1188..ab1e7c7e 100644--- a/backend/app/repositories/ai_trace.repository.go+++ b/backend/app/repositories/ai_trace.repository.go@@ -10,7 +10,6 @@ import ("strings""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/models"@@ -19,7 +18,7 @@ import (type aiTraceRepository struct{}func (r *aiTraceRepository) InsertAsync(ctx context.Context, lines []models.AiTrace) error {- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),"INSERT INTO ai_traces (id, project_id, recorded_at, duration, status_code, model, response_model, provider, operation, input_tokens, output_tokens, total_tokens, cached_tokens, reasoning_tokens, input_cost, output_cost, total_cost, trace_name, user_id, finish_reason, server_name, app_version, storage_key, attributes, distributed_trace_id, is_root)")if err != nil {return errdiff --git a/backend/app/repositories/endpoint.repository.go b/backend/app/repositories/endpoint.repository.goindex a7b898b5..c76caa7a 100644--- a/backend/app/repositories/endpoint.repository.go+++ b/backend/app/repositories/endpoint.repository.go@@ -11,14 +11,13 @@ import ("errors""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")type endpointRepository struct{}func (e *endpointRepository) InsertAsync(ctx context.Context, lines []models.Endpoint) error {- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)")+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)")if err != nil {return err}diff --git a/backend/app/repositories/exception_stack_trace.repository.go b/backend/app/repositories/exception_stack_trace.repository.goindex 32831e25..9a3fd9c2 100644--- a/backend/app/repositories/exception_stack_trace.repository.go+++ b/backend/app/repositories/exception_stack_trace.repository.go@@ -12,14 +12,13 @@ import ("strings""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")type exceptionStackTraceRepository struct{}func (e *exceptionStackTraceRepository) InsertAsync(ctx context.Context, lines []models.ExceptionStackTrace) error {- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)")+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)")if err != nil {return err}diff --git a/backend/app/repositories/fired_notification.repository.go b/backend/app/repositories/fired_notification.repository.goindex ca6e0ae6..83c0baa1 100644--- a/backend/app/repositories/fired_notification.repository.go+++ b/backend/app/repositories/fired_notification.repository.go@@ -6,7 +6,6 @@ import ("context""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/chdb")@@ -31,7 +30,7 @@ type firedNotificationRepository struct{}func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotification) error {batch, err := chdb.Conn.PrepareBatch(- clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),+ chdb.BatchCtx(),"INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, fired_at)",)if err != nil {diff --git a/backend/app/repositories/log_record.repository.go b/backend/app/repositories/log_record.repository.goindex 6ac66db3..a9a386e1 100644--- a/backend/app/repositories/log_record.repository.go+++ b/backend/app/repositories/log_record.repository.go@@ -7,7 +7,6 @@ import ("strings""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/chdb"@@ -48,7 +47,7 @@ func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.}batch, err := chdb.Conn.PrepareBatch(- clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),+ chdb.BatchCtx(),"INSERT INTO log_records (id, project_id, timestamp, trace_id, span_id, trace_flags, severity_text, severity_number, service_name, body, resource_schema_url, resource_attributes, scope_schema_url, scope_name, scope_version, scope_attributes, log_attributes)",)if err != nil {diff --git a/backend/app/repositories/metric_point.repository.go b/backend/app/repositories/metric_point.repository.goindex d931385f..76ca39e9 100644--- a/backend/app/repositories/metric_point.repository.go+++ b/backend/app/repositories/metric_point.repository.go@@ -8,14 +8,13 @@ import ("context""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")type metricPointRepository struct{}func (r *metricPointRepository) InsertAsync(ctx context.Context, points []models.MetricPoint) error {- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO metric_points (project_id, name, value, tags, recorded_at)")+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO metric_points (project_id, name, value, tags, recorded_at)")if err != nil {return err}diff --git a/backend/app/repositories/session.repository.go b/backend/app/repositories/session.repository.goindex 5a22526a..fb50ac1c 100644--- a/backend/app/repositories/session.repository.go+++ b/backend/app/repositories/session.repository.go@@ -9,7 +9,6 @@ import ("strings""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/models"@@ -21,7 +20,7 @@ func (r *sessionRepository) Upsert(ctx context.Context, sessions []models.Sessioif len(sessions) == 0 {return nil}- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),"INSERT INTO sessions (id, project_id, started_at, ended_at, duration, client_ip, attributes, app_version, server_name, distributed_trace_id, version)")if err != nil {return errdiff --git a/backend/app/repositories/session_recording.repository.go b/backend/app/repositories/session_recording.repository.goindex e6ba2358..19223e9c 100644--- a/backend/app/repositories/session_recording.repository.go+++ b/backend/app/repositories/session_recording.repository.go@@ -9,7 +9,6 @@ import ("database/sql""errors"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")@@ -19,7 +18,7 @@ func (r *sessionRecordingRepository) InsertAsync(ctx context.Context, recordingsif len(recordings) == 0 {return nil}- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO session_recordings (id, project_id, exception_id, session_id, segment_index, file_path, recorded_at)")+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO session_recordings (id, project_id, exception_id, session_id, segment_index, file_path, recorded_at)")if err != nil {return err}diff --git a/backend/app/repositories/span.repository.go b/backend/app/repositories/span.repository.goindex caa9a4f2..f55ff9e4 100644--- a/backend/app/repositories/span.repository.go+++ b/backend/app/repositories/span.repository.go@@ -7,7 +7,6 @@ import ("github.com/tracewayapp/traceway/backend/app/models""context"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")@@ -18,7 +17,7 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) ereturn nil}- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),"INSERT INTO spans (id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id)")if err != nil {return errdiff --git a/backend/app/repositories/task.repository.go b/backend/app/repositories/task.repository.goindex 14bc51e8..048e65d2 100644--- a/backend/app/repositories/task.repository.go+++ b/backend/app/repositories/task.repository.go@@ -11,14 +11,13 @@ import ("errors""time"- "github.com/ClickHouse/clickhouse-go/v2""github.com/google/uuid")type taskRepository struct{}func (e *taskRepository) InsertAsync(ctx context.Context, lines []models.Task) error {- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)")+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)")if err != nil {return err}diff --git a/benchmarks/README.md b/benchmarks/README.mdindex 876c4130..26dc039c 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -52,6 +52,58 @@ other. Each folder is wiped on each dispatch of its scenario.Don't read the throughput number as "the dashboard stays usable at Nitems/sec" — that's what read-probe is for.+## Two insert paths: sync vs async ClickHouse++Every backend `PrepareBatch` call funnels through `chdb.BatchCtx()`, which+reads `CH_ASYNC_INSERT` once at startup. **This is a per-dispatch toggle**+— a single matrix entry runs one insert mode; comparison requires two+dispatches.++| | `CH_ASYNC_INSERT=0` (default) | `CH_ASYNC_INSERT=1` |+|--------------------------|-------------------------------|----------------------|+| Driver flag | `clickhouse.WithAsync(false)` | `clickhouse.WithAsync(true)` |+| Part-per-request | 1 part per insert (worst case for MergeTree) | Batched server-side, parts amortised |+| What it measures | Pessimistic ceiling — what you get if you point a stock OTel collector at the default Traceway config | Realistic production ceiling — what a tuned deployment gets |+| Result filename | `<tier>-<mode>-<signal>-<scenario>.json` | `<tier>-<mode>-<signal>-<scenario>-async.json` |+| Headline bar in charts | Solid | Solid (sibling bar in chart-headline-<signal>.png when both passes are present) |++Quote both numbers when reporting. The sync number tells you the floor your+users hit before tuning; the async number tells you the headroom.++**Triggering async-insert locally:**++```bash+# Default (sync) — no env var needed+./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans++# Async — set CH_ASYNC_INSERT=1 in the parent env. run-matrix-entry.sh+# propagates it via the 7th positional arg "async" to sut-bootstrap.sh,+# which exports it into the SUT's docker-compose env so chdb.BatchCtx()+# reads the right value at startup.+CH_ASYNC_INSERT=1 ./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans --async+```++`run-local.sh --async` is sugar that just sets `CH_ASYNC_INSERT=1` and+threads the `async` arg into `run-matrix-entry.sh`. Direct script call:++```bash+./benchmarks/scripts/run-matrix-entry.sh ccx13 pgch spans 30m benchmarks/results-throughput "" async+# ^tier ^mode ^sig ^dur ^out-dir ^smoke ^async+```++**Triggering async-insert in CI:**++In the GitHub Actions `workflow_dispatch` form, flip the `async_insert`+boolean to `true`. The workflow propagates it through to+`run-matrix-entry.sh` and tags both the artifact and the committed JSON+filenames with `-async`. Dispatch twice (once off, once on) and the+aggregate job's chart renderer paints sibling bars in+`chart-headline-<signal>.png`.++`managed-ch` mode also honours `CH_ASYNC_INSERT` — useful for measuring+how much of the managed offering's headroom is currently being left on+the table by the conservative sync default.+## How a run worksPer matrix entry (one tier × one mode × one signal):@@ -64,17 +116,26 @@ Per matrix entry (one tier × one mode × one signal):4. `loadgen-bootstrap.sh` cross-compiles the loadgen, pushes it to theloadgen box, runs it with `--signal <spans|metrics|logs>` against theSUT's *private* IP.-5. The loadgen runs a three-phase ramp:+5. The loadgen runs a three-phase ramp. After every step it GETs+ `/api/health/deep` and embeds a `ch` block in the step's JSON+ (`partsCount`, `partsByTable`, `activeMerges`, `longestMergeSec`,+ `errorsRecent`, `memoryUsageBytes`, `uptimeSec`). This is how the+ bench sees ClickHouse-side pressure — the HTTP-only signal alone+ can't distinguish "SUT cliffed on CPU" from "MergeTree threw+ `Too many parts`".- **Phase 1 — batch-size ramp.** Single client at a fixed 5 req/sec.Batch sizes step through `256,1024,4096,8192,16384`. Each step holds for`--step-duration` (default 2 min). Stops at the first failing step.- - **Inter-phase cooldown.** Phase 1's last step typically runs the SUT- near saturation; the loadgen sleeps `--inter-phase-cooldown-seconds`- (default 30 s) before starting Phase 2 so the SUT can drain its- queues (CH merges, PG WAL, HTTP handler pool). Without this pause,- pgch runs reliably produced "0 OK / 0 errors" Phase 2 because new- requests sat on the SUT-side TCP backlog while the SUT was still- digesting Phase 1's final wave.+ - **Inter-phase merge-idle wait.** Phase 1's last step typically runs the+ SUT near saturation; CH merge horizons are minutes, not seconds. The+ loadgen polls `/api/health/deep` every 5 s until `activeMerges == 0`+ and `partsCount` is stable (within ±5%) for two consecutive polls, or+ `--max-merge-idle-wait` (default 5 m) expires. The old fixed+ `--inter-phase-cooldown-seconds` flag still works (additive pre-wait)+ but defaults to 0 — it's deprecated in favour of polling CH directly.+ Without this gate, pgch runs reliably produced "0 OK / 0 errors"+ Phase 2 because new requests sat on the SUT-side TCP backlog while+ CH was still merging Phase 1's parts.- **Phase 2 — request-rate ramp (collector shape).** Batch size fixed at`min(Phase 1 winner, --phase2-batch-cap=16384)`. Request rates stepthrough `1,5,25,100,400`. When the coarse ramp finds the first failing@@ -83,7 +144,7 @@ Per matrix entry (one tier × one mode × one signal):within `--phase2-bisect-tolerance` (default 20%). So `5→25` (a 5× jump)gets refined into something like `5, 15, 10, 12` until the gap is<20% of the last passing rate.- - **Inter-phase cooldown** before Phase 3 too.+ - **Merge-idle wait** before Phase 3 too.- **Phase 3 — request-rate ramp (SDK-fleet shape).** Batch size fixed at`--phase3-batch-size` (default **100**, matching typical language-SDK`BatchSpanProcessor` output rather than the collector's 8192). Request@@ -109,13 +170,13 @@ Per matrix entry (one tier × one mode × one signal):reports; per-phase numbers stay in the JSON for shape-specific analysis.7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.-### SUT crash recovery+### SUT crash recovery & restart detection`pgch` and `managed-ch` containers declare `restart: on-failure:3` in theircompose files. When ClickHouse OOMs under Phase 1's heaviest step (typicalon `ccx33` + `pgch` once Phase 2 climbs past `batch=16384 × rate=100`), Docker-auto-restarts the crashed service within seconds. After each inter-phase-cooldown the loadgen then polls `GET <target>/health` for up to+auto-restarts the crashed service within seconds. After each merge-idle+wait the loadgen polls `GET <target>/health` for up to`--sut-health-timeout-seconds` (default 60 s) to confirm the SUT actuallycame back. If `/health` never returns 200, the loadgen skips the remainingphases and writes the partial JSON via the existing per-step checkpoint —@@ -124,6 +185,18 @@ prevents a permanently-broken SUT from restart-looping forever; if thethird restart also crashes, the container stays down and the health-pollfires the clean-abort path.+**Detecting silent CH restarts.** Compose's `restart` policy used to mean+a crashed-and-recovered ClickHouse was invisible: the SUT came back up and+the bench kept running, producing a "passing" report that hid the+incident. The bench now tracks CH's `uptime()` across every step. If+step N's uptime is lower than step N-1's uptime plus the elapsed step+window, ClickHouse restarted mid-step. The step is marked failed with+`failReason: CH restarted mid-step (...)`, `chRestarted: true` is set on+the top-level JSON, the run skips remaining phases, and the loadgen+exits non-zero. The chart renderer hatches the affected headline bar+in red with a `CH↻` annotation so the dispatch report flags the run as+invalid rather than burying the restart in a passing number.+After all matrix entries finish, `chart.py` renders the full chart suite(headline bars, Phase 1 / 2 / 3 ramps, Pareto, tier scaling, cliff grid,batch efficiency, signal mix — plus the read-probe equivalents in the@@ -198,6 +271,12 @@ vice-versa.# Switch to the read-probe scenario (writes to benchmarks/results-probe/)./benchmarks/scripts/run-local.sh --scenario read-probe++# Async-insert pass — same matrix but with CH_ASYNC_INSERT=1 on the SUT.+# Output filenames pick up a -async suffix so paired bars render in+# chart-headline-<signal>.png. Dispatch the default (sync) pass first,+# then this one, into the same results folder.+./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans --async```## Running from GitHub Actions@@ -278,13 +357,14 @@ benchmarks/docker-compose.pgch.ymldocker-compose.managed-ch.yml # External CH; Postgres still localloadgen/ # The Go binary that generates load (OTLP-only)- main.go # CLI + orchestration+ main.go # CLI + orchestration + merge-idle waitingest.go # Worker pool driving the selected signal senderotlp_common.go # Shared OTLP helpers + signalSender interfaceotlp_spans.go # ExportTraceServiceRequest builderotlp_metrics.go # ExportMetricsServiceRequest builder (Gauge)otlp_logs.go # ExportLogsServiceRequest builder- ramp.go # Two-phase ramp (batch size, then request rate)+ ramp.go # Three-phase ramp + CH-restart detection+ ch_snapshot.go # GET /api/health/deep -> per-step ch{} blockstats.go # Latency tracker (percentiles via sort)util.go, log.go # Misc helpersscripts/diff --git a/benchmarks/compose/docker-compose.managed-ch.yml b/benchmarks/compose/docker-compose.managed-ch.ymlindex e1d52ce8..b6caf2c0 100644--- a/benchmarks/compose/docker-compose.managed-ch.yml+++ b/benchmarks/compose/docker-compose.managed-ch.yml@@ -19,6 +19,7 @@ services:POSTGRES_SSLMODE: "disable"JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"GIN_MODE: "release"+ CH_ASYNC_INSERT: "${CH_ASYNC_INSERT:-0}"depends_on:postgres:condition: service_healthydiff --git a/benchmarks/compose/docker-compose.pgch.yml b/benchmarks/compose/docker-compose.pgch.ymlindex 6ba6e55f..11357768 100644--- a/benchmarks/compose/docker-compose.pgch.yml+++ b/benchmarks/compose/docker-compose.pgch.yml@@ -19,6 +19,7 @@ services:POSTGRES_SSLMODE: "disable"JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"GIN_MODE: "release"+ CH_ASYNC_INSERT: "${CH_ASYNC_INSERT:-0}"depends_on:clickhouse:condition: service_healthydiff --git a/benchmarks/loadgen/ch_snapshot.go b/benchmarks/loadgen/ch_snapshot.gonew file mode 100644index 00000000..74084f2e--- /dev/null+++ b/benchmarks/loadgen/ch_snapshot.go@@ -0,0 +1,100 @@+package main++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "time"+)++type tableParts struct {+ Table string `json:"table"`+ Parts int64 `json:"parts"`+}++type chError struct {+ Name string `json:"name"`+ Value int64 `json:"value"`+ LastErrorTime string `json:"lastErrorTime,omitempty"`+}++type chSnapshot struct {+ Reachable bool `json:"reachable"`+ UptimeSec int64 `json:"uptimeSec"`+ PartsCount int64 `json:"partsCount"`+ PartsByTable []tableParts `json:"partsByTable,omitempty"`+ ActiveMerges int64 `json:"activeMerges"`+ LongestMergeSec float64 `json:"longestMergeSec"`+ ErrorsRecent []chError `json:"errorsRecent,omitempty"`+ MemoryUsageBytes int64 `json:"memoryUsageBytes,omitempty"`+ MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`+}++// healthDeepBody mirrors the backend HealthDeepResponse with its JSON tags. The+// backend uses `chReachable`/`chUptimeSec`; we expose those as `reachable`/+// `uptimeSec` in our embedded snapshot so the bench JSON stays consistent with+// other loadgen fields.+type healthDeepBody struct {+ CHReachable bool `json:"chReachable"`+ CHUptimeSec int64 `json:"chUptimeSec"`+ PartsCount int64 `json:"partsCount"`+ PartsByTable []tableParts `json:"partsByTable"`+ ActiveMerges int64 `json:"activeMerges"`+ LongestMergeSec float64 `json:"longestMergeSec"`+ ErrorsRecent []chError `json:"errorsRecent"`+ MemoryUsageBytes int64 `json:"memoryUsageBytes"`+ MemoryTotalBytes int64 `json:"memoryTotalBytes"`+}++// fetchCHSnapshot pings the backend's /health/deep endpoint and translates the+// payload into a chSnapshot. On any error (timeout, transport, non-2xx, decode)+// it returns a snapshot with Reachable=false and logs to stderrPrefix() — the+// caller does NOT fail the step on a missing snapshot. A 503 from the backend+// (CH unreachable but backend alive) is still parsed and embedded so the JSON+// captures the chReachable=false signal.+func fetchCHSnapshot(ctx context.Context, cfg config, client *http.Client) chSnapshot {+ if cfg.jwt == "" {+ return chSnapshot{}+ }++ reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()++ req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, cfg.target+"/api/health/deep", nil)+ if err != nil {+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: build request failed: %v\n", err)+ return chSnapshot{}+ }+ req.Header.Set("Authorization", "Bearer "+cfg.jwt)++ resp, err := client.Do(req)+ if err != nil {+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: http error: %v\n", err)+ return chSnapshot{}+ }+ defer resp.Body.Close()++ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusServiceUnavailable {+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: unexpected status %d\n", resp.StatusCode)+ return chSnapshot{}+ }++ var body healthDeepBody+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: decode failed: %v\n", err)+ return chSnapshot{}+ }++ return chSnapshot{+ Reachable: body.CHReachable,+ UptimeSec: body.CHUptimeSec,+ PartsCount: body.PartsCount,+ PartsByTable: body.PartsByTable,+ ActiveMerges: body.ActiveMerges,+ LongestMergeSec: body.LongestMergeSec,+ ErrorsRecent: body.ErrorsRecent,+ MemoryUsageBytes: body.MemoryUsageBytes,+ MemoryTotalBytes: body.MemoryTotalBytes,+ }+}diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.goindex e3fabead..a8f8d4d6 100644--- a/benchmarks/loadgen/main.go+++ b/benchmarks/loadgen/main.go@@ -33,6 +33,7 @@ type config struct {softCliffRatio float64stepDrainSeconds time.DurationinterPhaseCooldownSeconds time.Duration+ maxMergeIdleWait time.DurationsutHealthTimeoutSeconds time.Durationphase2BisectMaxSteps intphase2BisectTolerance float64@@ -72,7 +73,8 @@ func main() {flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if combined (HTTP error + OTLP rejected) item rate exceeds this")flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")- flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 30*time.Second, "Pause between Phase 1 and Phase 2 to let the SUT drain queues and recover from Phase 1's final-step load. 0 disables.")+ flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 0, "DEPRECATED: replaced by --max-merge-idle-wait, which polls CH directly. Non-zero values still apply as an additional pre-wait before merge-idle polling. 0 (default) disables.")+ flag.DurationVar(&cfg.maxMergeIdleWait, "max-merge-idle-wait", 5*time.Minute, "Between phases, poll /health/deep until activeMerges==0 and partsCount is stable for two consecutive polls, or this timeout elapses. 0 disables (skip merge-idle wait entirely).")flag.DurationVar(&cfg.sutHealthTimeoutSeconds, "sut-health-timeout-seconds", 60*time.Second, "After each inter-phase cooldown, poll GET <target>/health for up to this long. If the SUT doesn't return 200 in time, remaining phases are skipped and the partial JSON is written via the existing checkpoint. 0 disables the check.")flag.IntVar(&cfg.phase2BisectMaxSteps, "phase2-bisect-max-steps", 3, "After Phase 2 finds the cliff, run up to this many bisection steps between the last passing and first failing rate to narrow the cliff. 0 disables.")flag.Float64Var(&cfg.phase2BisectTolerance, "phase2-bisect-tolerance", 0.20, "Bisection stops when (firstFailRate-lastPassRate)/lastPassRate falls below this fraction (e.g. 0.20 = stop when the cliff is pinned to within 20% of the last passing rate).")@@ -173,6 +175,7 @@ func main() {// kill of the writer process itself) never see a half-written file.writeCheckpoint := func() {out.EndedAt = time.Now().UTC().Format(time.RFC3339)+ out.ChRestarted = chRestarted.Load()out.computeHeadline()if err := writeReportAtomic(cfg.reportOut, &out); err != nil {fmt.Fprintf(os.Stderr, "checkpoint write failed: %v\n", err)@@ -181,24 +184,21 @@ func main() {switch cfg.scenario {case "throughput":- phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {+ phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, httpClient, func(p phaseResult) {out.Phase1 = &pwriteCheckpoint()})out.Phase1 = &phase1- // Cool-down between phases: Phase 1's last step often runs the SUT at- // 70-99% of capacity, leaving its internal queues (CH merges, PG WAL,- // HTTP handler pool) saturated. Jumping straight into Phase 2 with the- // SUT still digesting that wave produces "0 OK / 0 errors" garbage- // because new requests sit on the SUT-side TCP backlog without ever- // reaching a handler or timing out.- if cfg.interPhaseCooldownSeconds > 0 {- fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)- select {- case <-time.After(cfg.interPhaseCooldownSeconds):- case <-ctx.Done():- }+ if chRestarted.Load() {+ fmt.Fprintf(stderrPrefix(), "CH restart observed in Phase 1 — skipping remaining phases\n")+ break}+ // Between phases: Phase 1's last step often runs the SUT at 70-99% of+ // capacity, leaving CH parts queues unmerged and PG/HTTP pools saturated.+ // Jumping straight into Phase 2 produces garbage data — instead, poll+ // /health/deep until CH merges are idle and parts count has stabilized.+ preWaitCooldown(ctx, cfg)+ waitForMergesIdle(ctx, cfg, httpClient, "phase 1 -> phase 2")// Verify the SUT survived Phase 1 before launching Phase 2. If the// compose restart policy didn't bring the SUT back, skip remaining// phases — the final writeCheckpoint after the switch captures@@ -207,27 +207,28 @@ func main() {fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 1 cooldown — skipping Phase 2/3: %v\n", err)break}- phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1, func(p phaseResult) {+ phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, httpClient, phase1, func(p phaseResult) {out.Phase2 = &pwriteCheckpoint()})out.Phase2 = &phase2+ if chRestarted.Load() {+ fmt.Fprintf(stderrPrefix(), "CH restart observed in Phase 2 — skipping Phase 3\n")+ break+ }// Phase 3 — small-batch high-rate (SDK-fleet shape). Independent of// Phase 1/2 results, so it runs even if Phase 2 produced no passing- // steps. Same cooldown before it as between 1 and 2.- if cfg.interPhaseCooldownSeconds > 0 && ctx.Err() == nil {- fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)- select {- case <-time.After(cfg.interPhaseCooldownSeconds):- case <-ctx.Done():- }+ // steps. Same merge-idle wait as between 1 and 2.+ if ctx.Err() == nil {+ preWaitCooldown(ctx, cfg)+ waitForMergesIdle(ctx, cfg, httpClient, "phase 2 -> phase 3")}if err := waitForSutHealthy(ctx, cfg.target, cfg.sutHealthTimeoutSeconds); err != nil {fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 2 cooldown — skipping Phase 3: %v\n", err)break}- phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {+ phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, httpClient, func(p phaseResult) {out.Phase3 = &pwriteCheckpoint()})@@ -252,6 +253,11 @@ func main() {fmt.Fprintf(os.Stderr, "wrote %s: signal=%s max fill level passed = %d rows\n",cfg.reportOut, cfg.signal, out.MaxFillLevelPassed)}++ if chRestarted.Load() {+ fmt.Fprintln(os.Stderr, "FAILED: ClickHouse restarted during the run; treating result as invalid")+ os.Exit(1)+ }}// waitForSutHealthy polls GET <target>/health until it returns 200 or the@@ -299,6 +305,99 @@ func waitForSutHealthy(ctx context.Context, target string, timeout time.Duration}}+// preWaitCooldown honours the deprecated --inter-phase-cooldown-seconds flag+// when set. Most invocations leave it at 0 and rely entirely on+// waitForMergesIdle; this hook stays so existing local scripts that still pass+// the old flag don't silently skip the wait.+func preWaitCooldown(ctx context.Context, cfg config) {+ if cfg.interPhaseCooldownSeconds <= 0 {+ return+ }+ fmt.Fprintf(stderrPrefix(), "inter-phase cooldown (deprecated flag): %v\n", cfg.interPhaseCooldownSeconds)+ select {+ case <-time.After(cfg.interPhaseCooldownSeconds):+ case <-ctx.Done():+ }+}++// waitForMergesIdle polls /health/deep every 5s until activeMerges == 0 AND+// partsCount has been stable (within ±5%) for two consecutive polls, OR the+// configured timeout elapses. Returns without error in all cases — the wait is+// best-effort observability, not a verdict; if CH never settles we just move+// on and the next phase's results will show it.+func waitForMergesIdle(ctx context.Context, cfg config, client *http.Client, label string) {+ if cfg.maxMergeIdleWait <= 0 {+ return+ }++ // SQLite mode has no ClickHouse — /health/deep returns chReachable:false+ // and there are no merges to wait for. Without this fast-path the loop+ // would burn the full --max-merge-idle-wait (default 5m) between every+ // phase doing nothing useful.+ if first := fetchCHSnapshot(ctx, cfg, client); !first.Reachable {+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: CH not reachable (sqlite mode or backend down) — skipping wait\n", label)+ return+ }++ deadline := time.Now().Add(cfg.maxMergeIdleWait)+ pollEvery := 5 * time.Second+ stableThreshold := 0.05+ requiredStable := 2++ var prevParts int64 = -1+ stable := 0+ for {+ if ctx.Err() != nil {+ return+ }+ snap := fetchCHSnapshot(ctx, cfg, client)+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: reachable=%t activeMerges=%d partsCount=%d longestMerge=%.1fs\n",+ label, snap.Reachable, snap.ActiveMerges, snap.PartsCount, snap.LongestMergeSec)++ if snap.Reachable && snap.ActiveMerges == 0 {+ if prevParts >= 0 {+ delta := float64(snap.PartsCount-prevParts) / float64(maxInt64(prevParts, 1))+ if delta < 0 {+ delta = -delta+ }+ if delta <= stableThreshold {+ stable+++ if stable >= requiredStable {+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: settled (parts=%d)\n", label, snap.PartsCount)+ return+ }+ } else {+ stable = 0+ }+ }+ prevParts = snap.PartsCount+ } else {+ stable = 0+ if snap.Reachable {+ prevParts = snap.PartsCount+ }+ }++ if time.Now().After(deadline) {+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: timeout after %v (activeMerges=%d partsCount=%d) — advancing anyway\n",+ label, cfg.maxMergeIdleWait, snap.ActiveMerges, snap.PartsCount)+ return+ }+ select {+ case <-time.After(pollEvery):+ case <-ctx.Done():+ return+ }+ }+}++func maxInt64(a, b int64) int64 {+ if a > b {+ return a+ }+ return b+}+// writeReportAtomic encodes the report into a sibling .tmp file and renames// it over the destination, so a kill or disk-full mid-write can't leave a// half-encoded file. Called after every step.diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.goindex 50b97350..33eab851 100644--- a/benchmarks/loadgen/ramp.go+++ b/benchmarks/loadgen/ramp.go@@ -3,9 +3,20 @@ package mainimport ("context""fmt"+ "net/http"+ "sync/atomic""time")+// Cross-phase state for detecting CH restarts. Single-run CLI; package-level+// state is fine. lastCHUptime stores the most recent step's CH uptime (0+// before the first reachable snapshot); chRestarted is set sticky-true once a+// restart is observed so main() can mark the run failed and exit non-zero.+var (+ lastCHUptime atomic.Int64+ chRestarted atomic.Bool+)+type stepResult struct {Step int `json:"step"`BatchSize int `json:"batchSize"`@@ -14,6 +25,7 @@ type stepResult struct {ActualItemsPerSec float64 `json:"actualItemsPerSec"`Rejected int64 `json:"rejected"`Ingest latencySnapshot `json:"ingest"`+ CH chSnapshot `json:"ch"`Passed bool `json:"passed"`FailReason string `json:"failReason,omitempty"`}@@ -40,6 +52,7 @@ type finalReport struct {ReadProbe *readProbeResult `json:"readProbe,omitempty"`MaxSustainableItemsPerSec float64 `json:"maxSustainableItemsPerSec,omitempty"`MaxFillLevelPassed int64 `json:"maxFillLevelPassed,omitempty"`+ ChRestarted bool `json:"chRestarted,omitempty"`}func (r *finalReport) computeHeadline() {@@ -83,7 +96,7 @@ type phaseCheckpoint func(phaseResult)// size step by step. Stops at the first failing step. Returns a phaseResult// whose MaxBatchSize is the largest batch that passed. Calls `checkpoint`// after each step (after both pass and fail) so partial state is durable.-func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {+func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {res := phaseResult{Kind: "batch-size-ramp",FixedRequestRate: cfg.phase1FixedRate,@@ -96,7 +109,7 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *labreak}ing.SetBatchSize(batch)- s := runOneStep(ctx, cfg, ing, ingest, idx+1, batch, cfg.phase1FixedRate)+ s := runOneStep(ctx, cfg, ing, ingest, client, idx+1, batch, cfg.phase1FixedRate)res.Steps = append(res.Steps, s)fmt.Fprintf(stderrPrefix(), "phase1 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)@@ -118,7 +131,7 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la// and grows request rate step by step. Phase 2's purpose: at the largest// single-request payload the SUT can absorb (collector-shape), find the// req/sec ceiling.-func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {+func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {batch := phase1.MaxBatchSizeif batch <= 0 {batch = cfg.phase2BatchCap@@ -126,7 +139,7 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *if batch > cfg.phase2BatchCap {batch = cfg.phase2BatchCap}- return runRateRamp(ctx, cfg, ing, ingest, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)+ return runRateRamp(ctx, cfg, ing, ingest, client, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)}// runSmallBatchRateRamp holds batchSize fixed at cfg.phase3BatchSize (default@@ -136,14 +149,14 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *// SDK fleets in the wild send hundreds-to-thousands of small batches per// second — language-SDK BatchSpanProcessor defaults are 512 per batch on a// 5 s rotation, often much less in practice; this phase covers that regime.-func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {- return runRateRamp(ctx, cfg, ing, ingest, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)+func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {+ return runRateRamp(ctx, cfg, ing, ingest, client, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)}// runRateRamp is the shared engine for Phase 2 and Phase 3: hold batch// fixed, ramp request rate, bisect after the first failing step. logPrefix// is just for stderr messages ("phase2" / "phase3").-func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {+func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {res := phaseResult{Kind: kind,FixedBatchSize: batch,@@ -163,7 +176,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latency}stepNo++ing.SetRequestRate(rate)- s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, rate)+ s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, rate)res.Steps = append(res.Steps, s)fmt.Fprintf(stderrPrefix(), "%s step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)@@ -196,7 +209,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencymid := (lastPassRate + firstFailRate) / 2stepNo++ing.SetRequestRate(mid)- s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, mid)+ s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, mid)res.Steps = append(res.Steps, s)fmt.Fprintf(stderrPrefix(), "%s bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)@@ -220,7 +233,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latency// latency + item counters. The drain window matters: without it, every step// boundary cancels ~workerCount in-flight requests mid-flight, inflating the// recorded error count and depressing the OK count.-func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, stepNo, batchSize int, requestRate float64) stepResult {+func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, stepNo, batchSize int, requestRate float64) stepResult {ingest.SnapshotAndReset()ing.SnapshotAndResetItems()@@ -241,6 +254,22 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTsnap := ingest.SnapshotAndReset()attempted, rejected := ing.SnapshotAndResetItems()+ ch := fetchCHSnapshot(ctx, cfg, client)++ // Restart detection: if ClickHouse's uptime is materially less than what+ // it was at the previous step's snapshot, the CH process restarted during+ // this step. The compose restart policy may have brought it back, but the+ // numbers from this step (and the preceding one) are no longer+ // interpretable, so the run is marked failed and remaining phases skipped.+ prev := lastCHUptime.Load()+ restartedThisStep := false+ if ch.Reachable && ch.UptimeSec > 0 {+ if prev > 0 && ch.UptimeSec < prev+int64(elapsed.Seconds()) {+ restartedThisStep = true+ chRestarted.Store(true)+ }+ lastCHUptime.Store(ch.UptimeSec)+ }var attemptedIps, actualIps float64if elapsed > 0 {@@ -258,6 +287,10 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyT}passed, reason := evaluateStep(cfg, snap, attempted, rejected, requestRate, elapsed)+ if restartedThisStep {+ passed = false+ reason = fmt.Sprintf("CH restarted mid-step (uptime %ds < previous %ds + step %ds)", ch.UptimeSec, prev, int64(elapsed.Seconds()))+ }return stepResult{Step: stepNo,@@ -267,6 +300,7 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTActualItemsPerSec: actualIps,Rejected: rejected,Ingest: snap,+ CH: ch,Passed: passed,FailReason: reason,}diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.pyindex aef98509..2f87ff8e 100755--- a/benchmarks/scripts/chart.py+++ b/benchmarks/scripts/chart.py@@ -204,6 +204,26 @@ def phase1_p99_at_batch(run: dict, batch_size: int) -> float | None:return None+def worst_phase_for_run(run: dict) -> str | None:+ """Pick the phase that produced the lowest passing actualItemsPerSec, or the+ phase that first failed. Used by the CH-pressure chart so the visualisation+ targets the phase where the SUT actually struggled."""+ failing = first_failure(run)+ if failing:+ return failing[0]+ worst_phase = None+ worst_value = float("inf")+ for phase_key in ("phase1", "phase2", "phase3"):+ for step in phase_steps(run, phase_key):+ if not step.get("passed"):+ continue+ actual = step.get("actualItemsPerSec", 0) or 0+ if actual < worst_value:+ worst_value = actual+ worst_phase = phase_key+ return worst_phase++# ---------- Throughput charts ----------@@ -226,6 +246,7 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:width = 0.8 / max(len(modes_present), 1)x_centers = list(range(len(tiers_present)))+ any_ch_restart = Falsefor mi, mode in enumerate(modes_present):for ti, tier in enumerate(tiers_present):match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]@@ -234,14 +255,22 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:continuerun = match[0]value = run.get("maxSustainableItemsPerSec", 0) or 0+ ch_restart = bool(run.get("chRestarted"))+ if ch_restart:+ any_ch_restart = Trueif value > 0:- ax.bar(x, value, width=width, color=mode_color(mode))+ bar_kwargs = {"width": width, "color": mode_color(mode)}+ if ch_restart:+ bar_kwargs.update({"edgecolor": "#aa3333", "hatch": "xx", "linewidth": 1.2})+ ax.bar(x, value, **bar_kwargs)src = headline_source(run)phase_tag = f" ({PHASE_LABEL[src[0]]})" if src else ""+ restart_tag = " CH↻" if ch_restart else ""ax.annotate(- f"{int(value):,}{phase_tag}",+ f"{int(value):,}{phase_tag}{restart_tag}",xy=(x, value), xytext=(0, 4), textcoords="offset points",ha="center", va="bottom", fontsize=8,+ color="#aa3333" if ch_restart else "black",)else:ax.bar(x, ghost_height, width=width, color="#dddddd",@@ -267,6 +296,8 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:legend_handles = [Patch(facecolor=mode_color(m), label=m) for m in modes_present]legend_handles.append(Patch(facecolor="#dddddd", edgecolor="#999999", hatch="///", label="all phases failed"))+ if any_ch_restart:+ legend_handles.append(Patch(facecolor="white", edgecolor="#aa3333", hatch="xx", label="CH restart during run (result invalid)"))ax.legend(handles=legend_handles, title="DB mode", loc="upper left")ax.grid(axis="y", linestyle=":", alpha=0.4)@@ -715,6 +746,70 @@ def render_batch_efficiency(runs: list[dict], signal: str, out: Path) -> None:plt.close(fig)+def render_ch_pressure(runs: list[dict], signal: str, out: Path) -> None:+ """For each (tier, mode) of a signal, plot CH parts count and active merges+ across the steps of its worst-performing phase. Makes the CH-side story —+ "parts piled up before the cliff" or "merges were idle and the cliff was+ backend-side" — visible at a glance."""+ sig_runs = iter_runs(runs, signal, "throughput")+ sig_runs = [r for r in sig_runs if any(+ (step.get("ch") or {}).get("reachable")+ for phase_key in ("phase1", "phase2", "phase3")+ for step in phase_steps(r, phase_key)+ )]+ if not sig_runs:+ return++ cols = min(len(sig_runs), 3)+ rows = (len(sig_runs) + cols - 1) // cols+ fig, axes = plt.subplots(rows, cols, figsize=(cols * 5.5, rows * 3.5), squeeze=False)++ for idx, run in enumerate(sig_runs):+ r, c = divmod(idx, cols)+ ax_parts = axes[r][c]+ ax_merges = ax_parts.twinx()++ phase_key = worst_phase_for_run(run)+ if not phase_key:+ ax_parts.set_visible(False)+ ax_merges.set_visible(False)+ continue+ steps = phase_steps(run, phase_key)+ if not steps:+ ax_parts.set_visible(False)+ ax_merges.set_visible(False)+ continue++ xs = [s.get("step", i + 1) for i, s in enumerate(steps)]+ parts = [(s.get("ch") or {}).get("partsCount", 0) for s in steps]+ merges = [(s.get("ch") or {}).get("activeMerges", 0) for s in steps]+ first_failed_step = next((s.get("step") for s in steps if not s.get("passed")), None)++ ax_parts.plot(xs, parts, marker="o", color="#1f77b4", label="parts")+ ax_merges.plot(xs, merges, marker="s", color="#d62728", linestyle="--", label="active merges")++ if first_failed_step is not None:+ ax_parts.axvline(first_failed_step, color="#aa3333", linestyle=":", alpha=0.6)++ title = f"{run_label(run)} — {PHASE_LABEL[phase_key]}"+ if run.get("chRestarted"):+ title += " (CH↻)"+ ax_parts.set_title(title, fontsize=10)+ ax_parts.set_xlabel("step")+ ax_parts.set_ylabel("parts (total, active)", color="#1f77b4")+ ax_merges.set_ylabel("active merges", color="#d62728")+ ax_parts.grid(True, linestyle=":", alpha=0.4)++ for idx in range(len(sig_runs), rows * cols):+ r, c = divmod(idx, cols)+ axes[r][c].set_visible(False)++ fig.suptitle(f"ClickHouse pressure during the worst-performing phase — {signal}", y=1.02)+ fig.tight_layout()+ fig.savefig(out, dpi=130, bbox_inches="tight")+ plt.close(fig)++def render_signal_mix(runs: list[dict], out: Path) -> None:"""Cross-signal throughput comparison: bars per (tier, mode), one bar per signal."""thr_runs = [r for r in runs if r.get("scenario", "throughput") == "throughput"]@@ -1213,6 +1308,7 @@ def main() -> int:render_tier_scaling(runs, signal, results_dir / f"chart-tier-scaling-{signal}.png")render_cliff_grid(runs, signal, results_dir / f"chart-cliff-{signal}.png")render_batch_efficiency(runs, signal, results_dir / f"chart-batch-efficiency-{signal}.png")+ render_ch_pressure(runs, signal, results_dir / f"chart-ch-pressure-{signal}.png")# Read-probe charts (no-op if no read-probe runs for this signal)render_readprobe_headline(runs, signal, results_dir / f"chart-readprobe-headline-{signal}.png")diff --git a/benchmarks/scripts/run-local.sh b/benchmarks/scripts/run-local.shindex 02734a03..457a3783 100755--- a/benchmarks/scripts/run-local.sh+++ b/benchmarks/scripts/run-local.sh@@ -13,6 +13,7 @@# run-local.sh --scenario read-probe # same matrix but ingest-and-probe-reads# run-local.sh --smoke # 1 tier, 1 mode, 1 signal, short steps (~5 min)# run-local.sh --tier ccx13 --mode sqlite --signal spans+# run-local.sh --async # set CH_ASYNC_INSERT=1; -async suffix on output files# run-local.sh --dry-run # validate env + print plan, no provisioning## Output:@@ -33,6 +34,7 @@ SCENARIO="throughput"DURATION="30m"SMOKE=0DRY_RUN=0+ASYNC=0usage() {sed -n '1,23p' "$0"; exit 2@@ -46,6 +48,7 @@ while [[ $# -gt 0 ]]; do--scenario) SCENARIO="$2"; shift 2 ;;--duration) DURATION="$2"; shift 2 ;;--smoke) SMOKE=1; shift ;;+ --async) ASYNC=1; shift ;;--dry-run) DRY_RUN=1; shift ;;-h|--help) usage ;;*) echo "unknown flag: $1" >&2; usage ;;@@ -114,11 +117,17 @@ fi# strategy.matrix instead.smoke_arg=""[[ "${SMOKE}" -eq 1 ]] && smoke_arg="smoke"+async_arg=""+if [[ "${ASYNC}" -eq 1 ]]; then+ async_arg="async"+ export CH_ASYNC_INSERT=1+ echo "CH_ASYNC_INSERT=1 — async-insert benchmark pass; output files will get -async suffix" >&2+fifailures=()for e in "${plan[@]}"; doIFS='|' read -r tier mode signal <<<"${e}"- if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${signal}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}"; then+ if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${signal}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}" "${async_arg}"; thenecho "FAIL: ${tier}/${mode}/${signal}" >&2failures+=("${tier}/${mode}/${signal}")fidiff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.shindex 0f25a09b..684c5613 100755--- a/benchmarks/scripts/run-matrix-entry.sh+++ b/benchmarks/scripts/run-matrix-entry.sh@@ -4,7 +4,7 @@# GitHub workflow; treat this as the single source of truth for matrix-entry# orchestration.#-# Usage: run-matrix-entry.sh <tier> <mode> <signal> <duration> <out-dir> [smoke]+# Usage: run-matrix-entry.sh <tier> <mode> <signal> <duration> <out-dir> [smoke] [async]# <tier> ccx13 | ccx23 | ccx33 | ccx43# <mode> sqlite | pgch# <signal> spans | metrics | logs@@ -12,16 +12,18 @@# <out-dir> Directory to write <tier>-<mode>-<signal>.json into# [smoke] "smoke" to enable short-step overrides (--phase1-batch-sizes# 256,1024 --phase2-request-rates 1,5 --step-duration 15s).-# Optional.+# Optional; pass "" to skip when also passing [async].+# [async] "async" to set CH_ASYNC_INSERT=1 on the SUT. Output filename+# gains a -async suffix when set. Optional.set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"if [[ $# -lt 5 ]]; then- echo "usage: $0 <tier> <mode> <signal> <duration> <out-dir> [smoke]" >&2+ echo "usage: $0 <tier> <mode> <signal> <duration> <out-dir> [smoke] [async]" >&2exit 2fi-TIER="$1"; MODE="$2"; SIGNAL="$3"; DURATION="$4"; OUT_DIR="$5"; SMOKE="${6:-}"+TIER="$1"; MODE="$2"; SIGNAL="$3"; DURATION="$4"; OUT_DIR="$5"; SMOKE="${6:-}"; ASYNC_FLAG="${7:-}"LOCATION="${BENCH_LOCATION:-nbg1}"SCENARIO="${BENCH_SCENARIO:-throughput}"@@ -65,7 +67,14 @@ SUT_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPublicIp')SUT_PRIVATE_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPrivateIp')LOADGEN_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.loadgenPublicIp')-# 2. Bring up the backend on the SUT.+# 2. Bring up the backend on the SUT. CH_ASYNC_INSERT propagates through to+# the docker compose env via sut-bootstrap.sh.+async_suffix=""+if [[ "${ASYNC_FLAG}" == "async" ]]; then+ export CH_ASYNC_INSERT=1+ async_suffix="-async"+ echo "CH_ASYNC_INSERT=1 (async-insert benchmark pass)" >&2+fi"${SCRIPT_DIR}/sut-bootstrap.sh" "${SUT_PUBLIC_IP}" "${MODE}"# 3. Run the loadgen, pulling JSON back into OUT_DIR.@@ -78,7 +87,7 @@ if [[ "${SMOKE}" == "smoke" ]]; thenfifi-OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}.json"+OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}${async_suffix}.json""${SCRIPT_DIR}/loadgen-bootstrap.sh" \"${LOADGEN_PUBLIC_IP}" \"${SUT_PRIVATE_IP}" \@@ -91,4 +100,4 @@ OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}.json""${extra_args[@]}"# Trap handles teardown — no explicit call needed.-echo "matrix entry ${TIER}-${MODE}-${SIGNAL}-${SCENARIO} complete -> ${OUT_PATH}" >&2+echo "matrix entry ${TIER}-${MODE}-${SIGNAL}-${SCENARIO}${async_suffix} complete -> ${OUT_PATH}" >&2diff --git a/benchmarks/scripts/sut-bootstrap.sh b/benchmarks/scripts/sut-bootstrap.shindex a4118dbd..c5fa285a 100755--- a/benchmarks/scripts/sut-bootstrap.sh+++ b/benchmarks/scripts/sut-bootstrap.sh@@ -85,8 +85,9 @@ EOFcompose_args=( --env-file benchmarks/compose/managed-ch.env "${compose_args[@]}" )fi-echo "bringing up compose stack (mode=${MODE}) on ${SUT_IP}" >&2-bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 docker compose ${compose_args[*]} up -d --build"+CH_ASYNC_INSERT_VAL="${CH_ASYNC_INSERT:-0}"+echo "bringing up compose stack (mode=${MODE}, CH_ASYNC_INSERT=${CH_ASYNC_INSERT_VAL}) on ${SUT_IP}" >&2+bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 CH_ASYNC_INSERT=${CH_ASYNC_INSERT_VAL} docker compose ${compose_args[*]} up -d --build"echo "polling /health on ${SUT_IP}" >&2deadline=$(( $(date +%s) + 600 )) # cold compose build can hit 10 min on small tiers