Skip to content

The only tool you need to know what is happening and how to fix it.

benchmarks

dduxf224c69fd6f2
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.yml
index 3b8d3f4c..bbb15f7d 100644
--- a/.github/workflows/benchmark-hardware.yml
+++ b/.github/workflows/benchmark-hardware.yml
@@ -33,6 +33,11 @@ on:
description: 'Hetzner datacenter (nbg1, fsn1, hel1, ash, hil)'
required: true
default: 'nbg1'
+ async_insert:
+ description: 'When true, sets CH_ASYNC_INSERT=1 on the SUT for this dispatch. Result files gain a -async suffix so paired bars can be rendered alongside the default (off) pass.'
+ required: true
+ type: boolean
+ default: false
concurrency:
group: benchmark-hardware
@@ -119,12 +124,17 @@ jobs:
run: ./benchmarks/scripts/preflight.sh
- name: Run matrix entry (${{ matrix.tier }} / ${{ matrix.mode }} / ${{ matrix.signal }})
- run: ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ matrix.signal }}" "${{ github.event.inputs.duration }}" benchmarks/_artifact
+ run: |
+ async_arg=""
+ if [[ "${{ github.event.inputs.async_insert }}" == "true" ]]; then
+ async_arg="async"
+ fi
+ ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ matrix.signal }}" "${{ github.event.inputs.duration }}" benchmarks/_artifact "" "${async_arg}"
- uses: actions/upload-artifact@v4
if: always()
with:
- name: result-${{ matrix.tier }}-${{ matrix.mode }}-${{ matrix.signal }}
+ name: result-${{ matrix.tier }}-${{ matrix.mode }}-${{ matrix.signal }}${{ github.event.inputs.async_insert == 'true' && '-async' || '' }}
path: benchmarks/_artifact/*.json
if-no-files-found: warn
diff --git a/backend/app/chdb/batch_ctx.go b/backend/app/chdb/batch_ctx.go
new file mode 100644
index 00000000..c5d58a5e
--- /dev/null
+++ b/backend/app/chdb/batch_ctx.go
@@ -0,0 +1,19 @@
+//go:build pgch
+
+package chdb
+
+import (
+ "context"
+ "os"
+
+ "github.com/ClickHouse/clickhouse-go/v2"
+)
+
+var asyncInsertEnabled = os.Getenv("CH_ASYNC_INSERT") == "1"
+
+// BatchCtx is the canonical context for clickhouse-go PrepareBatch calls.
+// async_insert is off by default; benchmark dispatches set CH_ASYNC_INSERT=1
+// to measure the realistic production ceiling.
+func BatchCtx() context.Context {
+ return clickhouse.Context(context.Background(), clickhouse.WithAsync(asyncInsertEnabled))
+}
diff --git a/backend/app/controllers/health_deep.go b/backend/app/controllers/health_deep.go
new file mode 100644
index 00000000..efbb6d69
--- /dev/null
+++ b/backend/app/controllers/health_deep.go
@@ -0,0 +1,43 @@
+package controllers
+
+import (
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+)
+
+type healthDeepController struct{}
+
+type TableParts struct {
+ Table string `json:"table"`
+ Parts int64 `json:"parts"`
+}
+
+type CHError struct {
+ Name string `json:"name"`
+ Value int64 `json:"value"`
+ LastErrorTime string `json:"lastErrorTime,omitempty"`
+}
+
+type HealthDeepResponse struct {
+ CHReachable bool `json:"chReachable"`
+ CHUptimeSec int64 `json:"chUptimeSec"`
+ PartsCount int64 `json:"partsCount"`
+ PartsByTable []TableParts `json:"partsByTable,omitempty"`
+ ActiveMerges int64 `json:"activeMerges"`
+ LongestMergeSec float64 `json:"longestMergeSec"`
+ ErrorsRecent []CHError `json:"errorsRecent,omitempty"`
+ MemoryUsageBytes int64 `json:"memoryUsageBytes,omitempty"`
+ MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`
+}
+
+func (h healthDeepController) Get(c *gin.Context) {
+ resp := fetchCHHealth(c.Request.Context())
+ if !resp.CHReachable {
+ c.JSON(http.StatusServiceUnavailable, resp)
+ return
+ }
+ c.JSON(http.StatusOK, resp)
+}
+
+var HealthDeepController = healthDeepController{}
diff --git a/backend/app/controllers/health_deep_pgch.go b/backend/app/controllers/health_deep_pgch.go
new file mode 100644
index 00000000..173e224b
--- /dev/null
+++ b/backend/app/controllers/health_deep_pgch.go
@@ -0,0 +1,77 @@
+//go:build pgch
+
+package controllers
+
+import (
+ "context"
+ "time"
+
+ "github.com/tracewayapp/traceway/backend/app/chdb"
+)
+
+func fetchCHHealth(parent context.Context) HealthDeepResponse {
+ ctx, cancel := context.WithTimeout(parent, 5*time.Second)
+ defer cancel()
+
+ resp := HealthDeepResponse{}
+
+ if err := chdb.Conn.Exec(ctx, "SELECT 1"); err != nil {
+ resp.CHReachable = false
+ return resp
+ }
+ resp.CHReachable = true
+
+ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(uptime())"); row != nil {
+ var uptime int64
+ if err := row.Scan(&uptime); err == nil {
+ resp.CHUptimeSec = uptime
+ }
+ }
+
+ if rows, err := chdb.Conn.Query(ctx, "SELECT table, sum(rows > 0) AS parts FROM system.parts WHERE active GROUP BY table ORDER BY parts DESC"); err == nil {
+ for rows.Next() {
+ var pt TableParts
+ if err := rows.Scan(&pt.Table, &pt.Parts); err != nil {
+ continue
+ }
+ resp.PartsByTable = append(resp.PartsByTable, pt)
+ resp.PartsCount += pt.Parts
+ }
+ rows.Close()
+ }
+
+ if row := chdb.Conn.QueryRow(ctx, "SELECT count(), coalesce(max(elapsed), 0) FROM system.merges"); row != nil {
+ var n int64
+ var longest float64
+ if err := row.Scan(&n, &longest); err == nil {
+ resp.ActiveMerges = n
+ resp.LongestMergeSec = longest
+ }
+ }
+
+ if rows, err := chdb.Conn.Query(ctx, "SELECT name, value, toString(last_error_time) FROM system.errors WHERE value > 0 ORDER BY last_error_time DESC LIMIT 20"); err == nil {
+ for rows.Next() {
+ var e CHError
+ if err := rows.Scan(&e.Name, &e.Value, &e.LastErrorTime); err != nil {
+ continue
+ }
+ resp.ErrorsRecent = append(resp.ErrorsRecent, e)
+ }
+ rows.Close()
+ }
+
+ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(value) FROM system.asynchronous_metrics WHERE metric = 'MemoryResident'"); row != nil {
+ var v int64
+ if err := row.Scan(&v); err == nil {
+ resp.MemoryUsageBytes = v
+ }
+ }
+ if row := chdb.Conn.QueryRow(ctx, "SELECT toInt64(value) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryTotal'"); row != nil {
+ var v int64
+ if err := row.Scan(&v); err == nil {
+ resp.MemoryTotalBytes = v
+ }
+ }
+
+ return resp
+}
diff --git a/backend/app/controllers/health_deep_sqlite.go b/backend/app/controllers/health_deep_sqlite.go
new file mode 100644
index 00000000..ace2d896
--- /dev/null
+++ b/backend/app/controllers/health_deep_sqlite.go
@@ -0,0 +1,9 @@
+//go:build !pgch
+
+package controllers
+
+import "context"
+
+func fetchCHHealth(_ context.Context) HealthDeepResponse {
+ return HealthDeepResponse{CHReachable: false}
+}
diff --git a/backend/app/controllers/routes.go b/backend/app/controllers/routes.go
index 8d85eb83..f1ecef61 100644
--- a/backend/app/controllers/routes.go
+++ b/backend/app/controllers/routes.go
@@ -44,6 +44,8 @@ func RegisterControllers(router *gin.RouterGroup) {
router.PUT("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.UpdateProject)
router.DELETE("/projects", middleware.UseAppAuth, middleware.RequireProjectAccess, middleware.RequireWriteAccess, ProjectController.DeleteProject)
+ router.GET("/health/deep", middleware.UseAppAuth, HealthDeepController.Get)
+
// Dashboard endpoints (projectId in query param)
router.POST("/stats", middleware.UseAppAuth, middleware.RequireProjectAccess, MetricRecordController.FindHomepageStats)
router.GET("/dashboard", middleware.UseAppAuth, middleware.RequireProjectAccess, DashboardController.GetDashboard)
diff --git a/backend/app/repositories/ai_trace.repository.go b/backend/app/repositories/ai_trace.repository.go
index 031b1188..ab1e7c7e 100644
--- a/backend/app/repositories/ai_trace.repository.go
+++ b/backend/app/repositories/ai_trace.repository.go
@@ -10,7 +10,6 @@ import (
"strings"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
"github.com/tracewayapp/traceway/backend/app/models"
@@ -19,7 +18,7 @@ import (
type aiTraceRepository struct{}
func (r *aiTraceRepository) InsertAsync(ctx context.Context, lines []models.AiTrace) error {
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),
"INSERT INTO ai_traces (id, project_id, recorded_at, duration, status_code, model, response_model, provider, operation, input_tokens, output_tokens, total_tokens, cached_tokens, reasoning_tokens, input_cost, output_cost, total_cost, trace_name, user_id, finish_reason, server_name, app_version, storage_key, attributes, distributed_trace_id, is_root)")
if err != nil {
return err
diff --git a/backend/app/repositories/endpoint.repository.go b/backend/app/repositories/endpoint.repository.go
index a7b898b5..c76caa7a 100644
--- a/backend/app/repositories/endpoint.repository.go
+++ b/backend/app/repositories/endpoint.repository.go
@@ -11,14 +11,13 @@ import (
"errors"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
type endpointRepository struct{}
func (e *endpointRepository) InsertAsync(ctx context.Context, lines []models.Endpoint) error {
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)")
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)")
if err != nil {
return err
}
diff --git a/backend/app/repositories/exception_stack_trace.repository.go b/backend/app/repositories/exception_stack_trace.repository.go
index 32831e25..9a3fd9c2 100644
--- a/backend/app/repositories/exception_stack_trace.repository.go
+++ b/backend/app/repositories/exception_stack_trace.repository.go
@@ -12,14 +12,13 @@ import (
"strings"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
type exceptionStackTraceRepository struct{}
func (e *exceptionStackTraceRepository) InsertAsync(ctx context.Context, lines []models.ExceptionStackTrace) error {
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)")
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)")
if err != nil {
return err
}
diff --git a/backend/app/repositories/fired_notification.repository.go b/backend/app/repositories/fired_notification.repository.go
index ca6e0ae6..83c0baa1 100644
--- a/backend/app/repositories/fired_notification.repository.go
+++ b/backend/app/repositories/fired_notification.repository.go
@@ -6,7 +6,6 @@ import (
"context"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
)
@@ -31,7 +30,7 @@ type firedNotificationRepository struct{}
func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotification) error {
batch, err := chdb.Conn.PrepareBatch(
- clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),
+ chdb.BatchCtx(),
"INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, fired_at)",
)
if err != nil {
diff --git a/backend/app/repositories/log_record.repository.go b/backend/app/repositories/log_record.repository.go
index 6ac66db3..a9a386e1 100644
--- a/backend/app/repositories/log_record.repository.go
+++ b/backend/app/repositories/log_record.repository.go
@@ -7,7 +7,6 @@ import (
"strings"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
@@ -48,7 +47,7 @@ func (r *logRecordRepository) InsertAsync(ctx context.Context, records []models.
}
batch, err := chdb.Conn.PrepareBatch(
- clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),
+ chdb.BatchCtx(),
"INSERT INTO log_records (id, project_id, timestamp, trace_id, span_id, trace_flags, severity_text, severity_number, service_name, body, resource_schema_url, resource_attributes, scope_schema_url, scope_name, scope_version, scope_attributes, log_attributes)",
)
if err != nil {
diff --git a/backend/app/repositories/metric_point.repository.go b/backend/app/repositories/metric_point.repository.go
index d931385f..76ca39e9 100644
--- a/backend/app/repositories/metric_point.repository.go
+++ b/backend/app/repositories/metric_point.repository.go
@@ -8,14 +8,13 @@ import (
"context"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
type metricPointRepository struct{}
func (r *metricPointRepository) InsertAsync(ctx context.Context, points []models.MetricPoint) error {
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO metric_points (project_id, name, value, tags, recorded_at)")
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO metric_points (project_id, name, value, tags, recorded_at)")
if err != nil {
return err
}
diff --git a/backend/app/repositories/session.repository.go b/backend/app/repositories/session.repository.go
index 5a22526a..fb50ac1c 100644
--- a/backend/app/repositories/session.repository.go
+++ b/backend/app/repositories/session.repository.go
@@ -9,7 +9,6 @@ import (
"strings"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
"github.com/tracewayapp/traceway/backend/app/models"
@@ -21,7 +20,7 @@ func (r *sessionRepository) Upsert(ctx context.Context, sessions []models.Sessio
if len(sessions) == 0 {
return nil
}
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),
"INSERT INTO sessions (id, project_id, started_at, ended_at, duration, client_ip, attributes, app_version, server_name, distributed_trace_id, version)")
if err != nil {
return err
diff --git a/backend/app/repositories/session_recording.repository.go b/backend/app/repositories/session_recording.repository.go
index e6ba2358..19223e9c 100644
--- a/backend/app/repositories/session_recording.repository.go
+++ b/backend/app/repositories/session_recording.repository.go
@@ -9,7 +9,6 @@ import (
"database/sql"
"errors"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
@@ -19,7 +18,7 @@ func (r *sessionRecordingRepository) InsertAsync(ctx context.Context, recordings
if len(recordings) == 0 {
return nil
}
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO session_recordings (id, project_id, exception_id, session_id, segment_index, file_path, recorded_at)")
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO session_recordings (id, project_id, exception_id, session_id, segment_index, file_path, recorded_at)")
if err != nil {
return err
}
diff --git a/backend/app/repositories/span.repository.go b/backend/app/repositories/span.repository.go
index caa9a4f2..f55ff9e4 100644
--- a/backend/app/repositories/span.repository.go
+++ b/backend/app/repositories/span.repository.go
@@ -7,7 +7,6 @@ import (
"github.com/tracewayapp/traceway/backend/app/models"
"context"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
@@ -18,7 +17,7 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) e
return nil
}
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)),
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),
"INSERT INTO spans (id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id)")
if err != nil {
return err
diff --git a/backend/app/repositories/task.repository.go b/backend/app/repositories/task.repository.go
index 14bc51e8..048e65d2 100644
--- a/backend/app/repositories/task.repository.go
+++ b/backend/app/repositories/task.repository.go
@@ -11,14 +11,13 @@ import (
"errors"
"time"
- "github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
)
type taskRepository struct{}
func (e *taskRepository) InsertAsync(ctx context.Context, lines []models.Task) error {
- batch, err := chdb.Conn.PrepareBatch(clickhouse.Context(context.Background(), clickhouse.WithAsync(false)), "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)")
+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)")
if err != nil {
return err
}
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 876c4130..26dc039c 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -52,6 +52,58 @@ other. Each folder is wiped on each dispatch of its scenario.
Don't read the throughput number as "the dashboard stays usable at N
items/sec" — that's what read-probe is for.
+## Two insert paths: sync vs async ClickHouse
+
+Every backend `PrepareBatch` call funnels through `chdb.BatchCtx()`, which
+reads `CH_ASYNC_INSERT` once at startup. **This is a per-dispatch toggle**
+— a single matrix entry runs one insert mode; comparison requires two
+dispatches.
+
+| | `CH_ASYNC_INSERT=0` (default) | `CH_ASYNC_INSERT=1` |
+|--------------------------|-------------------------------|----------------------|
+| Driver flag | `clickhouse.WithAsync(false)` | `clickhouse.WithAsync(true)` |
+| Part-per-request | 1 part per insert (worst case for MergeTree) | Batched server-side, parts amortised |
+| What it measures | Pessimistic ceiling — what you get if you point a stock OTel collector at the default Traceway config | Realistic production ceiling — what a tuned deployment gets |
+| Result filename | `<tier>-<mode>-<signal>-<scenario>.json` | `<tier>-<mode>-<signal>-<scenario>-async.json` |
+| Headline bar in charts | Solid | Solid (sibling bar in chart-headline-<signal>.png when both passes are present) |
+
+Quote both numbers when reporting. The sync number tells you the floor your
+users hit before tuning; the async number tells you the headroom.
+
+**Triggering async-insert locally:**
+
+```bash
+# Default (sync) — no env var needed
+./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans
+
+# Async — set CH_ASYNC_INSERT=1 in the parent env. run-matrix-entry.sh
+# propagates it via the 7th positional arg "async" to sut-bootstrap.sh,
+# which exports it into the SUT's docker-compose env so chdb.BatchCtx()
+# reads the right value at startup.
+CH_ASYNC_INSERT=1 ./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans --async
+```
+
+`run-local.sh --async` is sugar that just sets `CH_ASYNC_INSERT=1` and
+threads the `async` arg into `run-matrix-entry.sh`. Direct script call:
+
+```bash
+./benchmarks/scripts/run-matrix-entry.sh ccx13 pgch spans 30m benchmarks/results-throughput "" async
+# ^tier ^mode ^sig ^dur ^out-dir ^smoke ^async
+```
+
+**Triggering async-insert in CI:**
+
+In the GitHub Actions `workflow_dispatch` form, flip the `async_insert`
+boolean to `true`. The workflow propagates it through to
+`run-matrix-entry.sh` and tags both the artifact and the committed JSON
+filenames with `-async`. Dispatch twice (once off, once on) and the
+aggregate job's chart renderer paints sibling bars in
+`chart-headline-<signal>.png`.
+
+`managed-ch` mode also honours `CH_ASYNC_INSERT` — useful for measuring
+how much of the managed offering's headroom is currently being left on
+the table by the conservative sync default.
+
## How a run works
Per matrix entry (one tier × one mode × one signal):
@@ -64,17 +116,26 @@ Per matrix entry (one tier × one mode × one signal):
4. `loadgen-bootstrap.sh` cross-compiles the loadgen, pushes it to the
loadgen box, runs it with `--signal <spans|metrics|logs>` against the
SUT's *private* IP.
-5. The loadgen runs a three-phase ramp:
+5. The loadgen runs a three-phase ramp. After every step it GETs
+ `/api/health/deep` and embeds a `ch` block in the step's JSON
+ (`partsCount`, `partsByTable`, `activeMerges`, `longestMergeSec`,
+ `errorsRecent`, `memoryUsageBytes`, `uptimeSec`). This is how the
+ bench sees ClickHouse-side pressure — the HTTP-only signal alone
+ can't distinguish "SUT cliffed on CPU" from "MergeTree threw
+ `Too many parts`".
- **Phase 1 — batch-size ramp.** Single client at a fixed 5 req/sec.
Batch sizes step through `256,1024,4096,8192,16384`. Each step holds for
`--step-duration` (default 2 min). Stops at the first failing step.
- - **Inter-phase cooldown.** Phase 1's last step typically runs the SUT
- near saturation; the loadgen sleeps `--inter-phase-cooldown-seconds`
- (default 30 s) before starting Phase 2 so the SUT can drain its
- queues (CH merges, PG WAL, HTTP handler pool). Without this pause,
- pgch runs reliably produced "0 OK / 0 errors" Phase 2 because new
- requests sat on the SUT-side TCP backlog while the SUT was still
- digesting Phase 1's final wave.
+ - **Inter-phase merge-idle wait.** Phase 1's last step typically runs the
+ SUT near saturation; CH merge horizons are minutes, not seconds. The
+ loadgen polls `/api/health/deep` every 5 s until `activeMerges == 0`
+ and `partsCount` is stable (within ±5%) for two consecutive polls, or
+ `--max-merge-idle-wait` (default 5 m) expires. The old fixed
+ `--inter-phase-cooldown-seconds` flag still works (additive pre-wait)
+ but defaults to 0 — it's deprecated in favour of polling CH directly.
+ Without this gate, pgch runs reliably produced "0 OK / 0 errors"
+ Phase 2 because new requests sat on the SUT-side TCP backlog while
+ CH was still merging Phase 1's parts.
- **Phase 2 — request-rate ramp (collector shape).** Batch size fixed at
`min(Phase 1 winner, --phase2-batch-cap=16384)`. Request rates step
through `1,5,25,100,400`. When the coarse ramp finds the first failing
@@ -83,7 +144,7 @@ Per matrix entry (one tier × one mode × one signal):
within `--phase2-bisect-tolerance` (default 20%). So `5→25` (a 5× jump)
gets refined into something like `5, 15, 10, 12` until the gap is
<20% of the last passing rate.
- - **Inter-phase cooldown** before Phase 3 too.
+ - **Merge-idle wait** before Phase 3 too.
- **Phase 3 — request-rate ramp (SDK-fleet shape).** Batch size fixed at
`--phase3-batch-size` (default **100**, matching typical language-SDK
`BatchSpanProcessor` output rather than the collector's 8192). Request
@@ -109,13 +170,13 @@ Per matrix entry (one tier × one mode × one signal):
reports; per-phase numbers stay in the JSON for shape-specific analysis.
7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.
-### SUT crash recovery
+### SUT crash recovery & restart detection
`pgch` and `managed-ch` containers declare `restart: on-failure:3` in their
compose files. When ClickHouse OOMs under Phase 1's heaviest step (typical
on `ccx33` + `pgch` once Phase 2 climbs past `batch=16384 × rate=100`), Docker
-auto-restarts the crashed service within seconds. After each inter-phase
-cooldown the loadgen then polls `GET <target>/health` for up to
+auto-restarts the crashed service within seconds. After each merge-idle
+wait the loadgen polls `GET <target>/health` for up to
`--sut-health-timeout-seconds` (default 60 s) to confirm the SUT actually
came back. If `/health` never returns 200, the loadgen skips the remaining
phases and writes the partial JSON via the existing per-step checkpoint —
@@ -124,6 +185,18 @@ prevents a permanently-broken SUT from restart-looping forever; if the
third restart also crashes, the container stays down and the health-poll
fires the clean-abort path.
+**Detecting silent CH restarts.** Compose's `restart` policy used to mean
+a crashed-and-recovered ClickHouse was invisible: the SUT came back up and
+the bench kept running, producing a "passing" report that hid the
+incident. The bench now tracks CH's `uptime()` across every step. If
+step N's uptime is lower than step N-1's uptime plus the elapsed step
+window, ClickHouse restarted mid-step. The step is marked failed with
+`failReason: CH restarted mid-step (...)`, `chRestarted: true` is set on
+the top-level JSON, the run skips remaining phases, and the loadgen
+exits non-zero. The chart renderer hatches the affected headline bar
+in red with a `CH↻` annotation so the dispatch report flags the run as
+invalid rather than burying the restart in a passing number.
+
After all matrix entries finish, `chart.py` renders the full chart suite
(headline bars, Phase 1 / 2 / 3 ramps, Pareto, tier scaling, cliff grid,
batch efficiency, signal mix — plus the read-probe equivalents in the
@@ -198,6 +271,12 @@ vice-versa.
# Switch to the read-probe scenario (writes to benchmarks/results-probe/)
./benchmarks/scripts/run-local.sh --scenario read-probe
+
+# Async-insert pass — same matrix but with CH_ASYNC_INSERT=1 on the SUT.
+# Output filenames pick up a -async suffix so paired bars render in
+# chart-headline-<signal>.png. Dispatch the default (sync) pass first,
+# then this one, into the same results folder.
+./benchmarks/scripts/run-local.sh --tier ccx13 --mode pgch --signal spans --async
```
## Running from GitHub Actions
@@ -278,13 +357,14 @@ benchmarks/
docker-compose.pgch.yml
docker-compose.managed-ch.yml # External CH; Postgres still local
loadgen/ # The Go binary that generates load (OTLP-only)
- main.go # CLI + orchestration
+ main.go # CLI + orchestration + merge-idle wait
ingest.go # Worker pool driving the selected signal sender
otlp_common.go # Shared OTLP helpers + signalSender interface
otlp_spans.go # ExportTraceServiceRequest builder
otlp_metrics.go # ExportMetricsServiceRequest builder (Gauge)
otlp_logs.go # ExportLogsServiceRequest builder
- ramp.go # Two-phase ramp (batch size, then request rate)
+ ramp.go # Three-phase ramp + CH-restart detection
+ ch_snapshot.go # GET /api/health/deep -> per-step ch{} block
stats.go # Latency tracker (percentiles via sort)
util.go, log.go # Misc helpers
scripts/
diff --git a/benchmarks/compose/docker-compose.managed-ch.yml b/benchmarks/compose/docker-compose.managed-ch.yml
index e1d52ce8..b6caf2c0 100644
--- a/benchmarks/compose/docker-compose.managed-ch.yml
+++ b/benchmarks/compose/docker-compose.managed-ch.yml
@@ -19,6 +19,7 @@ services:
POSTGRES_SSLMODE: "disable"
JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"
GIN_MODE: "release"
+ CH_ASYNC_INSERT: "${CH_ASYNC_INSERT:-0}"
depends_on:
postgres:
condition: service_healthy
diff --git a/benchmarks/compose/docker-compose.pgch.yml b/benchmarks/compose/docker-compose.pgch.yml
index 6ba6e55f..11357768 100644
--- a/benchmarks/compose/docker-compose.pgch.yml
+++ b/benchmarks/compose/docker-compose.pgch.yml
@@ -19,6 +19,7 @@ services:
POSTGRES_SSLMODE: "disable"
JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"
GIN_MODE: "release"
+ CH_ASYNC_INSERT: "${CH_ASYNC_INSERT:-0}"
depends_on:
clickhouse:
condition: service_healthy
diff --git a/benchmarks/loadgen/ch_snapshot.go b/benchmarks/loadgen/ch_snapshot.go
new file mode 100644
index 00000000..74084f2e
--- /dev/null
+++ b/benchmarks/loadgen/ch_snapshot.go
@@ -0,0 +1,100 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "time"
+)
+
+type tableParts struct {
+ Table string `json:"table"`
+ Parts int64 `json:"parts"`
+}
+
+type chError struct {
+ Name string `json:"name"`
+ Value int64 `json:"value"`
+ LastErrorTime string `json:"lastErrorTime,omitempty"`
+}
+
+type chSnapshot struct {
+ Reachable bool `json:"reachable"`
+ UptimeSec int64 `json:"uptimeSec"`
+ PartsCount int64 `json:"partsCount"`
+ PartsByTable []tableParts `json:"partsByTable,omitempty"`
+ ActiveMerges int64 `json:"activeMerges"`
+ LongestMergeSec float64 `json:"longestMergeSec"`
+ ErrorsRecent []chError `json:"errorsRecent,omitempty"`
+ MemoryUsageBytes int64 `json:"memoryUsageBytes,omitempty"`
+ MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`
+}
+
+// healthDeepBody mirrors the backend HealthDeepResponse with its JSON tags. The
+// backend uses `chReachable`/`chUptimeSec`; we expose those as `reachable`/
+// `uptimeSec` in our embedded snapshot so the bench JSON stays consistent with
+// other loadgen fields.
+type healthDeepBody struct {
+ CHReachable bool `json:"chReachable"`
+ CHUptimeSec int64 `json:"chUptimeSec"`
+ PartsCount int64 `json:"partsCount"`
+ PartsByTable []tableParts `json:"partsByTable"`
+ ActiveMerges int64 `json:"activeMerges"`
+ LongestMergeSec float64 `json:"longestMergeSec"`
+ ErrorsRecent []chError `json:"errorsRecent"`
+ MemoryUsageBytes int64 `json:"memoryUsageBytes"`
+ MemoryTotalBytes int64 `json:"memoryTotalBytes"`
+}
+
+// fetchCHSnapshot pings the backend's /health/deep endpoint and translates the
+// payload into a chSnapshot. On any error (timeout, transport, non-2xx, decode)
+// it returns a snapshot with Reachable=false and logs to stderrPrefix() — the
+// caller does NOT fail the step on a missing snapshot. A 503 from the backend
+// (CH unreachable but backend alive) is still parsed and embedded so the JSON
+// captures the chReachable=false signal.
+func fetchCHSnapshot(ctx context.Context, cfg config, client *http.Client) chSnapshot {
+ if cfg.jwt == "" {
+ return chSnapshot{}
+ }
+
+ reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, cfg.target+"/api/health/deep", nil)
+ if err != nil {
+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: build request failed: %v\n", err)
+ return chSnapshot{}
+ }
+ req.Header.Set("Authorization", "Bearer "+cfg.jwt)
+
+ resp, err := client.Do(req)
+ if err != nil {
+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: http error: %v\n", err)
+ return chSnapshot{}
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusServiceUnavailable {
+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: unexpected status %d\n", resp.StatusCode)
+ return chSnapshot{}
+ }
+
+ var body healthDeepBody
+ if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+ fmt.Fprintf(stderrPrefix(), "fetchCHSnapshot: decode failed: %v\n", err)
+ return chSnapshot{}
+ }
+
+ return chSnapshot{
+ Reachable: body.CHReachable,
+ UptimeSec: body.CHUptimeSec,
+ PartsCount: body.PartsCount,
+ PartsByTable: body.PartsByTable,
+ ActiveMerges: body.ActiveMerges,
+ LongestMergeSec: body.LongestMergeSec,
+ ErrorsRecent: body.ErrorsRecent,
+ MemoryUsageBytes: body.MemoryUsageBytes,
+ MemoryTotalBytes: body.MemoryTotalBytes,
+ }
+}
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
index e3fabead..a8f8d4d6 100644
--- a/benchmarks/loadgen/main.go
+++ b/benchmarks/loadgen/main.go
@@ -33,6 +33,7 @@ type config struct {
softCliffRatio float64
stepDrainSeconds time.Duration
interPhaseCooldownSeconds time.Duration
+ maxMergeIdleWait time.Duration
sutHealthTimeoutSeconds time.Duration
phase2BisectMaxSteps int
phase2BisectTolerance float64
@@ -72,7 +73,8 @@ func main() {
flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if combined (HTTP error + OTLP rejected) item rate exceeds this")
flag.Float64Var(&cfg.softCliffRatio, "soft-cliff-ratio", 0.70, "Step fails when achieved req-rate is below this fraction of target — catches saturated-but-not-erroring cliffs. 0 disables.")
flag.DurationVar(&cfg.stepDrainSeconds, "step-drain-seconds", 10*time.Second, "After step duration expires, wait up to this long for in-flight HTTP requests to complete before hard-canceling (reduces error-count noise at boundaries)")
- flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 30*time.Second, "Pause between Phase 1 and Phase 2 to let the SUT drain queues and recover from Phase 1's final-step load. 0 disables.")
+ flag.DurationVar(&cfg.interPhaseCooldownSeconds, "inter-phase-cooldown-seconds", 0, "DEPRECATED: replaced by --max-merge-idle-wait, which polls CH directly. Non-zero values still apply as an additional pre-wait before merge-idle polling. 0 (default) disables.")
+ flag.DurationVar(&cfg.maxMergeIdleWait, "max-merge-idle-wait", 5*time.Minute, "Between phases, poll /health/deep until activeMerges==0 and partsCount is stable for two consecutive polls, or this timeout elapses. 0 disables (skip merge-idle wait entirely).")
flag.DurationVar(&cfg.sutHealthTimeoutSeconds, "sut-health-timeout-seconds", 60*time.Second, "After each inter-phase cooldown, poll GET <target>/health for up to this long. If the SUT doesn't return 200 in time, remaining phases are skipped and the partial JSON is written via the existing checkpoint. 0 disables the check.")
flag.IntVar(&cfg.phase2BisectMaxSteps, "phase2-bisect-max-steps", 3, "After Phase 2 finds the cliff, run up to this many bisection steps between the last passing and first failing rate to narrow the cliff. 0 disables.")
flag.Float64Var(&cfg.phase2BisectTolerance, "phase2-bisect-tolerance", 0.20, "Bisection stops when (firstFailRate-lastPassRate)/lastPassRate falls below this fraction (e.g. 0.20 = stop when the cliff is pinned to within 20% of the last passing rate).")
@@ -173,6 +175,7 @@ func main() {
// kill of the writer process itself) never see a half-written file.
writeCheckpoint := func() {
out.EndedAt = time.Now().UTC().Format(time.RFC3339)
+ out.ChRestarted = chRestarted.Load()
out.computeHeadline()
if err := writeReportAtomic(cfg.reportOut, &out); err != nil {
fmt.Fprintf(os.Stderr, "checkpoint write failed: %v\n", err)
@@ -181,24 +184,21 @@ func main() {
switch cfg.scenario {
case "throughput":
- phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {
+ phase1 := runBatchSizeRamp(ctx, cfg, ing, ingestStats, httpClient, func(p phaseResult) {
out.Phase1 = &p
writeCheckpoint()
})
out.Phase1 = &phase1
- // Cool-down between phases: Phase 1's last step often runs the SUT at
- // 70-99% of capacity, leaving its internal queues (CH merges, PG WAL,
- // HTTP handler pool) saturated. Jumping straight into Phase 2 with the
- // SUT still digesting that wave produces "0 OK / 0 errors" garbage
- // because new requests sit on the SUT-side TCP backlog without ever
- // reaching a handler or timing out.
- if cfg.interPhaseCooldownSeconds > 0 {
- fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)
- select {
- case <-time.After(cfg.interPhaseCooldownSeconds):
- case <-ctx.Done():
- }
+ if chRestarted.Load() {
+ fmt.Fprintf(stderrPrefix(), "CH restart observed in Phase 1 — skipping remaining phases\n")
+ break
}
+ // Between phases: Phase 1's last step often runs the SUT at 70-99% of
+ // capacity, leaving CH parts queues unmerged and PG/HTTP pools saturated.
+ // Jumping straight into Phase 2 produces garbage data — instead, poll
+ // /health/deep until CH merges are idle and parts count has stabilized.
+ preWaitCooldown(ctx, cfg)
+ waitForMergesIdle(ctx, cfg, httpClient, "phase 1 -> phase 2")
// Verify the SUT survived Phase 1 before launching Phase 2. If the
// compose restart policy didn't bring the SUT back, skip remaining
// phases — the final writeCheckpoint after the switch captures
@@ -207,27 +207,28 @@ func main() {
fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 1 cooldown — skipping Phase 2/3: %v\n", err)
break
}
- phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, phase1, func(p phaseResult) {
+ phase2 := runRequestRateRamp(ctx, cfg, ing, ingestStats, httpClient, phase1, func(p phaseResult) {
out.Phase2 = &p
writeCheckpoint()
})
out.Phase2 = &phase2
+ if chRestarted.Load() {
+ fmt.Fprintf(stderrPrefix(), "CH restart observed in Phase 2 — skipping Phase 3\n")
+ break
+ }
// Phase 3 — small-batch high-rate (SDK-fleet shape). Independent of
// Phase 1/2 results, so it runs even if Phase 2 produced no passing
- // steps. Same cooldown before it as between 1 and 2.
- if cfg.interPhaseCooldownSeconds > 0 && ctx.Err() == nil {
- fmt.Fprintf(stderrPrefix(), "inter-phase cooldown: %v\n", cfg.interPhaseCooldownSeconds)
- select {
- case <-time.After(cfg.interPhaseCooldownSeconds):
- case <-ctx.Done():
- }
+ // steps. Same merge-idle wait as between 1 and 2.
+ if ctx.Err() == nil {
+ preWaitCooldown(ctx, cfg)
+ waitForMergesIdle(ctx, cfg, httpClient, "phase 2 -> phase 3")
}
if err := waitForSutHealthy(ctx, cfg.target, cfg.sutHealthTimeoutSeconds); err != nil {
fmt.Fprintf(stderrPrefix(), "SUT unhealthy after Phase 2 cooldown — skipping Phase 3: %v\n", err)
break
}
- phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, func(p phaseResult) {
+ phase3 := runSmallBatchRateRamp(ctx, cfg, ing, ingestStats, httpClient, func(p phaseResult) {
out.Phase3 = &p
writeCheckpoint()
})
@@ -252,6 +253,11 @@ func main() {
fmt.Fprintf(os.Stderr, "wrote %s: signal=%s max fill level passed = %d rows\n",
cfg.reportOut, cfg.signal, out.MaxFillLevelPassed)
}
+
+ if chRestarted.Load() {
+ fmt.Fprintln(os.Stderr, "FAILED: ClickHouse restarted during the run; treating result as invalid")
+ os.Exit(1)
+ }
}
// waitForSutHealthy polls GET <target>/health until it returns 200 or the
@@ -299,6 +305,99 @@ func waitForSutHealthy(ctx context.Context, target string, timeout time.Duration
}
}
+// preWaitCooldown honours the deprecated --inter-phase-cooldown-seconds flag
+// when set. Most invocations leave it at 0 and rely entirely on
+// waitForMergesIdle; this hook stays so existing local scripts that still pass
+// the old flag don't silently skip the wait.
+func preWaitCooldown(ctx context.Context, cfg config) {
+ if cfg.interPhaseCooldownSeconds <= 0 {
+ return
+ }
+ fmt.Fprintf(stderrPrefix(), "inter-phase cooldown (deprecated flag): %v\n", cfg.interPhaseCooldownSeconds)
+ select {
+ case <-time.After(cfg.interPhaseCooldownSeconds):
+ case <-ctx.Done():
+ }
+}
+
+// waitForMergesIdle polls /health/deep every 5s until activeMerges == 0 AND
+// partsCount has been stable (within ±5%) for two consecutive polls, OR the
+// configured timeout elapses. Returns without error in all cases — the wait is
+// best-effort observability, not a verdict; if CH never settles we just move
+// on and the next phase's results will show it.
+func waitForMergesIdle(ctx context.Context, cfg config, client *http.Client, label string) {
+ if cfg.maxMergeIdleWait <= 0 {
+ return
+ }
+
+ // SQLite mode has no ClickHouse — /health/deep returns chReachable:false
+ // and there are no merges to wait for. Without this fast-path the loop
+ // would burn the full --max-merge-idle-wait (default 5m) between every
+ // phase doing nothing useful.
+ if first := fetchCHSnapshot(ctx, cfg, client); !first.Reachable {
+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: CH not reachable (sqlite mode or backend down) — skipping wait\n", label)
+ return
+ }
+
+ deadline := time.Now().Add(cfg.maxMergeIdleWait)
+ pollEvery := 5 * time.Second
+ stableThreshold := 0.05
+ requiredStable := 2
+
+ var prevParts int64 = -1
+ stable := 0
+ for {
+ if ctx.Err() != nil {
+ return
+ }
+ snap := fetchCHSnapshot(ctx, cfg, client)
+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: reachable=%t activeMerges=%d partsCount=%d longestMerge=%.1fs\n",
+ label, snap.Reachable, snap.ActiveMerges, snap.PartsCount, snap.LongestMergeSec)
+
+ if snap.Reachable && snap.ActiveMerges == 0 {
+ if prevParts >= 0 {
+ delta := float64(snap.PartsCount-prevParts) / float64(maxInt64(prevParts, 1))
+ if delta < 0 {
+ delta = -delta
+ }
+ if delta <= stableThreshold {
+ stable++
+ if stable >= requiredStable {
+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: settled (parts=%d)\n", label, snap.PartsCount)
+ return
+ }
+ } else {
+ stable = 0
+ }
+ }
+ prevParts = snap.PartsCount
+ } else {
+ stable = 0
+ if snap.Reachable {
+ prevParts = snap.PartsCount
+ }
+ }
+
+ if time.Now().After(deadline) {
+ fmt.Fprintf(stderrPrefix(), "merge-idle [%s]: timeout after %v (activeMerges=%d partsCount=%d) — advancing anyway\n",
+ label, cfg.maxMergeIdleWait, snap.ActiveMerges, snap.PartsCount)
+ return
+ }
+ select {
+ case <-time.After(pollEvery):
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func maxInt64(a, b int64) int64 {
+ if a > b {
+ return a
+ }
+ return b
+}
+
// writeReportAtomic encodes the report into a sibling .tmp file and renames
// it over the destination, so a kill or disk-full mid-write can't leave a
// half-encoded file. Called after every step.
diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.go
index 50b97350..33eab851 100644
--- a/benchmarks/loadgen/ramp.go
+++ b/benchmarks/loadgen/ramp.go
@@ -3,9 +3,20 @@ package main
import (
"context"
"fmt"
+ "net/http"
+ "sync/atomic"
"time"
)
+// Cross-phase state for detecting CH restarts. Single-run CLI; package-level
+// state is fine. lastCHUptime stores the most recent step's CH uptime (0
+// before the first reachable snapshot); chRestarted is set sticky-true once a
+// restart is observed so main() can mark the run failed and exit non-zero.
+var (
+ lastCHUptime atomic.Int64
+ chRestarted atomic.Bool
+)
+
type stepResult struct {
Step int `json:"step"`
BatchSize int `json:"batchSize"`
@@ -14,6 +25,7 @@ type stepResult struct {
ActualItemsPerSec float64 `json:"actualItemsPerSec"`
Rejected int64 `json:"rejected"`
Ingest latencySnapshot `json:"ingest"`
+ CH chSnapshot `json:"ch"`
Passed bool `json:"passed"`
FailReason string `json:"failReason,omitempty"`
}
@@ -40,6 +52,7 @@ type finalReport struct {
ReadProbe *readProbeResult `json:"readProbe,omitempty"`
MaxSustainableItemsPerSec float64 `json:"maxSustainableItemsPerSec,omitempty"`
MaxFillLevelPassed int64 `json:"maxFillLevelPassed,omitempty"`
+ ChRestarted bool `json:"chRestarted,omitempty"`
}
func (r *finalReport) computeHeadline() {
@@ -83,7 +96,7 @@ type phaseCheckpoint func(phaseResult)
// size step by step. Stops at the first failing step. Returns a phaseResult
// whose MaxBatchSize is the largest batch that passed. Calls `checkpoint`
// after each step (after both pass and fail) so partial state is durable.
-func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {
+func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
Kind: "batch-size-ramp",
FixedRequestRate: cfg.phase1FixedRate,
@@ -96,7 +109,7 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la
break
}
ing.SetBatchSize(batch)
- s := runOneStep(ctx, cfg, ing, ingest, idx+1, batch, cfg.phase1FixedRate)
+ s := runOneStep(ctx, cfg, ing, ingest, client, idx+1, batch, cfg.phase1FixedRate)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "phase1 step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
@@ -118,7 +131,7 @@ func runBatchSizeRamp(ctx context.Context, cfg config, ing *ingester, ingest *la
// and grows request rate step by step. Phase 2's purpose: at the largest
// single-request payload the SUT can absorb (collector-shape), find the
// req/sec ceiling.
-func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {
+func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, phase1 phaseResult, checkpoint phaseCheckpoint) phaseResult {
batch := phase1.MaxBatchSize
if batch <= 0 {
batch = cfg.phase2BatchCap
@@ -126,7 +139,7 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
if batch > cfg.phase2BatchCap {
batch = cfg.phase2BatchCap
}
- return runRateRamp(ctx, cfg, ing, ingest, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)
+ return runRateRamp(ctx, cfg, ing, ingest, client, batch, cfg.phase2RequestRates, "request-rate-ramp", "phase2", checkpoint)
}
// runSmallBatchRateRamp holds batchSize fixed at cfg.phase3BatchSize (default
@@ -136,14 +149,14 @@ func runRequestRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *
// SDK fleets in the wild send hundreds-to-thousands of small batches per
// second — language-SDK BatchSpanProcessor defaults are 512 per batch on a
// 5 s rotation, often much less in practice; this phase covers that regime.
-func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, checkpoint phaseCheckpoint) phaseResult {
- return runRateRamp(ctx, cfg, ing, ingest, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)
+func runSmallBatchRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, checkpoint phaseCheckpoint) phaseResult {
+ return runRateRamp(ctx, cfg, ing, ingest, client, cfg.phase3BatchSize, cfg.phase3RequestRates, "small-batch-rate-ramp", "phase3", checkpoint)
}
// runRateRamp is the shared engine for Phase 2 and Phase 3: hold batch
// fixed, ramp request rate, bisect after the first failing step. logPrefix
// is just for stderr messages ("phase2" / "phase3").
-func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {
+func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, batch int, rates []float64, kind, logPrefix string, checkpoint phaseCheckpoint) phaseResult {
res := phaseResult{
Kind: kind,
FixedBatchSize: batch,
@@ -163,7 +176,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latency
}
stepNo++
ing.SetRequestRate(rate)
- s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, rate)
+ s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, rate)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "%s step %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
@@ -196,7 +209,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latency
mid := (lastPassRate + firstFailRate) / 2
stepNo++
ing.SetRequestRate(mid)
- s := runOneStep(ctx, cfg, ing, ingest, stepNo, batch, mid)
+ s := runOneStep(ctx, cfg, ing, ingest, client, stepNo, batch, mid)
res.Steps = append(res.Steps, s)
fmt.Fprintf(stderrPrefix(), "%s bisect %d: batch=%d rate=%.1f items/s=%.0f p99=%.0fms err=%.2f%% passed=%t %s\n",
logPrefix, s.Step, s.BatchSize, s.RequestRate, s.ActualItemsPerSec, s.Ingest.P99, s.Ingest.ErrRate*100, s.Passed, s.FailReason)
@@ -220,7 +233,7 @@ func runRateRamp(ctx context.Context, cfg config, ing *ingester, ingest *latency
// latency + item counters. The drain window matters: without it, every step
// boundary cancels ~workerCount in-flight requests mid-flight, inflating the
// recorded error count and depressing the OK count.
-func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, stepNo, batchSize int, requestRate float64) stepResult {
+func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyTracker, client *http.Client, stepNo, batchSize int, requestRate float64) stepResult {
ingest.SnapshotAndReset()
ing.SnapshotAndResetItems()
@@ -241,6 +254,22 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyT
snap := ingest.SnapshotAndReset()
attempted, rejected := ing.SnapshotAndResetItems()
+ ch := fetchCHSnapshot(ctx, cfg, client)
+
+ // Restart detection: if ClickHouse's uptime is materially less than what
+ // it was at the previous step's snapshot, the CH process restarted during
+ // this step. The compose restart policy may have brought it back, but the
+ // numbers from this step (and the preceding one) are no longer
+ // interpretable, so the run is marked failed and remaining phases skipped.
+ prev := lastCHUptime.Load()
+ restartedThisStep := false
+ if ch.Reachable && ch.UptimeSec > 0 {
+ if prev > 0 && ch.UptimeSec < prev+int64(elapsed.Seconds()) {
+ restartedThisStep = true
+ chRestarted.Store(true)
+ }
+ lastCHUptime.Store(ch.UptimeSec)
+ }
var attemptedIps, actualIps float64
if elapsed > 0 {
@@ -258,6 +287,10 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyT
}
passed, reason := evaluateStep(cfg, snap, attempted, rejected, requestRate, elapsed)
+ if restartedThisStep {
+ passed = false
+ reason = fmt.Sprintf("CH restarted mid-step (uptime %ds < previous %ds + step %ds)", ch.UptimeSec, prev, int64(elapsed.Seconds()))
+ }
return stepResult{
Step: stepNo,
@@ -267,6 +300,7 @@ func runOneStep(ctx context.Context, cfg config, ing *ingester, ingest *latencyT
ActualItemsPerSec: actualIps,
Rejected: rejected,
Ingest: snap,
+ CH: ch,
Passed: passed,
FailReason: reason,
}
diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.py
index aef98509..2f87ff8e 100755
--- a/benchmarks/scripts/chart.py
+++ b/benchmarks/scripts/chart.py
@@ -204,6 +204,26 @@ def phase1_p99_at_batch(run: dict, batch_size: int) -> float | None:
return None
+def worst_phase_for_run(run: dict) -> str | None:
+ """Pick the phase that produced the lowest passing actualItemsPerSec, or the
+ phase that first failed. Used by the CH-pressure chart so the visualisation
+ targets the phase where the SUT actually struggled."""
+ failing = first_failure(run)
+ if failing:
+ return failing[0]
+ worst_phase = None
+ worst_value = float("inf")
+ for phase_key in ("phase1", "phase2", "phase3"):
+ for step in phase_steps(run, phase_key):
+ if not step.get("passed"):
+ continue
+ actual = step.get("actualItemsPerSec", 0) or 0
+ if actual < worst_value:
+ worst_value = actual
+ worst_phase = phase_key
+ return worst_phase
+
+
# ---------- Throughput charts ----------
@@ -226,6 +246,7 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:
width = 0.8 / max(len(modes_present), 1)
x_centers = list(range(len(tiers_present)))
+ any_ch_restart = False
for mi, mode in enumerate(modes_present):
for ti, tier in enumerate(tiers_present):
match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]
@@ -234,14 +255,22 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:
continue
run = match[0]
value = run.get("maxSustainableItemsPerSec", 0) or 0
+ ch_restart = bool(run.get("chRestarted"))
+ if ch_restart:
+ any_ch_restart = True
if value > 0:
- ax.bar(x, value, width=width, color=mode_color(mode))
+ bar_kwargs = {"width": width, "color": mode_color(mode)}
+ if ch_restart:
+ bar_kwargs.update({"edgecolor": "#aa3333", "hatch": "xx", "linewidth": 1.2})
+ ax.bar(x, value, **bar_kwargs)
src = headline_source(run)
phase_tag = f" ({PHASE_LABEL[src[0]]})" if src else ""
+ restart_tag = " CH↻" if ch_restart else ""
ax.annotate(
- f"{int(value):,}{phase_tag}",
+ f"{int(value):,}{phase_tag}{restart_tag}",
xy=(x, value), xytext=(0, 4), textcoords="offset points",
ha="center", va="bottom", fontsize=8,
+ color="#aa3333" if ch_restart else "black",
)
else:
ax.bar(x, ghost_height, width=width, color="#dddddd",
@@ -267,6 +296,8 @@ def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:
legend_handles = [Patch(facecolor=mode_color(m), label=m) for m in modes_present]
legend_handles.append(Patch(facecolor="#dddddd", edgecolor="#999999", hatch="///", label="all phases failed"))
+ if any_ch_restart:
+ legend_handles.append(Patch(facecolor="white", edgecolor="#aa3333", hatch="xx", label="CH restart during run (result invalid)"))
ax.legend(handles=legend_handles, title="DB mode", loc="upper left")
ax.grid(axis="y", linestyle=":", alpha=0.4)
@@ -715,6 +746,70 @@ def render_batch_efficiency(runs: list[dict], signal: str, out: Path) -> None:
plt.close(fig)
+def render_ch_pressure(runs: list[dict], signal: str, out: Path) -> None:
+ """For each (tier, mode) of a signal, plot CH parts count and active merges
+ across the steps of its worst-performing phase. Makes the CH-side story —
+ "parts piled up before the cliff" or "merges were idle and the cliff was
+ backend-side" — visible at a glance."""
+ sig_runs = iter_runs(runs, signal, "throughput")
+ sig_runs = [r for r in sig_runs if any(
+ (step.get("ch") or {}).get("reachable")
+ for phase_key in ("phase1", "phase2", "phase3")
+ for step in phase_steps(r, phase_key)
+ )]
+ if not sig_runs:
+ return
+
+ cols = min(len(sig_runs), 3)
+ rows = (len(sig_runs) + cols - 1) // cols
+ fig, axes = plt.subplots(rows, cols, figsize=(cols * 5.5, rows * 3.5), squeeze=False)
+
+ for idx, run in enumerate(sig_runs):
+ r, c = divmod(idx, cols)
+ ax_parts = axes[r][c]
+ ax_merges = ax_parts.twinx()
+
+ phase_key = worst_phase_for_run(run)
+ if not phase_key:
+ ax_parts.set_visible(False)
+ ax_merges.set_visible(False)
+ continue
+ steps = phase_steps(run, phase_key)
+ if not steps:
+ ax_parts.set_visible(False)
+ ax_merges.set_visible(False)
+ continue
+
+ xs = [s.get("step", i + 1) for i, s in enumerate(steps)]
+ parts = [(s.get("ch") or {}).get("partsCount", 0) for s in steps]
+ merges = [(s.get("ch") or {}).get("activeMerges", 0) for s in steps]
+ first_failed_step = next((s.get("step") for s in steps if not s.get("passed")), None)
+
+ ax_parts.plot(xs, parts, marker="o", color="#1f77b4", label="parts")
+ ax_merges.plot(xs, merges, marker="s", color="#d62728", linestyle="--", label="active merges")
+
+ if first_failed_step is not None:
+ ax_parts.axvline(first_failed_step, color="#aa3333", linestyle=":", alpha=0.6)
+
+ title = f"{run_label(run)} — {PHASE_LABEL[phase_key]}"
+ if run.get("chRestarted"):
+ title += " (CH↻)"
+ ax_parts.set_title(title, fontsize=10)
+ ax_parts.set_xlabel("step")
+ ax_parts.set_ylabel("parts (total, active)", color="#1f77b4")
+ ax_merges.set_ylabel("active merges", color="#d62728")
+ ax_parts.grid(True, linestyle=":", alpha=0.4)
+
+ for idx in range(len(sig_runs), rows * cols):
+ r, c = divmod(idx, cols)
+ axes[r][c].set_visible(False)
+
+ fig.suptitle(f"ClickHouse pressure during the worst-performing phase — {signal}", y=1.02)
+ fig.tight_layout()
+ fig.savefig(out, dpi=130, bbox_inches="tight")
+ plt.close(fig)
+
+
def render_signal_mix(runs: list[dict], out: Path) -> None:
"""Cross-signal throughput comparison: bars per (tier, mode), one bar per signal."""
thr_runs = [r for r in runs if r.get("scenario", "throughput") == "throughput"]
@@ -1213,6 +1308,7 @@ def main() -> int:
render_tier_scaling(runs, signal, results_dir / f"chart-tier-scaling-{signal}.png")
render_cliff_grid(runs, signal, results_dir / f"chart-cliff-{signal}.png")
render_batch_efficiency(runs, signal, results_dir / f"chart-batch-efficiency-{signal}.png")
+ render_ch_pressure(runs, signal, results_dir / f"chart-ch-pressure-{signal}.png")
# Read-probe charts (no-op if no read-probe runs for this signal)
render_readprobe_headline(runs, signal, results_dir / f"chart-readprobe-headline-{signal}.png")
diff --git a/benchmarks/scripts/run-local.sh b/benchmarks/scripts/run-local.sh
index 02734a03..457a3783 100755
--- a/benchmarks/scripts/run-local.sh
+++ b/benchmarks/scripts/run-local.sh
@@ -13,6 +13,7 @@
# run-local.sh --scenario read-probe # same matrix but ingest-and-probe-reads
# run-local.sh --smoke # 1 tier, 1 mode, 1 signal, short steps (~5 min)
# run-local.sh --tier ccx13 --mode sqlite --signal spans
+# run-local.sh --async # set CH_ASYNC_INSERT=1; -async suffix on output files
# run-local.sh --dry-run # validate env + print plan, no provisioning
#
# Output:
@@ -33,6 +34,7 @@ SCENARIO="throughput"
DURATION="30m"
SMOKE=0
DRY_RUN=0
+ASYNC=0
usage() {
sed -n '1,23p' "$0"; exit 2
@@ -46,6 +48,7 @@ while [[ $# -gt 0 ]]; do
--scenario) SCENARIO="$2"; shift 2 ;;
--duration) DURATION="$2"; shift 2 ;;
--smoke) SMOKE=1; shift ;;
+ --async) ASYNC=1; shift ;;
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage ;;
*) echo "unknown flag: $1" >&2; usage ;;
@@ -114,11 +117,17 @@ fi
# strategy.matrix instead.
smoke_arg=""
[[ "${SMOKE}" -eq 1 ]] && smoke_arg="smoke"
+async_arg=""
+if [[ "${ASYNC}" -eq 1 ]]; then
+ async_arg="async"
+ export CH_ASYNC_INSERT=1
+ echo "CH_ASYNC_INSERT=1 — async-insert benchmark pass; output files will get -async suffix" >&2
+fi
failures=()
for e in "${plan[@]}"; do
IFS='|' read -r tier mode signal <<<"${e}"
- if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${signal}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}"; then
+ if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${signal}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}" "${async_arg}"; then
echo "FAIL: ${tier}/${mode}/${signal}" >&2
failures+=("${tier}/${mode}/${signal}")
fi
diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.sh
index 0f25a09b..684c5613 100755
--- a/benchmarks/scripts/run-matrix-entry.sh
+++ b/benchmarks/scripts/run-matrix-entry.sh
@@ -4,7 +4,7 @@
# GitHub workflow; treat this as the single source of truth for matrix-entry
# orchestration.
#
-# Usage: run-matrix-entry.sh <tier> <mode> <signal> <duration> <out-dir> [smoke]
+# Usage: run-matrix-entry.sh <tier> <mode> <signal> <duration> <out-dir> [smoke] [async]
# <tier> ccx13 | ccx23 | ccx33 | ccx43
# <mode> sqlite | pgch
# <signal> spans | metrics | logs
@@ -12,16 +12,18 @@
# <out-dir> Directory to write <tier>-<mode>-<signal>.json into
# [smoke] "smoke" to enable short-step overrides (--phase1-batch-sizes
# 256,1024 --phase2-request-rates 1,5 --step-duration 15s).
-# Optional.
+# Optional; pass "" to skip when also passing [async].
+# [async] "async" to set CH_ASYNC_INSERT=1 on the SUT. Output filename
+# gains a -async suffix when set. Optional.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [[ $# -lt 5 ]]; then
- echo "usage: $0 <tier> <mode> <signal> <duration> <out-dir> [smoke]" >&2
+ echo "usage: $0 <tier> <mode> <signal> <duration> <out-dir> [smoke] [async]" >&2
exit 2
fi
-TIER="$1"; MODE="$2"; SIGNAL="$3"; DURATION="$4"; OUT_DIR="$5"; SMOKE="${6:-}"
+TIER="$1"; MODE="$2"; SIGNAL="$3"; DURATION="$4"; OUT_DIR="$5"; SMOKE="${6:-}"; ASYNC_FLAG="${7:-}"
LOCATION="${BENCH_LOCATION:-nbg1}"
SCENARIO="${BENCH_SCENARIO:-throughput}"
@@ -65,7 +67,14 @@ SUT_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPublicIp')
SUT_PRIVATE_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPrivateIp')
LOADGEN_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.loadgenPublicIp')
-# 2. Bring up the backend on the SUT.
+# 2. Bring up the backend on the SUT. CH_ASYNC_INSERT propagates through to
+# the docker compose env via sut-bootstrap.sh.
+async_suffix=""
+if [[ "${ASYNC_FLAG}" == "async" ]]; then
+ export CH_ASYNC_INSERT=1
+ async_suffix="-async"
+ echo "CH_ASYNC_INSERT=1 (async-insert benchmark pass)" >&2
+fi
"${SCRIPT_DIR}/sut-bootstrap.sh" "${SUT_PUBLIC_IP}" "${MODE}"
# 3. Run the loadgen, pulling JSON back into OUT_DIR.
@@ -78,7 +87,7 @@ if [[ "${SMOKE}" == "smoke" ]]; then
fi
fi
-OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}.json"
+OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}${async_suffix}.json"
"${SCRIPT_DIR}/loadgen-bootstrap.sh" \
"${LOADGEN_PUBLIC_IP}" \
"${SUT_PRIVATE_IP}" \
@@ -91,4 +100,4 @@ OUT_PATH="${OUT_DIR}/${TIER}-${MODE}-${SIGNAL}-${SCENARIO}.json"
"${extra_args[@]}"
# Trap handles teardown — no explicit call needed.
-echo "matrix entry ${TIER}-${MODE}-${SIGNAL}-${SCENARIO} complete -> ${OUT_PATH}" >&2
+echo "matrix entry ${TIER}-${MODE}-${SIGNAL}-${SCENARIO}${async_suffix} complete -> ${OUT_PATH}" >&2
diff --git a/benchmarks/scripts/sut-bootstrap.sh b/benchmarks/scripts/sut-bootstrap.sh
index a4118dbd..c5fa285a 100755
--- a/benchmarks/scripts/sut-bootstrap.sh
+++ b/benchmarks/scripts/sut-bootstrap.sh
@@ -85,8 +85,9 @@ EOF
compose_args=( --env-file benchmarks/compose/managed-ch.env "${compose_args[@]}" )
fi
-echo "bringing up compose stack (mode=${MODE}) on ${SUT_IP}" >&2
-bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 docker compose ${compose_args[*]} up -d --build"
+CH_ASYNC_INSERT_VAL="${CH_ASYNC_INSERT:-0}"
+echo "bringing up compose stack (mode=${MODE}, CH_ASYNC_INSERT=${CH_ASYNC_INSERT_VAL}) on ${SUT_IP}" >&2
+bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 CH_ASYNC_INSERT=${CH_ASYNC_INSERT_VAL} docker compose ${compose_args[*]} up -d --build"
echo "polling /health on ${SUT_IP}" >&2
deadline=$(( $(date +%s) + 600 )) # cold compose build can hit 10 min on small tiers