Skip to content

The only tool you need to know what is happening and how to fix it.

benchmarks (#139)

narrow-adventure47de8fd4e5e5
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.yml
new file mode 100644
index 00000000..dff51bb9
--- /dev/null
+++ b/.github/workflows/benchmark-hardware.yml
@@ -0,0 +1,138 @@
+name: benchmark-hardware
+
+# Manual-only: this workflow provisions and tears down Hetzner servers, which
+# costs real money (about €0.30/run today). Triggering must be deliberate.
+on:
+ workflow_dispatch:
+ inputs:
+ tiers:
+ description: 'Comma-separated Hetzner tiers (e.g. ccx13,ccx23,ccx33,ccx43)'
+ required: true
+ default: 'ccx13,ccx23,ccx33,ccx43'
+ modes:
+ description: 'Comma-separated DB modes (sqlite,pgch)'
+ required: true
+ default: 'sqlite,pgch'
+ duration:
+ description: 'Per-matrix-entry loadgen duration (e.g. 30m, 5m)'
+ required: true
+ default: '30m'
+ location:
+ description: 'Hetzner datacenter (nbg1, fsn1, hel1, ash, hil)'
+ required: true
+ default: 'nbg1'
+
+concurrency:
+ group: benchmark-hardware
+ cancel-in-progress: false
+
+permissions:
+ contents: write # aggregate job commits the results back to main
+
+jobs:
+ setup:
+ runs-on: ubuntu-latest
+ outputs:
+ matrix: ${{ steps.plan.outputs.matrix }}
+ steps:
+ - id: plan
+ name: Build matrix from inputs
+ run: |
+ set -euo pipefail
+ tiers="${{ github.event.inputs.tiers }}"
+ modes="${{ github.event.inputs.modes }}"
+ jq -nc \
+ --arg tiers "${tiers}" \
+ --arg modes "${modes}" \
+ '{include: [($tiers | split(",")) as $T | ($modes | split(",")) as $M | $T[] as $t | $M[] as $m | {tier: $t, mode: $m}]}' \
+ > matrix.json
+ cat matrix.json
+ echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"
+
+ run:
+ needs: setup
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ max-parallel: 2
+ matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
+ env:
+ HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+ BENCHMARK_SSH_KEY: ${{ runner.temp }}/benchmark-key
+ BENCH_LOCATION: ${{ github.event.inputs.location }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install hcloud CLI
+ run: |
+ curl -fsSL https://github.com/hetznercloud/cli/releases/download/v1.50.0/hcloud-linux-amd64.tar.gz \
+ | sudo tar -xz -C /usr/local/bin hcloud
+
+ - name: Install matplotlib (chart render is in aggregate job, but preflight checks it)
+ run: pip install --quiet matplotlib
+
+ - name: Materialize SSH key
+ run: |
+ umask 077
+ printf '%s\n' "${{ secrets.BENCHMARK_SSH_PRIVATE_KEY }}" > "${BENCHMARK_SSH_KEY}"
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: '1.25'
+
+ - name: Preflight
+ run: ./benchmarks/scripts/preflight.sh
+
+ - name: Run matrix entry (${{ matrix.tier }} / ${{ matrix.mode }})
+ run: ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ github.event.inputs.duration }}" benchmarks/results/_artifact
+
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: result-${{ matrix.tier }}-${{ matrix.mode }}
+ path: benchmarks/results/_artifact/*.json
+ if-no-files-found: warn
+
+ aggregate:
+ needs: run
+ if: always()
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ # Need history so the bot commit doesn't trample anything.
+ fetch-depth: 0
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ - uses: actions/download-artifact@v4
+ with:
+ path: /tmp/results-raw
+
+ - name: Stage into benchmarks/results/<date>/
+ id: stage
+ run: |
+ set -euo pipefail
+ date_tag="$(date -u +%Y-%m-%d)"
+ dst="benchmarks/results/${date_tag}"
+ mkdir -p "${dst}"
+ find /tmp/results-raw -name '*.json' -exec cp {} "${dst}/" \;
+ echo "dst=${dst}" >> "$GITHUB_OUTPUT"
+ ls -la "${dst}"
+
+ - name: Render charts
+ run: |
+ pip install --quiet matplotlib
+ python3 benchmarks/scripts/chart.py "${{ steps.stage.outputs.dst }}"
+
+ - name: Commit results
+ run: |
+ set -euo pipefail
+ git config user.name "traceway-bot"
+ git config user.email "[email protected]"
+ git add "${{ steps.stage.outputs.dst }}"
+ if git diff --cached --quiet; then
+ echo "nothing to commit"
+ exit 0
+ fi
+ git commit -m "bench: hardware benchmark $(date -u +%Y-%m-%d)"
+ git push
diff --git a/.gitignore b/.gitignore
index 125b3c2e..1702ee87 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,11 @@ backend/static/dist/
/examples/hono-otel/node_modules
/examples/nestjs-otel/node_modules
/website/.wrangler
+
+# Benchmark tooling: built binaries and laptop-only result dirs.
+# Real CI-produced result dirs (e.g. benchmarks/results/2026-05-12/) are
+# committed; the -local suffix marks dev runs we never want to push.
+/benchmarks/loadgen/loadgen
+/benchmarks/loadgen/loadgen-linux-amd64
+/benchmarks/loadgen/loadgen-linux-arm64
+/benchmarks/results/*-local/
diff --git a/benchmarks/LOCAL.md b/benchmarks/LOCAL.md
new file mode 100644
index 00000000..64fe6f75
--- /dev/null
+++ b/benchmarks/LOCAL.md
@@ -0,0 +1,360 @@
+# Running benchmarks locally
+
+Step-by-step recipe for running the loadgen against a backend on your laptop —
+no Hetzner, no GitHub Actions. Useful for: iterating on the loadgen itself,
+checking that a backend change didn't regress single-machine throughput, or
+just understanding the tooling before spending real money on Hetzner.
+
+> Laptop numbers are not comparable to Hetzner numbers. Use this for tooling
+> validation and relative regression checks, not for the published chart.
+
+---
+
+## Prerequisites
+
+Install once:
+
+```bash
+# macOS
+brew install docker jq go
+
+# Confirm Docker is running
+docker info >/dev/null && echo "docker: ok"
+```
+
+You also need Python 3 with matplotlib. The cleanest path on macOS (system
+Python is PEP-668 locked and refuses `pip install`):
+
+```bash
+# Pick one of:
+
+# Option A — venv in /tmp (wiped on restart, fine for ad-hoc smoke tests)
+python3 -m venv /tmp/bench-venv && /tmp/bench-venv/bin/pip install -q matplotlib
+
+# Option B — venv inside the repo (persists across restarts; gitignored)
+python3 -m venv .venv-bench && .venv-bench/bin/pip install -q matplotlib
+```
+
+The rest of this doc assumes option B, so swap `/tmp/bench-venv` for
+`.venv-bench` if you went with option A.
+
+---
+
+## Pick a backend mode
+
+Two compose stacks, each pinned to one of the project's existing Dockerfiles:
+
+| Mode | Compose file | Image | What it tests |
+|------|--------------|-------|---------------|
+| `sqlite` | `benchmarks/compose/docker-compose.sqlite.yml` | `Dockerfile.sqlite` | Single-binary backend with embedded SQLite. Fast to bring up, lower ceiling. |
+| `pgch` | `benchmarks/compose/docker-compose.pgch.yml` | `Dockerfile.minimal` + clickhouse + postgres | Full prod-shape stack. Slower first build, much higher ceiling. |
+
+Both expose port **8087** on the host (override with `BENCH_PORT=8088 docker
+compose -f ... up`).
+
+First-time builds:
+- `sqlite` mode: ~3–6 min (npm install + Go build).
+- `pgch` mode: ~6–10 min (above + ClickHouse + Postgres image pulls).
+
+Subsequent runs reuse the Docker layer cache and start in seconds.
+
+---
+
+## The 6-step run
+
+These are the exact commands. Open a terminal at the repo root:
+
+```bash
+cd /Users/dusanstanojevic/Documents/workspace/traceway
+```
+
+### 1. Bring up the backend on a clean volume
+
+```bash
+# Always tear down with -v first. Reusing a volume across runs means the next
+# loadgen run starts with old data, AND can hit "database disk image is
+# malformed" if the previous container was killed mid-write.
+docker compose -f benchmarks/compose/docker-compose.sqlite.yml down -v 2>/dev/null
+docker compose -f benchmarks/compose/docker-compose.sqlite.yml up -d --build
+```
+
+For `pgch` mode swap the filename to `docker-compose.pgch.yml` here and in the
+remaining steps.
+
+### 2. Wait for `/health` to return 200
+
+```bash
+until curl -sf http://localhost:8087/health >/dev/null; do echo "waiting..."; sleep 2; done
+echo "ready"
+```
+
+If this loops for more than ~3 min, something is wrong. Inspect:
+
+```bash
+docker compose -f benchmarks/compose/docker-compose.sqlite.yml logs --tail=80 traceway
+```
+
+### 3. Register a fresh user + project; capture creds
+
+```bash
+eval "$(benchmarks/scripts/seed-project.sh http://localhost:8087 \
+ | jq -r '"export JWT=\(.jwt) PROJECT_TOKEN=\(.projectToken) PROJECT_ID=\(.projectId)"')"
+echo "PROJECT_ID=$PROJECT_ID"
+```
+
+`seed-project.sh` hits `/api/register`. On self-hosted backends only one
+organization is allowed, so this only works on a **fresh** DB — step 1's `down
+-v` is what guarantees that.
+
+### 4. Run the loadgen
+
+Pick numbers appropriate for the mode you booted in step 1:
+
+**SQLite (laptop):**
+
+```bash
+mkdir -p /tmp/bench-localhost && rm -f /tmp/bench-localhost/*.json
+benchmarks/loadgen/loadgen \
+ --target http://localhost:8087 \
+ --token "$PROJECT_TOKEN" --jwt "$JWT" --project-id "$PROJECT_ID" \
+ --duration 5m --step-duration 45s \
+ --initial-eps 100 --max-eps 800 \
+ --report-out /tmp/bench-localhost/local-sqlite.json \
+ --tier local --mode sqlite
+```
+
+**pgch (laptop):**
+
+```bash
+benchmarks/loadgen/loadgen \
+ --target http://localhost:8087 \
+ --token "$PROJECT_TOKEN" --jwt "$JWT" --project-id "$PROJECT_ID" \
+ --duration 5m --step-duration 45s \
+ --initial-eps 500 --max-eps 4000 \
+ --report-out /tmp/bench-localhost/local-pgch.json \
+ --tier local --mode pgch
+```
+
+On stderr you'll see one line per step like:
+
+```
+step 1: target=300 actual=306 ingest_p99=92ms ingest_err=0.00% passed=true
+step 2: target=600 actual=600 ingest_p99=17ms ingest_err=0.00% passed=false read endpoint "endpoints_grouped" p99 = 3268ms > 3000ms threshold
+wrote /tmp/bench-localhost/local-sqlite.json: max sustainable EPS = 306 (2 step(s))
+```
+
+The run finishes when either (a) a step fails the breaking-point check, (b)
+`--max-eps` is exceeded, or (c) `--duration` expires.
+
+### 5. Render the chart + summary
+
+```bash
+.venv-bench/bin/python benchmarks/scripts/chart.py /tmp/bench-localhost/
+open /tmp/bench-localhost/chart.png
+open /tmp/bench-localhost/chart-detail.png
+cat /tmp/bench-localhost/summary.md
+```
+
+A one-entry result renders fine; it just won't have multiple bars/lines to
+compare against.
+
+### 6. Tear down
+
+```bash
+docker compose -f benchmarks/compose/docker-compose.sqlite.yml down -v
+```
+
+The `-v` deletes the named volume so the next run is guaranteed clean.
+
+---
+
+## Reading the output
+
+Each step record in the JSON looks like:
+
+```json
+{
+ "step": 1,
+ "targetEps": 300, // Synthetic event units the ramp asked for. NOT requests/sec — see "What an EPS is" below.
+ "actualEps": 306.6, // What the loadgen actually pushed on the wire (same unit).
+ "ingest": { // Latency/errors for /api/report + OTLP combined
+ "p50": 3.8, "p95": 12.5, "p99": 92.3,
+ "ok": 1380, "errors": 0, "errRate": 0
+ },
+ "reads": { // Per dashboard read endpoint
+ "endpoints_grouped": { "p50": 1120, "p95": 1462, "p99": 1509, "ok": 36, ... },
+ "dashboard_overview": { ... },
+ "exceptions_grouped": { ... },
+ "logs": { ... },
+ "session_recording": { ... }
+ },
+ "passed": true,
+ "failReason": "" // populated when passed=false
+}
+```
+
+**A step passes** when *all three* hold:
+1. Ingest error rate ≤ 5% (5xx, timeouts).
+2. Every read endpoint p99 < 3000 ms.
+3. SUT was reachable for the whole step.
+
+`maxSustainableEps` in the JSON is the `actualEps` of the **last passing
+step** — that's the headline number for this configuration.
+
+If `target` and `actual` diverge a lot on a passing step, the backend is
+already saturated even though it's not erroring yet. The next step usually
+fails.
+
+If reads degrade before ingest, that's expected: aggregation reads
+(`endpoints_grouped`, `dashboard_overview`) compete with write locks/IO and
+will be the first to cliff on SQLite. Point lookups (`logs`,
+`session_recording`, `exceptions_grouped`) typically stay fast right up until
+the SUT gives up.
+
+---
+
+## What an EPS is (and isn't)
+
+`EPS` in the loadgen is **synthetic event units**, not requests/sec and not
+DB rows/sec. The conversion:
+
+```
+requests/sec = EPS / 10 # because tracesPerFrame = 10
+```
+
+So `--max-eps 800` actually means "push up to 80 HTTP requests/sec." The
+factor-of-10 fudge keeps the rate-limiter math clean (each `/api/report`
+frame carries 10 traces; the loadgen counts EPS as traces).
+
+### What goes in each request
+
+Each "frame" the loadgen sends is one HTTP POST, routed by `--report-share`
+(default 0.7 → 70 % go to `/api/report`, 30 % to OTLP).
+
+**`/api/report` frame** — gzipped JSON, built by `buildFrame()` in
+`benchmarks/loadgen/report.go`:
+
+| Field | Count per request | Notes |
+|---|---|---|
+| `traces` | **10 (fixed)** | One row per `transactions` |
+| `spans` (nested in traces) | **20–50** (each trace gets 2–5) | Random duration 1–200 ms, name from `{db.query, http.call, cache.get, render, json.encode}` |
+| `metrics` | **5 (fixed)** | `bench.metric.{cpu,mem,qps,lat,errs}`, random value 0–100 |
+| `sessions` | 0 / 1 — 5 % with recording, 16 % session-only | Random uuid + StartedAt |
+| `sessionRecordings` | 0 / 1 — 5 % chance | ~100 KB synthetic rrweb-shaped blob |
+| `stackTraces` | 0 / 1 — 2 % chance | Hardcoded short stack trace |
+
+Trace fields: random endpoint from a fixed 13-item list (`GET /api/users`,
+`POST /api/orders`, `GET /api/products/:id`, …), duration 10–1000 ms, random
+status code, body size 200–8200 bytes.
+
+**OTLP frame** — protobuf, built by `sendOTLPTraces()` in
+`benchmarks/loadgen/otlp.go`:
+
+| Field | Count per request | Notes |
+|---|---|---|
+| `spans` | **20 (fixed, `otlpSpansPerRequest`)** | Random 16-byte traceId + 8-byte spanId, http.method=GET, status code, route from the same 13-endpoint pool |
+
+No metrics, sessions, or exceptions on this path.
+
+### Row growth per second at 400 EPS
+
+At 400 EPS the loadgen fires **40 requests/sec total**: ~28 to `/api/report`,
+~12 to OTLP. That fans out into:
+
+| Table | Rows/sec |
+|---|---|
+| `transactions` | ~280 (28 × 10) |
+| `spans` (from `/api/report`) | ~980 (28 × ~35) |
+| `spans` (from OTLP) | ~240 (12 × 20) |
+| `metric_points` | ~140 (28 × 5) |
+| `sessions` | ~6 (28 × 21 %) |
+| `session_recordings` | ~1.4 (28 × 5 %) |
+| `exception_stack_traces` | ~0.56 (28 × 2 %) |
+
+So **"400 EPS" ≈ 1,500 rows/sec** across the database, not 400. Keep this
+mapping in mind when comparing the chart's headline number to your real
+workload.
+
+### Data variety
+
+Everything is uniform random within bounded ranges — deliberately flat:
+
+- 13 distinct endpoint paths → `GROUP BY endpoint` aggregations always
+ produce ~13 groups
+- 5 span names, 5 metric names
+- All UUIDs are fresh per record (no hash clustering)
+- Durations: uniform 10–1000 ms → percentile queries hit a smooth
+ distribution
+- No diurnal traffic, no power-law endpoint skew, no correlated bursts
+
+This isolates the backend's raw throughput from traffic-shape noise. It is
+**not** representative of real production telemetry; don't read the numbers
+as "this is what users will experience" — they're "this is the floor the
+backend can sustain on synthetic, evenly-distributed load."
+
+---
+
+## Tuning the run
+
+| Flag | Default | When to change |
+|------|---------|----------------|
+| `--initial-eps` | 100 | Raise close to the expected ceiling so you don't waste the first few steps far below saturation. |
+| `--max-eps` | 50000 | Lower it to bound runtime; raise to find the ceiling on a beefier box. |
+| `--step-duration` | 2m | Shorten for smoke checks (45s), keep at 2m for real measurements. |
+| `--duration` | 30m | Hard cap on total wall time. |
+| `--report-share` | 0.7 | Fraction of ingest sent via `/api/report` vs OTLP. Lower it to test OTLP-heavy workloads. |
+| `--read-rps` | 1 | Read traffic per endpoint per second. Raise to model a busier dashboard. |
+| `--ingest-err-threshold` | 0.05 | Allowed ingest error rate before a step fails. |
+| `--read-p99-threshold-ms` | 3000 | The dashboard latency budget. |
+
+The ramp doubles target each step (100→200→400→800→…), so the cliff
+resolution is coarse by design — for a precise number, run a second pass with
+`--initial-eps` set near the known cliff and shorter steps.
+
+---
+
+## Common pitfalls
+
+- **`database disk image is malformed (11)`** in the container logs.
+ Previous run was killed mid-write and the SQLite WAL didn't recover. Run
+ step 1 again with `down -v`.
+
+- **`actual` is ~10× lower than `target` on every step.**
+ You're running an older loadgen binary. Rebuild:
+ ```bash
+ (cd benchmarks/loadgen && go build -o loadgen .)
+ ```
+
+- **`seed-project.sh` returns `409 Conflict: organization already exists`.**
+ The DB isn't fresh — go back to step 1 with `down -v`.
+
+- **Port 8087 already in use.**
+ Pick another: `BENCH_PORT=8088 docker compose -f ... up -d --build`, then
+ use `http://localhost:8088` everywhere.
+
+- **`/tmp/bench-localhost/` gone after a restart.**
+ macOS empties `/tmp` on reboot. Use `~/bench-localhost/` or a path inside
+ the repo if you want results to survive.
+
+- **Loadgen run was killed; some Docker resources still up.**
+ The compose stack is yours to manage — it won't auto-tear-down. Run step 6.
+
+---
+
+## Going beyond local
+
+Once the local run works end-to-end, move to the Hetzner ladder:
+
+```bash
+# Validate Hetzner credentials, no money spent
+./benchmarks/scripts/run-local.sh --dry-run
+
+# ~5 min, ~€0.02 — one tier, one mode, on real Hetzner
+./benchmarks/scripts/run-local.sh --smoke
+
+# Full matrix — 4 tiers x 2 modes, ~1 h, ~€1.20
+./benchmarks/scripts/run-local.sh
+```
+
+Prereqs and detail in [README.md](./README.md). The same `run-matrix-entry.sh`
+script powers both the laptop path and the GitHub Actions workflow.
diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 00000000..f6cd550d
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,172 @@
+# benchmarks/
+
+Hardware-vs-breaking-point benchmark for the Traceway backend. Provisions
+Hetzner Cloud servers, hammers the backend with realistic ingest while
+measuring read latency, and produces a chart of sustainable throughput per
+hardware tier and DB mode.
+
+## What it answers
+
+> On hardware tier **X** with DB config **Y**, you can sustain **N** traces +
+> spans + session-replays per second while reads remain responsive.
+
+Two DB modes are tested:
+- **sqlite** — single-binary Traceway with embedded SQLite (`Dockerfile.sqlite`).
+- **pgch** — full ClickHouse + Postgres stack (`Dockerfile.minimal`).
+
+Four hardware tiers, all Hetzner CCX (dedicated vCPU) so neighbor noise doesn't
+pollute the read-latency signal:
+
+| Tier | vCPU | RAM | Disk |
+|-------|------|-------|-------------|
+| CCX13 | 2 | 8 GB | 80 GB NVMe |
+| CCX23 | 4 | 16 GB | 160 GB NVMe |
+| CCX33 | 8 | 32 GB | 240 GB NVMe |
+| CCX43 | 16 | 64 GB | 360 GB NVMe |
+
+## How a run works
+
+Per matrix entry (one tier × one mode):
+
+1. `hetzner-up.sh` provisions a SUT box + a CAX11 loadgen box on a private network (override with `LOADGEN_TIER`).
+2. `sut-bootstrap.sh` rsyncs the repo, installs Docker, brings up the right
+ `docker-compose.<mode>.yml`, waits for `/health`.
+3. `seed-project.sh` registers a fresh user + project, captures the JWT and
+ project token.
+4. `loadgen-bootstrap.sh` cross-compiles the loadgen for linux/amd64, pushes
+ it to the loadgen box, runs it against the SUT's *private* IP.
+5. The loadgen ramps target ingest in doubling steps (100 -> 200 -> 400 -> ...)
+ while a constant read mix probes the dashboard endpoints. Each step holds
+ for `--step-duration` (default 2 min).
+6. A step "fails" when ingest error rate > 5%, OR any read endpoint P99 > 3000ms.
+ Max sustainable EPS is the actual rate of the last passing step.
+7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.
+
+After all matrix entries finish, `chart.py` renders `chart.png` (bar) +
+`chart-detail.png` (line: read P99 vs ingest EPS) + `summary.md`.
+
+## Running from your laptop
+
+### One-time setup
+
+1. **Install tooling**: `hcloud`, `jq`, `python3`, Go 1.25+, `rsync`. On macOS:
+ `brew install hcloud jq rsync go`.
+2. **Install matplotlib in a venv** (system Python is usually PEP 668 locked):
+ ```bash
+ python3 -m venv .venv
+ source .venv/bin/activate
+ pip install matplotlib
+ ```
+3. **Generate an SSH key** specifically for benchmarks and upload its public
+ half to Hetzner under the name `benchmark-key`:
+ ```bash
+ ssh-keygen -t ed25519 -f ~/.ssh/hetzner_benchmark -C benchmark-key
+ chmod 600 ~/.ssh/hetzner_benchmark
+ hcloud ssh-key create --name benchmark-key --public-key-from-file ~/.ssh/hetzner_benchmark.pub
+ ```
+4. **Export creds** (use `direnv` or a sourced `.envrc.local`, never commit):
+ ```bash
+ export HCLOUD_TOKEN=...
+ export BENCHMARK_SSH_KEY=~/.ssh/hetzner_benchmark
+ ```
+
+### Smoke (cheap — ~5 min, ~€0.02)
+
+```bash
+./benchmarks/scripts/run-local.sh --smoke
+```
+
+One tier (ccx13), one mode (sqlite), short steps. If this works the full
+matrix works.
+
+### Full matrix (~1 h, ~€1.20)
+
+```bash
+./benchmarks/scripts/run-local.sh
+```
+
+4 tiers × 2 modes, default `--duration 30m`. Output goes to
+`benchmarks/results/<YYYY-MM-DD>-local/` — the `-local` suffix exists so
+dev runs never get confused with CI runs and never get accidentally committed.
+
+### Other useful invocations
+
+```bash
+# Validate environment without provisioning anything (free)
+./benchmarks/scripts/run-local.sh --dry-run
+
+# Re-run just one tier with default duration
+./benchmarks/scripts/run-local.sh --tier ccx23 --mode pgch
+
+# Override the per-entry runtime
+./benchmarks/scripts/run-local.sh --tier ccx13 --duration 10m
+
+# Drop the "-local" suffix so the result dir matches the CI naming convention
+# (use this only when you want to commit a one-off re-benchmark)
+./benchmarks/scripts/run-local.sh --tier ccx23 --commit
+```
+
+## Running from GitHub Actions
+
+`.github/workflows/benchmark-hardware.yml`, `workflow_dispatch` only. Inputs
+mirror the local flags. The workflow YAML is a thin wrapper around the same
+`run-matrix-entry.sh` script the local path uses — if it works locally, it
+works in CI.
+
+Required GitHub secrets:
+- `HCLOUD_TOKEN`
+- `BENCHMARK_SSH_PRIVATE_KEY` — the private key matching the Hetzner-side
+ `benchmark-key`.
+
+After the matrix completes, an `aggregate` job downloads all artifacts, runs
+`chart.py`, and commits `benchmarks/results/<YYYY-MM-DD>/` to `main` via a bot
+commit. No PR — it's a generated artifact.
+
+## Layout
+
+```
+benchmarks/
+ compose/ # SUT-side docker compose, one per mode
+ docker-compose.sqlite.yml
+ docker-compose.pgch.yml
+ loadgen/ # The Go binary that generates load
+ main.go # CLI + orchestration
+ ingest.go # Stream A — stepped ingest worker pool
+ report.go # /api/report JSON+gzip sender
+ otlp.go # OTLP/HTTP protobuf sender
+ reads.go # Stream B — constant read mix
+ ramp.go # Step controller + breaking-point logic
+ stats.go # Latency tracker (percentiles via sort)
+ util.go, log.go # Misc helpers
+ scripts/
+ run-local.sh # ★ Laptop entry point
+ run-matrix-entry.sh # One matrix cycle (used by local + CI)
+ preflight.sh # Validates env before any provisioning
+ hetzner-up.sh # hcloud server create
+ hetzner-down.sh # hcloud server delete (idempotent)
+ sut-bootstrap.sh # Installs Docker, brings up backend
+ loadgen-bootstrap.sh # Cross-compiles + runs loadgen
+ seed-project.sh # /api/register -> JWT + project token JSON
+ chart.py # matplotlib renderer
+ _ssh.sh # Shared ssh/rsync helpers
+ results/ # Committed bench results live here
+```
+
+## Failure modes & debugging
+
+- **A run leaves Hetzner servers behind.** `hetzner-down.sh` is idempotent and
+ callable directly: `./benchmarks/scripts/hetzner-down.sh <run-id>`. Run IDs
+ appear in the orchestrator's stderr at the start of each matrix entry. Also
+ visible in `hcloud server list` — anything tagged `bench=true` is safe to
+ delete.
+- **`preflight.sh` complains about `benchmark-key`.** The Hetzner-side SSH key
+ resource is named `benchmark-key` regardless of what you named the file
+ locally. Re-upload via `hcloud ssh-key create --name benchmark-key
+ --public-key-from-file <your-pub-key>`.
+- **Docker compose build is slow on tiny tiers.** First-time `docker compose
+ up --build` on a CCX13 takes 5–8 minutes (npm install + Go build).
+ `sut-bootstrap.sh` waits up to 10 minutes for `/health` before giving up.
+ Subsequent runs reuse the Docker layer cache via the persistent volumes.
+- **All steps pass on the highest tier.** Increase `--max-eps` (default 50000)
+ if you want to find the ceiling on big boxes. The default is meant to keep
+ one-tier runs bounded; on CCX43 with PG/CH the ceiling can be well above 10k EPS.
diff --git a/benchmarks/compose/docker-compose.pgch.yml b/benchmarks/compose/docker-compose.pgch.yml
new file mode 100644
index 00000000..87e55be4
--- /dev/null
+++ b/benchmarks/compose/docker-compose.pgch.yml
@@ -0,0 +1,72 @@
+services:
+ traceway:
+ build:
+ context: ../..
+ dockerfile: Dockerfile.minimal
+ ports:
+ - "${BENCH_PORT:-8087}:80"
+ environment:
+ CLICKHOUSE_SERVER: "clickhouse:9000"
+ CLICKHOUSE_DATABASE: "traceway"
+ CLICKHOUSE_USERNAME: "default"
+ CLICKHOUSE_PASSWORD: "clickhouse"
+ CLICKHOUSE_TLS: "false"
+ POSTGRES_HOST: "postgres"
+ POSTGRES_PORT: "5432"
+ POSTGRES_DATABASE: "traceway"
+ POSTGRES_USERNAME: "traceway"
+ POSTGRES_PASSWORD: "traceway"
+ POSTGRES_SSLMODE: "disable"
+ JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"
+ GIN_MODE: "release"
+ depends_on:
+ clickhouse:
+ condition: service_healthy
+ postgres:
+ condition: service_healthy
+ restart: "no"
+ healthcheck:
+ test: ["CMD", "wget", "-qO-", "http://localhost/health"]
+ interval: 5s
+ timeout: 3s
+ start_period: 10s
+ retries: 20
+
+ clickhouse:
+ image: clickhouse/clickhouse-server:24.8-alpine
+ environment:
+ CLICKHOUSE_DB: traceway
+ CLICKHOUSE_PASSWORD: clickhouse
+ volumes:
+ - clickhouse-bench-data:/var/lib/clickhouse
+ healthcheck:
+ test: ["CMD", "clickhouse-client", "--password", "clickhouse", "--query", "SELECT 1"]
+ interval: 5s
+ timeout: 3s
+ start_period: 10s
+ retries: 20
+ restart: "no"
+ ulimits:
+ nofile:
+ soft: 262144
+ hard: 262144
+
+ postgres:
+ image: postgres:17
+ environment:
+ POSTGRES_USER: traceway
+ POSTGRES_PASSWORD: traceway
+ POSTGRES_DB: traceway
+ volumes:
+ - postgres-bench-data:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U traceway"]
+ interval: 5s
+ timeout: 3s
+ start_period: 10s
+ retries: 20
+ restart: "no"
+
+volumes:
+ clickhouse-bench-data:
+ postgres-bench-data:
diff --git a/benchmarks/compose/docker-compose.sqlite.yml b/benchmarks/compose/docker-compose.sqlite.yml
new file mode 100644
index 00000000..a140b529
--- /dev/null
+++ b/benchmarks/compose/docker-compose.sqlite.yml
@@ -0,0 +1,24 @@
+services:
+ traceway:
+ build:
+ context: ../..
+ dockerfile: Dockerfile.sqlite
+ ports:
+ - "${BENCH_PORT:-8087}:80"
+ environment:
+ JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"
+ SQLITE_RETENTION_DAYS: "0"
+ SESSION_RECORDING_RETENTION_DAYS: "0"
+ GIN_MODE: "release"
+ volumes:
+ - traceway-bench-sqlite-data:/data
+ restart: "no"
+ healthcheck:
+ test: ["CMD", "wget", "-qO-", "http://localhost/health"]
+ interval: 5s
+ timeout: 3s
+ start_period: 10s
+ retries: 20
+
+volumes:
+ traceway-bench-sqlite-data:
diff --git a/benchmarks/loadgen/go.mod b/benchmarks/loadgen/go.mod
new file mode 100644
index 00000000..7b72cd8a
--- /dev/null
+++ b/benchmarks/loadgen/go.mod
@@ -0,0 +1,20 @@
+module bench-loadgen
+
+go 1.25
+
+require (
+ github.com/google/uuid v1.6.0
+ go.opentelemetry.io/proto/otlp v1.5.0
+ golang.org/x/time v0.7.0
+ google.golang.org/protobuf v1.36.1
+)
+
+require (
+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
+ golang.org/x/net v0.33.0 // indirect
+ golang.org/x/sys v0.29.0 // indirect
+ golang.org/x/text v0.21.0 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
+ google.golang.org/grpc v1.69.2 // indirect
+)
diff --git a/benchmarks/loadgen/go.sum b/benchmarks/loadgen/go.sum
new file mode 100644
index 00000000..bd2d2ffe
--- /dev/null
+++ b/benchmarks/loadgen/go.sum
@@ -0,0 +1,40 @@
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ=
+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
+go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
+go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
+go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
+go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA=
+google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
+google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
+google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
+google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
+google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
diff --git a/benchmarks/loadgen/ingest.go b/benchmarks/loadgen/ingest.go
new file mode 100644
index 00000000..7adfc625
--- /dev/null
+++ b/benchmarks/loadgen/ingest.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+ "context"
+ "math/rand"
+ "net/http"
+ "sync"
+ "sync/atomic"
+
+ "golang.org/x/time/rate"
+)
+
+const (
+ // One frame carries 10 traces, so frame rate = targetEPS / tracesPerFrame.
+ // Matches Go SDK behavior where frames rotate every ~5s with whatever has
+ // accumulated; benchmark uses a fixed batch size for predictable accounting.
+ tracesPerFrame = 10
+ ingestWorkers = 256
+ recordingChance = 0.05 // 5% of frames carry a session recording segment
+ exceptionChance = 0.02 // 2% of frames carry an exception
+)
+
+type ingester struct {
+ cfg config
+ client *http.Client
+ stats *latencyTracker
+ limiter *rate.Limiter
+ // rng pool: each worker grabs one to avoid contending on the global source
+ rngPool sync.Pool
+ // sessionsCreated counts sessions sent via /api/report so the reader can
+ // query a real session ID for the recording playback endpoint.
+ sessionsCreated atomic.Int64
+ sampleSession atomic.Value // string — most recently seeded session id
+}
+
+func newIngester(cfg config, client *http.Client, stats *latencyTracker) *ingester {
+ framesPerSec := cfg.initialEPS / tracesPerFrame
+ if framesPerSec < 1 {
+ framesPerSec = 1
+ }
+ return &ingester{
+ cfg: cfg,
+ client: client,
+ stats: stats,
+ limiter: rate.NewLimiter(rate.Limit(framesPerSec), int(framesPerSec)+1),
+ rngPool: sync.Pool{New: func() any { return rand.New(rand.NewSource(rand.Int63())) }},
+ }
+}
+
+func (i *ingester) SetRate(eps float64) {
+ fps := eps / tracesPerFrame
+ if fps < 1 {
+ fps = 1
+ }
+ i.limiter.SetLimit(rate.Limit(fps))
+ i.limiter.SetBurst(int(fps) + 1)
+}
+
+func (i *ingester) SampleSessionId() string {
+ v := i.sampleSession.Load()
+ if v == nil {
+ return ""
+ }
+ return v.(string)
+}
+
+func (i *ingester) run(ctx context.Context) {
+ var wg sync.WaitGroup
+ wg.Add(ingestWorkers)
+ for w := 0; w < ingestWorkers; w++ {
+ go func() {
+ defer wg.Done()
+ rng := i.rngPool.Get().(*rand.Rand)
+ defer i.rngPool.Put(rng)
+ for {
+ if err := i.limiter.Wait(ctx); err != nil {
+ return
+ }
+ i.sendOne(ctx, rng)
+ }
+ }()
+ }
+ wg.Wait()
+}
+
+func (i *ingester) sendOne(ctx context.Context, rng *rand.Rand) {
+ if rng.Float64() < i.cfg.reportShare {
+ sessionId := sendReportFrame(ctx, i.client, i.cfg, rng, i.stats)
+ if sessionId != "" {
+ i.sampleSession.Store(sessionId)
+ i.sessionsCreated.Add(1)
+ }
+ } else {
+ sendOTLPTraces(ctx, i.client, i.cfg, rng, i.stats)
+ }
+}
diff --git a/benchmarks/loadgen/log.go b/benchmarks/loadgen/log.go
new file mode 100644
index 00000000..652c07e6
--- /dev/null
+++ b/benchmarks/loadgen/log.go
@@ -0,0 +1,25 @@
+package main
+
+import (
+ "io"
+ "os"
+ "sync"
+)
+
+var stderrMu sync.Mutex
+
+// stderrPrefix returns a single shared writer for human-readable progress logs.
+// Wrapping os.Stderr through a tiny adapter so concurrent goroutines never
+// interleave half a line — important because the ramp loop and ingest worker
+// pool both want to print status updates while running.
+func stderrPrefix() io.Writer {
+ return &lockedStderr{}
+}
+
+type lockedStderr struct{}
+
+func (l *lockedStderr) Write(p []byte) (int, error) {
+ stderrMu.Lock()
+ defer stderrMu.Unlock()
+ return os.Stderr.Write(p)
+}
diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.go
new file mode 100644
index 00000000..6fa92e7f
--- /dev/null
+++ b/benchmarks/loadgen/main.go
@@ -0,0 +1,114 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+)
+
+type config struct {
+ target string
+ projectToken string
+ jwt string
+ projectId string
+ duration time.Duration
+ stepDuration time.Duration
+ initialEPS float64
+ maxEPS float64
+ reportShare float64
+ readRPS float64
+ ingestErrThreshold float64
+ readP99ThresholdMs float64
+ reportOut string
+ tier string
+ mode string
+}
+
+func main() {
+ var cfg config
+
+ flag.StringVar(&cfg.target, "target", "", "Base URL of the system under test (e.g. http://10.0.0.2 or http://localhost:8087)")
+ flag.StringVar(&cfg.projectToken, "token", "", "Project bearer token for /api/report + OTLP")
+ flag.StringVar(&cfg.jwt, "jwt", "", "JWT for dashboard read endpoints")
+ flag.StringVar(&cfg.projectId, "project-id", "", "Project UUID (passed as ?projectId= on read endpoints)")
+ flag.DurationVar(&cfg.duration, "duration", 30*time.Minute, "Total run duration cap")
+ flag.DurationVar(&cfg.stepDuration, "step-duration", 2*time.Minute, "Per-step hold time")
+ flag.Float64Var(&cfg.initialEPS, "initial-eps", 100, "Starting target traces/sec")
+ flag.Float64Var(&cfg.maxEPS, "max-eps", 50000, "Hard cap on target traces/sec")
+ flag.Float64Var(&cfg.reportShare, "report-share", 0.7, "Fraction of ingest sent via /api/report (rest via OTLP)")
+ flag.Float64Var(&cfg.readRPS, "read-rps", 1.0, "Read RPS per endpoint")
+ flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if ingest error rate exceeds this")
+ flag.Float64Var(&cfg.readP99ThresholdMs, "read-p99-threshold-ms", 3000, "Step fails if any read endpoint p99 (ms) exceeds this")
+ flag.StringVar(&cfg.reportOut, "report-out", "", "Path to write JSON results (required)")
+ flag.StringVar(&cfg.tier, "tier", "local", "Hardware tier label embedded in output (e.g. ccx13)")
+ flag.StringVar(&cfg.mode, "mode", "unknown", "DB mode label embedded in output (sqlite | pgch)")
+ flag.Parse()
+
+ if cfg.target == "" || cfg.projectToken == "" || cfg.reportOut == "" {
+ fmt.Fprintln(os.Stderr, "missing required flag: --target, --token, --report-out")
+ flag.Usage()
+ os.Exit(2)
+ }
+
+ ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+ defer cancel()
+
+ deadline := time.Now().Add(cfg.duration)
+ ctx, cancelDeadline := context.WithDeadline(ctx, deadline)
+ defer cancelDeadline()
+
+ httpClient := &http.Client{
+ Timeout: 15 * time.Second,
+ Transport: &http.Transport{
+ MaxIdleConns: 200,
+ MaxIdleConnsPerHost: 100,
+ IdleConnTimeout: 60 * time.Second,
+ },
+ }
+
+ ingestStats := newLatencyTracker()
+ readStats := map[string]*latencyTracker{
+ "endpoints_grouped": newLatencyTracker(),
+ "exceptions_grouped": newLatencyTracker(),
+ "dashboard_overview": newLatencyTracker(),
+ "session_recording": newLatencyTracker(),
+ "logs": newLatencyTracker(),
+ }
+
+ ingester := newIngester(cfg, httpClient, ingestStats)
+ reader := newReader(cfg, httpClient, readStats)
+ reader.attachIngester(ingester)
+
+ go ingester.run(ctx)
+ go reader.run(ctx)
+
+ results := runRamp(ctx, cfg, ingester, ingestStats, readStats)
+
+ out := finalReport{
+ Tier: cfg.tier,
+ Mode: cfg.mode,
+ StartedAt: time.Now().UTC().Add(-cfg.duration).Format(time.RFC3339),
+ EndedAt: time.Now().UTC().Format(time.RFC3339),
+ Steps: results,
+ }
+ out.computeBreakingPoint()
+
+ f, err := os.Create(cfg.reportOut)
+ if err != nil {
+ log.Fatalf("create %s: %v", cfg.reportOut, err)
+ }
+ defer f.Close()
+ enc := json.NewEncoder(f)
+ enc.SetIndent("", " ")
+ if err := enc.Encode(&out); err != nil {
+ log.Fatalf("write %s: %v", cfg.reportOut, err)
+ }
+ fmt.Fprintf(os.Stderr, "wrote %s: max sustainable EPS = %.0f (%d step(s))\n", cfg.reportOut, out.MaxSustainableEPS, len(out.Steps))
+}
diff --git a/benchmarks/loadgen/otlp.go b/benchmarks/loadgen/otlp.go
new file mode 100644
index 00000000..4f3933e3
--- /dev/null
+++ b/benchmarks/loadgen/otlp.go
@@ -0,0 +1,107 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "fmt"
+ "net/http"
+ mathrand "math/rand"
+ "time"
+
+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"
+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"
+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
+ "google.golang.org/protobuf/proto"
+)
+
+const otlpSpansPerRequest = 20
+
+// sendOTLPTraces marshals an ExportTraceServiceRequest with otlpSpansPerRequest
+// spans into protobuf, POSTs to /api/otel/v1/traces, and records latency.
+// Protobuf (not protojson) because that's what production exporters send — it
+// exercises the backend's protobuf decode path which is non-trivial CPU work
+// that would be hidden if we sent JSON.
+func sendOTLPTraces(ctx context.Context, client *http.Client, cfg config, rng *mathrand.Rand, stats *latencyTracker) {
+ now := time.Now().UTC()
+ resourceAttrs := []*commonpb.KeyValue{
+ strAttr("service.name", "bench-loadgen"),
+ strAttr("service.version", "1.0.0"),
+ strAttr("deployment.environment", "bench"),
+ }
+
+ spans := make([]*tracepb.Span, otlpSpansPerRequest)
+ for i := range spans {
+ traceId := make([]byte, 16)
+ spanId := make([]byte, 8)
+ _, _ = rand.Read(traceId)
+ _, _ = rand.Read(spanId)
+
+ startNs := now.Add(-time.Duration(rng.Intn(500)) * time.Millisecond).UnixNano()
+ duration := time.Duration(10+rng.Intn(990)) * time.Millisecond
+ endNs := startNs + int64(duration)
+
+ spans[i] = &tracepb.Span{
+ TraceId: traceId,
+ SpanId: spanId,
+ Name: endpointPaths[rng.Intn(len(endpointPaths))],
+ Kind: tracepb.Span_SPAN_KIND_SERVER,
+ StartTimeUnixNano: uint64(startNs),
+ EndTimeUnixNano: uint64(endNs),
+ Attributes: []*commonpb.KeyValue{
+ strAttr("http.method", "GET"),
+ intAttr("http.status_code", int64(pickStatus(rng))),
+ strAttr("http.route", endpointPaths[rng.Intn(len(endpointPaths))]),
+ },
+ Status: &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK},
+ }
+ }
+
+ req := &coltracepb.ExportTraceServiceRequest{
+ ResourceSpans: []*tracepb.ResourceSpans{{
+ Resource: &resourcepb.Resource{Attributes: resourceAttrs},
+ ScopeSpans: []*tracepb.ScopeSpans{{
+ Scope: &commonpb.InstrumentationScope{Name: "bench-loadgen", Version: "1.0.0"},
+ Spans: spans,
+ }},
+ }},
+ }
+
+ body, err := proto.Marshal(req)
+ if err != nil {
+ stats.Record(0, err)
+ return
+ }
+
+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.target+"/api/otel/v1/traces", bytes.NewReader(body))
+ if err != nil {
+ stats.Record(0, err)
+ return
+ }
+ httpReq.Header.Set("Content-Type", "application/x-protobuf")
+ httpReq.Header.Set("Authorization", "Bearer "+cfg.projectToken)
+
+ start := time.Now()
+ resp, err := client.Do(httpReq)
+ elapsed := time.Since(start)
+ if err != nil {
+ stats.Record(elapsed.Seconds()*1000, err)
+ return
+ }
+ defer resp.Body.Close()
+ _, _ = readAndDiscard(resp.Body)
+ if resp.StatusCode >= 400 {
+ stats.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))
+ return
+ }
+ stats.Record(elapsed.Seconds()*1000, nil)
+}
+
+func strAttr(k, v string) *commonpb.KeyValue {
+ return &commonpb.KeyValue{Key: k, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: v}}}
+}
+
+func intAttr(k string, v int64) *commonpb.KeyValue {
+ return &commonpb.KeyValue{Key: k, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: v}}}
+}
diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.go
new file mode 100644
index 00000000..6d5ac620
--- /dev/null
+++ b/benchmarks/loadgen/ramp.go
@@ -0,0 +1,124 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "time"
+)
+
+type stepResult struct {
+ Step int `json:"step"`
+ TargetEPS float64 `json:"targetEps"`
+ ActualEPS float64 `json:"actualEps"`
+ Ingest latencySnapshot `json:"ingest"`
+ Reads map[string]latencySnapshot `json:"reads"`
+ Passed bool `json:"passed"`
+ FailReason string `json:"failReason,omitempty"`
+}
+
+type finalReport struct {
+ Tier string `json:"tier"`
+ Mode string `json:"mode"`
+ StartedAt string `json:"startedAt"`
+ EndedAt string `json:"endedAt"`
+ MaxSustainableEPS float64 `json:"maxSustainableEps"`
+ Steps []stepResult `json:"steps"`
+}
+
+func (r *finalReport) computeBreakingPoint() {
+ for i := len(r.Steps) - 1; i >= 0; i-- {
+ if r.Steps[i].Passed {
+ r.MaxSustainableEPS = r.Steps[i].ActualEPS
+ return
+ }
+ }
+}
+
+// runRamp drives the EPS ramp: hold a target rate for stepDuration, snapshot
+// stats, evaluate the breaking-point criteria, and either advance or stop.
+// Doubling cadence (100 -> 200 -> 400 -> ...) gets us to the cliff in <10 steps
+// for any realistic SUT, keeping total runtime bounded.
+func runRamp(
+ ctx context.Context,
+ cfg config,
+ ingester *ingester,
+ ingest *latencyTracker,
+ reads map[string]*latencyTracker,
+) []stepResult {
+ results := []stepResult{}
+ target := cfg.initialEPS
+
+ for stepNo := 1; target <= cfg.maxEPS; stepNo++ {
+ ingester.SetRate(target)
+
+ // Reset trackers at start of step so we measure only the new rate's
+ // steady state, not the ramp-up transient from the previous step.
+ ingest.SnapshotAndReset()
+ for _, t := range reads {
+ t.SnapshotAndReset()
+ }
+
+ stepCtx, cancel := context.WithTimeout(ctx, cfg.stepDuration)
+ start := time.Now()
+ <-stepCtx.Done()
+ cancel()
+ elapsed := time.Since(start)
+
+ ingestSnap := ingest.SnapshotAndReset()
+ readSnaps := map[string]latencySnapshot{}
+ for name, t := range reads {
+ readSnaps[name] = t.SnapshotAndReset()
+ }
+
+ // Both target and actual must be in the same unit (traces/sec).
+ // Rate limiter ticks per request (= per frame), so requests/sec × the
+ // batch size gives the trace throughput on the wire. OTLP requests
+ // carry spans rather than traces; we count them at the same rate
+ // because the user-facing question is "events/sec the system swallowed"
+ // regardless of which protocol carried them.
+ actualEPS := 0.0
+ if elapsed > 0 {
+ actualEPS = float64(ingestSnap.OK+ingestSnap.Errors) / elapsed.Seconds() * tracesPerFrame
+ }
+
+ passed, reason := evaluate(cfg, ingestSnap, readSnaps)
+
+ res := stepResult{
+ Step: stepNo,
+ TargetEPS: target,
+ ActualEPS: actualEPS,
+ Ingest: ingestSnap,
+ Reads: readSnaps,
+ Passed: passed,
+ FailReason: reason,
+ }
+ results = append(results, res)
+ fmt.Fprintf(stderrPrefix(), "step %d: target=%.0f actual=%.0f ingest_p99=%.0fms ingest_err=%.2f%% passed=%t %s\n",
+ stepNo, target, actualEPS, ingestSnap.P99, ingestSnap.ErrRate*100, passed, reason)
+
+ if !passed || ctx.Err() != nil {
+ return results
+ }
+
+ target *= 2
+ }
+ return results
+}
+
+func evaluate(cfg config, ingest latencySnapshot, reads map[string]latencySnapshot) (bool, string) {
+ if ingest.ErrRate > cfg.ingestErrThreshold {
+ return false, fmt.Sprintf("ingest error rate %.2f%% > %.2f%% threshold", ingest.ErrRate*100, cfg.ingestErrThreshold*100)
+ }
+ for name, s := range reads {
+ // If a read endpoint never returned a single OK response during this
+ // step the SUT is effectively unavailable for reads; treat that as a
+ // hard fail even though P99 reads as 0 from the empty histogram.
+ if s.OK == 0 && s.Errors > 0 {
+ return false, fmt.Sprintf("read endpoint %q served zero OK responses", name)
+ }
+ if s.P99 > cfg.readP99ThresholdMs {
+ return false, fmt.Sprintf("read endpoint %q p99 = %.0fms > %.0fms threshold", name, s.P99, cfg.readP99ThresholdMs)
+ }
+ }
+ return true, ""
+}
diff --git a/benchmarks/loadgen/reads.go b/benchmarks/loadgen/reads.go
new file mode 100644
index 00000000..d77cb6ec
--- /dev/null
+++ b/benchmarks/loadgen/reads.go
@@ -0,0 +1,204 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "sync"
+ "time"
+)
+
+type reader struct {
+ cfg config
+ client *http.Client
+ stats map[string]*latencyTracker
+ ingester *ingester
+}
+
+func newReader(cfg config, client *http.Client, stats map[string]*latencyTracker) *reader {
+ return &reader{cfg: cfg, client: client, stats: stats}
+}
+
+// attachIngester gives the reader access to a sample session id so it can hit
+// /api/sessions/:id/recording with an id that actually exists.
+func (r *reader) attachIngester(i *ingester) { r.ingester = i }
+
+type readSpec struct {
+ name string
+ method string
+ path string
+ body func() any
+}
+
+func (r *reader) specs() []readSpec {
+ last24h := func() (string, string) {
+ now := time.Now().UTC()
+ return now.Add(-24 * time.Hour).Format(time.RFC3339), now.Format(time.RFC3339)
+ }
+ last1h := func() (string, string) {
+ now := time.Now().UTC()
+ return now.Add(-1 * time.Hour).Format(time.RFC3339), now.Format(time.RFC3339)
+ }
+ return []readSpec{
+ {
+ name: "endpoints_grouped", method: "POST", path: "/api/endpoints/grouped",
+ body: func() any {
+ f, t := last24h()
+ return map[string]any{
+ "fromDate": f,
+ "toDate": t,
+ "orderBy": "count",
+ "sortDirection": "desc",
+ "pagination": map[string]int{"page": 1, "pageSize": 50},
+ "search": "",
+ }
+ },
+ },
+ {
+ name: "exceptions_grouped", method: "POST", path: "/api/exception-stack-traces",
+ body: func() any {
+ f, t := last24h()
+ return map[string]any{
+ "fromDate": f,
+ "toDate": t,
+ "orderBy": "count desc",
+ "pagination": map[string]int{"page": 1, "pageSize": 50},
+ "search": "",
+ "searchType": "all",
+ "includeArchived": false,
+ }
+ },
+ },
+ {name: "dashboard_overview", method: "GET", path: "/api/dashboard/overview"},
+ {name: "logs", method: "POST", path: "/api/logs", body: func() any {
+ f, t := last1h()
+ return map[string]any{
+ "fromDate": f,
+ "toDate": t,
+ "orderBy": "timestamp desc",
+ "pagination": map[string]int{"page": 1, "pageSize": 50},
+ }
+ }},
+ // session_recording handled separately because the URL depends on a
+ // live session id surfaced by the ingester.
+ }
+}
+
+func (r *reader) run(ctx context.Context) {
+ var wg sync.WaitGroup
+ for _, spec := range r.specs() {
+ spec := spec
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ r.loopSpec(ctx, spec)
+ }()
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ r.loopRecording(ctx)
+ }()
+ wg.Wait()
+}
+
+func (r *reader) loopSpec(ctx context.Context, spec readSpec) {
+ interval := time.Duration(float64(time.Second) / r.cfg.readRPS)
+ if interval < 10*time.Millisecond {
+ interval = 10 * time.Millisecond
+ }
+ t := time.NewTicker(interval)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ r.doOne(ctx, spec)
+ }
+ }
+}
+
+func (r *reader) loopRecording(ctx context.Context) {
+ interval := time.Duration(float64(time.Second) / r.cfg.readRPS)
+ if interval < 10*time.Millisecond {
+ interval = 10 * time.Millisecond
+ }
+ t := time.NewTicker(interval)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ sid := ""
+ if r.ingester != nil {
+ sid = r.ingester.SampleSessionId()
+ }
+ if sid == "" {
+ // Nothing seeded yet — quietly skip; don't record as ok or err.
+ continue
+ }
+ r.doOne(ctx, readSpec{
+ name: "session_recording",
+ method: "GET",
+ path: "/api/sessions/" + sid + "/recording",
+ })
+ }
+ }
+}
+
+func (r *reader) doOne(ctx context.Context, spec readSpec) {
+ tracker, ok := r.stats[spec.name]
+ if !ok {
+ return
+ }
+
+ u, err := url.Parse(r.cfg.target + spec.path)
+ if err != nil {
+ tracker.Record(0, err)
+ return
+ }
+ q := u.Query()
+ q.Set("projectId", r.cfg.projectId)
+ u.RawQuery = q.Encode()
+
+ var body []byte
+ if spec.body != nil {
+ body, err = json.Marshal(spec.body())
+ if err != nil {
+ tracker.Record(0, err)
+ return
+ }
+ }
+
+ req, err := http.NewRequestWithContext(ctx, spec.method, u.String(), bytes.NewReader(body))
+ if err != nil {
+ tracker.Record(0, err)
+ return
+ }
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ if r.cfg.jwt != "" {
+ req.Header.Set("Authorization", "Bearer "+r.cfg.jwt)
+ }
+
+ start := time.Now()
+ resp, err := r.client.Do(req)
+ elapsed := time.Since(start)
+ if err != nil {
+ tracker.Record(elapsed.Seconds()*1000, err)
+ return
+ }
+ defer resp.Body.Close()
+ _, _ = readAndDiscard(resp.Body)
+ if resp.StatusCode >= 400 {
+ tracker.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))
+ return
+ }
+ tracker.Record(elapsed.Seconds()*1000, nil)
+}
diff --git a/benchmarks/loadgen/report.go b/benchmarks/loadgen/report.go
new file mode 100644
index 00000000..d56cd7f6
--- /dev/null
+++ b/benchmarks/loadgen/report.go
@@ -0,0 +1,285 @@
+package main
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "net/http"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+type rTrace struct {
+ Id string `json:"id"`
+ Endpoint string `json:"endpoint"`
+ Duration int64 `json:"duration"` // nanoseconds
+ RecordedAt time.Time `json:"recordedAt"`
+ StatusCode int `json:"statusCode"`
+ BodySize int `json:"bodySize"`
+ ClientIP string `json:"clientIP"`
+ Attributes map[string]string `json:"attributes"`
+ Spans []rSpan `json:"spans"`
+ IsTask bool `json:"isTask"`
+ DistributedTraceId string `json:"distributedTraceId"`
+}
+
+type rSpan struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ StartTime time.Time `json:"startTime"`
+ Duration int64 `json:"duration"`
+}
+
+type rSession struct {
+ Id string `json:"id"`
+ StartedAt time.Time `json:"startedAt"`
+ ClientIP string `json:"clientIP"`
+ Attributes map[string]string `json:"attributes"`
+ DistributedTraceId string `json:"distributedTraceId,omitempty"`
+}
+
+type rRecording struct {
+ ExceptionId string `json:"exceptionId"`
+ SessionId string `json:"sessionId,omitempty"`
+ SegmentIndex int32 `json:"segmentIndex"`
+ Events json.RawMessage `json:"events"`
+ StartedAt *time.Time `json:"startedAt,omitempty"`
+ EndedAt *time.Time `json:"endedAt,omitempty"`
+}
+
+type rException struct {
+ IsTask bool `json:"isTask"`
+ StackTrace string `json:"stackTrace"`
+ RecordedAt time.Time `json:"recordedAt"`
+ Attributes map[string]string `json:"attributes"`
+ IsMessage bool `json:"isMessage"`
+}
+
+type rMetric struct {
+ Name string `json:"name"`
+ Value float64 `json:"value"`
+ RecordedAt time.Time `json:"recordedAt"`
+ Tags map[string]string `json:"tags,omitempty"`
+}
+
+type rFrame struct {
+ StackTraces []rException `json:"stackTraces"`
+ Metrics []rMetric `json:"metrics"`
+ Traces []rTrace `json:"traces"`
+ SessionRecordings []rRecording `json:"sessionRecordings"`
+ Sessions []rSession `json:"sessions"`
+}
+
+type rReport struct {
+ CollectionFrames []rFrame `json:"collectionFrames"`
+ AppVersion string `json:"appVersion"`
+ ServerName string `json:"serverName"`
+}
+
+var endpointPaths = []string{
+ "GET /api/users", "POST /api/users", "GET /api/orders", "POST /api/orders",
+ "GET /api/products/:id", "PUT /api/products/:id", "GET /api/search",
+ "POST /api/checkout", "GET /api/cart", "DELETE /api/cart/:id",
+ "GET /api/auth/me", "POST /api/auth/login", "GET /api/health",
+}
+
+// sendReportFrame builds one CollectionFrame, POSTs it gzipped to /api/report,
+// records latency on the ingest tracker, and returns a sample session id (when
+// one was included in the frame) so the reader can later query its recording.
+func sendReportFrame(ctx context.Context, client *http.Client, cfg config, rng *rand.Rand, stats *latencyTracker) string {
+ frame := buildFrame(rng)
+
+ body, err := encodeFrame(rReport{
+ CollectionFrames: []rFrame{frame},
+ AppVersion: "bench-1.0.0",
+ ServerName: fmt.Sprintf("bench-%s-%s", cfg.tier, cfg.mode),
+ })
+ if err != nil {
+ stats.Record(0, err)
+ return ""
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.target+"/api/report", bytes.NewReader(body))
+ if err != nil {
+ stats.Record(0, err)
+ return ""
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Content-Encoding", "gzip")
+ req.Header.Set("Authorization", "Bearer "+cfg.projectToken)
+
+ start := time.Now()
+ resp, err := client.Do(req)
+ elapsed := time.Since(start)
+ if err != nil {
+ stats.Record(elapsed.Seconds()*1000, err)
+ return ""
+ }
+ defer resp.Body.Close()
+ // Drain so the connection can be reused — keep-alive is critical at high
+ // frame rates; not draining leaves connections in CLOSE_WAIT.
+ _, _ = readAndDiscard(resp.Body)
+
+ if resp.StatusCode >= 400 {
+ stats.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))
+ return ""
+ }
+ stats.Record(elapsed.Seconds()*1000, nil)
+
+ if len(frame.Sessions) > 0 {
+ return frame.Sessions[0].Id
+ }
+ return ""
+}
+
+func buildFrame(rng *rand.Rand) rFrame {
+ now := time.Now().UTC()
+ frame := rFrame{
+ Traces: make([]rTrace, 0, tracesPerFrame),
+ Metrics: make([]rMetric, 0, 5),
+ Sessions: nil,
+ StackTraces: nil,
+ }
+
+ for i := 0; i < tracesPerFrame; i++ {
+ t := rTrace{
+ Id: uuid.NewString(),
+ Endpoint: endpointPaths[rng.Intn(len(endpointPaths))],
+ Duration: int64(time.Duration(10+rng.Intn(990)) * time.Millisecond), // 10-1000ms
+ RecordedAt: now.Add(-time.Duration(rng.Intn(5000)) * time.Millisecond),
+ StatusCode: pickStatus(rng),
+ BodySize: 200 + rng.Intn(8000),
+ ClientIP: "192.0.2.1",
+ Attributes: map[string]string{"region": "bench", "tier": "x"},
+ Spans: buildSpans(rng, now),
+ IsTask: rng.Float64() < 0.10,
+ }
+ frame.Traces = append(frame.Traces, t)
+ }
+
+ for i := 0; i < 5; i++ {
+ frame.Metrics = append(frame.Metrics, rMetric{
+ Name: "bench.metric." + []string{"cpu", "mem", "qps", "lat", "errs"}[i],
+ Value: rng.Float64() * 100,
+ RecordedAt: now,
+ })
+ }
+
+ if rng.Float64() < recordingChance {
+ sid := uuid.NewString()
+ started := now.Add(-2 * time.Second)
+ ended := now
+ frame.Sessions = []rSession{{
+ Id: sid,
+ StartedAt: started,
+ ClientIP: "192.0.2.1",
+ Attributes: map[string]string{"browser": "bench"},
+ }}
+ frame.SessionRecordings = []rRecording{{
+ ExceptionId: uuid.NewString(),
+ SessionId: sid,
+ SegmentIndex: 0,
+ Events: fakeRRWeb(rng),
+ StartedAt: &started,
+ EndedAt: &ended,
+ }}
+ } else if rng.Float64() < 0.2 {
+ // Even without a recording, ship occasional sessions so the read mix
+ // always has at least one session id to query.
+ sid := uuid.NewString()
+ frame.Sessions = []rSession{{
+ Id: sid,
+ StartedAt: now,
+ ClientIP: "192.0.2.1",
+ Attributes: map[string]string{"browser": "bench"},
+ }}
+ }
+
+ if rng.Float64() < exceptionChance {
+ frame.StackTraces = []rException{{
+ IsTask: false,
+ StackTrace: "RuntimeError: bench-synthetic\n at bench:1\n at bench:2",
+ RecordedAt: now,
+ Attributes: map[string]string{"env": "bench"},
+ }}
+ }
+
+ return frame
+}
+
+func buildSpans(rng *rand.Rand, now time.Time) []rSpan {
+ n := 2 + rng.Intn(4) // 2-5 spans
+ spans := make([]rSpan, n)
+ for i := range spans {
+ spans[i] = rSpan{
+ Id: uuid.NewString(),
+ Name: []string{"db.query", "http.call", "cache.get", "render", "json.encode"}[rng.Intn(5)],
+ StartTime: now.Add(-time.Duration(rng.Intn(500)) * time.Millisecond),
+ Duration: int64(time.Duration(1+rng.Intn(200)) * time.Millisecond),
+ }
+ }
+ return spans
+}
+
+func pickStatus(rng *rand.Rand) int {
+ r := rng.Float64()
+ switch {
+ case r < 0.92:
+ return 200
+ case r < 0.95:
+ return 201
+ case r < 0.98:
+ return 404
+ default:
+ return 500
+ }
+}
+
+// fakeRRWeb produces a ~100KB JSON array shaped like rrweb events. The backend
+// stores this opaquely in S3/disk without inspecting the structure, so a valid
+// JSON array of arbitrary size is sufficient to exercise the storage path.
+func fakeRRWeb(rng *rand.Rand) json.RawMessage {
+ var buf bytes.Buffer
+ buf.WriteString("[")
+ const targetBytes = 100_000
+ first := true
+ for buf.Len() < targetBytes {
+ if !first {
+ buf.WriteString(",")
+ }
+ first = false
+ fmt.Fprintf(&buf, `{"type":%d,"timestamp":%d,"data":{"href":"https://bench.local/page/%d","text":"%s"}}`,
+ 2+rng.Intn(4), time.Now().UnixMilli(), rng.Intn(10000), randomString(rng, 64))
+ }
+ buf.WriteString("]")
+ return buf.Bytes()
+}
+
+func randomString(rng *rand.Rand, n int) string {
+ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "
+ b := make([]byte, n)
+ for i := range b {
+ b[i] = charset[rng.Intn(len(charset))]
+ }
+ return string(b)
+}
+
+func encodeFrame(r rReport) ([]byte, error) {
+ js, err := json.Marshal(r)
+ if err != nil {
+ return nil, err
+ }
+ var gz bytes.Buffer
+ w := gzip.NewWriter(&gz)
+ if _, err := w.Write(js); err != nil {
+ return nil, err
+ }
+ if err := w.Close(); err != nil {
+ return nil, err
+ }
+ return gz.Bytes(), nil
+}
diff --git a/benchmarks/loadgen/stats.go b/benchmarks/loadgen/stats.go
new file mode 100644
index 00000000..13825297
--- /dev/null
+++ b/benchmarks/loadgen/stats.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+ "sort"
+ "sync"
+)
+
+type latencyTracker struct {
+ mu sync.Mutex
+ samples []float64
+ ok int64
+ errors int64
+}
+
+func newLatencyTracker() *latencyTracker {
+ return &latencyTracker{samples: make([]float64, 0, 8192)}
+}
+
+func (l *latencyTracker) Record(milliseconds float64, err error) {
+ l.mu.Lock()
+ if err != nil {
+ l.errors++
+ } else {
+ l.ok++
+ l.samples = append(l.samples, milliseconds)
+ }
+ l.mu.Unlock()
+}
+
+type latencySnapshot struct {
+ P50 float64 `json:"p50"`
+ P95 float64 `json:"p95"`
+ P99 float64 `json:"p99"`
+ OK int64 `json:"ok"`
+ Errors int64 `json:"errors"`
+ ErrRate float64 `json:"errRate"`
+}
+
+func (l *latencyTracker) SnapshotAndReset() latencySnapshot {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+
+ s := latencySnapshot{OK: l.ok, Errors: l.errors}
+ total := l.ok + l.errors
+ if total > 0 {
+ s.ErrRate = float64(l.errors) / float64(total)
+ }
+ if len(l.samples) > 0 {
+ sort.Float64s(l.samples)
+ s.P50 = percentile(l.samples, 0.50)
+ s.P95 = percentile(l.samples, 0.95)
+ s.P99 = percentile(l.samples, 0.99)
+ }
+
+ l.samples = l.samples[:0]
+ l.ok = 0
+ l.errors = 0
+ return s
+}
+
+func percentile(sorted []float64, q float64) float64 {
+ if len(sorted) == 0 {
+ return 0
+ }
+ idx := int(float64(len(sorted)-1) * q)
+ if idx < 0 {
+ idx = 0
+ }
+ if idx >= len(sorted) {
+ idx = len(sorted) - 1
+ }
+ return sorted[idx]
+}
diff --git a/benchmarks/loadgen/util.go b/benchmarks/loadgen/util.go
new file mode 100644
index 00000000..9d18a042
--- /dev/null
+++ b/benchmarks/loadgen/util.go
@@ -0,0 +1,7 @@
+package main
+
+import "io"
+
+func readAndDiscard(r io.Reader) (int64, error) {
+ return io.Copy(io.Discard, r)
+}
diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/benchmarks/scripts/_ssh.sh b/benchmarks/scripts/_ssh.sh
new file mode 100755
index 00000000..198b75d1
--- /dev/null
+++ b/benchmarks/scripts/_ssh.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Shared SSH/rsync invocation helpers. Source this from other scripts:
+# source "$(dirname "$0")/_ssh.sh"
+# Requires BENCHMARK_SSH_KEY env var pointing at a private key file.
+
+ssh_opts=(
+ -o StrictHostKeyChecking=accept-new
+ -o UserKnownHostsFile=/dev/null
+ -o LogLevel=ERROR
+ -o ServerAliveInterval=30
+ -o ConnectTimeout=15
+)
+
+if [[ -n "${BENCHMARK_SSH_KEY:-}" ]]; then
+ ssh_opts+=(-i "${BENCHMARK_SSH_KEY}")
+fi
+
+bench_ssh() {
+ local host="$1"; shift
+ ssh "${ssh_opts[@]}" "root@${host}" "$@"
+}
+
+bench_scp() {
+ scp "${ssh_opts[@]}" "$@"
+}
+
+bench_rsync() {
+ rsync -az --delete \
+ -e "ssh ${ssh_opts[*]}" \
+ "$@"
+}
+
+# Block until ssh-into-box succeeds or timeout expires (default 180s). Hetzner
+# boxes take ~15-30s to finish cloud-init; the first ssh attempt usually fails
+# even though hcloud reports the server as "running".
+wait_for_ssh() {
+ local host="$1" timeout="${2:-180}"
+ local deadline=$(( $(date +%s) + timeout ))
+ while [[ $(date +%s) -lt ${deadline} ]]; do
+ if bench_ssh "${host}" -o ConnectTimeout=5 true 2>/dev/null; then
+ return 0
+ fi
+ sleep 5
+ done
+ echo "ssh to ${host} not ready after ${timeout}s" >&2
+ return 1
+}
diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.py
new file mode 100755
index 00000000..cdb8f321
--- /dev/null
+++ b/benchmarks/scripts/chart.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Render hardware-vs-breaking-point charts from a directory of loadgen JSONs.
+
+Inputs: a directory containing N JSON files (one per matrix entry), each in
+the schema emitted by benchmarks/loadgen/. Order-independent.
+
+Outputs (written into the same directory):
+ - chart.png Bar: max sustainable EPS per tier, two bars per tier
+ (sqlite vs pgch). The headline.
+ - chart-detail.png Lines: read P99 (ms) vs ingest EPS, one line per
+ (tier, mode). Shows how reads degrade as ingest climbs.
+ - summary.md Markdown table for human consumption / docs source.
+"""
+
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+# Stable left-to-right ordering of tiers regardless of how the matrix runs.
+TIER_ORDER = ["ccx13", "ccx23", "ccx33", "ccx43"]
+MODE_ORDER = ["sqlite", "pgch"]
+MODE_COLORS = {"sqlite": "#4f9fff", "pgch": "#ff9f4f"}
+READ_P99_BUDGET_MS = 3000
+
+
+def load_runs(results_dir: Path) -> list[dict]:
+ runs = []
+ for p in sorted(results_dir.glob("*.json")):
+ with p.open() as fh:
+ doc = json.load(fh)
+ doc["_path"] = p
+ runs.append(doc)
+ return runs
+
+
+def tier_rank(t: str) -> int:
+ return TIER_ORDER.index(t) if t in TIER_ORDER else len(TIER_ORDER)
+
+
+def mode_rank(m: str) -> int:
+ return MODE_ORDER.index(m) if m in MODE_ORDER else len(MODE_ORDER)
+
+
+def render_bar(runs: list[dict], out: Path) -> None:
+ fig, ax = plt.subplots(figsize=(9, 5))
+ tiers_present = sorted({r["tier"] for r in runs}, key=tier_rank)
+ modes_present = sorted({r["mode"] for r in runs}, key=mode_rank)
+
+ width = 0.8 / max(len(modes_present), 1)
+ x = list(range(len(tiers_present)))
+
+ for mi, mode in enumerate(modes_present):
+ ys = []
+ for tier in tiers_present:
+ match = [r for r in runs if r["tier"] == tier and r["mode"] == mode]
+ ys.append(match[0]["maxSustainableEps"] if match else 0)
+ offsets = [xi + (mi - (len(modes_present) - 1) / 2) * width for xi in x]
+ bars = ax.bar(offsets, ys, width=width, label=mode, color=MODE_COLORS.get(mode, "#777"))
+ for b, y in zip(bars, ys):
+ ax.annotate(f"{int(y):,}", xy=(b.get_x() + b.get_width() / 2, y),
+ xytext=(0, 3), textcoords="offset points",
+ ha="center", va="bottom", fontsize=9)
+
+ ax.set_xticks(x)
+ ax.set_xticklabels(tiers_present)
+ ax.set_ylabel("Max sustainable traces/sec")
+ ax.set_title("Traceway: max sustainable ingest by hardware tier\n"
+ f"(failure = ingest err > 5% OR any read p99 > {READ_P99_BUDGET_MS}ms)")
+ ax.legend(title="DB mode")
+ ax.grid(axis="y", linestyle=":", alpha=0.4)
+ fig.tight_layout()
+ fig.savefig(out, dpi=130)
+ plt.close(fig)
+
+
+def render_detail(runs: list[dict], out: Path) -> None:
+ fig, ax = plt.subplots(figsize=(10, 6))
+
+ for run in sorted(runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):
+ xs, ys = [], []
+ for s in run.get("steps", []):
+ if not s.get("reads"):
+ continue
+ xs.append(s["actualEps"])
+ # Worst read p99 across all read endpoints — that's what defines the cliff.
+ ys.append(max((rd.get("p99", 0) for rd in s["reads"].values()), default=0))
+ if xs:
+ label = f"{run['tier']} / {run['mode']}"
+ ax.plot(xs, ys, marker="o", label=label)
+
+ ax.axhline(READ_P99_BUDGET_MS, linestyle="--", color="#cc3333", alpha=0.6,
+ label=f"P99 budget ({READ_P99_BUDGET_MS}ms)")
+ ax.set_xlabel("Actual ingest rate (traces/sec)")
+ ax.set_ylabel("Worst read endpoint P99 (ms)")
+ ax.set_title("Read latency degradation under increasing ingest load")
+ ax.set_xscale("log")
+ ax.set_yscale("log")
+ ax.grid(True, which="both", linestyle=":", alpha=0.4)
+ ax.legend(loc="upper left", fontsize=9)
+ fig.tight_layout()
+ fig.savefig(out, dpi=130)
+ plt.close(fig)
+
+
+def render_summary(runs: list[dict], out: Path) -> None:
+ lines = ["# Traceway hardware benchmark — summary", ""]
+ lines.append(f"Runs analyzed: {len(runs)}")
+ lines.append("")
+ lines.append("| Tier | Mode | Max EPS | Ingest p99 @max (ms) | Worst read p99 @max (ms) | Steps |")
+ lines.append("|------|------|---------|----------------------|---------------------------|-------|")
+
+ for run in sorted(runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):
+ max_eps = int(run.get("maxSustainableEps", 0))
+ last_pass = None
+ for s in reversed(run.get("steps", [])):
+ if s.get("passed"):
+ last_pass = s
+ break
+ ingest_p99 = int(last_pass["ingest"]["p99"]) if last_pass else 0
+ worst_read = 0
+ if last_pass and last_pass.get("reads"):
+ worst_read = int(max(rd.get("p99", 0) for rd in last_pass["reads"].values()))
+ lines.append(f"| {run['tier']} | {run['mode']} | {max_eps:,} | {ingest_p99} | {worst_read} | {len(run.get('steps', []))} |")
+
+ lines.append("")
+ lines.append("Generated by `benchmarks/scripts/chart.py`. "
+ "Failure thresholds: ingest error rate > 5%, OR any read endpoint p99 > "
+ f"{READ_P99_BUDGET_MS}ms.")
+ out.write_text("\n".join(lines) + "\n")
+
+
+def main() -> int:
+ if len(sys.argv) < 2:
+ print("usage: chart.py <results-dir>", file=sys.stderr)
+ return 2
+ results_dir = Path(sys.argv[1])
+ if not results_dir.is_dir():
+ print(f"not a directory: {results_dir}", file=sys.stderr)
+ return 2
+
+ runs = load_runs(results_dir)
+ if not runs:
+ print(f"no *.json files in {results_dir}", file=sys.stderr)
+ return 1
+
+ render_bar(runs, results_dir / "chart.png")
+ render_detail(runs, results_dir / "chart-detail.png")
+ render_summary(runs, results_dir / "summary.md")
+ print(f"wrote {results_dir}/chart.png, chart-detail.png, summary.md")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/benchmarks/scripts/hetzner-down.sh b/benchmarks/scripts/hetzner-down.sh
new file mode 100755
index 00000000..b557ab49
--- /dev/null
+++ b/benchmarks/scripts/hetzner-down.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Tear down servers + network for a given run-id. Idempotent: missing resources
+# are not errors. Designed to be safe to call from a bash trap, including when
+# triggered mid-provisioning.
+#
+# Usage: hetzner-down.sh <run-id>
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+ echo "usage: $0 <run-id>" >&2
+ exit 2
+fi
+
+RUN_ID="$1"
+SUT_NAME="bench-sut-${RUN_ID}"
+LOADGEN_NAME="bench-loadgen-${RUN_ID}"
+NET_NAME="bench-net-${RUN_ID}"
+
+drop_server() {
+ local name="$1"
+ if hcloud server describe "${name}" >/dev/null 2>&1; then
+ echo "deleting server ${name}" >&2
+ hcloud server delete "${name}" >/dev/null || echo "warn: delete ${name} returned non-zero" >&2
+ fi
+}
+
+drop_server "${SUT_NAME}"
+drop_server "${LOADGEN_NAME}"
+
+if hcloud network describe "${NET_NAME}" >/dev/null 2>&1; then
+ echo "deleting network ${NET_NAME}" >&2
+ # Network delete requires no attached servers; we just deleted them above.
+ hcloud network delete "${NET_NAME}" >/dev/null || echo "warn: delete ${NET_NAME} returned non-zero" >&2
+fi
+
+echo "teardown of ${RUN_ID}: done" >&2
diff --git a/benchmarks/scripts/hetzner-up.sh b/benchmarks/scripts/hetzner-up.sh
new file mode 100755
index 00000000..8c62ccfe
--- /dev/null
+++ b/benchmarks/scripts/hetzner-up.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Provision a system-under-test (SUT) box + a load generator box + a private
+# network in Hetzner Cloud. Idempotent on its own outputs: re-running with the
+# same RUN_ID short-circuits if the resources already exist.
+#
+# Usage: hetzner-up.sh <tier> <run-id> [location]
+# <tier> ccx13 | ccx23 | ccx33 | ccx43
+# <run-id> short label, used to name the resources (e.g. 20260512-ccx13-sqlite-a1b2)
+# [location] Hetzner datacenter, default nbg1
+#
+# Emits a JSON line on stdout:
+# {"sutPublicIp":"...","sutPrivateIp":"...","loadgenPublicIp":"...","loadgenPrivateIp":"...","networkId":"...","runId":"..."}
+set -euo pipefail
+
+if [[ $# -lt 2 ]]; then
+ echo "usage: $0 <tier> <run-id> [location]" >&2
+ exit 2
+fi
+
+TIER="$1"
+RUN_ID="$2"
+LOCATION="${3:-nbg1}"
+LOADGEN_TIER="${LOADGEN_TIER:-cax11}"
+IMAGE="${BENCH_IMAGE:-debian-12}"
+NET_NAME="bench-net-${RUN_ID}"
+SUT_NAME="bench-sut-${RUN_ID}"
+LOADGEN_NAME="bench-loadgen-${RUN_ID}"
+
+# Create a /24 private network so SUT and loadgen can talk over 10.0.0.x.
+if ! hcloud network describe "${NET_NAME}" >/dev/null 2>&1; then
+ echo "creating network ${NET_NAME}" >&2
+ hcloud network create --name "${NET_NAME}" --ip-range 10.0.0.0/24 >/dev/null
+ hcloud network add-subnet "${NET_NAME}" --network-zone eu-central --type cloud --ip-range 10.0.0.0/24 >/dev/null
+fi
+NET_ID=$(hcloud network describe "${NET_NAME}" -o format='{{.ID}}')
+
+create_server() {
+ local name="$1" type="$2" private_ip="$3"
+ if hcloud server describe "${name}" >/dev/null 2>&1; then
+ echo "server ${name} already exists, reusing" >&2
+ return
+ fi
+ echo "creating server ${name} (${type}) in ${LOCATION}" >&2
+ hcloud server create \
+ --name "${name}" \
+ --type "${type}" \
+ --image "${IMAGE}" \
+ --location "${LOCATION}" \
+ --ssh-key benchmark-key \
+ --network "${NET_NAME}" \
+ --label "bench=true,run=${RUN_ID}" \
+ >/dev/null
+}
+
+create_server "${SUT_NAME}" "${TIER}" "10.0.0.2"
+create_server "${LOADGEN_NAME}" "${LOADGEN_TIER}" "10.0.0.3"
+
+# Hetzner doesn't let you pin the private IP at create time on shared networks
+# without --network-extra; we re-read both servers' assigned private IPs and
+# return them.
+get_field() {
+ local name="$1" field="$2"
+ hcloud server describe "${name}" -o format="{{${field}}}"
+}
+
+SUT_PUBLIC_IP=$(get_field "${SUT_NAME}" ".PublicNet.IPv4.IP")
+LOADGEN_PUBLIC_IP=$(get_field "${LOADGEN_NAME}" ".PublicNet.IPv4.IP")
+SUT_PRIVATE_IP=$(hcloud server describe "${SUT_NAME}" -o json | jq -r '.private_net[0].ip')
+LOADGEN_PRIVATE_IP=$(hcloud server describe "${LOADGEN_NAME}" -o json | jq -r '.private_net[0].ip')
+
+jq -nc \
+ --arg sutPub "${SUT_PUBLIC_IP}" \
+ --arg sutPriv "${SUT_PRIVATE_IP}" \
+ --arg lgPub "${LOADGEN_PUBLIC_IP}" \
+ --arg lgPriv "${LOADGEN_PRIVATE_IP}" \
+ --arg netId "${NET_ID}" \
+ --arg runId "${RUN_ID}" \
+ '{sutPublicIp: $sutPub, sutPrivateIp: $sutPriv, loadgenPublicIp: $lgPub, loadgenPrivateIp: $lgPriv, networkId: $netId, runId: $runId}'
diff --git a/benchmarks/scripts/loadgen-bootstrap.sh b/benchmarks/scripts/loadgen-bootstrap.sh
new file mode 100755
index 00000000..15417984
--- /dev/null
+++ b/benchmarks/scripts/loadgen-bootstrap.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Cross-compile the loadgen for linux/amd64, scp it to the loadgen box, seed a
+# project on the SUT, run the loadgen pointed at the SUT's private IP, and pull
+# the result JSON back.
+#
+# Usage: loadgen-bootstrap.sh <loadgen-public-ip> <sut-private-ip> <sut-public-ip> <duration> <tier> <mode> <out-path> [extra-loadgen-args...]
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=_ssh.sh
+source "${SCRIPT_DIR}/_ssh.sh"
+
+if [[ $# -lt 7 ]]; then
+ echo "usage: $0 <loadgen-public-ip> <sut-private-ip> <sut-public-ip> <duration> <tier> <mode> <out-path> [extra-loadgen-args...]" >&2
+ exit 2
+fi
+LG_IP="$1"; SUT_PRIVATE_IP="$2"; SUT_PUBLIC_IP="$3"; DURATION="$4"; TIER="$5"; MODE="$6"; OUT_PATH="$7"
+shift 7
+
+# 1. Seed a project on the SUT (over its PUBLIC IP from the orchestrator). The
+# project token + JWT + project id are then handed to the loadgen, which
+# talks to the SUT over the PRIVATE network for the actual benchmark.
+echo "seeding project via http://${SUT_PUBLIC_IP}/api/register" >&2
+SEED_JSON=$("${SCRIPT_DIR}/seed-project.sh" "http://${SUT_PUBLIC_IP}")
+JWT=$(printf '%s' "${SEED_JSON}" | jq -r '.jwt')
+TOKEN=$(printf '%s' "${SEED_JSON}" | jq -r '.projectToken')
+PROJECT_ID=$(printf '%s' "${SEED_JSON}" | jq -r '.projectId')
+
+# 2. Wait for SSH on the loadgen box and detect its arch so we cross-compile
+# correctly (CAX* tiers are arm64, CX*/CPX* are amd64).
+echo "waiting for ssh on loadgen ${LG_IP}" >&2
+wait_for_ssh "${LG_IP}"
+LG_UNAME=$(bench_ssh "${LG_IP}" "uname -m" | tr -d '\r\n')
+case "${LG_UNAME}" in
+ x86_64) GOARCH=amd64 ;;
+ aarch64) GOARCH=arm64 ;;
+ *) echo "unknown loadgen arch '${LG_UNAME}'" >&2; exit 1 ;;
+esac
+
+# 3. Cross-compile loadgen on the orchestrator (laptop / GH runner) for the
+# loadgen box's arch.
+echo "cross-compiling loadgen for linux/${GOARCH}" >&2
+(
+ cd "${REPO_ROOT}/benchmarks/loadgen"
+ GOOS=linux GOARCH="${GOARCH}" CGO_ENABLED=0 go build -o "loadgen-linux-${GOARCH}" .
+)
+
+bench_ssh "${LG_IP}" "mkdir -p /root/loadgen"
+bench_scp "${REPO_ROOT}/benchmarks/loadgen/loadgen-linux-${GOARCH}" "root@${LG_IP}:/root/loadgen/loadgen"
+bench_ssh "${LG_IP}" "chmod +x /root/loadgen/loadgen"
+
+# 4. Run the benchmark, streaming stderr back so progress is visible. The
+# loadgen writes JSON to /root/loadgen/result.json on the loadgen box;
+# we scp it home afterwards.
+echo "running loadgen on ${LG_IP} -> http://${SUT_PRIVATE_IP} (tier=${TIER} mode=${MODE} duration=${DURATION})" >&2
+bench_ssh "${LG_IP}" /root/loadgen/loadgen \
+ --target "http://${SUT_PRIVATE_IP}" \
+ --token "${TOKEN}" \
+ --jwt "${JWT}" \
+ --project-id "${PROJECT_ID}" \
+ --duration "${DURATION}" \
+ --tier "${TIER}" \
+ --mode "${MODE}" \
+ --report-out /root/loadgen/result.json \
+ "$@"
+
+mkdir -p "$(dirname "${OUT_PATH}")"
+bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}"
+echo "wrote ${OUT_PATH}" >&2
diff --git a/benchmarks/scripts/preflight.sh b/benchmarks/scripts/preflight.sh
new file mode 100755
index 00000000..49d0753d
--- /dev/null
+++ b/benchmarks/scripts/preflight.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Preflight checks for the benchmark tooling. Fails loud and early so we don't
+# half-provision a Hetzner box and then discover matplotlib is missing.
+#
+# Required env vars: HCLOUD_TOKEN, BENCHMARK_SSH_KEY (path to private key file)
+# Required commands: hcloud, jq, python3, ssh, scp, curl, go
+# Required python: matplotlib (in whatever python3 we'll invoke chart.py with)
+# Required Hetzner: an SSH key uploaded as "benchmark-key"
+
+set -euo pipefail
+
+fail=0
+warn() { printf 'WARN: %s\n' "$*" >&2; }
+err() { printf 'FAIL: %s\n' "$*" >&2; fail=1; }
+
+# --- Commands on PATH -------------------------------------------------------
+
+for cmd in hcloud jq python3 ssh scp curl go; do
+ if ! command -v "${cmd}" >/dev/null 2>&1; then
+ err "missing command: ${cmd}"
+ fi
+done
+
+# --- Python: matplotlib for chart rendering ---------------------------------
+
+if command -v python3 >/dev/null 2>&1; then
+ if ! python3 -c "import matplotlib" >/dev/null 2>&1; then
+ err "python3 cannot import matplotlib. Install in a venv: python3 -m venv .venv && .venv/bin/pip install matplotlib; then export PATH=\$PWD/.venv/bin:\$PATH"
+ fi
+fi
+
+# --- Hetzner credentials ----------------------------------------------------
+
+if [[ -z "${HCLOUD_TOKEN:-}" ]]; then
+ err "HCLOUD_TOKEN not set. Export your Hetzner Cloud API token."
+else
+ if ! hcloud server-type list >/dev/null 2>&1; then
+ err "HCLOUD_TOKEN is set but 'hcloud server-type list' failed — token invalid or revoked?"
+ fi
+fi
+
+# --- SSH key ---------------------------------------------------------------
+
+if [[ -z "${BENCHMARK_SSH_KEY:-}" ]]; then
+ warn "BENCHMARK_SSH_KEY not set; will skip key-on-disk checks. Hetzner-side key 'benchmark-key' is still required."
+else
+ if [[ ! -f "${BENCHMARK_SSH_KEY}" ]]; then
+ err "BENCHMARK_SSH_KEY=${BENCHMARK_SSH_KEY} does not exist"
+ elif [[ "$(stat -f '%Lp' "${BENCHMARK_SSH_KEY}" 2>/dev/null || stat -c '%a' "${BENCHMARK_SSH_KEY}" 2>/dev/null)" != "600" ]]; then
+ warn "BENCHMARK_SSH_KEY permissions are not 600; ssh may refuse to use it"
+ fi
+fi
+
+if command -v hcloud >/dev/null 2>&1 && [[ -n "${HCLOUD_TOKEN:-}" ]]; then
+ if ! hcloud ssh-key describe benchmark-key >/dev/null 2>&1; then
+ err "no SSH key named 'benchmark-key' in your Hetzner project. Upload the matching public key via: hcloud ssh-key create --name benchmark-key --public-key-from-file ~/.ssh/hetzner_benchmark.pub"
+ fi
+fi
+
+if [[ "${fail}" -ne 0 ]]; then
+ echo >&2
+ echo "preflight failed. fix the issues above before running." >&2
+ exit 1
+fi
+echo "preflight: OK" >&2
diff --git a/benchmarks/scripts/run-local.sh b/benchmarks/scripts/run-local.sh
new file mode 100755
index 00000000..3123d277
--- /dev/null
+++ b/benchmarks/scripts/run-local.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# Run the hardware benchmark from your laptop, end-to-end, against real Hetzner.
+# The GitHub Action is a thin wrapper around run-matrix-entry.sh; this script
+# is the developer-facing equivalent — same orchestration, run from anywhere.
+#
+# Required env:
+# HCLOUD_TOKEN Hetzner Cloud API token
+# BENCHMARK_SSH_KEY Path to the private key matching the Hetzner-side
+# SSH key named 'benchmark-key'.
+#
+# Common usage:
+# run-local.sh # full matrix (4 tiers x 2 modes)
+# run-local.sh --smoke # 1 tier, 1 mode, short steps (~5 min)
+# run-local.sh --tier ccx13 --mode sqlite
+# run-local.sh --dry-run # validate env + print plan, no provisioning
+# run-local.sh --commit # drop the "-local" suffix on the output dir
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+TIERS="ccx13,ccx23,ccx33,ccx43"
+MODES="sqlite,pgch"
+DURATION="30m"
+SMOKE=0
+DRY_RUN=0
+COMMIT=0
+
+usage() {
+ sed -n '1,16p' "$0"; exit 2
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --tier) TIERS="$2"; shift 2 ;;
+ --mode) MODES="$2"; shift 2 ;;
+ --duration) DURATION="$2"; shift 2 ;;
+ --smoke) SMOKE=1; shift ;;
+ --dry-run) DRY_RUN=1; shift ;;
+ --commit) COMMIT=1; shift ;;
+ -h|--help) usage ;;
+ *) echo "unknown flag: $1" >&2; usage ;;
+ esac
+done
+
+if [[ "${SMOKE}" -eq 1 ]]; then
+ TIERS="ccx13"
+ MODES="sqlite"
+ DURATION="3m"
+ echo "smoke mode: tier=ccx13 mode=sqlite duration=${DURATION} stepped 100->400 eps" >&2
+fi
+
+# Preflight first — fail fast on missing tooling.
+"${SCRIPT_DIR}/preflight.sh"
+
+date_tag="$(date -u +%Y-%m-%d)"
+if [[ "${COMMIT}" -eq 1 ]]; then
+ OUT_DIR="${REPO_ROOT}/benchmarks/results/${date_tag}"
+else
+ OUT_DIR="${REPO_ROOT}/benchmarks/results/${date_tag}-local"
+fi
+mkdir -p "${OUT_DIR}"
+echo "results dir: ${OUT_DIR}" >&2
+
+# Plan: explode tiers x modes.
+plan=()
+IFS=',' read -ra TIER_ARR <<<"${TIERS}"
+IFS=',' read -ra MODE_ARR <<<"${MODES}"
+for t in "${TIER_ARR[@]}"; do
+ for m in "${MODE_ARR[@]}"; do
+ plan+=("${t}|${m}")
+ done
+done
+
+echo "plan (${#plan[@]} entries):" >&2
+for e in "${plan[@]}"; do
+ echo " - ${e//|/ x }" >&2
+done
+
+if [[ "${DRY_RUN}" -eq 1 ]]; then
+ echo "dry-run: would call run-matrix-entry.sh ${#plan[@]} time(s) with duration=${DURATION}; no servers will be created." >&2
+ exit 0
+fi
+
+# Sequential execution. Local runs prioritize simplicity and low concurrent
+# Hetzner spend over wall-clock speed; the GH workflow parallelizes with
+# strategy.matrix instead.
+smoke_arg=""
+[[ "${SMOKE}" -eq 1 ]] && smoke_arg="smoke"
+
+failures=()
+for e in "${plan[@]}"; do
+ tier="${e%%|*}"; mode="${e##*|}"
+ if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}"; then
+ echo "FAIL: ${tier}/${mode}" >&2
+ failures+=("${tier}/${mode}")
+ fi
+done
+
+# Render charts once everything (or at least something) is done.
+if compgen -G "${OUT_DIR}/*.json" >/dev/null; then
+ echo "rendering charts" >&2
+ python3 "${SCRIPT_DIR}/chart.py" "${OUT_DIR}"
+else
+ echo "no JSON files were produced; skipping chart render" >&2
+fi
+
+if [[ ${#failures[@]} -gt 0 ]]; then
+ echo "FINISHED with failures: ${failures[*]}" >&2
+ exit 1
+fi
+echo "FINISHED. Results in ${OUT_DIR}" >&2
diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.sh
new file mode 100755
index 00000000..35e31323
--- /dev/null
+++ b/benchmarks/scripts/run-matrix-entry.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Run ONE (tier, mode) cycle end-to-end: provision Hetzner -> bootstrap SUT ->
+# run loadgen -> fetch JSON -> tear down. Called by both run-local.sh and the
+# GitHub workflow; treat this as the single source of truth for matrix-entry
+# orchestration.
+#
+# Usage: run-matrix-entry.sh <tier> <mode> <duration> <out-dir> [smoke]
+# <tier> ccx13 | ccx23 | ccx33 | ccx43
+# <mode> sqlite | pgch
+# <duration> Loadgen total runtime, e.g. 30m, 3m
+# <out-dir> Directory to write <tier>-<mode>.json into
+# [smoke] "smoke" to enable short-step overrides (--initial-eps 100
+# --max-eps 400 --step-duration 1m). Optional.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+if [[ $# -lt 4 ]]; then
+ echo "usage: $0 <tier> <mode> <duration> <out-dir> [smoke]" >&2
+ exit 2
+fi
+TIER="$1"; MODE="$2"; DURATION="$3"; OUT_DIR="$4"; SMOKE="${5:-}"
+LOCATION="${BENCH_LOCATION:-nbg1}"
+
+RUN_ID="$(date -u +%Y%m%d-%H%M%S)-${TIER}-${MODE}-$RANDOM"
+echo "=== run-matrix-entry tier=${TIER} mode=${MODE} duration=${DURATION} run_id=${RUN_ID} ===" >&2
+
+mkdir -p "${OUT_DIR}"
+
+# Always tear down — even on failure, even on Ctrl-C. The trap is set BEFORE
+# any hcloud create call so a failure mid-provision still cleans up.
+cleanup() {
+ local rc=$?
+ echo "--- teardown for ${RUN_ID} (exit=${rc}) ---" >&2
+ "${SCRIPT_DIR}/hetzner-down.sh" "${RUN_ID}" || true
+ exit "${rc}"
+}
+trap cleanup EXIT INT TERM
+
+# 1. Provision.
+INFRA_JSON=$("${SCRIPT_DIR}/hetzner-up.sh" "${TIER}" "${RUN_ID}" "${LOCATION}")
+echo "infra: ${INFRA_JSON}" >&2
+SUT_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPublicIp')
+SUT_PRIVATE_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPrivateIp')
+LOADGEN_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.loadgenPublicIp')
+
+# 2. Bring up the backend on the SUT.
+"${SCRIPT_DIR}/sut-bootstrap.sh" "${SUT_PUBLIC_IP}" "${MODE}"
+
+# 3. Run the loadgen, pulling JSON back into OUT_DIR.
+extra_args=()
+if [[ "${SMOKE}" == "smoke" ]]; then
+ extra_args+=( --initial-eps 100 --max-eps 400 --step-duration 1m )
+fi
+
+OUT_PATH="${OUT_DIR}/${TIER}-${MODE}.json"
+"${SCRIPT_DIR}/loadgen-bootstrap.sh" \
+ "${LOADGEN_PUBLIC_IP}" \
+ "${SUT_PRIVATE_IP}" \
+ "${SUT_PUBLIC_IP}" \
+ "${DURATION}" \
+ "${TIER}" \
+ "${MODE}" \
+ "${OUT_PATH}" \
+ "${extra_args[@]}"
+
+# Trap handles teardown — no explicit call needed.
+echo "matrix entry ${TIER}-${MODE} complete -> ${OUT_PATH}" >&2
diff --git a/benchmarks/scripts/seed-project.sh b/benchmarks/scripts/seed-project.sh
new file mode 100755
index 00000000..3b63b0da
--- /dev/null
+++ b/benchmarks/scripts/seed-project.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Register a fresh user + org + project against a Traceway backend.
+#
+# Emits a single JSON line on stdout (everything else on stderr) so callers can
+# capture cleanly:
+# eval "$(seed-project.sh http://localhost:8087 | jq -r '@sh "JWT=\(.jwt) PROJECT_TOKEN=\(.projectToken) PROJECT_ID=\(.projectId)"')"
+#
+# Output schema:
+# { "jwt": "<JWT for dashboard read endpoints>",
+# "projectToken": "<bearer token for /api/report + OTLP ingestion>",
+# "projectId": "<UUID; pass as ?projectId= on read endpoints>" }
+#
+# Usage: seed-project.sh <base-url>
+# <base-url> Backend root, e.g. http://localhost:8087 or http://10.0.0.2
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+ echo "usage: $0 <base-url>" >&2
+ exit 2
+fi
+
+BASE_URL="${1%/}"
+EMAIL="bench-$(date +%s)[email protected]"
+PASSWORD="bench-pw-not-secret-12345"
+
+response=$(curl -sS --fail-with-body -X POST "${BASE_URL}/api/register" \
+ -H "Content-Type: application/json" \
+ -d "$(cat <<JSON
+{
+ "email": "${EMAIL}",
+ "name": "bench",
+ "password": "${PASSWORD}",
+ "organizationName": "bench-org",
+ "timezone": "UTC",
+ "projectName": "bench-project",
+ "framework": "custom"
+}
+JSON
+)")
+
+echo "registered ${EMAIL}" >&2
+
+out=$(printf '%s' "${response}" | jq -c '{jwt: .token, projectToken: .project.token, projectId: .project.id}')
+if [[ "$(printf '%s' "${out}" | jq -r '.projectToken')" == "null" ]]; then
+ echo "ERROR: no project token in response: ${response}" >&2
+ exit 1
+fi
+
+printf '%s\n' "${out}"
diff --git a/benchmarks/scripts/sut-bootstrap.sh b/benchmarks/scripts/sut-bootstrap.sh
new file mode 100755
index 00000000..1f488938
--- /dev/null
+++ b/benchmarks/scripts/sut-bootstrap.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Bring up the Traceway backend on a freshly-provisioned Hetzner box.
+#
+# 1. Wait for SSH.
+# 2. Install Docker if missing.
+# 3. rsync the repo to /opt/traceway (sans heavy junk).
+# 4. `docker compose -f benchmarks/compose/docker-compose.<mode>.yml up -d --build`
+# 5. Poll /health until 200.
+#
+# Usage: sut-bootstrap.sh <sut-public-ip> <mode>
+# <mode> sqlite | pgch
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=_ssh.sh
+source "${SCRIPT_DIR}/_ssh.sh"
+
+if [[ $# -lt 2 ]]; then
+ echo "usage: $0 <sut-public-ip> <mode>" >&2
+ exit 2
+fi
+SUT_IP="$1"
+MODE="$2"
+
+case "${MODE}" in
+ sqlite|pgch) ;;
+ *) echo "mode must be sqlite or pgch, got: ${MODE}" >&2; exit 2 ;;
+esac
+
+echo "waiting for ssh on ${SUT_IP}" >&2
+wait_for_ssh "${SUT_IP}"
+
+echo "installing docker on ${SUT_IP}" >&2
+bench_ssh "${SUT_IP}" 'bash -s' <<'REMOTE'
+set -euo pipefail
+if ! command -v docker >/dev/null 2>&1; then
+ curl -fsSL https://get.docker.com | sh
+fi
+# compose plugin lives at /usr/libexec/docker/cli-plugins after get.docker.com.
+docker compose version >/dev/null || { echo "docker compose plugin missing" >&2; exit 1; }
+mkdir -p /opt/traceway
+REMOTE
+
+echo "rsyncing source to ${SUT_IP}:/opt/traceway" >&2
+bench_rsync \
+ --exclude '.git' \
+ --exclude 'node_modules' \
+ --exclude 'frontend/build' \
+ --exclude 'frontend/.svelte-kit' \
+ --exclude 'benchmarks/loadgen/loadgen' \
+ --exclude 'benchmarks/results' \
+ --exclude 'traceway.db*' \
+ --exclude 'traceway_telemetry.db*' \
+ --exclude 'backend/storage' \
+ "${REPO_ROOT}/" "root@${SUT_IP}:/opt/traceway/"
+
+echo "bringing up compose stack (mode=${MODE}) on ${SUT_IP}" >&2
+bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 docker compose -f benchmarks/compose/docker-compose.${MODE}.yml up -d --build"
+
+echo "polling /health on ${SUT_IP}" >&2
+deadline=$(( $(date +%s) + 600 )) # cold compose build can hit 10 min on small tiers
+while [[ $(date +%s) -lt ${deadline} ]]; do
+ if curl -sf --max-time 5 "http://${SUT_IP}/health" >/dev/null 2>&1; then
+ echo "SUT ${SUT_IP} healthy (mode=${MODE})" >&2
+ exit 0
+ fi
+ sleep 5
+done
+echo "SUT ${SUT_IP} never reported healthy" >&2
+bench_ssh "${SUT_IP}" "cd /opt/traceway && docker compose -f benchmarks/compose/docker-compose.${MODE}.yml logs --tail=80" >&2 || true
+exit 1