The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
benchmarks (#139)
narrow-adventure47de8fd4e5e5
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.ymlnew file mode 100644index 00000000..dff51bb9--- /dev/null+++ b/.github/workflows/benchmark-hardware.yml@@ -0,0 +1,138 @@+name: benchmark-hardware++# Manual-only: this workflow provisions and tears down Hetzner servers, which+# costs real money (about €0.30/run today). Triggering must be deliberate.+on:+ workflow_dispatch:+ inputs:+ tiers:+ description: 'Comma-separated Hetzner tiers (e.g. ccx13,ccx23,ccx33,ccx43)'+ required: true+ default: 'ccx13,ccx23,ccx33,ccx43'+ modes:+ description: 'Comma-separated DB modes (sqlite,pgch)'+ required: true+ default: 'sqlite,pgch'+ duration:+ description: 'Per-matrix-entry loadgen duration (e.g. 30m, 5m)'+ required: true+ default: '30m'+ location:+ description: 'Hetzner datacenter (nbg1, fsn1, hel1, ash, hil)'+ required: true+ default: 'nbg1'++concurrency:+ group: benchmark-hardware+ cancel-in-progress: false++permissions:+ contents: write # aggregate job commits the results back to main++jobs:+ setup:+ runs-on: ubuntu-latest+ outputs:+ matrix: ${{ steps.plan.outputs.matrix }}+ steps:+ - id: plan+ name: Build matrix from inputs+ run: |+ set -euo pipefail+ tiers="${{ github.event.inputs.tiers }}"+ modes="${{ github.event.inputs.modes }}"+ jq -nc \+ --arg tiers "${tiers}" \+ --arg modes "${modes}" \+ '{include: [($tiers | split(",")) as $T | ($modes | split(",")) as $M | $T[] as $t | $M[] as $m | {tier: $t, mode: $m}]}' \+ > matrix.json+ cat matrix.json+ echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"++ run:+ needs: setup+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ max-parallel: 2+ matrix: ${{ fromJson(needs.setup.outputs.matrix) }}+ env:+ HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}+ BENCHMARK_SSH_KEY: ${{ runner.temp }}/benchmark-key+ BENCH_LOCATION: ${{ github.event.inputs.location }}+ steps:+ - uses: actions/checkout@v4++ - name: Install hcloud CLI+ run: |+ curl -fsSL https://github.com/hetznercloud/cli/releases/download/v1.50.0/hcloud-linux-amd64.tar.gz \+ | sudo tar -xz -C /usr/local/bin hcloud++ - name: Install matplotlib (chart render is in aggregate job, but preflight checks it)+ run: pip install --quiet matplotlib++ - name: Materialize SSH key+ run: |+ umask 077+ printf '%s\n' "${{ secrets.BENCHMARK_SSH_PRIVATE_KEY }}" > "${BENCHMARK_SSH_KEY}"++ - uses: actions/setup-go@v5+ with:+ go-version: '1.25'++ - name: Preflight+ run: ./benchmarks/scripts/preflight.sh++ - name: Run matrix entry (${{ matrix.tier }} / ${{ matrix.mode }})+ run: ./benchmarks/scripts/run-matrix-entry.sh "${{ matrix.tier }}" "${{ matrix.mode }}" "${{ github.event.inputs.duration }}" benchmarks/results/_artifact++ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: result-${{ matrix.tier }}-${{ matrix.mode }}+ path: benchmarks/results/_artifact/*.json+ if-no-files-found: warn++ aggregate:+ needs: run+ if: always()+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ with:+ # Need history so the bot commit doesn't trample anything.+ fetch-depth: 0+ token: ${{ secrets.GITHUB_TOKEN }}++ - uses: actions/download-artifact@v4+ with:+ path: /tmp/results-raw++ - name: Stage into benchmarks/results/<date>/+ id: stage+ run: |+ set -euo pipefail+ date_tag="$(date -u +%Y-%m-%d)"+ dst="benchmarks/results/${date_tag}"+ mkdir -p "${dst}"+ find /tmp/results-raw -name '*.json' -exec cp {} "${dst}/" \;+ echo "dst=${dst}" >> "$GITHUB_OUTPUT"+ ls -la "${dst}"++ - name: Render charts+ run: |+ pip install --quiet matplotlib+ python3 benchmarks/scripts/chart.py "${{ steps.stage.outputs.dst }}"++ - name: Commit results+ run: |+ set -euo pipefail+ git config user.name "traceway-bot"+ git config user.email "[email protected]"+ git add "${{ steps.stage.outputs.dst }}"+ if git diff --cached --quiet; then+ echo "nothing to commit"+ exit 0+ fi+ git commit -m "bench: hardware benchmark $(date -u +%Y-%m-%d)"+ git pushdiff --git a/.gitignore b/.gitignoreindex 125b3c2e..1702ee87 100644--- a/.gitignore+++ b/.gitignore@@ -46,3 +46,11 @@ backend/static/dist//examples/hono-otel/node_modules/examples/nestjs-otel/node_modules/website/.wrangler++# Benchmark tooling: built binaries and laptop-only result dirs.+# Real CI-produced result dirs (e.g. benchmarks/results/2026-05-12/) are+# committed; the -local suffix marks dev runs we never want to push.+/benchmarks/loadgen/loadgen+/benchmarks/loadgen/loadgen-linux-amd64+/benchmarks/loadgen/loadgen-linux-arm64+/benchmarks/results/*-local/diff --git a/benchmarks/LOCAL.md b/benchmarks/LOCAL.mdnew file mode 100644index 00000000..64fe6f75--- /dev/null+++ b/benchmarks/LOCAL.md@@ -0,0 +1,360 @@+# Running benchmarks locally++Step-by-step recipe for running the loadgen against a backend on your laptop —+no Hetzner, no GitHub Actions. Useful for: iterating on the loadgen itself,+checking that a backend change didn't regress single-machine throughput, or+just understanding the tooling before spending real money on Hetzner.++> Laptop numbers are not comparable to Hetzner numbers. Use this for tooling+> validation and relative regression checks, not for the published chart.++---++## Prerequisites++Install once:++```bash+# macOS+brew install docker jq go++# Confirm Docker is running+docker info >/dev/null && echo "docker: ok"+```++You also need Python 3 with matplotlib. The cleanest path on macOS (system+Python is PEP-668 locked and refuses `pip install`):++```bash+# Pick one of:++# Option A — venv in /tmp (wiped on restart, fine for ad-hoc smoke tests)+python3 -m venv /tmp/bench-venv && /tmp/bench-venv/bin/pip install -q matplotlib++# Option B — venv inside the repo (persists across restarts; gitignored)+python3 -m venv .venv-bench && .venv-bench/bin/pip install -q matplotlib+```++The rest of this doc assumes option B, so swap `/tmp/bench-venv` for+`.venv-bench` if you went with option A.++---++## Pick a backend mode++Two compose stacks, each pinned to one of the project's existing Dockerfiles:++| Mode | Compose file | Image | What it tests |+|------|--------------|-------|---------------|+| `sqlite` | `benchmarks/compose/docker-compose.sqlite.yml` | `Dockerfile.sqlite` | Single-binary backend with embedded SQLite. Fast to bring up, lower ceiling. |+| `pgch` | `benchmarks/compose/docker-compose.pgch.yml` | `Dockerfile.minimal` + clickhouse + postgres | Full prod-shape stack. Slower first build, much higher ceiling. |++Both expose port **8087** on the host (override with `BENCH_PORT=8088 docker+compose -f ... up`).++First-time builds:+- `sqlite` mode: ~3–6 min (npm install + Go build).+- `pgch` mode: ~6–10 min (above + ClickHouse + Postgres image pulls).++Subsequent runs reuse the Docker layer cache and start in seconds.++---++## The 6-step run++These are the exact commands. Open a terminal at the repo root:++```bash+cd /Users/dusanstanojevic/Documents/workspace/traceway+```++### 1. Bring up the backend on a clean volume++```bash+# Always tear down with -v first. Reusing a volume across runs means the next+# loadgen run starts with old data, AND can hit "database disk image is+# malformed" if the previous container was killed mid-write.+docker compose -f benchmarks/compose/docker-compose.sqlite.yml down -v 2>/dev/null+docker compose -f benchmarks/compose/docker-compose.sqlite.yml up -d --build+```++For `pgch` mode swap the filename to `docker-compose.pgch.yml` here and in the+remaining steps.++### 2. Wait for `/health` to return 200++```bash+until curl -sf http://localhost:8087/health >/dev/null; do echo "waiting..."; sleep 2; done+echo "ready"+```++If this loops for more than ~3 min, something is wrong. Inspect:++```bash+docker compose -f benchmarks/compose/docker-compose.sqlite.yml logs --tail=80 traceway+```++### 3. Register a fresh user + project; capture creds++```bash+eval "$(benchmarks/scripts/seed-project.sh http://localhost:8087 \+ | jq -r '"export JWT=\(.jwt) PROJECT_TOKEN=\(.projectToken) PROJECT_ID=\(.projectId)"')"+echo "PROJECT_ID=$PROJECT_ID"+```++`seed-project.sh` hits `/api/register`. On self-hosted backends only one+organization is allowed, so this only works on a **fresh** DB — step 1's `down+-v` is what guarantees that.++### 4. Run the loadgen++Pick numbers appropriate for the mode you booted in step 1:++**SQLite (laptop):**++```bash+mkdir -p /tmp/bench-localhost && rm -f /tmp/bench-localhost/*.json+benchmarks/loadgen/loadgen \+ --target http://localhost:8087 \+ --token "$PROJECT_TOKEN" --jwt "$JWT" --project-id "$PROJECT_ID" \+ --duration 5m --step-duration 45s \+ --initial-eps 100 --max-eps 800 \+ --report-out /tmp/bench-localhost/local-sqlite.json \+ --tier local --mode sqlite+```++**pgch (laptop):**++```bash+benchmarks/loadgen/loadgen \+ --target http://localhost:8087 \+ --token "$PROJECT_TOKEN" --jwt "$JWT" --project-id "$PROJECT_ID" \+ --duration 5m --step-duration 45s \+ --initial-eps 500 --max-eps 4000 \+ --report-out /tmp/bench-localhost/local-pgch.json \+ --tier local --mode pgch+```++On stderr you'll see one line per step like:++```+step 1: target=300 actual=306 ingest_p99=92ms ingest_err=0.00% passed=true+step 2: target=600 actual=600 ingest_p99=17ms ingest_err=0.00% passed=false read endpoint "endpoints_grouped" p99 = 3268ms > 3000ms threshold+wrote /tmp/bench-localhost/local-sqlite.json: max sustainable EPS = 306 (2 step(s))+```++The run finishes when either (a) a step fails the breaking-point check, (b)+`--max-eps` is exceeded, or (c) `--duration` expires.++### 5. Render the chart + summary++```bash+.venv-bench/bin/python benchmarks/scripts/chart.py /tmp/bench-localhost/+open /tmp/bench-localhost/chart.png+open /tmp/bench-localhost/chart-detail.png+cat /tmp/bench-localhost/summary.md+```++A one-entry result renders fine; it just won't have multiple bars/lines to+compare against.++### 6. Tear down++```bash+docker compose -f benchmarks/compose/docker-compose.sqlite.yml down -v+```++The `-v` deletes the named volume so the next run is guaranteed clean.++---++## Reading the output++Each step record in the JSON looks like:++```json+{+ "step": 1,+ "targetEps": 300, // Synthetic event units the ramp asked for. NOT requests/sec — see "What an EPS is" below.+ "actualEps": 306.6, // What the loadgen actually pushed on the wire (same unit).+ "ingest": { // Latency/errors for /api/report + OTLP combined+ "p50": 3.8, "p95": 12.5, "p99": 92.3,+ "ok": 1380, "errors": 0, "errRate": 0+ },+ "reads": { // Per dashboard read endpoint+ "endpoints_grouped": { "p50": 1120, "p95": 1462, "p99": 1509, "ok": 36, ... },+ "dashboard_overview": { ... },+ "exceptions_grouped": { ... },+ "logs": { ... },+ "session_recording": { ... }+ },+ "passed": true,+ "failReason": "" // populated when passed=false+}+```++**A step passes** when *all three* hold:+1. Ingest error rate ≤ 5% (5xx, timeouts).+2. Every read endpoint p99 < 3000 ms.+3. SUT was reachable for the whole step.++`maxSustainableEps` in the JSON is the `actualEps` of the **last passing+step** — that's the headline number for this configuration.++If `target` and `actual` diverge a lot on a passing step, the backend is+already saturated even though it's not erroring yet. The next step usually+fails.++If reads degrade before ingest, that's expected: aggregation reads+(`endpoints_grouped`, `dashboard_overview`) compete with write locks/IO and+will be the first to cliff on SQLite. Point lookups (`logs`,+`session_recording`, `exceptions_grouped`) typically stay fast right up until+the SUT gives up.++---++## What an EPS is (and isn't)++`EPS` in the loadgen is **synthetic event units**, not requests/sec and not+DB rows/sec. The conversion:++```+requests/sec = EPS / 10 # because tracesPerFrame = 10+```++So `--max-eps 800` actually means "push up to 80 HTTP requests/sec." The+factor-of-10 fudge keeps the rate-limiter math clean (each `/api/report`+frame carries 10 traces; the loadgen counts EPS as traces).++### What goes in each request++Each "frame" the loadgen sends is one HTTP POST, routed by `--report-share`+(default 0.7 → 70 % go to `/api/report`, 30 % to OTLP).++**`/api/report` frame** — gzipped JSON, built by `buildFrame()` in+`benchmarks/loadgen/report.go`:++| Field | Count per request | Notes |+|---|---|---|+| `traces` | **10 (fixed)** | One row per `transactions` |+| `spans` (nested in traces) | **20–50** (each trace gets 2–5) | Random duration 1–200 ms, name from `{db.query, http.call, cache.get, render, json.encode}` |+| `metrics` | **5 (fixed)** | `bench.metric.{cpu,mem,qps,lat,errs}`, random value 0–100 |+| `sessions` | 0 / 1 — 5 % with recording, 16 % session-only | Random uuid + StartedAt |+| `sessionRecordings` | 0 / 1 — 5 % chance | ~100 KB synthetic rrweb-shaped blob |+| `stackTraces` | 0 / 1 — 2 % chance | Hardcoded short stack trace |++Trace fields: random endpoint from a fixed 13-item list (`GET /api/users`,+`POST /api/orders`, `GET /api/products/:id`, …), duration 10–1000 ms, random+status code, body size 200–8200 bytes.++**OTLP frame** — protobuf, built by `sendOTLPTraces()` in+`benchmarks/loadgen/otlp.go`:++| Field | Count per request | Notes |+|---|---|---|+| `spans` | **20 (fixed, `otlpSpansPerRequest`)** | Random 16-byte traceId + 8-byte spanId, http.method=GET, status code, route from the same 13-endpoint pool |++No metrics, sessions, or exceptions on this path.++### Row growth per second at 400 EPS++At 400 EPS the loadgen fires **40 requests/sec total**: ~28 to `/api/report`,+~12 to OTLP. That fans out into:++| Table | Rows/sec |+|---|---|+| `transactions` | ~280 (28 × 10) |+| `spans` (from `/api/report`) | ~980 (28 × ~35) |+| `spans` (from OTLP) | ~240 (12 × 20) |+| `metric_points` | ~140 (28 × 5) |+| `sessions` | ~6 (28 × 21 %) |+| `session_recordings` | ~1.4 (28 × 5 %) |+| `exception_stack_traces` | ~0.56 (28 × 2 %) |++So **"400 EPS" ≈ 1,500 rows/sec** across the database, not 400. Keep this+mapping in mind when comparing the chart's headline number to your real+workload.++### Data variety++Everything is uniform random within bounded ranges — deliberately flat:++- 13 distinct endpoint paths → `GROUP BY endpoint` aggregations always+ produce ~13 groups+- 5 span names, 5 metric names+- All UUIDs are fresh per record (no hash clustering)+- Durations: uniform 10–1000 ms → percentile queries hit a smooth+ distribution+- No diurnal traffic, no power-law endpoint skew, no correlated bursts++This isolates the backend's raw throughput from traffic-shape noise. It is+**not** representative of real production telemetry; don't read the numbers+as "this is what users will experience" — they're "this is the floor the+backend can sustain on synthetic, evenly-distributed load."++---++## Tuning the run++| Flag | Default | When to change |+|------|---------|----------------|+| `--initial-eps` | 100 | Raise close to the expected ceiling so you don't waste the first few steps far below saturation. |+| `--max-eps` | 50000 | Lower it to bound runtime; raise to find the ceiling on a beefier box. |+| `--step-duration` | 2m | Shorten for smoke checks (45s), keep at 2m for real measurements. |+| `--duration` | 30m | Hard cap on total wall time. |+| `--report-share` | 0.7 | Fraction of ingest sent via `/api/report` vs OTLP. Lower it to test OTLP-heavy workloads. |+| `--read-rps` | 1 | Read traffic per endpoint per second. Raise to model a busier dashboard. |+| `--ingest-err-threshold` | 0.05 | Allowed ingest error rate before a step fails. |+| `--read-p99-threshold-ms` | 3000 | The dashboard latency budget. |++The ramp doubles target each step (100→200→400→800→…), so the cliff+resolution is coarse by design — for a precise number, run a second pass with+`--initial-eps` set near the known cliff and shorter steps.++---++## Common pitfalls++- **`database disk image is malformed (11)`** in the container logs.+ Previous run was killed mid-write and the SQLite WAL didn't recover. Run+ step 1 again with `down -v`.++- **`actual` is ~10× lower than `target` on every step.**+ You're running an older loadgen binary. Rebuild:+ ```bash+ (cd benchmarks/loadgen && go build -o loadgen .)+ ```++- **`seed-project.sh` returns `409 Conflict: organization already exists`.**+ The DB isn't fresh — go back to step 1 with `down -v`.++- **Port 8087 already in use.**+ Pick another: `BENCH_PORT=8088 docker compose -f ... up -d --build`, then+ use `http://localhost:8088` everywhere.++- **`/tmp/bench-localhost/` gone after a restart.**+ macOS empties `/tmp` on reboot. Use `~/bench-localhost/` or a path inside+ the repo if you want results to survive.++- **Loadgen run was killed; some Docker resources still up.**+ The compose stack is yours to manage — it won't auto-tear-down. Run step 6.++---++## Going beyond local++Once the local run works end-to-end, move to the Hetzner ladder:++```bash+# Validate Hetzner credentials, no money spent+./benchmarks/scripts/run-local.sh --dry-run++# ~5 min, ~€0.02 — one tier, one mode, on real Hetzner+./benchmarks/scripts/run-local.sh --smoke++# Full matrix — 4 tiers x 2 modes, ~1 h, ~€1.20+./benchmarks/scripts/run-local.sh+```++Prereqs and detail in [README.md](./README.md). The same `run-matrix-entry.sh`+script powers both the laptop path and the GitHub Actions workflow.diff --git a/benchmarks/README.md b/benchmarks/README.mdnew file mode 100644index 00000000..f6cd550d--- /dev/null+++ b/benchmarks/README.md@@ -0,0 +1,172 @@+# benchmarks/++Hardware-vs-breaking-point benchmark for the Traceway backend. Provisions+Hetzner Cloud servers, hammers the backend with realistic ingest while+measuring read latency, and produces a chart of sustainable throughput per+hardware tier and DB mode.++## What it answers++> On hardware tier **X** with DB config **Y**, you can sustain **N** traces ++> spans + session-replays per second while reads remain responsive.++Two DB modes are tested:+- **sqlite** — single-binary Traceway with embedded SQLite (`Dockerfile.sqlite`).+- **pgch** — full ClickHouse + Postgres stack (`Dockerfile.minimal`).++Four hardware tiers, all Hetzner CCX (dedicated vCPU) so neighbor noise doesn't+pollute the read-latency signal:++| Tier | vCPU | RAM | Disk |+|-------|------|-------|-------------|+| CCX13 | 2 | 8 GB | 80 GB NVMe |+| CCX23 | 4 | 16 GB | 160 GB NVMe |+| CCX33 | 8 | 32 GB | 240 GB NVMe |+| CCX43 | 16 | 64 GB | 360 GB NVMe |++## How a run works++Per matrix entry (one tier × one mode):++1. `hetzner-up.sh` provisions a SUT box + a CAX11 loadgen box on a private network (override with `LOADGEN_TIER`).+2. `sut-bootstrap.sh` rsyncs the repo, installs Docker, brings up the right+ `docker-compose.<mode>.yml`, waits for `/health`.+3. `seed-project.sh` registers a fresh user + project, captures the JWT and+ project token.+4. `loadgen-bootstrap.sh` cross-compiles the loadgen for linux/amd64, pushes+ it to the loadgen box, runs it against the SUT's *private* IP.+5. The loadgen ramps target ingest in doubling steps (100 -> 200 -> 400 -> ...)+ while a constant read mix probes the dashboard endpoints. Each step holds+ for `--step-duration` (default 2 min).+6. A step "fails" when ingest error rate > 5%, OR any read endpoint P99 > 3000ms.+ Max sustainable EPS is the actual rate of the last passing step.+7. `hetzner-down.sh` deletes everything via a bash `trap` — even on Ctrl-C.++After all matrix entries finish, `chart.py` renders `chart.png` (bar) ++`chart-detail.png` (line: read P99 vs ingest EPS) + `summary.md`.++## Running from your laptop++### One-time setup++1. **Install tooling**: `hcloud`, `jq`, `python3`, Go 1.25+, `rsync`. On macOS:+ `brew install hcloud jq rsync go`.+2. **Install matplotlib in a venv** (system Python is usually PEP 668 locked):+ ```bash+ python3 -m venv .venv+ source .venv/bin/activate+ pip install matplotlib+ ```+3. **Generate an SSH key** specifically for benchmarks and upload its public+ half to Hetzner under the name `benchmark-key`:+ ```bash+ ssh-keygen -t ed25519 -f ~/.ssh/hetzner_benchmark -C benchmark-key+ chmod 600 ~/.ssh/hetzner_benchmark+ hcloud ssh-key create --name benchmark-key --public-key-from-file ~/.ssh/hetzner_benchmark.pub+ ```+4. **Export creds** (use `direnv` or a sourced `.envrc.local`, never commit):+ ```bash+ export HCLOUD_TOKEN=...+ export BENCHMARK_SSH_KEY=~/.ssh/hetzner_benchmark+ ```++### Smoke (cheap — ~5 min, ~€0.02)++```bash+./benchmarks/scripts/run-local.sh --smoke+```++One tier (ccx13), one mode (sqlite), short steps. If this works the full+matrix works.++### Full matrix (~1 h, ~€1.20)++```bash+./benchmarks/scripts/run-local.sh+```++4 tiers × 2 modes, default `--duration 30m`. Output goes to+`benchmarks/results/<YYYY-MM-DD>-local/` — the `-local` suffix exists so+dev runs never get confused with CI runs and never get accidentally committed.++### Other useful invocations++```bash+# Validate environment without provisioning anything (free)+./benchmarks/scripts/run-local.sh --dry-run++# Re-run just one tier with default duration+./benchmarks/scripts/run-local.sh --tier ccx23 --mode pgch++# Override the per-entry runtime+./benchmarks/scripts/run-local.sh --tier ccx13 --duration 10m++# Drop the "-local" suffix so the result dir matches the CI naming convention+# (use this only when you want to commit a one-off re-benchmark)+./benchmarks/scripts/run-local.sh --tier ccx23 --commit+```++## Running from GitHub Actions++`.github/workflows/benchmark-hardware.yml`, `workflow_dispatch` only. Inputs+mirror the local flags. The workflow YAML is a thin wrapper around the same+`run-matrix-entry.sh` script the local path uses — if it works locally, it+works in CI.++Required GitHub secrets:+- `HCLOUD_TOKEN`+- `BENCHMARK_SSH_PRIVATE_KEY` — the private key matching the Hetzner-side+ `benchmark-key`.++After the matrix completes, an `aggregate` job downloads all artifacts, runs+`chart.py`, and commits `benchmarks/results/<YYYY-MM-DD>/` to `main` via a bot+commit. No PR — it's a generated artifact.++## Layout++```+benchmarks/+ compose/ # SUT-side docker compose, one per mode+ docker-compose.sqlite.yml+ docker-compose.pgch.yml+ loadgen/ # The Go binary that generates load+ main.go # CLI + orchestration+ ingest.go # Stream A — stepped ingest worker pool+ report.go # /api/report JSON+gzip sender+ otlp.go # OTLP/HTTP protobuf sender+ reads.go # Stream B — constant read mix+ ramp.go # Step controller + breaking-point logic+ stats.go # Latency tracker (percentiles via sort)+ util.go, log.go # Misc helpers+ scripts/+ run-local.sh # ★ Laptop entry point+ run-matrix-entry.sh # One matrix cycle (used by local + CI)+ preflight.sh # Validates env before any provisioning+ hetzner-up.sh # hcloud server create+ hetzner-down.sh # hcloud server delete (idempotent)+ sut-bootstrap.sh # Installs Docker, brings up backend+ loadgen-bootstrap.sh # Cross-compiles + runs loadgen+ seed-project.sh # /api/register -> JWT + project token JSON+ chart.py # matplotlib renderer+ _ssh.sh # Shared ssh/rsync helpers+ results/ # Committed bench results live here+```++## Failure modes & debugging++- **A run leaves Hetzner servers behind.** `hetzner-down.sh` is idempotent and+ callable directly: `./benchmarks/scripts/hetzner-down.sh <run-id>`. Run IDs+ appear in the orchestrator's stderr at the start of each matrix entry. Also+ visible in `hcloud server list` — anything tagged `bench=true` is safe to+ delete.+- **`preflight.sh` complains about `benchmark-key`.** The Hetzner-side SSH key+ resource is named `benchmark-key` regardless of what you named the file+ locally. Re-upload via `hcloud ssh-key create --name benchmark-key+ --public-key-from-file <your-pub-key>`.+- **Docker compose build is slow on tiny tiers.** First-time `docker compose+ up --build` on a CCX13 takes 5–8 minutes (npm install + Go build).+ `sut-bootstrap.sh` waits up to 10 minutes for `/health` before giving up.+ Subsequent runs reuse the Docker layer cache via the persistent volumes.+- **All steps pass on the highest tier.** Increase `--max-eps` (default 50000)+ if you want to find the ceiling on big boxes. The default is meant to keep+ one-tier runs bounded; on CCX43 with PG/CH the ceiling can be well above 10k EPS.diff --git a/benchmarks/compose/docker-compose.pgch.yml b/benchmarks/compose/docker-compose.pgch.ymlnew file mode 100644index 00000000..87e55be4--- /dev/null+++ b/benchmarks/compose/docker-compose.pgch.yml@@ -0,0 +1,72 @@+services:+ traceway:+ build:+ context: ../..+ dockerfile: Dockerfile.minimal+ ports:+ - "${BENCH_PORT:-8087}:80"+ environment:+ CLICKHOUSE_SERVER: "clickhouse:9000"+ CLICKHOUSE_DATABASE: "traceway"+ CLICKHOUSE_USERNAME: "default"+ CLICKHOUSE_PASSWORD: "clickhouse"+ CLICKHOUSE_TLS: "false"+ POSTGRES_HOST: "postgres"+ POSTGRES_PORT: "5432"+ POSTGRES_DATABASE: "traceway"+ POSTGRES_USERNAME: "traceway"+ POSTGRES_PASSWORD: "traceway"+ POSTGRES_SSLMODE: "disable"+ JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"+ GIN_MODE: "release"+ depends_on:+ clickhouse:+ condition: service_healthy+ postgres:+ condition: service_healthy+ restart: "no"+ healthcheck:+ test: ["CMD", "wget", "-qO-", "http://localhost/health"]+ interval: 5s+ timeout: 3s+ start_period: 10s+ retries: 20++ clickhouse:+ image: clickhouse/clickhouse-server:24.8-alpine+ environment:+ CLICKHOUSE_DB: traceway+ CLICKHOUSE_PASSWORD: clickhouse+ volumes:+ - clickhouse-bench-data:/var/lib/clickhouse+ healthcheck:+ test: ["CMD", "clickhouse-client", "--password", "clickhouse", "--query", "SELECT 1"]+ interval: 5s+ timeout: 3s+ start_period: 10s+ retries: 20+ restart: "no"+ ulimits:+ nofile:+ soft: 262144+ hard: 262144++ postgres:+ image: postgres:17+ environment:+ POSTGRES_USER: traceway+ POSTGRES_PASSWORD: traceway+ POSTGRES_DB: traceway+ volumes:+ - postgres-bench-data:/var/lib/postgresql/data+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U traceway"]+ interval: 5s+ timeout: 3s+ start_period: 10s+ retries: 20+ restart: "no"++volumes:+ clickhouse-bench-data:+ postgres-bench-data:diff --git a/benchmarks/compose/docker-compose.sqlite.yml b/benchmarks/compose/docker-compose.sqlite.ymlnew file mode 100644index 00000000..a140b529--- /dev/null+++ b/benchmarks/compose/docker-compose.sqlite.yml@@ -0,0 +1,24 @@+services:+ traceway:+ build:+ context: ../..+ dockerfile: Dockerfile.sqlite+ ports:+ - "${BENCH_PORT:-8087}:80"+ environment:+ JWT_SECRET: "bench-jwt-secret-not-for-production-use-only-32chars+"+ SQLITE_RETENTION_DAYS: "0"+ SESSION_RECORDING_RETENTION_DAYS: "0"+ GIN_MODE: "release"+ volumes:+ - traceway-bench-sqlite-data:/data+ restart: "no"+ healthcheck:+ test: ["CMD", "wget", "-qO-", "http://localhost/health"]+ interval: 5s+ timeout: 3s+ start_period: 10s+ retries: 20++volumes:+ traceway-bench-sqlite-data:diff --git a/benchmarks/loadgen/go.mod b/benchmarks/loadgen/go.modnew file mode 100644index 00000000..7b72cd8a--- /dev/null+++ b/benchmarks/loadgen/go.mod@@ -0,0 +1,20 @@+module bench-loadgen++go 1.25++require (+ github.com/google/uuid v1.6.0+ go.opentelemetry.io/proto/otlp v1.5.0+ golang.org/x/time v0.7.0+ google.golang.org/protobuf v1.36.1+)++require (+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect+ golang.org/x/net v0.33.0 // indirect+ golang.org/x/sys v0.29.0 // indirect+ golang.org/x/text v0.21.0 // indirect+ google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect+ google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect+ google.golang.org/grpc v1.69.2 // indirect+)diff --git a/benchmarks/loadgen/go.sum b/benchmarks/loadgen/go.sumnew file mode 100644index 00000000..bd2d2ffe--- /dev/null+++ b/benchmarks/loadgen/go.sum@@ -0,0 +1,40 @@+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ=+go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=+go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=+go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=+go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=+go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=+go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=+go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=+go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=+go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=+go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=+go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=+go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=+google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA=+google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY=+google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw=+google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=+google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=+google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=+google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=+google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=diff --git a/benchmarks/loadgen/ingest.go b/benchmarks/loadgen/ingest.gonew file mode 100644index 00000000..7adfc625--- /dev/null+++ b/benchmarks/loadgen/ingest.go@@ -0,0 +1,96 @@+package main++import (+ "context"+ "math/rand"+ "net/http"+ "sync"+ "sync/atomic"++ "golang.org/x/time/rate"+)++const (+ // One frame carries 10 traces, so frame rate = targetEPS / tracesPerFrame.+ // Matches Go SDK behavior where frames rotate every ~5s with whatever has+ // accumulated; benchmark uses a fixed batch size for predictable accounting.+ tracesPerFrame = 10+ ingestWorkers = 256+ recordingChance = 0.05 // 5% of frames carry a session recording segment+ exceptionChance = 0.02 // 2% of frames carry an exception+)++type ingester struct {+ cfg config+ client *http.Client+ stats *latencyTracker+ limiter *rate.Limiter+ // rng pool: each worker grabs one to avoid contending on the global source+ rngPool sync.Pool+ // sessionsCreated counts sessions sent via /api/report so the reader can+ // query a real session ID for the recording playback endpoint.+ sessionsCreated atomic.Int64+ sampleSession atomic.Value // string — most recently seeded session id+}++func newIngester(cfg config, client *http.Client, stats *latencyTracker) *ingester {+ framesPerSec := cfg.initialEPS / tracesPerFrame+ if framesPerSec < 1 {+ framesPerSec = 1+ }+ return &ingester{+ cfg: cfg,+ client: client,+ stats: stats,+ limiter: rate.NewLimiter(rate.Limit(framesPerSec), int(framesPerSec)+1),+ rngPool: sync.Pool{New: func() any { return rand.New(rand.NewSource(rand.Int63())) }},+ }+}++func (i *ingester) SetRate(eps float64) {+ fps := eps / tracesPerFrame+ if fps < 1 {+ fps = 1+ }+ i.limiter.SetLimit(rate.Limit(fps))+ i.limiter.SetBurst(int(fps) + 1)+}++func (i *ingester) SampleSessionId() string {+ v := i.sampleSession.Load()+ if v == nil {+ return ""+ }+ return v.(string)+}++func (i *ingester) run(ctx context.Context) {+ var wg sync.WaitGroup+ wg.Add(ingestWorkers)+ for w := 0; w < ingestWorkers; w++ {+ go func() {+ defer wg.Done()+ rng := i.rngPool.Get().(*rand.Rand)+ defer i.rngPool.Put(rng)+ for {+ if err := i.limiter.Wait(ctx); err != nil {+ return+ }+ i.sendOne(ctx, rng)+ }+ }()+ }+ wg.Wait()+}++func (i *ingester) sendOne(ctx context.Context, rng *rand.Rand) {+ if rng.Float64() < i.cfg.reportShare {+ sessionId := sendReportFrame(ctx, i.client, i.cfg, rng, i.stats)+ if sessionId != "" {+ i.sampleSession.Store(sessionId)+ i.sessionsCreated.Add(1)+ }+ } else {+ sendOTLPTraces(ctx, i.client, i.cfg, rng, i.stats)+ }+}diff --git a/benchmarks/loadgen/log.go b/benchmarks/loadgen/log.gonew file mode 100644index 00000000..652c07e6--- /dev/null+++ b/benchmarks/loadgen/log.go@@ -0,0 +1,25 @@+package main++import (+ "io"+ "os"+ "sync"+)++var stderrMu sync.Mutex++// stderrPrefix returns a single shared writer for human-readable progress logs.+// Wrapping os.Stderr through a tiny adapter so concurrent goroutines never+// interleave half a line — important because the ramp loop and ingest worker+// pool both want to print status updates while running.+func stderrPrefix() io.Writer {+ return &lockedStderr{}+}++type lockedStderr struct{}++func (l *lockedStderr) Write(p []byte) (int, error) {+ stderrMu.Lock()+ defer stderrMu.Unlock()+ return os.Stderr.Write(p)+}diff --git a/benchmarks/loadgen/main.go b/benchmarks/loadgen/main.gonew file mode 100644index 00000000..6fa92e7f--- /dev/null+++ b/benchmarks/loadgen/main.go@@ -0,0 +1,114 @@+package main++import (+ "context"+ "encoding/json"+ "flag"+ "fmt"+ "log"+ "net/http"+ "os"+ "os/signal"+ "syscall"+ "time"+)++type config struct {+ target string+ projectToken string+ jwt string+ projectId string+ duration time.Duration+ stepDuration time.Duration+ initialEPS float64+ maxEPS float64+ reportShare float64+ readRPS float64+ ingestErrThreshold float64+ readP99ThresholdMs float64+ reportOut string+ tier string+ mode string+}++func main() {+ var cfg config++ flag.StringVar(&cfg.target, "target", "", "Base URL of the system under test (e.g. http://10.0.0.2 or http://localhost:8087)")+ flag.StringVar(&cfg.projectToken, "token", "", "Project bearer token for /api/report + OTLP")+ flag.StringVar(&cfg.jwt, "jwt", "", "JWT for dashboard read endpoints")+ flag.StringVar(&cfg.projectId, "project-id", "", "Project UUID (passed as ?projectId= on read endpoints)")+ flag.DurationVar(&cfg.duration, "duration", 30*time.Minute, "Total run duration cap")+ flag.DurationVar(&cfg.stepDuration, "step-duration", 2*time.Minute, "Per-step hold time")+ flag.Float64Var(&cfg.initialEPS, "initial-eps", 100, "Starting target traces/sec")+ flag.Float64Var(&cfg.maxEPS, "max-eps", 50000, "Hard cap on target traces/sec")+ flag.Float64Var(&cfg.reportShare, "report-share", 0.7, "Fraction of ingest sent via /api/report (rest via OTLP)")+ flag.Float64Var(&cfg.readRPS, "read-rps", 1.0, "Read RPS per endpoint")+ flag.Float64Var(&cfg.ingestErrThreshold, "ingest-err-threshold", 0.05, "Step fails if ingest error rate exceeds this")+ flag.Float64Var(&cfg.readP99ThresholdMs, "read-p99-threshold-ms", 3000, "Step fails if any read endpoint p99 (ms) exceeds this")+ flag.StringVar(&cfg.reportOut, "report-out", "", "Path to write JSON results (required)")+ flag.StringVar(&cfg.tier, "tier", "local", "Hardware tier label embedded in output (e.g. ccx13)")+ flag.StringVar(&cfg.mode, "mode", "unknown", "DB mode label embedded in output (sqlite | pgch)")+ flag.Parse()++ if cfg.target == "" || cfg.projectToken == "" || cfg.reportOut == "" {+ fmt.Fprintln(os.Stderr, "missing required flag: --target, --token, --report-out")+ flag.Usage()+ os.Exit(2)+ }++ ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)+ defer cancel()++ deadline := time.Now().Add(cfg.duration)+ ctx, cancelDeadline := context.WithDeadline(ctx, deadline)+ defer cancelDeadline()++ httpClient := &http.Client{+ Timeout: 15 * time.Second,+ Transport: &http.Transport{+ MaxIdleConns: 200,+ MaxIdleConnsPerHost: 100,+ IdleConnTimeout: 60 * time.Second,+ },+ }++ ingestStats := newLatencyTracker()+ readStats := map[string]*latencyTracker{+ "endpoints_grouped": newLatencyTracker(),+ "exceptions_grouped": newLatencyTracker(),+ "dashboard_overview": newLatencyTracker(),+ "session_recording": newLatencyTracker(),+ "logs": newLatencyTracker(),+ }++ ingester := newIngester(cfg, httpClient, ingestStats)+ reader := newReader(cfg, httpClient, readStats)+ reader.attachIngester(ingester)++ go ingester.run(ctx)+ go reader.run(ctx)++ results := runRamp(ctx, cfg, ingester, ingestStats, readStats)++ out := finalReport{+ Tier: cfg.tier,+ Mode: cfg.mode,+ StartedAt: time.Now().UTC().Add(-cfg.duration).Format(time.RFC3339),+ EndedAt: time.Now().UTC().Format(time.RFC3339),+ Steps: results,+ }+ out.computeBreakingPoint()++ f, err := os.Create(cfg.reportOut)+ if err != nil {+ log.Fatalf("create %s: %v", cfg.reportOut, err)+ }+ defer f.Close()+ enc := json.NewEncoder(f)+ enc.SetIndent("", " ")+ if err := enc.Encode(&out); err != nil {+ log.Fatalf("write %s: %v", cfg.reportOut, err)+ }+ fmt.Fprintf(os.Stderr, "wrote %s: max sustainable EPS = %.0f (%d step(s))\n", cfg.reportOut, out.MaxSustainableEPS, len(out.Steps))+}diff --git a/benchmarks/loadgen/otlp.go b/benchmarks/loadgen/otlp.gonew file mode 100644index 00000000..4f3933e3--- /dev/null+++ b/benchmarks/loadgen/otlp.go@@ -0,0 +1,107 @@+package main++import (+ "bytes"+ "context"+ "crypto/rand"+ "fmt"+ "net/http"+ mathrand "math/rand"+ "time"++ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+ "google.golang.org/protobuf/proto"+)++const otlpSpansPerRequest = 20++// sendOTLPTraces marshals an ExportTraceServiceRequest with otlpSpansPerRequest+// spans into protobuf, POSTs to /api/otel/v1/traces, and records latency.+// Protobuf (not protojson) because that's what production exporters send — it+// exercises the backend's protobuf decode path which is non-trivial CPU work+// that would be hidden if we sent JSON.+func sendOTLPTraces(ctx context.Context, client *http.Client, cfg config, rng *mathrand.Rand, stats *latencyTracker) {+ now := time.Now().UTC()+ resourceAttrs := []*commonpb.KeyValue{+ strAttr("service.name", "bench-loadgen"),+ strAttr("service.version", "1.0.0"),+ strAttr("deployment.environment", "bench"),+ }++ spans := make([]*tracepb.Span, otlpSpansPerRequest)+ for i := range spans {+ traceId := make([]byte, 16)+ spanId := make([]byte, 8)+ _, _ = rand.Read(traceId)+ _, _ = rand.Read(spanId)++ startNs := now.Add(-time.Duration(rng.Intn(500)) * time.Millisecond).UnixNano()+ duration := time.Duration(10+rng.Intn(990)) * time.Millisecond+ endNs := startNs + int64(duration)++ spans[i] = &tracepb.Span{+ TraceId: traceId,+ SpanId: spanId,+ Name: endpointPaths[rng.Intn(len(endpointPaths))],+ Kind: tracepb.Span_SPAN_KIND_SERVER,+ StartTimeUnixNano: uint64(startNs),+ EndTimeUnixNano: uint64(endNs),+ Attributes: []*commonpb.KeyValue{+ strAttr("http.method", "GET"),+ intAttr("http.status_code", int64(pickStatus(rng))),+ strAttr("http.route", endpointPaths[rng.Intn(len(endpointPaths))]),+ },+ Status: &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK},+ }+ }++ req := &coltracepb.ExportTraceServiceRequest{+ ResourceSpans: []*tracepb.ResourceSpans{{+ Resource: &resourcepb.Resource{Attributes: resourceAttrs},+ ScopeSpans: []*tracepb.ScopeSpans{{+ Scope: &commonpb.InstrumentationScope{Name: "bench-loadgen", Version: "1.0.0"},+ Spans: spans,+ }},+ }},+ }++ body, err := proto.Marshal(req)+ if err != nil {+ stats.Record(0, err)+ return+ }++ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.target+"/api/otel/v1/traces", bytes.NewReader(body))+ if err != nil {+ stats.Record(0, err)+ return+ }+ httpReq.Header.Set("Content-Type", "application/x-protobuf")+ httpReq.Header.Set("Authorization", "Bearer "+cfg.projectToken)++ start := time.Now()+ resp, err := client.Do(httpReq)+ elapsed := time.Since(start)+ if err != nil {+ stats.Record(elapsed.Seconds()*1000, err)+ return+ }+ defer resp.Body.Close()+ _, _ = readAndDiscard(resp.Body)+ if resp.StatusCode >= 400 {+ stats.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))+ return+ }+ stats.Record(elapsed.Seconds()*1000, nil)+}++func strAttr(k, v string) *commonpb.KeyValue {+ return &commonpb.KeyValue{Key: k, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: v}}}+}++func intAttr(k string, v int64) *commonpb.KeyValue {+ return &commonpb.KeyValue{Key: k, Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_IntValue{IntValue: v}}}+}diff --git a/benchmarks/loadgen/ramp.go b/benchmarks/loadgen/ramp.gonew file mode 100644index 00000000..6d5ac620--- /dev/null+++ b/benchmarks/loadgen/ramp.go@@ -0,0 +1,124 @@+package main++import (+ "context"+ "fmt"+ "time"+)++type stepResult struct {+ Step int `json:"step"`+ TargetEPS float64 `json:"targetEps"`+ ActualEPS float64 `json:"actualEps"`+ Ingest latencySnapshot `json:"ingest"`+ Reads map[string]latencySnapshot `json:"reads"`+ Passed bool `json:"passed"`+ FailReason string `json:"failReason,omitempty"`+}++type finalReport struct {+ Tier string `json:"tier"`+ Mode string `json:"mode"`+ StartedAt string `json:"startedAt"`+ EndedAt string `json:"endedAt"`+ MaxSustainableEPS float64 `json:"maxSustainableEps"`+ Steps []stepResult `json:"steps"`+}++func (r *finalReport) computeBreakingPoint() {+ for i := len(r.Steps) - 1; i >= 0; i-- {+ if r.Steps[i].Passed {+ r.MaxSustainableEPS = r.Steps[i].ActualEPS+ return+ }+ }+}++// runRamp drives the EPS ramp: hold a target rate for stepDuration, snapshot+// stats, evaluate the breaking-point criteria, and either advance or stop.+// Doubling cadence (100 -> 200 -> 400 -> ...) gets us to the cliff in <10 steps+// for any realistic SUT, keeping total runtime bounded.+func runRamp(+ ctx context.Context,+ cfg config,+ ingester *ingester,+ ingest *latencyTracker,+ reads map[string]*latencyTracker,+) []stepResult {+ results := []stepResult{}+ target := cfg.initialEPS++ for stepNo := 1; target <= cfg.maxEPS; stepNo++ {+ ingester.SetRate(target)++ // Reset trackers at start of step so we measure only the new rate's+ // steady state, not the ramp-up transient from the previous step.+ ingest.SnapshotAndReset()+ for _, t := range reads {+ t.SnapshotAndReset()+ }++ stepCtx, cancel := context.WithTimeout(ctx, cfg.stepDuration)+ start := time.Now()+ <-stepCtx.Done()+ cancel()+ elapsed := time.Since(start)++ ingestSnap := ingest.SnapshotAndReset()+ readSnaps := map[string]latencySnapshot{}+ for name, t := range reads {+ readSnaps[name] = t.SnapshotAndReset()+ }++ // Both target and actual must be in the same unit (traces/sec).+ // Rate limiter ticks per request (= per frame), so requests/sec × the+ // batch size gives the trace throughput on the wire. OTLP requests+ // carry spans rather than traces; we count them at the same rate+ // because the user-facing question is "events/sec the system swallowed"+ // regardless of which protocol carried them.+ actualEPS := 0.0+ if elapsed > 0 {+ actualEPS = float64(ingestSnap.OK+ingestSnap.Errors) / elapsed.Seconds() * tracesPerFrame+ }++ passed, reason := evaluate(cfg, ingestSnap, readSnaps)++ res := stepResult{+ Step: stepNo,+ TargetEPS: target,+ ActualEPS: actualEPS,+ Ingest: ingestSnap,+ Reads: readSnaps,+ Passed: passed,+ FailReason: reason,+ }+ results = append(results, res)+ fmt.Fprintf(stderrPrefix(), "step %d: target=%.0f actual=%.0f ingest_p99=%.0fms ingest_err=%.2f%% passed=%t %s\n",+ stepNo, target, actualEPS, ingestSnap.P99, ingestSnap.ErrRate*100, passed, reason)++ if !passed || ctx.Err() != nil {+ return results+ }++ target *= 2+ }+ return results+}++func evaluate(cfg config, ingest latencySnapshot, reads map[string]latencySnapshot) (bool, string) {+ if ingest.ErrRate > cfg.ingestErrThreshold {+ return false, fmt.Sprintf("ingest error rate %.2f%% > %.2f%% threshold", ingest.ErrRate*100, cfg.ingestErrThreshold*100)+ }+ for name, s := range reads {+ // If a read endpoint never returned a single OK response during this+ // step the SUT is effectively unavailable for reads; treat that as a+ // hard fail even though P99 reads as 0 from the empty histogram.+ if s.OK == 0 && s.Errors > 0 {+ return false, fmt.Sprintf("read endpoint %q served zero OK responses", name)+ }+ if s.P99 > cfg.readP99ThresholdMs {+ return false, fmt.Sprintf("read endpoint %q p99 = %.0fms > %.0fms threshold", name, s.P99, cfg.readP99ThresholdMs)+ }+ }+ return true, ""+}diff --git a/benchmarks/loadgen/reads.go b/benchmarks/loadgen/reads.gonew file mode 100644index 00000000..d77cb6ec--- /dev/null+++ b/benchmarks/loadgen/reads.go@@ -0,0 +1,204 @@+package main++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/url"+ "sync"+ "time"+)++type reader struct {+ cfg config+ client *http.Client+ stats map[string]*latencyTracker+ ingester *ingester+}++func newReader(cfg config, client *http.Client, stats map[string]*latencyTracker) *reader {+ return &reader{cfg: cfg, client: client, stats: stats}+}++// attachIngester gives the reader access to a sample session id so it can hit+// /api/sessions/:id/recording with an id that actually exists.+func (r *reader) attachIngester(i *ingester) { r.ingester = i }++type readSpec struct {+ name string+ method string+ path string+ body func() any+}++func (r *reader) specs() []readSpec {+ last24h := func() (string, string) {+ now := time.Now().UTC()+ return now.Add(-24 * time.Hour).Format(time.RFC3339), now.Format(time.RFC3339)+ }+ last1h := func() (string, string) {+ now := time.Now().UTC()+ return now.Add(-1 * time.Hour).Format(time.RFC3339), now.Format(time.RFC3339)+ }+ return []readSpec{+ {+ name: "endpoints_grouped", method: "POST", path: "/api/endpoints/grouped",+ body: func() any {+ f, t := last24h()+ return map[string]any{+ "fromDate": f,+ "toDate": t,+ "orderBy": "count",+ "sortDirection": "desc",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ "search": "",+ }+ },+ },+ {+ name: "exceptions_grouped", method: "POST", path: "/api/exception-stack-traces",+ body: func() any {+ f, t := last24h()+ return map[string]any{+ "fromDate": f,+ "toDate": t,+ "orderBy": "count desc",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ "search": "",+ "searchType": "all",+ "includeArchived": false,+ }+ },+ },+ {name: "dashboard_overview", method: "GET", path: "/api/dashboard/overview"},+ {name: "logs", method: "POST", path: "/api/logs", body: func() any {+ f, t := last1h()+ return map[string]any{+ "fromDate": f,+ "toDate": t,+ "orderBy": "timestamp desc",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ }+ }},+ // session_recording handled separately because the URL depends on a+ // live session id surfaced by the ingester.+ }+}++func (r *reader) run(ctx context.Context) {+ var wg sync.WaitGroup+ for _, spec := range r.specs() {+ spec := spec+ wg.Add(1)+ go func() {+ defer wg.Done()+ r.loopSpec(ctx, spec)+ }()+ }+ wg.Add(1)+ go func() {+ defer wg.Done()+ r.loopRecording(ctx)+ }()+ wg.Wait()+}++func (r *reader) loopSpec(ctx context.Context, spec readSpec) {+ interval := time.Duration(float64(time.Second) / r.cfg.readRPS)+ if interval < 10*time.Millisecond {+ interval = 10 * time.Millisecond+ }+ t := time.NewTicker(interval)+ defer t.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-t.C:+ r.doOne(ctx, spec)+ }+ }+}++func (r *reader) loopRecording(ctx context.Context) {+ interval := time.Duration(float64(time.Second) / r.cfg.readRPS)+ if interval < 10*time.Millisecond {+ interval = 10 * time.Millisecond+ }+ t := time.NewTicker(interval)+ defer t.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-t.C:+ sid := ""+ if r.ingester != nil {+ sid = r.ingester.SampleSessionId()+ }+ if sid == "" {+ // Nothing seeded yet — quietly skip; don't record as ok or err.+ continue+ }+ r.doOne(ctx, readSpec{+ name: "session_recording",+ method: "GET",+ path: "/api/sessions/" + sid + "/recording",+ })+ }+ }+}++func (r *reader) doOne(ctx context.Context, spec readSpec) {+ tracker, ok := r.stats[spec.name]+ if !ok {+ return+ }++ u, err := url.Parse(r.cfg.target + spec.path)+ if err != nil {+ tracker.Record(0, err)+ return+ }+ q := u.Query()+ q.Set("projectId", r.cfg.projectId)+ u.RawQuery = q.Encode()++ var body []byte+ if spec.body != nil {+ body, err = json.Marshal(spec.body())+ if err != nil {+ tracker.Record(0, err)+ return+ }+ }++ req, err := http.NewRequestWithContext(ctx, spec.method, u.String(), bytes.NewReader(body))+ if err != nil {+ tracker.Record(0, err)+ return+ }+ if body != nil {+ req.Header.Set("Content-Type", "application/json")+ }+ if r.cfg.jwt != "" {+ req.Header.Set("Authorization", "Bearer "+r.cfg.jwt)+ }++ start := time.Now()+ resp, err := r.client.Do(req)+ elapsed := time.Since(start)+ if err != nil {+ tracker.Record(elapsed.Seconds()*1000, err)+ return+ }+ defer resp.Body.Close()+ _, _ = readAndDiscard(resp.Body)+ if resp.StatusCode >= 400 {+ tracker.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))+ return+ }+ tracker.Record(elapsed.Seconds()*1000, nil)+}diff --git a/benchmarks/loadgen/report.go b/benchmarks/loadgen/report.gonew file mode 100644index 00000000..d56cd7f6--- /dev/null+++ b/benchmarks/loadgen/report.go@@ -0,0 +1,285 @@+package main++import (+ "bytes"+ "compress/gzip"+ "context"+ "encoding/json"+ "fmt"+ "math/rand"+ "net/http"+ "time"++ "github.com/google/uuid"+)++type rTrace struct {+ Id string `json:"id"`+ Endpoint string `json:"endpoint"`+ Duration int64 `json:"duration"` // nanoseconds+ RecordedAt time.Time `json:"recordedAt"`+ StatusCode int `json:"statusCode"`+ BodySize int `json:"bodySize"`+ ClientIP string `json:"clientIP"`+ Attributes map[string]string `json:"attributes"`+ Spans []rSpan `json:"spans"`+ IsTask bool `json:"isTask"`+ DistributedTraceId string `json:"distributedTraceId"`+}++type rSpan struct {+ Id string `json:"id"`+ Name string `json:"name"`+ StartTime time.Time `json:"startTime"`+ Duration int64 `json:"duration"`+}++type rSession struct {+ Id string `json:"id"`+ StartedAt time.Time `json:"startedAt"`+ ClientIP string `json:"clientIP"`+ Attributes map[string]string `json:"attributes"`+ DistributedTraceId string `json:"distributedTraceId,omitempty"`+}++type rRecording struct {+ ExceptionId string `json:"exceptionId"`+ SessionId string `json:"sessionId,omitempty"`+ SegmentIndex int32 `json:"segmentIndex"`+ Events json.RawMessage `json:"events"`+ StartedAt *time.Time `json:"startedAt,omitempty"`+ EndedAt *time.Time `json:"endedAt,omitempty"`+}++type rException struct {+ IsTask bool `json:"isTask"`+ StackTrace string `json:"stackTrace"`+ RecordedAt time.Time `json:"recordedAt"`+ Attributes map[string]string `json:"attributes"`+ IsMessage bool `json:"isMessage"`+}++type rMetric struct {+ Name string `json:"name"`+ Value float64 `json:"value"`+ RecordedAt time.Time `json:"recordedAt"`+ Tags map[string]string `json:"tags,omitempty"`+}++type rFrame struct {+ StackTraces []rException `json:"stackTraces"`+ Metrics []rMetric `json:"metrics"`+ Traces []rTrace `json:"traces"`+ SessionRecordings []rRecording `json:"sessionRecordings"`+ Sessions []rSession `json:"sessions"`+}++type rReport struct {+ CollectionFrames []rFrame `json:"collectionFrames"`+ AppVersion string `json:"appVersion"`+ ServerName string `json:"serverName"`+}++var endpointPaths = []string{+ "GET /api/users", "POST /api/users", "GET /api/orders", "POST /api/orders",+ "GET /api/products/:id", "PUT /api/products/:id", "GET /api/search",+ "POST /api/checkout", "GET /api/cart", "DELETE /api/cart/:id",+ "GET /api/auth/me", "POST /api/auth/login", "GET /api/health",+}++// sendReportFrame builds one CollectionFrame, POSTs it gzipped to /api/report,+// records latency on the ingest tracker, and returns a sample session id (when+// one was included in the frame) so the reader can later query its recording.+func sendReportFrame(ctx context.Context, client *http.Client, cfg config, rng *rand.Rand, stats *latencyTracker) string {+ frame := buildFrame(rng)++ body, err := encodeFrame(rReport{+ CollectionFrames: []rFrame{frame},+ AppVersion: "bench-1.0.0",+ ServerName: fmt.Sprintf("bench-%s-%s", cfg.tier, cfg.mode),+ })+ if err != nil {+ stats.Record(0, err)+ return ""+ }++ req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.target+"/api/report", bytes.NewReader(body))+ if err != nil {+ stats.Record(0, err)+ return ""+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Content-Encoding", "gzip")+ req.Header.Set("Authorization", "Bearer "+cfg.projectToken)++ start := time.Now()+ resp, err := client.Do(req)+ elapsed := time.Since(start)+ if err != nil {+ stats.Record(elapsed.Seconds()*1000, err)+ return ""+ }+ defer resp.Body.Close()+ // Drain so the connection can be reused — keep-alive is critical at high+ // frame rates; not draining leaves connections in CLOSE_WAIT.+ _, _ = readAndDiscard(resp.Body)++ if resp.StatusCode >= 400 {+ stats.Record(elapsed.Seconds()*1000, fmt.Errorf("status %d", resp.StatusCode))+ return ""+ }+ stats.Record(elapsed.Seconds()*1000, nil)++ if len(frame.Sessions) > 0 {+ return frame.Sessions[0].Id+ }+ return ""+}++func buildFrame(rng *rand.Rand) rFrame {+ now := time.Now().UTC()+ frame := rFrame{+ Traces: make([]rTrace, 0, tracesPerFrame),+ Metrics: make([]rMetric, 0, 5),+ Sessions: nil,+ StackTraces: nil,+ }++ for i := 0; i < tracesPerFrame; i++ {+ t := rTrace{+ Id: uuid.NewString(),+ Endpoint: endpointPaths[rng.Intn(len(endpointPaths))],+ Duration: int64(time.Duration(10+rng.Intn(990)) * time.Millisecond), // 10-1000ms+ RecordedAt: now.Add(-time.Duration(rng.Intn(5000)) * time.Millisecond),+ StatusCode: pickStatus(rng),+ BodySize: 200 + rng.Intn(8000),+ ClientIP: "192.0.2.1",+ Attributes: map[string]string{"region": "bench", "tier": "x"},+ Spans: buildSpans(rng, now),+ IsTask: rng.Float64() < 0.10,+ }+ frame.Traces = append(frame.Traces, t)+ }++ for i := 0; i < 5; i++ {+ frame.Metrics = append(frame.Metrics, rMetric{+ Name: "bench.metric." + []string{"cpu", "mem", "qps", "lat", "errs"}[i],+ Value: rng.Float64() * 100,+ RecordedAt: now,+ })+ }++ if rng.Float64() < recordingChance {+ sid := uuid.NewString()+ started := now.Add(-2 * time.Second)+ ended := now+ frame.Sessions = []rSession{{+ Id: sid,+ StartedAt: started,+ ClientIP: "192.0.2.1",+ Attributes: map[string]string{"browser": "bench"},+ }}+ frame.SessionRecordings = []rRecording{{+ ExceptionId: uuid.NewString(),+ SessionId: sid,+ SegmentIndex: 0,+ Events: fakeRRWeb(rng),+ StartedAt: &started,+ EndedAt: &ended,+ }}+ } else if rng.Float64() < 0.2 {+ // Even without a recording, ship occasional sessions so the read mix+ // always has at least one session id to query.+ sid := uuid.NewString()+ frame.Sessions = []rSession{{+ Id: sid,+ StartedAt: now,+ ClientIP: "192.0.2.1",+ Attributes: map[string]string{"browser": "bench"},+ }}+ }++ if rng.Float64() < exceptionChance {+ frame.StackTraces = []rException{{+ IsTask: false,+ StackTrace: "RuntimeError: bench-synthetic\n at bench:1\n at bench:2",+ RecordedAt: now,+ Attributes: map[string]string{"env": "bench"},+ }}+ }++ return frame+}++func buildSpans(rng *rand.Rand, now time.Time) []rSpan {+ n := 2 + rng.Intn(4) // 2-5 spans+ spans := make([]rSpan, n)+ for i := range spans {+ spans[i] = rSpan{+ Id: uuid.NewString(),+ Name: []string{"db.query", "http.call", "cache.get", "render", "json.encode"}[rng.Intn(5)],+ StartTime: now.Add(-time.Duration(rng.Intn(500)) * time.Millisecond),+ Duration: int64(time.Duration(1+rng.Intn(200)) * time.Millisecond),+ }+ }+ return spans+}++func pickStatus(rng *rand.Rand) int {+ r := rng.Float64()+ switch {+ case r < 0.92:+ return 200+ case r < 0.95:+ return 201+ case r < 0.98:+ return 404+ default:+ return 500+ }+}++// fakeRRWeb produces a ~100KB JSON array shaped like rrweb events. The backend+// stores this opaquely in S3/disk without inspecting the structure, so a valid+// JSON array of arbitrary size is sufficient to exercise the storage path.+func fakeRRWeb(rng *rand.Rand) json.RawMessage {+ var buf bytes.Buffer+ buf.WriteString("[")+ const targetBytes = 100_000+ first := true+ for buf.Len() < targetBytes {+ if !first {+ buf.WriteString(",")+ }+ first = false+ fmt.Fprintf(&buf, `{"type":%d,"timestamp":%d,"data":{"href":"https://bench.local/page/%d","text":"%s"}}`,+ 2+rng.Intn(4), time.Now().UnixMilli(), rng.Intn(10000), randomString(rng, 64))+ }+ buf.WriteString("]")+ return buf.Bytes()+}++func randomString(rng *rand.Rand, n int) string {+ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "+ b := make([]byte, n)+ for i := range b {+ b[i] = charset[rng.Intn(len(charset))]+ }+ return string(b)+}++func encodeFrame(r rReport) ([]byte, error) {+ js, err := json.Marshal(r)+ if err != nil {+ return nil, err+ }+ var gz bytes.Buffer+ w := gzip.NewWriter(&gz)+ if _, err := w.Write(js); err != nil {+ return nil, err+ }+ if err := w.Close(); err != nil {+ return nil, err+ }+ return gz.Bytes(), nil+}diff --git a/benchmarks/loadgen/stats.go b/benchmarks/loadgen/stats.gonew file mode 100644index 00000000..13825297--- /dev/null+++ b/benchmarks/loadgen/stats.go@@ -0,0 +1,73 @@+package main++import (+ "sort"+ "sync"+)++type latencyTracker struct {+ mu sync.Mutex+ samples []float64+ ok int64+ errors int64+}++func newLatencyTracker() *latencyTracker {+ return &latencyTracker{samples: make([]float64, 0, 8192)}+}++func (l *latencyTracker) Record(milliseconds float64, err error) {+ l.mu.Lock()+ if err != nil {+ l.errors+++ } else {+ l.ok+++ l.samples = append(l.samples, milliseconds)+ }+ l.mu.Unlock()+}++type latencySnapshot struct {+ P50 float64 `json:"p50"`+ P95 float64 `json:"p95"`+ P99 float64 `json:"p99"`+ OK int64 `json:"ok"`+ Errors int64 `json:"errors"`+ ErrRate float64 `json:"errRate"`+}++func (l *latencyTracker) SnapshotAndReset() latencySnapshot {+ l.mu.Lock()+ defer l.mu.Unlock()++ s := latencySnapshot{OK: l.ok, Errors: l.errors}+ total := l.ok + l.errors+ if total > 0 {+ s.ErrRate = float64(l.errors) / float64(total)+ }+ if len(l.samples) > 0 {+ sort.Float64s(l.samples)+ s.P50 = percentile(l.samples, 0.50)+ s.P95 = percentile(l.samples, 0.95)+ s.P99 = percentile(l.samples, 0.99)+ }++ l.samples = l.samples[:0]+ l.ok = 0+ l.errors = 0+ return s+}++func percentile(sorted []float64, q float64) float64 {+ if len(sorted) == 0 {+ return 0+ }+ idx := int(float64(len(sorted)-1) * q)+ if idx < 0 {+ idx = 0+ }+ if idx >= len(sorted) {+ idx = len(sorted) - 1+ }+ return sorted[idx]+}diff --git a/benchmarks/loadgen/util.go b/benchmarks/loadgen/util.gonew file mode 100644index 00000000..9d18a042--- /dev/null+++ b/benchmarks/loadgen/util.go@@ -0,0 +1,7 @@+package main++import "io"++func readAndDiscard(r io.Reader) (int64, error) {+ return io.Copy(io.Discard, r)+}diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeepnew file mode 100644index 00000000..e69de29bdiff --git a/benchmarks/scripts/_ssh.sh b/benchmarks/scripts/_ssh.shnew file mode 100755index 00000000..198b75d1--- /dev/null+++ b/benchmarks/scripts/_ssh.sh@@ -0,0 +1,47 @@+#!/usr/bin/env bash+# Shared SSH/rsync invocation helpers. Source this from other scripts:+# source "$(dirname "$0")/_ssh.sh"+# Requires BENCHMARK_SSH_KEY env var pointing at a private key file.++ssh_opts=(+ -o StrictHostKeyChecking=accept-new+ -o UserKnownHostsFile=/dev/null+ -o LogLevel=ERROR+ -o ServerAliveInterval=30+ -o ConnectTimeout=15+)++if [[ -n "${BENCHMARK_SSH_KEY:-}" ]]; then+ ssh_opts+=(-i "${BENCHMARK_SSH_KEY}")+fi++bench_ssh() {+ local host="$1"; shift+ ssh "${ssh_opts[@]}" "root@${host}" "$@"+}++bench_scp() {+ scp "${ssh_opts[@]}" "$@"+}++bench_rsync() {+ rsync -az --delete \+ -e "ssh ${ssh_opts[*]}" \+ "$@"+}++# Block until ssh-into-box succeeds or timeout expires (default 180s). Hetzner+# boxes take ~15-30s to finish cloud-init; the first ssh attempt usually fails+# even though hcloud reports the server as "running".+wait_for_ssh() {+ local host="$1" timeout="${2:-180}"+ local deadline=$(( $(date +%s) + timeout ))+ while [[ $(date +%s) -lt ${deadline} ]]; do+ if bench_ssh "${host}" -o ConnectTimeout=5 true 2>/dev/null; then+ return 0+ fi+ sleep 5+ done+ echo "ssh to ${host} not ready after ${timeout}s" >&2+ return 1+}diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.pynew file mode 100755index 00000000..cdb8f321--- /dev/null+++ b/benchmarks/scripts/chart.py@@ -0,0 +1,160 @@+#!/usr/bin/env python3+"""Render hardware-vs-breaking-point charts from a directory of loadgen JSONs.++Inputs: a directory containing N JSON files (one per matrix entry), each in+the schema emitted by benchmarks/loadgen/. Order-independent.++Outputs (written into the same directory):+ - chart.png Bar: max sustainable EPS per tier, two bars per tier+ (sqlite vs pgch). The headline.+ - chart-detail.png Lines: read P99 (ms) vs ingest EPS, one line per+ (tier, mode). Shows how reads degrade as ingest climbs.+ - summary.md Markdown table for human consumption / docs source.+"""++import json+import sys+from collections import defaultdict+from pathlib import Path++import matplotlib++matplotlib.use("Agg")+import matplotlib.pyplot as plt++# Stable left-to-right ordering of tiers regardless of how the matrix runs.+TIER_ORDER = ["ccx13", "ccx23", "ccx33", "ccx43"]+MODE_ORDER = ["sqlite", "pgch"]+MODE_COLORS = {"sqlite": "#4f9fff", "pgch": "#ff9f4f"}+READ_P99_BUDGET_MS = 3000+++def load_runs(results_dir: Path) -> list[dict]:+ runs = []+ for p in sorted(results_dir.glob("*.json")):+ with p.open() as fh:+ doc = json.load(fh)+ doc["_path"] = p+ runs.append(doc)+ return runs+++def tier_rank(t: str) -> int:+ return TIER_ORDER.index(t) if t in TIER_ORDER else len(TIER_ORDER)+++def mode_rank(m: str) -> int:+ return MODE_ORDER.index(m) if m in MODE_ORDER else len(MODE_ORDER)+++def render_bar(runs: list[dict], out: Path) -> None:+ fig, ax = plt.subplots(figsize=(9, 5))+ tiers_present = sorted({r["tier"] for r in runs}, key=tier_rank)+ modes_present = sorted({r["mode"] for r in runs}, key=mode_rank)++ width = 0.8 / max(len(modes_present), 1)+ x = list(range(len(tiers_present)))++ for mi, mode in enumerate(modes_present):+ ys = []+ for tier in tiers_present:+ match = [r for r in runs if r["tier"] == tier and r["mode"] == mode]+ ys.append(match[0]["maxSustainableEps"] if match else 0)+ offsets = [xi + (mi - (len(modes_present) - 1) / 2) * width for xi in x]+ bars = ax.bar(offsets, ys, width=width, label=mode, color=MODE_COLORS.get(mode, "#777"))+ for b, y in zip(bars, ys):+ ax.annotate(f"{int(y):,}", xy=(b.get_x() + b.get_width() / 2, y),+ xytext=(0, 3), textcoords="offset points",+ ha="center", va="bottom", fontsize=9)++ ax.set_xticks(x)+ ax.set_xticklabels(tiers_present)+ ax.set_ylabel("Max sustainable traces/sec")+ ax.set_title("Traceway: max sustainable ingest by hardware tier\n"+ f"(failure = ingest err > 5% OR any read p99 > {READ_P99_BUDGET_MS}ms)")+ ax.legend(title="DB mode")+ ax.grid(axis="y", linestyle=":", alpha=0.4)+ fig.tight_layout()+ fig.savefig(out, dpi=130)+ plt.close(fig)+++def render_detail(runs: list[dict], out: Path) -> None:+ fig, ax = plt.subplots(figsize=(10, 6))++ for run in sorted(runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):+ xs, ys = [], []+ for s in run.get("steps", []):+ if not s.get("reads"):+ continue+ xs.append(s["actualEps"])+ # Worst read p99 across all read endpoints — that's what defines the cliff.+ ys.append(max((rd.get("p99", 0) for rd in s["reads"].values()), default=0))+ if xs:+ label = f"{run['tier']} / {run['mode']}"+ ax.plot(xs, ys, marker="o", label=label)++ ax.axhline(READ_P99_BUDGET_MS, linestyle="--", color="#cc3333", alpha=0.6,+ label=f"P99 budget ({READ_P99_BUDGET_MS}ms)")+ ax.set_xlabel("Actual ingest rate (traces/sec)")+ ax.set_ylabel("Worst read endpoint P99 (ms)")+ ax.set_title("Read latency degradation under increasing ingest load")+ ax.set_xscale("log")+ ax.set_yscale("log")+ ax.grid(True, which="both", linestyle=":", alpha=0.4)+ ax.legend(loc="upper left", fontsize=9)+ fig.tight_layout()+ fig.savefig(out, dpi=130)+ plt.close(fig)+++def render_summary(runs: list[dict], out: Path) -> None:+ lines = ["# Traceway hardware benchmark — summary", ""]+ lines.append(f"Runs analyzed: {len(runs)}")+ lines.append("")+ lines.append("| Tier | Mode | Max EPS | Ingest p99 @max (ms) | Worst read p99 @max (ms) | Steps |")+ lines.append("|------|------|---------|----------------------|---------------------------|-------|")++ for run in sorted(runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):+ max_eps = int(run.get("maxSustainableEps", 0))+ last_pass = None+ for s in reversed(run.get("steps", [])):+ if s.get("passed"):+ last_pass = s+ break+ ingest_p99 = int(last_pass["ingest"]["p99"]) if last_pass else 0+ worst_read = 0+ if last_pass and last_pass.get("reads"):+ worst_read = int(max(rd.get("p99", 0) for rd in last_pass["reads"].values()))+ lines.append(f"| {run['tier']} | {run['mode']} | {max_eps:,} | {ingest_p99} | {worst_read} | {len(run.get('steps', []))} |")++ lines.append("")+ lines.append("Generated by `benchmarks/scripts/chart.py`. "+ "Failure thresholds: ingest error rate > 5%, OR any read endpoint p99 > "+ f"{READ_P99_BUDGET_MS}ms.")+ out.write_text("\n".join(lines) + "\n")+++def main() -> int:+ if len(sys.argv) < 2:+ print("usage: chart.py <results-dir>", file=sys.stderr)+ return 2+ results_dir = Path(sys.argv[1])+ if not results_dir.is_dir():+ print(f"not a directory: {results_dir}", file=sys.stderr)+ return 2++ runs = load_runs(results_dir)+ if not runs:+ print(f"no *.json files in {results_dir}", file=sys.stderr)+ return 1++ render_bar(runs, results_dir / "chart.png")+ render_detail(runs, results_dir / "chart-detail.png")+ render_summary(runs, results_dir / "summary.md")+ print(f"wrote {results_dir}/chart.png, chart-detail.png, summary.md")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/benchmarks/scripts/hetzner-down.sh b/benchmarks/scripts/hetzner-down.shnew file mode 100755index 00000000..b557ab49--- /dev/null+++ b/benchmarks/scripts/hetzner-down.sh@@ -0,0 +1,36 @@+#!/usr/bin/env bash+# Tear down servers + network for a given run-id. Idempotent: missing resources+# are not errors. Designed to be safe to call from a bash trap, including when+# triggered mid-provisioning.+#+# Usage: hetzner-down.sh <run-id>+set -euo pipefail++if [[ $# -lt 1 ]]; then+ echo "usage: $0 <run-id>" >&2+ exit 2+fi++RUN_ID="$1"+SUT_NAME="bench-sut-${RUN_ID}"+LOADGEN_NAME="bench-loadgen-${RUN_ID}"+NET_NAME="bench-net-${RUN_ID}"++drop_server() {+ local name="$1"+ if hcloud server describe "${name}" >/dev/null 2>&1; then+ echo "deleting server ${name}" >&2+ hcloud server delete "${name}" >/dev/null || echo "warn: delete ${name} returned non-zero" >&2+ fi+}++drop_server "${SUT_NAME}"+drop_server "${LOADGEN_NAME}"++if hcloud network describe "${NET_NAME}" >/dev/null 2>&1; then+ echo "deleting network ${NET_NAME}" >&2+ # Network delete requires no attached servers; we just deleted them above.+ hcloud network delete "${NET_NAME}" >/dev/null || echo "warn: delete ${NET_NAME} returned non-zero" >&2+fi++echo "teardown of ${RUN_ID}: done" >&2diff --git a/benchmarks/scripts/hetzner-up.sh b/benchmarks/scripts/hetzner-up.shnew file mode 100755index 00000000..8c62ccfe--- /dev/null+++ b/benchmarks/scripts/hetzner-up.sh@@ -0,0 +1,78 @@+#!/usr/bin/env bash+# Provision a system-under-test (SUT) box + a load generator box + a private+# network in Hetzner Cloud. Idempotent on its own outputs: re-running with the+# same RUN_ID short-circuits if the resources already exist.+#+# Usage: hetzner-up.sh <tier> <run-id> [location]+# <tier> ccx13 | ccx23 | ccx33 | ccx43+# <run-id> short label, used to name the resources (e.g. 20260512-ccx13-sqlite-a1b2)+# [location] Hetzner datacenter, default nbg1+#+# Emits a JSON line on stdout:+# {"sutPublicIp":"...","sutPrivateIp":"...","loadgenPublicIp":"...","loadgenPrivateIp":"...","networkId":"...","runId":"..."}+set -euo pipefail++if [[ $# -lt 2 ]]; then+ echo "usage: $0 <tier> <run-id> [location]" >&2+ exit 2+fi++TIER="$1"+RUN_ID="$2"+LOCATION="${3:-nbg1}"+LOADGEN_TIER="${LOADGEN_TIER:-cax11}"+IMAGE="${BENCH_IMAGE:-debian-12}"+NET_NAME="bench-net-${RUN_ID}"+SUT_NAME="bench-sut-${RUN_ID}"+LOADGEN_NAME="bench-loadgen-${RUN_ID}"++# Create a /24 private network so SUT and loadgen can talk over 10.0.0.x.+if ! hcloud network describe "${NET_NAME}" >/dev/null 2>&1; then+ echo "creating network ${NET_NAME}" >&2+ hcloud network create --name "${NET_NAME}" --ip-range 10.0.0.0/24 >/dev/null+ hcloud network add-subnet "${NET_NAME}" --network-zone eu-central --type cloud --ip-range 10.0.0.0/24 >/dev/null+fi+NET_ID=$(hcloud network describe "${NET_NAME}" -o format='{{.ID}}')++create_server() {+ local name="$1" type="$2" private_ip="$3"+ if hcloud server describe "${name}" >/dev/null 2>&1; then+ echo "server ${name} already exists, reusing" >&2+ return+ fi+ echo "creating server ${name} (${type}) in ${LOCATION}" >&2+ hcloud server create \+ --name "${name}" \+ --type "${type}" \+ --image "${IMAGE}" \+ --location "${LOCATION}" \+ --ssh-key benchmark-key \+ --network "${NET_NAME}" \+ --label "bench=true,run=${RUN_ID}" \+ >/dev/null+}++create_server "${SUT_NAME}" "${TIER}" "10.0.0.2"+create_server "${LOADGEN_NAME}" "${LOADGEN_TIER}" "10.0.0.3"++# Hetzner doesn't let you pin the private IP at create time on shared networks+# without --network-extra; we re-read both servers' assigned private IPs and+# return them.+get_field() {+ local name="$1" field="$2"+ hcloud server describe "${name}" -o format="{{${field}}}"+}++SUT_PUBLIC_IP=$(get_field "${SUT_NAME}" ".PublicNet.IPv4.IP")+LOADGEN_PUBLIC_IP=$(get_field "${LOADGEN_NAME}" ".PublicNet.IPv4.IP")+SUT_PRIVATE_IP=$(hcloud server describe "${SUT_NAME}" -o json | jq -r '.private_net[0].ip')+LOADGEN_PRIVATE_IP=$(hcloud server describe "${LOADGEN_NAME}" -o json | jq -r '.private_net[0].ip')++jq -nc \+ --arg sutPub "${SUT_PUBLIC_IP}" \+ --arg sutPriv "${SUT_PRIVATE_IP}" \+ --arg lgPub "${LOADGEN_PUBLIC_IP}" \+ --arg lgPriv "${LOADGEN_PRIVATE_IP}" \+ --arg netId "${NET_ID}" \+ --arg runId "${RUN_ID}" \+ '{sutPublicIp: $sutPub, sutPrivateIp: $sutPriv, loadgenPublicIp: $lgPub, loadgenPrivateIp: $lgPriv, networkId: $netId, runId: $runId}'diff --git a/benchmarks/scripts/loadgen-bootstrap.sh b/benchmarks/scripts/loadgen-bootstrap.shnew file mode 100755index 00000000..15417984--- /dev/null+++ b/benchmarks/scripts/loadgen-bootstrap.sh@@ -0,0 +1,70 @@+#!/usr/bin/env bash+# Cross-compile the loadgen for linux/amd64, scp it to the loadgen box, seed a+# project on the SUT, run the loadgen pointed at the SUT's private IP, and pull+# the result JSON back.+#+# Usage: loadgen-bootstrap.sh <loadgen-public-ip> <sut-private-ip> <sut-public-ip> <duration> <tier> <mode> <out-path> [extra-loadgen-args...]+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"+# shellcheck source=_ssh.sh+source "${SCRIPT_DIR}/_ssh.sh"++if [[ $# -lt 7 ]]; then+ echo "usage: $0 <loadgen-public-ip> <sut-private-ip> <sut-public-ip> <duration> <tier> <mode> <out-path> [extra-loadgen-args...]" >&2+ exit 2+fi+LG_IP="$1"; SUT_PRIVATE_IP="$2"; SUT_PUBLIC_IP="$3"; DURATION="$4"; TIER="$5"; MODE="$6"; OUT_PATH="$7"+shift 7++# 1. Seed a project on the SUT (over its PUBLIC IP from the orchestrator). The+# project token + JWT + project id are then handed to the loadgen, which+# talks to the SUT over the PRIVATE network for the actual benchmark.+echo "seeding project via http://${SUT_PUBLIC_IP}/api/register" >&2+SEED_JSON=$("${SCRIPT_DIR}/seed-project.sh" "http://${SUT_PUBLIC_IP}")+JWT=$(printf '%s' "${SEED_JSON}" | jq -r '.jwt')+TOKEN=$(printf '%s' "${SEED_JSON}" | jq -r '.projectToken')+PROJECT_ID=$(printf '%s' "${SEED_JSON}" | jq -r '.projectId')++# 2. Wait for SSH on the loadgen box and detect its arch so we cross-compile+# correctly (CAX* tiers are arm64, CX*/CPX* are amd64).+echo "waiting for ssh on loadgen ${LG_IP}" >&2+wait_for_ssh "${LG_IP}"+LG_UNAME=$(bench_ssh "${LG_IP}" "uname -m" | tr -d '\r\n')+case "${LG_UNAME}" in+ x86_64) GOARCH=amd64 ;;+ aarch64) GOARCH=arm64 ;;+ *) echo "unknown loadgen arch '${LG_UNAME}'" >&2; exit 1 ;;+esac++# 3. Cross-compile loadgen on the orchestrator (laptop / GH runner) for the+# loadgen box's arch.+echo "cross-compiling loadgen for linux/${GOARCH}" >&2+(+ cd "${REPO_ROOT}/benchmarks/loadgen"+ GOOS=linux GOARCH="${GOARCH}" CGO_ENABLED=0 go build -o "loadgen-linux-${GOARCH}" .+)++bench_ssh "${LG_IP}" "mkdir -p /root/loadgen"+bench_scp "${REPO_ROOT}/benchmarks/loadgen/loadgen-linux-${GOARCH}" "root@${LG_IP}:/root/loadgen/loadgen"+bench_ssh "${LG_IP}" "chmod +x /root/loadgen/loadgen"++# 4. Run the benchmark, streaming stderr back so progress is visible. The+# loadgen writes JSON to /root/loadgen/result.json on the loadgen box;+# we scp it home afterwards.+echo "running loadgen on ${LG_IP} -> http://${SUT_PRIVATE_IP} (tier=${TIER} mode=${MODE} duration=${DURATION})" >&2+bench_ssh "${LG_IP}" /root/loadgen/loadgen \+ --target "http://${SUT_PRIVATE_IP}" \+ --token "${TOKEN}" \+ --jwt "${JWT}" \+ --project-id "${PROJECT_ID}" \+ --duration "${DURATION}" \+ --tier "${TIER}" \+ --mode "${MODE}" \+ --report-out /root/loadgen/result.json \+ "$@"++mkdir -p "$(dirname "${OUT_PATH}")"+bench_scp "root@${LG_IP}:/root/loadgen/result.json" "${OUT_PATH}"+echo "wrote ${OUT_PATH}" >&2diff --git a/benchmarks/scripts/preflight.sh b/benchmarks/scripts/preflight.shnew file mode 100755index 00000000..49d0753d--- /dev/null+++ b/benchmarks/scripts/preflight.sh@@ -0,0 +1,65 @@+#!/usr/bin/env bash+# Preflight checks for the benchmark tooling. Fails loud and early so we don't+# half-provision a Hetzner box and then discover matplotlib is missing.+#+# Required env vars: HCLOUD_TOKEN, BENCHMARK_SSH_KEY (path to private key file)+# Required commands: hcloud, jq, python3, ssh, scp, curl, go+# Required python: matplotlib (in whatever python3 we'll invoke chart.py with)+# Required Hetzner: an SSH key uploaded as "benchmark-key"++set -euo pipefail++fail=0+warn() { printf 'WARN: %s\n' "$*" >&2; }+err() { printf 'FAIL: %s\n' "$*" >&2; fail=1; }++# --- Commands on PATH -------------------------------------------------------++for cmd in hcloud jq python3 ssh scp curl go; do+ if ! command -v "${cmd}" >/dev/null 2>&1; then+ err "missing command: ${cmd}"+ fi+done++# --- Python: matplotlib for chart rendering ---------------------------------++if command -v python3 >/dev/null 2>&1; then+ if ! python3 -c "import matplotlib" >/dev/null 2>&1; then+ err "python3 cannot import matplotlib. Install in a venv: python3 -m venv .venv && .venv/bin/pip install matplotlib; then export PATH=\$PWD/.venv/bin:\$PATH"+ fi+fi++# --- Hetzner credentials ----------------------------------------------------++if [[ -z "${HCLOUD_TOKEN:-}" ]]; then+ err "HCLOUD_TOKEN not set. Export your Hetzner Cloud API token."+else+ if ! hcloud server-type list >/dev/null 2>&1; then+ err "HCLOUD_TOKEN is set but 'hcloud server-type list' failed — token invalid or revoked?"+ fi+fi++# --- SSH key ---------------------------------------------------------------++if [[ -z "${BENCHMARK_SSH_KEY:-}" ]]; then+ warn "BENCHMARK_SSH_KEY not set; will skip key-on-disk checks. Hetzner-side key 'benchmark-key' is still required."+else+ if [[ ! -f "${BENCHMARK_SSH_KEY}" ]]; then+ err "BENCHMARK_SSH_KEY=${BENCHMARK_SSH_KEY} does not exist"+ elif [[ "$(stat -f '%Lp' "${BENCHMARK_SSH_KEY}" 2>/dev/null || stat -c '%a' "${BENCHMARK_SSH_KEY}" 2>/dev/null)" != "600" ]]; then+ warn "BENCHMARK_SSH_KEY permissions are not 600; ssh may refuse to use it"+ fi+fi++if command -v hcloud >/dev/null 2>&1 && [[ -n "${HCLOUD_TOKEN:-}" ]]; then+ if ! hcloud ssh-key describe benchmark-key >/dev/null 2>&1; then+ err "no SSH key named 'benchmark-key' in your Hetzner project. Upload the matching public key via: hcloud ssh-key create --name benchmark-key --public-key-from-file ~/.ssh/hetzner_benchmark.pub"+ fi+fi++if [[ "${fail}" -ne 0 ]]; then+ echo >&2+ echo "preflight failed. fix the issues above before running." >&2+ exit 1+fi+echo "preflight: OK" >&2diff --git a/benchmarks/scripts/run-local.sh b/benchmarks/scripts/run-local.shnew file mode 100755index 00000000..3123d277--- /dev/null+++ b/benchmarks/scripts/run-local.sh@@ -0,0 +1,112 @@+#!/usr/bin/env bash+# Run the hardware benchmark from your laptop, end-to-end, against real Hetzner.+# The GitHub Action is a thin wrapper around run-matrix-entry.sh; this script+# is the developer-facing equivalent — same orchestration, run from anywhere.+#+# Required env:+# HCLOUD_TOKEN Hetzner Cloud API token+# BENCHMARK_SSH_KEY Path to the private key matching the Hetzner-side+# SSH key named 'benchmark-key'.+#+# Common usage:+# run-local.sh # full matrix (4 tiers x 2 modes)+# run-local.sh --smoke # 1 tier, 1 mode, short steps (~5 min)+# run-local.sh --tier ccx13 --mode sqlite+# run-local.sh --dry-run # validate env + print plan, no provisioning+# run-local.sh --commit # drop the "-local" suffix on the output dir+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"++TIERS="ccx13,ccx23,ccx33,ccx43"+MODES="sqlite,pgch"+DURATION="30m"+SMOKE=0+DRY_RUN=0+COMMIT=0++usage() {+ sed -n '1,16p' "$0"; exit 2+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --tier) TIERS="$2"; shift 2 ;;+ --mode) MODES="$2"; shift 2 ;;+ --duration) DURATION="$2"; shift 2 ;;+ --smoke) SMOKE=1; shift ;;+ --dry-run) DRY_RUN=1; shift ;;+ --commit) COMMIT=1; shift ;;+ -h|--help) usage ;;+ *) echo "unknown flag: $1" >&2; usage ;;+ esac+done++if [[ "${SMOKE}" -eq 1 ]]; then+ TIERS="ccx13"+ MODES="sqlite"+ DURATION="3m"+ echo "smoke mode: tier=ccx13 mode=sqlite duration=${DURATION} stepped 100->400 eps" >&2+fi++# Preflight first — fail fast on missing tooling.+"${SCRIPT_DIR}/preflight.sh"++date_tag="$(date -u +%Y-%m-%d)"+if [[ "${COMMIT}" -eq 1 ]]; then+ OUT_DIR="${REPO_ROOT}/benchmarks/results/${date_tag}"+else+ OUT_DIR="${REPO_ROOT}/benchmarks/results/${date_tag}-local"+fi+mkdir -p "${OUT_DIR}"+echo "results dir: ${OUT_DIR}" >&2++# Plan: explode tiers x modes.+plan=()+IFS=',' read -ra TIER_ARR <<<"${TIERS}"+IFS=',' read -ra MODE_ARR <<<"${MODES}"+for t in "${TIER_ARR[@]}"; do+ for m in "${MODE_ARR[@]}"; do+ plan+=("${t}|${m}")+ done+done++echo "plan (${#plan[@]} entries):" >&2+for e in "${plan[@]}"; do+ echo " - ${e//|/ x }" >&2+done++if [[ "${DRY_RUN}" -eq 1 ]]; then+ echo "dry-run: would call run-matrix-entry.sh ${#plan[@]} time(s) with duration=${DURATION}; no servers will be created." >&2+ exit 0+fi++# Sequential execution. Local runs prioritize simplicity and low concurrent+# Hetzner spend over wall-clock speed; the GH workflow parallelizes with+# strategy.matrix instead.+smoke_arg=""+[[ "${SMOKE}" -eq 1 ]] && smoke_arg="smoke"++failures=()+for e in "${plan[@]}"; do+ tier="${e%%|*}"; mode="${e##*|}"+ if ! "${SCRIPT_DIR}/run-matrix-entry.sh" "${tier}" "${mode}" "${DURATION}" "${OUT_DIR}" "${smoke_arg}"; then+ echo "FAIL: ${tier}/${mode}" >&2+ failures+=("${tier}/${mode}")+ fi+done++# Render charts once everything (or at least something) is done.+if compgen -G "${OUT_DIR}/*.json" >/dev/null; then+ echo "rendering charts" >&2+ python3 "${SCRIPT_DIR}/chart.py" "${OUT_DIR}"+else+ echo "no JSON files were produced; skipping chart render" >&2+fi++if [[ ${#failures[@]} -gt 0 ]]; then+ echo "FINISHED with failures: ${failures[*]}" >&2+ exit 1+fi+echo "FINISHED. Results in ${OUT_DIR}" >&2diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.shnew file mode 100755index 00000000..35e31323--- /dev/null+++ b/benchmarks/scripts/run-matrix-entry.sh@@ -0,0 +1,68 @@+#!/usr/bin/env bash+# Run ONE (tier, mode) cycle end-to-end: provision Hetzner -> bootstrap SUT ->+# run loadgen -> fetch JSON -> tear down. Called by both run-local.sh and the+# GitHub workflow; treat this as the single source of truth for matrix-entry+# orchestration.+#+# Usage: run-matrix-entry.sh <tier> <mode> <duration> <out-dir> [smoke]+# <tier> ccx13 | ccx23 | ccx33 | ccx43+# <mode> sqlite | pgch+# <duration> Loadgen total runtime, e.g. 30m, 3m+# <out-dir> Directory to write <tier>-<mode>.json into+# [smoke] "smoke" to enable short-step overrides (--initial-eps 100+# --max-eps 400 --step-duration 1m). Optional.+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"++if [[ $# -lt 4 ]]; then+ echo "usage: $0 <tier> <mode> <duration> <out-dir> [smoke]" >&2+ exit 2+fi+TIER="$1"; MODE="$2"; DURATION="$3"; OUT_DIR="$4"; SMOKE="${5:-}"+LOCATION="${BENCH_LOCATION:-nbg1}"++RUN_ID="$(date -u +%Y%m%d-%H%M%S)-${TIER}-${MODE}-$RANDOM"+echo "=== run-matrix-entry tier=${TIER} mode=${MODE} duration=${DURATION} run_id=${RUN_ID} ===" >&2++mkdir -p "${OUT_DIR}"++# Always tear down — even on failure, even on Ctrl-C. The trap is set BEFORE+# any hcloud create call so a failure mid-provision still cleans up.+cleanup() {+ local rc=$?+ echo "--- teardown for ${RUN_ID} (exit=${rc}) ---" >&2+ "${SCRIPT_DIR}/hetzner-down.sh" "${RUN_ID}" || true+ exit "${rc}"+}+trap cleanup EXIT INT TERM++# 1. Provision.+INFRA_JSON=$("${SCRIPT_DIR}/hetzner-up.sh" "${TIER}" "${RUN_ID}" "${LOCATION}")+echo "infra: ${INFRA_JSON}" >&2+SUT_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPublicIp')+SUT_PRIVATE_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.sutPrivateIp')+LOADGEN_PUBLIC_IP=$(printf '%s' "${INFRA_JSON}" | jq -r '.loadgenPublicIp')++# 2. Bring up the backend on the SUT.+"${SCRIPT_DIR}/sut-bootstrap.sh" "${SUT_PUBLIC_IP}" "${MODE}"++# 3. Run the loadgen, pulling JSON back into OUT_DIR.+extra_args=()+if [[ "${SMOKE}" == "smoke" ]]; then+ extra_args+=( --initial-eps 100 --max-eps 400 --step-duration 1m )+fi++OUT_PATH="${OUT_DIR}/${TIER}-${MODE}.json"+"${SCRIPT_DIR}/loadgen-bootstrap.sh" \+ "${LOADGEN_PUBLIC_IP}" \+ "${SUT_PRIVATE_IP}" \+ "${SUT_PUBLIC_IP}" \+ "${DURATION}" \+ "${TIER}" \+ "${MODE}" \+ "${OUT_PATH}" \+ "${extra_args[@]}"++# Trap handles teardown — no explicit call needed.+echo "matrix entry ${TIER}-${MODE} complete -> ${OUT_PATH}" >&2diff --git a/benchmarks/scripts/seed-project.sh b/benchmarks/scripts/seed-project.shnew file mode 100755index 00000000..3b63b0da--- /dev/null+++ b/benchmarks/scripts/seed-project.sh@@ -0,0 +1,49 @@+#!/usr/bin/env bash+# Register a fresh user + org + project against a Traceway backend.+#+# Emits a single JSON line on stdout (everything else on stderr) so callers can+# capture cleanly:+# eval "$(seed-project.sh http://localhost:8087 | jq -r '@sh "JWT=\(.jwt) PROJECT_TOKEN=\(.projectToken) PROJECT_ID=\(.projectId)"')"+#+# Output schema:+# { "jwt": "<JWT for dashboard read endpoints>",+# "projectToken": "<bearer token for /api/report + OTLP ingestion>",+# "projectId": "<UUID; pass as ?projectId= on read endpoints>" }+#+# Usage: seed-project.sh <base-url>+# <base-url> Backend root, e.g. http://localhost:8087 or http://10.0.0.2+set -euo pipefail++if [[ $# -lt 1 ]]; then+ echo "usage: $0 <base-url>" >&2+ exit 2+fi++BASE_URL="${1%/}"+EMAIL="bench-$(date +%s)[email protected]"+PASSWORD="bench-pw-not-secret-12345"++response=$(curl -sS --fail-with-body -X POST "${BASE_URL}/api/register" \+ -H "Content-Type: application/json" \+ -d "$(cat <<JSON+{+ "email": "${EMAIL}",+ "name": "bench",+ "password": "${PASSWORD}",+ "organizationName": "bench-org",+ "timezone": "UTC",+ "projectName": "bench-project",+ "framework": "custom"+}+JSON+)")++echo "registered ${EMAIL}" >&2++out=$(printf '%s' "${response}" | jq -c '{jwt: .token, projectToken: .project.token, projectId: .project.id}')+if [[ "$(printf '%s' "${out}" | jq -r '.projectToken')" == "null" ]]; then+ echo "ERROR: no project token in response: ${response}" >&2+ exit 1+fi++printf '%s\n' "${out}"diff --git a/benchmarks/scripts/sut-bootstrap.sh b/benchmarks/scripts/sut-bootstrap.shnew file mode 100755index 00000000..1f488938--- /dev/null+++ b/benchmarks/scripts/sut-bootstrap.sh@@ -0,0 +1,72 @@+#!/usr/bin/env bash+# Bring up the Traceway backend on a freshly-provisioned Hetzner box.+#+# 1. Wait for SSH.+# 2. Install Docker if missing.+# 3. rsync the repo to /opt/traceway (sans heavy junk).+# 4. `docker compose -f benchmarks/compose/docker-compose.<mode>.yml up -d --build`+# 5. Poll /health until 200.+#+# Usage: sut-bootstrap.sh <sut-public-ip> <mode>+# <mode> sqlite | pgch+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"+# shellcheck source=_ssh.sh+source "${SCRIPT_DIR}/_ssh.sh"++if [[ $# -lt 2 ]]; then+ echo "usage: $0 <sut-public-ip> <mode>" >&2+ exit 2+fi+SUT_IP="$1"+MODE="$2"++case "${MODE}" in+ sqlite|pgch) ;;+ *) echo "mode must be sqlite or pgch, got: ${MODE}" >&2; exit 2 ;;+esac++echo "waiting for ssh on ${SUT_IP}" >&2+wait_for_ssh "${SUT_IP}"++echo "installing docker on ${SUT_IP}" >&2+bench_ssh "${SUT_IP}" 'bash -s' <<'REMOTE'+set -euo pipefail+if ! command -v docker >/dev/null 2>&1; then+ curl -fsSL https://get.docker.com | sh+fi+# compose plugin lives at /usr/libexec/docker/cli-plugins after get.docker.com.+docker compose version >/dev/null || { echo "docker compose plugin missing" >&2; exit 1; }+mkdir -p /opt/traceway+REMOTE++echo "rsyncing source to ${SUT_IP}:/opt/traceway" >&2+bench_rsync \+ --exclude '.git' \+ --exclude 'node_modules' \+ --exclude 'frontend/build' \+ --exclude 'frontend/.svelte-kit' \+ --exclude 'benchmarks/loadgen/loadgen' \+ --exclude 'benchmarks/results' \+ --exclude 'traceway.db*' \+ --exclude 'traceway_telemetry.db*' \+ --exclude 'backend/storage' \+ "${REPO_ROOT}/" "root@${SUT_IP}:/opt/traceway/"++echo "bringing up compose stack (mode=${MODE}) on ${SUT_IP}" >&2+bench_ssh "${SUT_IP}" "cd /opt/traceway && BENCH_PORT=80 docker compose -f benchmarks/compose/docker-compose.${MODE}.yml up -d --build"++echo "polling /health on ${SUT_IP}" >&2+deadline=$(( $(date +%s) + 600 )) # cold compose build can hit 10 min on small tiers+while [[ $(date +%s) -lt ${deadline} ]]; do+ if curl -sf --max-time 5 "http://${SUT_IP}/health" >/dev/null 2>&1; then+ echo "SUT ${SUT_IP} healthy (mode=${MODE})" >&2+ exit 0+ fi+ sleep 5+done+echo "SUT ${SUT_IP} never reported healthy" >&2+bench_ssh "${SUT_IP}" "cd /opt/traceway && docker compose -f benchmarks/compose/docker-compose.${MODE}.yml logs --tail=80" >&2 || true+exit 1