Skip to content

The only tool you need to know what is happening and how to fix it.

bench: read-probe 2026-07-20

traceway-bot4d1337ac264b
diff --git a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json
new file mode 100644
index 00000000..982e12ad
--- /dev/null
+++ b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json
@@ -0,0 +1,66 @@
+{
+ "tier": "ccx13",
+ "mode": "duckdb",
+ "signal": "logs",
+ "scenario": "read-probe",
+ "startedAt": "2026-07-20T13:38:52Z",
+ "endedAt": "2026-07-20T13:40:01Z",
+ "readProbe": {
+ "endpoints": [
+ {
+ "name": "logs-severity-error",
+ "path": "/api/logs"
+ },
+ {
+ "name": "logs-body-search",
+ "path": "/api/logs"
+ },
+ {
+ "name": "logs-trace-id",
+ "path": "/api/logs"
+ }
+ ],
+ "readPath": "/api/logs",
+ "readThresholdMs": 5000,
+ "settleSeconds": 10,
+ "fillBatchSize": 8192,
+ "fillRequestRate": 100,
+ "steps": [
+ {
+ "fillLevelTarget": 1000000,
+ "rowsIngested": 3129344,
+ "ingestSecondsElapsed": 36.173149845,
+ "probes": [
+ {
+ "name": "logs-severity-error",
+ "path": "/api/logs",
+ "latencyMs": 6005.229562,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ },
+ {
+ "name": "logs-body-search",
+ "path": "/api/logs",
+ "latencyMs": 6005.237126,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ },
+ {
+ "name": "logs-trace-id",
+ "path": "/api/logs",
+ "latencyMs": 6000.9449540000005,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ }
+ ],
+ "medianLatencyMs": 6005.229562,
+ "readLatencyMs": 6005.229562,
+ "readOk": false,
+ "passed": false,
+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded (latency 6005ms)"
+ }
+ ],
+ "maxFillLevelPassed": 0
+ },
+ "sutContainer": {}
+}
diff --git a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json
new file mode 100644
index 00000000..c24b8397
--- /dev/null
+++ b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json
@@ -0,0 +1,161 @@
+{
+ "tier": "ccx13",
+ "mode": "duckdb",
+ "signal": "metrics",
+ "scenario": "read-probe",
+ "startedAt": "2026-07-20T12:43:35Z",
+ "endedAt": "2026-07-20T13:32:30Z",
+ "readProbe": {
+ "endpoints": [
+ {
+ "name": "metrics-server",
+ "path": "/api/metrics/server"
+ },
+ {
+ "name": "metrics-application",
+ "path": "/api/metrics/application"
+ },
+ {
+ "name": "dashboard",
+ "path": "/api/dashboard"
+ }
+ ],
+ "readPath": "/api/metrics/server",
+ "readThresholdMs": 5000,
+ "settleSeconds": 10,
+ "fillBatchSize": 8192,
+ "fillRequestRate": 100,
+ "steps": [
+ {
+ "fillLevelTarget": 1000000,
+ "rowsIngested": 1155072,
+ "ingestSecondsElapsed": 3.266561102,
+ "probes": [
+ {
+ "name": "metrics-server",
+ "path": "/api/metrics/server",
+ "latencyMs": 78.3667,
+ "ok": true
+ },
+ {
+ "name": "metrics-application",
+ "path": "/api/metrics/application",
+ "latencyMs": 98.676076,
+ "ok": true
+ },
+ {
+ "name": "dashboard",
+ "path": "/api/dashboard",
+ "latencyMs": 147.198833,
+ "ok": true
+ }
+ ],
+ "medianLatencyMs": 98.676076,
+ "readLatencyMs": 98.676076,
+ "readOk": true,
+ "passed": true
+ },
+ {
+ "fillLevelTarget": 10000000,
+ "rowsIngested": 10289152,
+ "ingestSecondsElapsed": 29.544118968,
+ "probes": [
+ {
+ "name": "metrics-server",
+ "path": "/api/metrics/server",
+ "latencyMs": 257.961618,
+ "ok": true
+ },
+ {
+ "name": "metrics-application",
+ "path": "/api/metrics/application",
+ "latencyMs": 297.02590000000004,
+ "ok": true
+ },
+ {
+ "name": "dashboard",
+ "path": "/api/dashboard",
+ "latencyMs": 487.871677,
+ "ok": true
+ }
+ ],
+ "medianLatencyMs": 297.02590000000004,
+ "readLatencyMs": 297.02590000000004,
+ "readOk": true,
+ "passed": true
+ },
+ {
+ "fillLevelTarget": 100000000,
+ "rowsIngested": 100057088,
+ "ingestSecondsElapsed": 266.359219308,
+ "probes": [
+ {
+ "name": "metrics-server",
+ "path": "/api/metrics/server",
+ "latencyMs": 2223.881928,
+ "ok": true
+ },
+ {
+ "name": "metrics-application",
+ "path": "/api/metrics/application",
+ "latencyMs": 2710.946288,
+ "ok": true
+ },
+ {
+ "name": "dashboard",
+ "path": "/api/dashboard",
+ "latencyMs": 4254.424445,
+ "ok": true
+ }
+ ],
+ "medianLatencyMs": 2710.946288,
+ "readLatencyMs": 2710.946288,
+ "readOk": true,
+ "passed": true
+ },
+ {
+ "fillLevelTarget": 1000000000,
+ "rowsIngested": 1000087552,
+ "ingestSecondsElapsed": 2585.606908558,
+ "probes": [
+ {
+ "name": "metrics-server",
+ "path": "/api/metrics/server",
+ "latencyMs": 0.674826,
+ "ok": false,
+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ },
+ {
+ "name": "metrics-application",
+ "path": "/api/metrics/application",
+ "latencyMs": 0.400743,
+ "ok": false,
+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ },
+ {
+ "name": "dashboard",
+ "path": "/api/dashboard",
+ "latencyMs": 0.36321200000000003,
+ "ok": false,
+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ }
+ ],
+ "medianLatencyMs": 0.400743,
+ "readLatencyMs": 0.400743,
+ "readOk": false,
+ "passed": false,
+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"
+ }
+ ],
+ "maxFillLevelPassed": 100000000
+ },
+ "maxFillLevelPassed": 100000000,
+ "bytesOnDisk": 10048208896,
+ "storedRows": 1000087552,
+ "sutContainer": {
+ "status": "exited",
+ "oomKilled": true,
+ "exitCode": 137,
+ "restartCount": 3
+ }
+}
diff --git a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json
new file mode 100644
index 00000000..bf7db085
--- /dev/null
+++ b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json
@@ -0,0 +1,132 @@
+{
+ "tier": "ccx13",
+ "mode": "duckdb",
+ "signal": "spans",
+ "scenario": "read-probe",
+ "startedAt": "2026-07-20T12:31:52Z",
+ "endedAt": "2026-07-20T12:37:17Z",
+ "readProbe": {
+ "endpoints": [
+ {
+ "name": "endpoints-grouped",
+ "path": "/api/endpoints/grouped"
+ },
+ {
+ "name": "endpoints-chart",
+ "path": "/api/endpoints/chart"
+ },
+ {
+ "name": "exception-stack-traces",
+ "path": "/api/exception-stack-traces"
+ }
+ ],
+ "readPath": "/api/endpoints/grouped",
+ "readThresholdMs": 5000,
+ "settleSeconds": 10,
+ "fillBatchSize": 8192,
+ "fillRequestRate": 100,
+ "steps": [
+ {
+ "fillLevelTarget": 1000000,
+ "rowsIngested": 1818624,
+ "ingestSecondsElapsed": 30.656478277,
+ "probes": [
+ {
+ "name": "endpoints-grouped",
+ "path": "/api/endpoints/grouped",
+ "latencyMs": 354.24581900000004,
+ "ok": true
+ },
+ {
+ "name": "endpoints-chart",
+ "path": "/api/endpoints/chart",
+ "latencyMs": 325.926413,
+ "ok": true
+ },
+ {
+ "name": "exception-stack-traces",
+ "path": "/api/exception-stack-traces",
+ "latencyMs": 6.095135,
+ "ok": true
+ }
+ ],
+ "medianLatencyMs": 325.926413,
+ "readLatencyMs": 325.926413,
+ "readOk": true,
+ "passed": true
+ },
+ {
+ "fillLevelTarget": 10000000,
+ "rowsIngested": 10280960,
+ "ingestSecondsElapsed": 40.570054621,
+ "probes": [
+ {
+ "name": "endpoints-grouped",
+ "path": "/api/endpoints/grouped",
+ "latencyMs": 437.671008,
+ "ok": true
+ },
+ {
+ "name": "endpoints-chart",
+ "path": "/api/endpoints/chart",
+ "latencyMs": 304.181004,
+ "ok": true
+ },
+ {
+ "name": "exception-stack-traces",
+ "path": "/api/exception-stack-traces",
+ "latencyMs": 4.126855,
+ "ok": true
+ }
+ ],
+ "medianLatencyMs": 304.181004,
+ "readLatencyMs": 304.181004,
+ "readOk": true,
+ "passed": true
+ },
+ {
+ "fillLevelTarget": 100000000,
+ "rowsIngested": 103022592,
+ "ingestSecondsElapsed": 221.832406721,
+ "probes": [
+ {
+ "name": "endpoints-grouped",
+ "path": "/api/endpoints/grouped",
+ "latencyMs": 0.887397,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ },
+ {
+ "name": "endpoints-chart",
+ "path": "/api/endpoints/chart",
+ "latencyMs": 0.49261400000000005,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ },
+ {
+ "name": "exception-stack-traces",
+ "path": "/api/exception-stack-traces",
+ "latencyMs": 0.397733,
+ "ok": false,
+ "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ }
+ ],
+ "medianLatencyMs": 0.49261400000000005,
+ "readLatencyMs": 0.49261400000000005,
+ "readOk": false,
+ "passed": false,
+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"
+ }
+ ],
+ "maxFillLevelPassed": 10000000
+ },
+ "maxFillLevelPassed": 10000000,
+ "bytesOnDisk": 234864640,
+ "storedRows": 103022592,
+ "sutContainer": {
+ "status": "exited",
+ "oomKilled": true,
+ "exitCode": 137,
+ "restartCount": 3
+ }
+}
diff --git a/benchmarks/results-probe/chart-readprobe-cliff-logs.png b/benchmarks/results-probe/chart-readprobe-cliff-logs.png
new file mode 100644
index 00000000..50b07a1e
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png
new file mode 100644
index 00000000..7f1cabff
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-cliff-spans.png b/benchmarks/results-probe/chart-readprobe-cliff-spans.png
new file mode 100644
index 00000000..30d92b94
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-headline-logs.png b/benchmarks/results-probe/chart-readprobe-headline-logs.png
new file mode 100644
index 00000000..0fc5df24
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-headline-metrics.png b/benchmarks/results-probe/chart-readprobe-headline-metrics.png
new file mode 100644
index 00000000..2e243653
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-headline-spans.png b/benchmarks/results-probe/chart-readprobe-headline-spans.png
new file mode 100644
index 00000000..47023b94
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png
new file mode 100644
index 00000000..c44abd7d
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png
new file mode 100644
index 00000000..c0d6fdab
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png
new file mode 100644
index 00000000..e5e0b2aa
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-logs.png b/benchmarks/results-probe/chart-readprobe-logs.png
new file mode 100644
index 00000000..745aff0d
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-metrics.png b/benchmarks/results-probe/chart-readprobe-metrics.png
new file mode 100644
index 00000000..a9c5060d
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png
new file mode 100644
index 00000000..94b3d020
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png
new file mode 100644
index 00000000..0cc9e3e9
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png
new file mode 100644
index 00000000..034aa30a
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-signal-mix.png b/benchmarks/results-probe/chart-readprobe-signal-mix.png
new file mode 100644
index 00000000..6b6e866d
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-signal-mix.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-spans.png b/benchmarks/results-probe/chart-readprobe-spans.png
new file mode 100644
index 00000000..a3b7c902
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.png
new file mode 100644
index 00000000..a00622b5
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png
new file mode 100644
index 00000000..ea603aac
Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png differ
diff --git a/benchmarks/results-probe/summary.md b/benchmarks/results-probe/summary.md
new file mode 100644
index 00000000..18f92116
--- /dev/null
+++ b/benchmarks/results-probe/summary.md
@@ -0,0 +1,29 @@
+# Traceway hardware benchmark — summary
+
+> See [../charts.md](../charts.md) for a guide to reading these charts.
+
+Runs analyzed: 3
+
+## Read-probe scenario
+
+Failure: a read probe exceeded the configured threshold (default 5000ms) or returned an error. Max fill level passed is the largest row count at which the read still came back in time.
+
+### Spans — probing `/api/endpoints/grouped`
+
+| Tier | Mode | Max spans rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |
+|------|------|----------------|-------------------|--------------------|------------------------|
+| ccx13 | duckdb | 10,000,000 | 304 | 103,022,592 | 0 |
+
+### Metrics — probing `/api/metrics/application`
+
+| Tier | Mode | Max metric_points rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |
+|------|------|----------------|-------------------|--------------------|------------------------|
+| ccx13 | duckdb | 100,000,000 | 2710 | 1,000,087,552 | 0 |
+
+### Logs — probing `/api/logs`
+
+| Tier | Mode | Max log_records rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |
+|------|------|----------------|-------------------|--------------------|------------------------|
+| ccx13 | duckdb | 0 | 0 | 3,129,344 | 6005 |
+
+Generated by `benchmarks/scripts/chart.py`.