The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
bench: read-probe 2026-07-20
traceway-bot4d1337ac264b
diff --git a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.jsonnew file mode 100644index 00000000..982e12ad--- /dev/null+++ b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json@@ -0,0 +1,66 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "logs",+ "scenario": "read-probe",+ "startedAt": "2026-07-20T13:38:52Z",+ "endedAt": "2026-07-20T13:40:01Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs"+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs"+ }+ ],+ "readPath": "/api/logs",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 3129344,+ "ingestSecondsElapsed": 36.173149845,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 6005.229562,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 6005.237126,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 6000.9449540000005,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ }+ ],+ "medianLatencyMs": 6005.229562,+ "readLatencyMs": 6005.229562,+ "readOk": false,+ "passed": false,+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded (latency 6005ms)"+ }+ ],+ "maxFillLevelPassed": 0+ },+ "sutContainer": {}+}diff --git a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.jsonnew file mode 100644index 00000000..c24b8397--- /dev/null+++ b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json@@ -0,0 +1,161 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "metrics",+ "scenario": "read-probe",+ "startedAt": "2026-07-20T12:43:35Z",+ "endedAt": "2026-07-20T13:32:30Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard"+ }+ ],+ "readPath": "/api/metrics/server",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1155072,+ "ingestSecondsElapsed": 3.266561102,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 78.3667,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 98.676076,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 147.198833,+ "ok": true+ }+ ],+ "medianLatencyMs": 98.676076,+ "readLatencyMs": 98.676076,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10289152,+ "ingestSecondsElapsed": 29.544118968,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 257.961618,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 297.02590000000004,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 487.871677,+ "ok": true+ }+ ],+ "medianLatencyMs": 297.02590000000004,+ "readLatencyMs": 297.02590000000004,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 100057088,+ "ingestSecondsElapsed": 266.359219308,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 2223.881928,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 2710.946288,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 4254.424445,+ "ok": true+ }+ ],+ "medianLatencyMs": 2710.946288,+ "readLatencyMs": 2710.946288,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000000,+ "rowsIngested": 1000087552,+ "ingestSecondsElapsed": 2585.606908558,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 0.674826,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 0.400743,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 0.36321200000000003,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ }+ ],+ "medianLatencyMs": 0.400743,+ "readLatencyMs": 0.400743,+ "readOk": false,+ "passed": false,+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"+ }+ ],+ "maxFillLevelPassed": 100000000+ },+ "maxFillLevelPassed": 100000000,+ "bytesOnDisk": 10048208896,+ "storedRows": 1000087552,+ "sutContainer": {+ "status": "exited",+ "oomKilled": true,+ "exitCode": 137,+ "restartCount": 3+ }+}diff --git a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.jsonnew file mode 100644index 00000000..bf7db085--- /dev/null+++ b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json@@ -0,0 +1,132 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "spans",+ "scenario": "read-probe",+ "startedAt": "2026-07-20T12:31:52Z",+ "endedAt": "2026-07-20T12:37:17Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces"+ }+ ],+ "readPath": "/api/endpoints/grouped",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1818624,+ "ingestSecondsElapsed": 30.656478277,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 354.24581900000004,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 325.926413,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 6.095135,+ "ok": true+ }+ ],+ "medianLatencyMs": 325.926413,+ "readLatencyMs": 325.926413,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10280960,+ "ingestSecondsElapsed": 40.570054621,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 437.671008,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 304.181004,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 4.126855,+ "ok": true+ }+ ],+ "medianLatencyMs": 304.181004,+ "readLatencyMs": 304.181004,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 103022592,+ "ingestSecondsElapsed": 221.832406721,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 0.887397,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 0.49261400000000005,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 0.397733,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ }+ ],+ "medianLatencyMs": 0.49261400000000005,+ "readLatencyMs": 0.49261400000000005,+ "readOk": false,+ "passed": false,+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"+ }+ ],+ "maxFillLevelPassed": 10000000+ },+ "maxFillLevelPassed": 10000000,+ "bytesOnDisk": 234864640,+ "storedRows": 103022592,+ "sutContainer": {+ "status": "exited",+ "oomKilled": true,+ "exitCode": 137,+ "restartCount": 3+ }+}diff --git a/benchmarks/results-probe/chart-readprobe-cliff-logs.png b/benchmarks/results-probe/chart-readprobe-cliff-logs.pngnew file mode 100644index 00000000..50b07a1eBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png b/benchmarks/results-probe/chart-readprobe-cliff-metrics.pngnew file mode 100644index 00000000..7f1cabffBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-cliff-spans.png b/benchmarks/results-probe/chart-readprobe-cliff-spans.pngnew file mode 100644index 00000000..30d92b94Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-cliff-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-headline-logs.png b/benchmarks/results-probe/chart-readprobe-headline-logs.pngnew file mode 100644index 00000000..0fc5df24Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-headline-metrics.png b/benchmarks/results-probe/chart-readprobe-headline-metrics.pngnew file mode 100644index 00000000..2e243653Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-headline-spans.png b/benchmarks/results-probe/chart-readprobe-headline-spans.pngnew file mode 100644index 00000000..47023b94Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-headline-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.pngnew file mode 100644index 00000000..c44abd7dBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.pngnew file mode 100644index 00000000..c0d6fdabBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.pngnew file mode 100644index 00000000..e5e0b2aaBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-logs.png b/benchmarks/results-probe/chart-readprobe-logs.pngnew file mode 100644index 00000000..745aff0dBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-metrics.png b/benchmarks/results-probe/chart-readprobe-metrics.pngnew file mode 100644index 00000000..a9c5060dBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.pngnew file mode 100644index 00000000..94b3d020Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.pngnew file mode 100644index 00000000..0cc9e3e9Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.pngnew file mode 100644index 00000000..034aa30aBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-signal-mix.png b/benchmarks/results-probe/chart-readprobe-signal-mix.pngnew file mode 100644index 00000000..6b6e866dBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-signal-mix.png differdiff --git a/benchmarks/results-probe/chart-readprobe-spans.png b/benchmarks/results-probe/chart-readprobe-spans.pngnew file mode 100644index 00000000..a3b7c902Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.pngnew file mode 100644index 00000000..a00622b5Binary files /dev/null and b/benchmarks/results-probe/chart-readprobe-tier-scaling-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.pngnew file mode 100644index 00000000..ea603aacBinary files /dev/null and b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png differdiff --git a/benchmarks/results-probe/summary.md b/benchmarks/results-probe/summary.mdnew file mode 100644index 00000000..18f92116--- /dev/null+++ b/benchmarks/results-probe/summary.md@@ -0,0 +1,29 @@+# Traceway hardware benchmark — summary++> See [../charts.md](../charts.md) for a guide to reading these charts.++Runs analyzed: 3++## Read-probe scenario++Failure: a read probe exceeded the configured threshold (default 5000ms) or returned an error. Max fill level passed is the largest row count at which the read still came back in time.++### Spans — probing `/api/endpoints/grouped`++| Tier | Mode | Max spans rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |+|------|------|----------------|-------------------|--------------------|------------------------|+| ccx13 | duckdb | 10,000,000 | 304 | 103,022,592 | 0 |++### Metrics — probing `/api/metrics/application`++| Tier | Mode | Max metric_points rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |+|------|------|----------------|-------------------|--------------------|------------------------|+| ccx13 | duckdb | 100,000,000 | 2710 | 1,000,087,552 | 0 |++### Logs — probing `/api/logs`++| Tier | Mode | Max log_records rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |+|------|------|----------------|-------------------|--------------------|------------------------|+| ccx13 | duckdb | 0 | 0 | 3,129,344 | 6005 |++Generated by `benchmarks/scripts/chart.py`.