Skip to content

The only tool you need to know what is happening and how to fix it.

bench: read-probe 2026-07-21

traceway-botf17bdc4f2e8b
diff --git a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json
index 982e12ad..63778e66 100644
--- a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json
+++ b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json
@@ -3,8 +3,8 @@
"mode": "duckdb",
"signal": "logs",
"scenario": "read-probe",
- "startedAt": "2026-07-20T13:38:52Z",
- "endedAt": "2026-07-20T13:40:01Z",
+ "startedAt": "2026-07-21T10:18:05Z",
+ "endedAt": "2026-07-21T10:29:15Z",
"readProbe": {
"endpoints": [
{
@@ -29,35 +29,37 @@
{
"fillLevelTarget": 1000000,
"rowsIngested": 3129344,
- "ingestSecondsElapsed": 36.173149845,
+ "ingestSecondsElapsed": 36.427747303,
+ "digestSeconds": 600.42227659,
+ "digestTimedOut": true,
"probes": [
{
"name": "logs-severity-error",
"path": "/api/logs",
- "latencyMs": 6005.229562,
+ "latencyMs": 6000.205114,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"
},
{
"name": "logs-body-search",
"path": "/api/logs",
- "latencyMs": 6005.237126,
+ "latencyMs": 6005.645981,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"
},
{
"name": "logs-trace-id",
"path": "/api/logs",
- "latencyMs": 6000.9449540000005,
+ "latencyMs": 6005.5903180000005,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"
+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"
}
],
- "medianLatencyMs": 6005.229562,
- "readLatencyMs": 6005.229562,
+ "medianLatencyMs": 6005.5903180000005,
+ "readLatencyMs": 6005.5903180000005,
"readOk": false,
"passed": false,
- "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded (latency 6005ms)"
+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded (latency 6000ms)"
}
],
"maxFillLevelPassed": 0
diff --git a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json
index c24b8397..30d2e16b 100644
--- a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json
+++ b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json
@@ -3,8 +3,8 @@
"mode": "duckdb",
"signal": "metrics",
"scenario": "read-probe",
- "startedAt": "2026-07-20T12:43:35Z",
- "endedAt": "2026-07-20T13:32:30Z",
+ "startedAt": "2026-07-21T09:09:09Z",
+ "endedAt": "2026-07-21T10:11:34Z",
"readProbe": {
"endpoints": [
{
@@ -29,129 +29,134 @@
{
"fillLevelTarget": 1000000,
"rowsIngested": 1155072,
- "ingestSecondsElapsed": 3.266561102,
+ "ingestSecondsElapsed": 3.773906448,
+ "digestSeconds": 5.006256428,
"probes": [
{
"name": "metrics-server",
"path": "/api/metrics/server",
- "latencyMs": 78.3667,
+ "latencyMs": 156.70147400000002,
"ok": true
},
{
"name": "metrics-application",
"path": "/api/metrics/application",
- "latencyMs": 98.676076,
+ "latencyMs": 114.659787,
"ok": true
},
{
"name": "dashboard",
"path": "/api/dashboard",
- "latencyMs": 147.198833,
+ "latencyMs": 177.931375,
"ok": true
}
],
- "medianLatencyMs": 98.676076,
- "readLatencyMs": 98.676076,
+ "medianLatencyMs": 156.70147400000002,
+ "readLatencyMs": 156.70147400000002,
"readOk": true,
"passed": true
},
{
"fillLevelTarget": 10000000,
- "rowsIngested": 10289152,
- "ingestSecondsElapsed": 29.544118968,
+ "rowsIngested": 10084352,
+ "ingestSecondsElapsed": 32.145869453,
+ "digestSeconds": 5.006441803,
"probes": [
{
"name": "metrics-server",
"path": "/api/metrics/server",
- "latencyMs": 257.961618,
+ "latencyMs": 403.811263,
"ok": true
},
{
"name": "metrics-application",
"path": "/api/metrics/application",
- "latencyMs": 297.02590000000004,
+ "latencyMs": 430.051894,
"ok": true
},
{
"name": "dashboard",
"path": "/api/dashboard",
- "latencyMs": 487.871677,
+ "latencyMs": 678.7028899999999,
"ok": true
}
],
- "medianLatencyMs": 297.02590000000004,
- "readLatencyMs": 297.02590000000004,
+ "medianLatencyMs": 430.051894,
+ "readLatencyMs": 430.051894,
"readOk": true,
"passed": true
},
{
"fillLevelTarget": 100000000,
- "rowsIngested": 100057088,
- "ingestSecondsElapsed": 266.359219308,
+ "rowsIngested": 100360192,
+ "ingestSecondsElapsed": 306.935291402,
+ "digestSeconds": 5.003737541,
"probes": [
{
"name": "metrics-server",
"path": "/api/metrics/server",
- "latencyMs": 2223.881928,
+ "latencyMs": 2548.953915,
"ok": true
},
{
"name": "metrics-application",
"path": "/api/metrics/application",
- "latencyMs": 2710.946288,
+ "latencyMs": 3114.2537730000004,
"ok": true
},
{
"name": "dashboard",
"path": "/api/dashboard",
- "latencyMs": 4254.424445,
+ "latencyMs": 4930.21749,
"ok": true
}
],
- "medianLatencyMs": 2710.946288,
- "readLatencyMs": 2710.946288,
+ "medianLatencyMs": 3114.2537730000004,
+ "readLatencyMs": 3114.2537730000004,
"readOk": true,
"passed": true
},
{
"fillLevelTarget": 1000000000,
- "rowsIngested": 1000087552,
- "ingestSecondsElapsed": 2585.606908558,
+ "rowsIngested": 1000054784,
+ "ingestSecondsElapsed": 2734.611078861,
+ "digestSeconds": 600.465126817,
+ "digestTimedOut": true,
"probes": [
{
"name": "metrics-server",
"path": "/api/metrics/server",
- "latencyMs": 0.674826,
+ "latencyMs": 1.591393,
"ok": false,
- "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"
},
{
"name": "metrics-application",
"path": "/api/metrics/application",
- "latencyMs": 0.400743,
+ "latencyMs": 0.408595,
"ok": false,
- "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"
},
{
"name": "dashboard",
"path": "/api/dashboard",
- "latencyMs": 0.36321200000000003,
+ "latencyMs": 0.29740099999999997,
"ok": false,
- "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"
}
],
- "medianLatencyMs": 0.400743,
- "readLatencyMs": 0.400743,
+ "medianLatencyMs": 0.408595,
+ "readLatencyMs": 0.408595,
"readOk": false,
"passed": false,
- "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"
+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 2ms)"
}
],
"maxFillLevelPassed": 100000000
},
"maxFillLevelPassed": 100000000,
- "bytesOnDisk": 10048208896,
- "storedRows": 1000087552,
+ "bytesOnDisk": 8903426048,
+ "storedRows": 1000054784,
"sutContainer": {
"status": "exited",
"oomKilled": true,
diff --git a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json
index bf7db085..331721cb 100644
--- a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json
+++ b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json
@@ -3,8 +3,8 @@
"mode": "duckdb",
"signal": "spans",
"scenario": "read-probe",
- "startedAt": "2026-07-20T12:31:52Z",
- "endedAt": "2026-07-20T12:37:17Z",
+ "startedAt": "2026-07-21T08:48:02Z",
+ "endedAt": "2026-07-21T09:00:18Z",
"readProbe": {
"endpoints": [
{
@@ -28,105 +28,72 @@
"steps": [
{
"fillLevelTarget": 1000000,
- "rowsIngested": 1818624,
- "ingestSecondsElapsed": 30.656478277,
+ "rowsIngested": 1982464,
+ "ingestSecondsElapsed": 17.818738977,
+ "digestSeconds": 5.003859531,
"probes": [
{
"name": "endpoints-grouped",
"path": "/api/endpoints/grouped",
- "latencyMs": 354.24581900000004,
+ "latencyMs": 260.901057,
"ok": true
},
{
"name": "endpoints-chart",
"path": "/api/endpoints/chart",
- "latencyMs": 325.926413,
+ "latencyMs": 206.440555,
"ok": true
},
{
"name": "exception-stack-traces",
"path": "/api/exception-stack-traces",
- "latencyMs": 6.095135,
+ "latencyMs": 3.680037,
"ok": true
}
],
- "medianLatencyMs": 325.926413,
- "readLatencyMs": 325.926413,
+ "medianLatencyMs": 206.440555,
+ "readLatencyMs": 206.440555,
"readOk": true,
"passed": true
},
{
"fillLevelTarget": 10000000,
- "rowsIngested": 10280960,
- "ingestSecondsElapsed": 40.570054621,
+ "rowsIngested": 10518528,
+ "ingestSecondsElapsed": 69.130426492,
+ "digestSeconds": 600.46114678,
+ "digestTimedOut": true,
"probes": [
{
"name": "endpoints-grouped",
"path": "/api/endpoints/grouped",
- "latencyMs": 437.671008,
- "ok": true
- },
- {
- "name": "endpoints-chart",
- "path": "/api/endpoints/chart",
- "latencyMs": 304.181004,
- "ok": true
- },
- {
- "name": "exception-stack-traces",
- "path": "/api/exception-stack-traces",
- "latencyMs": 4.126855,
- "ok": true
- }
- ],
- "medianLatencyMs": 304.181004,
- "readLatencyMs": 304.181004,
- "readOk": true,
- "passed": true
- },
- {
- "fillLevelTarget": 100000000,
- "rowsIngested": 103022592,
- "ingestSecondsElapsed": 221.832406721,
- "probes": [
- {
- "name": "endpoints-grouped",
- "path": "/api/endpoints/grouped",
- "latencyMs": 0.887397,
+ "latencyMs": 6000.232965,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"
},
{
"name": "endpoints-chart",
"path": "/api/endpoints/chart",
- "latencyMs": 0.49261400000000005,
+ "latencyMs": 6005.662727,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"
},
{
"name": "exception-stack-traces",
"path": "/api/exception-stack-traces",
- "latencyMs": 0.397733,
+ "latencyMs": 6005.671618,
"ok": false,
- "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"
+ "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"
}
],
- "medianLatencyMs": 0.49261400000000005,
- "readLatencyMs": 0.49261400000000005,
+ "medianLatencyMs": 6005.662727,
+ "readLatencyMs": 6005.662727,
"readOk": false,
"passed": false,
- "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"
+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded (latency 6000ms)"
}
],
- "maxFillLevelPassed": 10000000
+ "maxFillLevelPassed": 1000000
},
- "maxFillLevelPassed": 10000000,
- "bytesOnDisk": 234864640,
- "storedRows": 103022592,
- "sutContainer": {
- "status": "exited",
- "oomKilled": true,
- "exitCode": 137,
- "restartCount": 3
- }
+ "maxFillLevelPassed": 1000000,
+ "sutContainer": {}
}
diff --git a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png
index 7f1cabff..afd423a7 100644
Binary files a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png and b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-cliff-spans.png b/benchmarks/results-probe/chart-readprobe-cliff-spans.png
index 30d92b94..cad43334 100644
Binary files a/benchmarks/results-probe/chart-readprobe-cliff-spans.png and b/benchmarks/results-probe/chart-readprobe-cliff-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-headline-metrics.png b/benchmarks/results-probe/chart-readprobe-headline-metrics.png
index 2e243653..fdf34a13 100644
Binary files a/benchmarks/results-probe/chart-readprobe-headline-metrics.png and b/benchmarks/results-probe/chart-readprobe-headline-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-headline-spans.png b/benchmarks/results-probe/chart-readprobe-headline-spans.png
index 47023b94..94debca9 100644
Binary files a/benchmarks/results-probe/chart-readprobe-headline-spans.png and b/benchmarks/results-probe/chart-readprobe-headline-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png
index c44abd7d..d8918d49 100644
Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png
index c0d6fdab..da2c500d 100644
Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png
index e5e0b2aa..f66549a2 100644
Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-logs.png b/benchmarks/results-probe/chart-readprobe-logs.png
index 745aff0d..668af26c 100644
Binary files a/benchmarks/results-probe/chart-readprobe-logs.png and b/benchmarks/results-probe/chart-readprobe-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-metrics.png b/benchmarks/results-probe/chart-readprobe-metrics.png
index a9c5060d..f6277094 100644
Binary files a/benchmarks/results-probe/chart-readprobe-metrics.png and b/benchmarks/results-probe/chart-readprobe-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png
index 94b3d020..f6fc6461 100644
Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png
index 0cc9e3e9..93199153 100644
Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png
index 034aa30a..436b7366 100644
Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-signal-mix.png b/benchmarks/results-probe/chart-readprobe-signal-mix.png
index 6b6e866d..ad2be50f 100644
Binary files a/benchmarks/results-probe/chart-readprobe-signal-mix.png and b/benchmarks/results-probe/chart-readprobe-signal-mix.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-spans.png b/benchmarks/results-probe/chart-readprobe-spans.png
index a3b7c902..734a5287 100644
Binary files a/benchmarks/results-probe/chart-readprobe-spans.png and b/benchmarks/results-probe/chart-readprobe-spans.png differ
diff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png
index ea603aac..9a4069e0 100644
Binary files a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png and b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png differ
diff --git a/benchmarks/results-probe/summary.md b/benchmarks/results-probe/summary.md
index 18f92116..0136bc6f 100644
--- a/benchmarks/results-probe/summary.md
+++ b/benchmarks/results-probe/summary.md
@@ -12,13 +12,13 @@ Failure: a read probe exceeded the configured threshold (default 5000ms) or retu
| Tier | Mode | Max spans rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |
|------|------|----------------|-------------------|--------------------|------------------------|
-| ccx13 | duckdb | 10,000,000 | 304 | 103,022,592 | 0 |
+| ccx13 | duckdb | 1,000,000 | 206 | 10,518,528 | 6005 |
### Metrics — probing `/api/metrics/application`
| Tier | Mode | Max metric_points rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |
|------|------|----------------|-------------------|--------------------|------------------------|
-| ccx13 | duckdb | 100,000,000 | 2710 | 1,000,087,552 | 0 |
+| ccx13 | duckdb | 100,000,000 | 3114 | 1,000,054,784 | 0 |
### Logs — probing `/api/logs`