The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
bench: read-probe 2026-07-21
traceway-botf17bdc4f2e8b
diff --git a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.jsonindex 982e12ad..63778e66 100644--- a/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json+++ b/benchmarks/results-probe/ccx13-duckdb-logs-read-probe.json@@ -3,8 +3,8 @@"mode": "duckdb","signal": "logs","scenario": "read-probe",- "startedAt": "2026-07-20T13:38:52Z",- "endedAt": "2026-07-20T13:40:01Z",+ "startedAt": "2026-07-21T10:18:05Z",+ "endedAt": "2026-07-21T10:29:15Z","readProbe": {"endpoints": [{@@ -29,35 +29,37 @@{"fillLevelTarget": 1000000,"rowsIngested": 3129344,- "ingestSecondsElapsed": 36.173149845,+ "ingestSecondsElapsed": 36.427747303,+ "digestSeconds": 600.42227659,+ "digestTimedOut": true,"probes": [{"name": "logs-severity-error","path": "/api/logs",- "latencyMs": 6005.229562,+ "latencyMs": 6000.205114,"ok": false,- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"},{"name": "logs-body-search","path": "/api/logs",- "latencyMs": 6005.237126,+ "latencyMs": 6005.645981,"ok": false,- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"},{"name": "logs-trace-id","path": "/api/logs",- "latencyMs": 6000.9449540000005,+ "latencyMs": 6005.5903180000005,"ok": false,- "error": "Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded"+ "error": "Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded"}],- "medianLatencyMs": 6005.229562,- "readLatencyMs": 6005.229562,+ "medianLatencyMs": 6005.5903180000005,+ "readLatencyMs": 6005.5903180000005,"readOk": false,"passed": false,- "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=6b28dd21-8b6c-45bf-90a1-c30a667593fb\": context deadline exceeded (latency 6005ms)"+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=c953ecec-e492-4c23-ad03-2ef4022a6769\": context deadline exceeded (latency 6000ms)"}],"maxFillLevelPassed": 0diff --git a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.jsonindex c24b8397..30d2e16b 100644--- a/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json+++ b/benchmarks/results-probe/ccx13-duckdb-metrics-read-probe.json@@ -3,8 +3,8 @@"mode": "duckdb","signal": "metrics","scenario": "read-probe",- "startedAt": "2026-07-20T12:43:35Z",- "endedAt": "2026-07-20T13:32:30Z",+ "startedAt": "2026-07-21T09:09:09Z",+ "endedAt": "2026-07-21T10:11:34Z","readProbe": {"endpoints": [{@@ -29,129 +29,134 @@{"fillLevelTarget": 1000000,"rowsIngested": 1155072,- "ingestSecondsElapsed": 3.266561102,+ "ingestSecondsElapsed": 3.773906448,+ "digestSeconds": 5.006256428,"probes": [{"name": "metrics-server","path": "/api/metrics/server",- "latencyMs": 78.3667,+ "latencyMs": 156.70147400000002,"ok": true},{"name": "metrics-application","path": "/api/metrics/application",- "latencyMs": 98.676076,+ "latencyMs": 114.659787,"ok": true},{"name": "dashboard","path": "/api/dashboard",- "latencyMs": 147.198833,+ "latencyMs": 177.931375,"ok": true}],- "medianLatencyMs": 98.676076,- "readLatencyMs": 98.676076,+ "medianLatencyMs": 156.70147400000002,+ "readLatencyMs": 156.70147400000002,"readOk": true,"passed": true},{"fillLevelTarget": 10000000,- "rowsIngested": 10289152,- "ingestSecondsElapsed": 29.544118968,+ "rowsIngested": 10084352,+ "ingestSecondsElapsed": 32.145869453,+ "digestSeconds": 5.006441803,"probes": [{"name": "metrics-server","path": "/api/metrics/server",- "latencyMs": 257.961618,+ "latencyMs": 403.811263,"ok": true},{"name": "metrics-application","path": "/api/metrics/application",- "latencyMs": 297.02590000000004,+ "latencyMs": 430.051894,"ok": true},{"name": "dashboard","path": "/api/dashboard",- "latencyMs": 487.871677,+ "latencyMs": 678.7028899999999,"ok": true}],- "medianLatencyMs": 297.02590000000004,- "readLatencyMs": 297.02590000000004,+ "medianLatencyMs": 430.051894,+ "readLatencyMs": 430.051894,"readOk": true,"passed": true},{"fillLevelTarget": 100000000,- "rowsIngested": 100057088,- "ingestSecondsElapsed": 266.359219308,+ "rowsIngested": 100360192,+ "ingestSecondsElapsed": 306.935291402,+ "digestSeconds": 5.003737541,"probes": [{"name": "metrics-server","path": "/api/metrics/server",- "latencyMs": 2223.881928,+ "latencyMs": 2548.953915,"ok": true},{"name": "metrics-application","path": "/api/metrics/application",- "latencyMs": 2710.946288,+ "latencyMs": 3114.2537730000004,"ok": true},{"name": "dashboard","path": "/api/dashboard",- "latencyMs": 4254.424445,+ "latencyMs": 4930.21749,"ok": true}],- "medianLatencyMs": 2710.946288,- "readLatencyMs": 2710.946288,+ "medianLatencyMs": 3114.2537730000004,+ "readLatencyMs": 3114.2537730000004,"readOk": true,"passed": true},{"fillLevelTarget": 1000000000,- "rowsIngested": 1000087552,- "ingestSecondsElapsed": 2585.606908558,+ "rowsIngested": 1000054784,+ "ingestSecondsElapsed": 2734.611078861,+ "digestSeconds": 600.465126817,+ "digestTimedOut": true,"probes": [{"name": "metrics-server","path": "/api/metrics/server",- "latencyMs": 0.674826,+ "latencyMs": 1.591393,"ok": false,- "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"},{"name": "metrics-application","path": "/api/metrics/application",- "latencyMs": 0.400743,+ "latencyMs": 0.408595,"ok": false,- "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"},{"name": "dashboard","path": "/api/dashboard",- "latencyMs": 0.36321200000000003,+ "latencyMs": 0.29740099999999997,"ok": false,- "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused"}],- "medianLatencyMs": 0.400743,- "readLatencyMs": 0.400743,+ "medianLatencyMs": 0.408595,+ "readLatencyMs": 0.408595,"readOk": false,"passed": false,- "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-19T13%3A32%3A30Z&projectId=851c4c17-e3f2-4bdf-b4bb-888931dd9798&toDate=2026-07-20T13%3A32%3A30Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T10%3A11%3A34Z&projectId=66c701a3-529c-4a28-8074-aae984066482&toDate=2026-07-21T10%3A11%3A34Z\": dial tcp 10.0.0.2:80: connect: connection refused (latency 2ms)"}],"maxFillLevelPassed": 100000000},"maxFillLevelPassed": 100000000,- "bytesOnDisk": 10048208896,- "storedRows": 1000087552,+ "bytesOnDisk": 8903426048,+ "storedRows": 1000054784,"sutContainer": {"status": "exited","oomKilled": true,diff --git a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.jsonindex bf7db085..331721cb 100644--- a/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json+++ b/benchmarks/results-probe/ccx13-duckdb-spans-read-probe.json@@ -3,8 +3,8 @@"mode": "duckdb","signal": "spans","scenario": "read-probe",- "startedAt": "2026-07-20T12:31:52Z",- "endedAt": "2026-07-20T12:37:17Z",+ "startedAt": "2026-07-21T08:48:02Z",+ "endedAt": "2026-07-21T09:00:18Z","readProbe": {"endpoints": [{@@ -28,105 +28,72 @@"steps": [{"fillLevelTarget": 1000000,- "rowsIngested": 1818624,- "ingestSecondsElapsed": 30.656478277,+ "rowsIngested": 1982464,+ "ingestSecondsElapsed": 17.818738977,+ "digestSeconds": 5.003859531,"probes": [{"name": "endpoints-grouped","path": "/api/endpoints/grouped",- "latencyMs": 354.24581900000004,+ "latencyMs": 260.901057,"ok": true},{"name": "endpoints-chart","path": "/api/endpoints/chart",- "latencyMs": 325.926413,+ "latencyMs": 206.440555,"ok": true},{"name": "exception-stack-traces","path": "/api/exception-stack-traces",- "latencyMs": 6.095135,+ "latencyMs": 3.680037,"ok": true}],- "medianLatencyMs": 325.926413,- "readLatencyMs": 325.926413,+ "medianLatencyMs": 206.440555,+ "readLatencyMs": 206.440555,"readOk": true,"passed": true},{"fillLevelTarget": 10000000,- "rowsIngested": 10280960,- "ingestSecondsElapsed": 40.570054621,+ "rowsIngested": 10518528,+ "ingestSecondsElapsed": 69.130426492,+ "digestSeconds": 600.46114678,+ "digestTimedOut": true,"probes": [{"name": "endpoints-grouped","path": "/api/endpoints/grouped",- "latencyMs": 437.671008,- "ok": true- },- {- "name": "endpoints-chart",- "path": "/api/endpoints/chart",- "latencyMs": 304.181004,- "ok": true- },- {- "name": "exception-stack-traces",- "path": "/api/exception-stack-traces",- "latencyMs": 4.126855,- "ok": true- }- ],- "medianLatencyMs": 304.181004,- "readLatencyMs": 304.181004,- "readOk": true,- "passed": true- },- {- "fillLevelTarget": 100000000,- "rowsIngested": 103022592,- "ingestSecondsElapsed": 221.832406721,- "probes": [- {- "name": "endpoints-grouped",- "path": "/api/endpoints/grouped",- "latencyMs": 0.887397,+ "latencyMs": 6000.232965,"ok": false,- "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"},{"name": "endpoints-chart","path": "/api/endpoints/chart",- "latencyMs": 0.49261400000000005,+ "latencyMs": 6005.662727,"ok": false,- "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"},{"name": "exception-stack-traces","path": "/api/exception-stack-traces",- "latencyMs": 0.397733,+ "latencyMs": 6005.671618,"ok": false,- "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused"+ "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded"}],- "medianLatencyMs": 0.49261400000000005,- "readLatencyMs": 0.49261400000000005,+ "medianLatencyMs": 6005.662727,+ "readLatencyMs": 6005.662727,"readOk": false,"passed": false,- "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=bf21b38f-8203-40d5-a2a7-db46354566fa\": dial tcp 10.0.0.2:80: connect: connection refused (latency 1ms)"+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=a621ac4a-7796-4851-bf00-df3631f87a0b\": context deadline exceeded (latency 6000ms)"}],- "maxFillLevelPassed": 10000000+ "maxFillLevelPassed": 1000000},- "maxFillLevelPassed": 10000000,- "bytesOnDisk": 234864640,- "storedRows": 103022592,- "sutContainer": {- "status": "exited",- "oomKilled": true,- "exitCode": 137,- "restartCount": 3- }+ "maxFillLevelPassed": 1000000,+ "sutContainer": {}}diff --git a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png b/benchmarks/results-probe/chart-readprobe-cliff-metrics.pngindex 7f1cabff..afd423a7 100644Binary files a/benchmarks/results-probe/chart-readprobe-cliff-metrics.png and b/benchmarks/results-probe/chart-readprobe-cliff-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-cliff-spans.png b/benchmarks/results-probe/chart-readprobe-cliff-spans.pngindex 30d92b94..cad43334 100644Binary files a/benchmarks/results-probe/chart-readprobe-cliff-spans.png and b/benchmarks/results-probe/chart-readprobe-cliff-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-headline-metrics.png b/benchmarks/results-probe/chart-readprobe-headline-metrics.pngindex 2e243653..fdf34a13 100644Binary files a/benchmarks/results-probe/chart-readprobe-headline-metrics.png and b/benchmarks/results-probe/chart-readprobe-headline-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-headline-spans.png b/benchmarks/results-probe/chart-readprobe-headline-spans.pngindex 47023b94..94debca9 100644Binary files a/benchmarks/results-probe/chart-readprobe-headline-spans.png and b/benchmarks/results-probe/chart-readprobe-headline-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.pngindex c44abd7d..d8918d49 100644Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.pngindex c0d6fdab..da2c500d 100644Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.pngindex e5e0b2aa..f66549a2 100644Binary files a/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png and b/benchmarks/results-probe/chart-readprobe-ingest-time-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-logs.png b/benchmarks/results-probe/chart-readprobe-logs.pngindex 745aff0d..668af26c 100644Binary files a/benchmarks/results-probe/chart-readprobe-logs.png and b/benchmarks/results-probe/chart-readprobe-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-metrics.png b/benchmarks/results-probe/chart-readprobe-metrics.pngindex a9c5060d..f6277094 100644Binary files a/benchmarks/results-probe/chart-readprobe-metrics.png and b/benchmarks/results-probe/chart-readprobe-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.pngindex 94b3d020..f6fc6461 100644Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-logs.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.pngindex 0cc9e3e9..93199153 100644Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-metrics.png differdiff --git a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.pngindex 034aa30a..436b7366 100644Binary files a/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png and b/benchmarks/results-probe/chart-readprobe-per-endpoint-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-signal-mix.png b/benchmarks/results-probe/chart-readprobe-signal-mix.pngindex 6b6e866d..ad2be50f 100644Binary files a/benchmarks/results-probe/chart-readprobe-signal-mix.png and b/benchmarks/results-probe/chart-readprobe-signal-mix.png differdiff --git a/benchmarks/results-probe/chart-readprobe-spans.png b/benchmarks/results-probe/chart-readprobe-spans.pngindex a3b7c902..734a5287 100644Binary files a/benchmarks/results-probe/chart-readprobe-spans.png and b/benchmarks/results-probe/chart-readprobe-spans.png differdiff --git a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.pngindex ea603aac..9a4069e0 100644Binary files a/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png and b/benchmarks/results-probe/chart-readprobe-tier-scaling-spans.png differdiff --git a/benchmarks/results-probe/summary.md b/benchmarks/results-probe/summary.mdindex 18f92116..0136bc6f 100644--- a/benchmarks/results-probe/summary.md+++ b/benchmarks/results-probe/summary.md@@ -12,13 +12,13 @@ Failure: a read probe exceeded the configured threshold (default 5000ms) or retu| Tier | Mode | Max spans rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) ||------|------|----------------|-------------------|--------------------|------------------------|-| ccx13 | duckdb | 10,000,000 | 304 | 103,022,592 | 0 |+| ccx13 | duckdb | 1,000,000 | 206 | 10,518,528 | 6005 |### Metrics — probing `/api/metrics/application`| Tier | Mode | Max metric_points rows passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) ||------|------|----------------|-------------------|--------------------|------------------------|-| ccx13 | duckdb | 100,000,000 | 2710 | 1,000,087,552 | 0 |+| ccx13 | duckdb | 100,000,000 | 3114 | 1,000,054,784 | 0 |### Logs — probing `/api/logs`