The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
bench: throughput 2026-05-23
traceway-botdfd9e746dcd0
diff --git a/benchmarks/results-throughput/ccx13-sqlite-metrics-throughput.json b/benchmarks/results-throughput/ccx13-sqlite-metrics-throughput.jsonindex 979363a3..191bc879 100644--- a/benchmarks/results-throughput/ccx13-sqlite-metrics-throughput.json+++ b/benchmarks/results-throughput/ccx13-sqlite-metrics-throughput.json@@ -3,8 +3,8 @@"mode": "sqlite","signal": "metrics","scenario": "throughput",- "startedAt": "2026-05-23T01:23:38Z",- "endedAt": "2026-05-23T01:46:29Z",+ "startedAt": "2026-05-23T04:58:05Z",+ "endedAt": "2026-05-23T05:22:42Z","phase1": {"kind": "batch-size-ramp","fixedRequestRate": 5,@@ -13,13 +13,13 @@"step": 1,"batchSize": 256,"requestRate": 5,- "attemptedItemsPerSec": 1281.9753500945953,- "actualItemsPerSec": 1281.9753500945953,+ "attemptedItemsPerSec": 1282.004034006758,+ "actualItemsPerSec": 1282.004034006758,"rejected": 0,"ingest": {- "p50": 14.237624,- "p95": 15.611144000000001,- "p99": 17.189861,+ "p50": 14.182811,+ "p95": 15.942796,+ "p99": 18.356655999999997,"ok": 601,"errors": 0,"errRate": 0@@ -37,14 +37,14 @@"step": 2,"batchSize": 1024,"requestRate": 5,- "attemptedItemsPerSec": 3864.5689564474715,- "actualItemsPerSec": 3864.5689564474715,+ "attemptedItemsPerSec": 3898.4833029201486,+ "actualItemsPerSec": 3898.4833029201486,"rejected": 0,"ingest": {- "p50": 46.126109,- "p95": 48.277471000000006,- "p99": 53.342400999999995,- "ok": 453,+ "p50": 46.60554,+ "p95": 49.190736,+ "p99": 53.169051,+ "ok": 457,"errors": 0,"errRate": 0},@@ -61,14 +61,14 @@"step": 3,"batchSize": 4096,"requestRate": 5,- "attemptedItemsPerSec": 15648.472536094168,- "actualItemsPerSec": 15648.472536094168,+ "attemptedItemsPerSec": 15547.156034271607,+ "actualItemsPerSec": 15547.156034271607,"rejected": 0,"ingest": {- "p50": 177.076233,- "p95": 183.78423899999999,- "p99": 194.51580800000002,- "ok": 459,+ "p50": 176.412991,+ "p95": 186.146252,+ "p99": 191.044352,+ "ok": 456,"errors": 0,"errRate": 0},@@ -85,16 +85,16 @@"step": 4,"batchSize": 8192,"requestRate": 5,- "attemptedItemsPerSec": 28545.682799874703,- "actualItemsPerSec": 1449.3393033048967,+ "attemptedItemsPerSec": 24758.511378158793,+ "actualItemsPerSec": 1147.7455605768978,"rejected": 0,"ingest": {- "p50": 13683.492944,- "p95": 26279.480776,- "p99": 28361.86245,- "ok": 23,- "errors": 430,- "errRate": 0.9492273730684326+ "p50": 12640.689540000001,+ "p95": 24442.002215,+ "p99": 24442.002215,+ "ok": 21,+ "errors": 432,+ "errRate": 0.9536423841059603},"ch": {"reachable": false,@@ -104,7 +104,7 @@"longestMergeSec": 0},"passed": false,- "failReason": "combined error rate 94.92% (http 94.92% + rejected 0.00%) \u003e 5.00% threshold"+ "failReason": "combined error rate 95.36% (http 95.36% + rejected 0.00%) \u003e 5.00% threshold"}],"maxBatchSize": 4096@@ -117,16 +117,16 @@"step": 1,"batchSize": 4096,"requestRate": 1,- "attemptedItemsPerSec": 3843.9109361497353,- "actualItemsPerSec": 535.6269337257828,+ "attemptedItemsPerSec": 4157.7885270983625,+ "actualItemsPerSec": 3203.5419798954595,"rejected": 0,"ingest": {- "p50": 22394.830422,- "p95": 29261.759722000003,- "p99": 29261.759722000003,- "ok": 17,- "errors": 105,- "errRate": 0.860655737704918+ "p50": 180.989608,+ "p95": 25037.939551,+ "p99": 28441.417234,+ "ok": 94,+ "errors": 28,+ "errRate": 0.22950819672131148},"ch": {"reachable": false,@@ -136,7 +136,7 @@"longestMergeSec": 0},"passed": false,- "failReason": "combined error rate 86.07% (http 86.07% + rejected 0.00%) \u003e 5.00% threshold"+ "failReason": "combined error rate 22.95% (http 22.95% + rejected 0.00%) \u003e 5.00% threshold"}]},@@ -148,13 +148,13 @@"step": 1,"batchSize": 100,"requestRate": 10,- "attemptedItemsPerSec": 1001.6133288872252,- "actualItemsPerSec": 1001.6133288872252,+ "attemptedItemsPerSec": 1001.6145903619727,+ "actualItemsPerSec": 1001.6145903619727,"rejected": 0,"ingest": {- "p50": 5.381842000000001,- "p95": 9.965987,- "p99": 11.167494,+ "p50": 5.385344,+ "p95": 10.277681000000001,+ "p99": 11.558016,"ok": 1202,"errors": 0,"errRate": 0@@ -172,13 +172,13 @@"step": 2,"batchSize": 100,"requestRate": 100,- "attemptedItemsPerSec": 9751.790436567333,- "actualItemsPerSec": 9751.790436567333,+ "attemptedItemsPerSec": 9751.726877729683,+ "actualItemsPerSec": 9751.726877729683,"rejected": 0,"ingest": {- "p50": 4.721463,- "p95": 10.129470999999999,- "p99": 12.524085000000001,+ "p50": 4.790314,+ "p95": 10.325179,+ "p99": 14.537877,"ok": 11703,"errors": 0,"errRate": 0@@ -196,14 +196,14 @@"step": 3,"batchSize": 100,"requestRate": 1000,- "attemptedItemsPerSec": 16747.523760034495,- "actualItemsPerSec": 16747.523760034495,+ "attemptedItemsPerSec": 16453.96628575246,+ "actualItemsPerSec": 16453.96628575246,"rejected": 0,"ingest": {- "p50": 3023.537745,- "p95": 3557.49289,- "p99": 3782.495661,- "ok": 20366,+ "p50": 3074.265864,+ "p95": 3629.118197,+ "p99": 3856.044782,+ "ok": 20005,"errors": 0,"errRate": 0},@@ -215,20 +215,20 @@"longestMergeSec": 0},"passed": false,- "failReason": "achieved 167.48 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"+ "failReason": "achieved 164.54 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"},{"step": 4,"batchSize": 100,"requestRate": 550,- "attemptedItemsPerSec": 16717.21696481755,- "actualItemsPerSec": 16717.21696481755,+ "attemptedItemsPerSec": 16541.70049989337,+ "actualItemsPerSec": 16541.70049989337,"rejected": 0,"ingest": {- "p50": 3045.92354,- "p95": 3578.602153,- "p99": 3819.647771,- "ok": 20316,+ "p50": 3077.627502,+ "p95": 3631.414504,+ "p99": 3886.4199360000002,+ "ok": 20106,"errors": 0,"errRate": 0},@@ -240,20 +240,20 @@"longestMergeSec": 0},"passed": false,- "failReason": "achieved 167.17 req/sec is below 70% of target 550.00 req/sec (workers saturated, SUT past cliff)"+ "failReason": "achieved 165.42 req/sec is below 70% of target 550.00 req/sec (workers saturated, SUT past cliff)"},{"step": 5,"batchSize": 100,"requestRate": 325,- "attemptedItemsPerSec": 16607.728580673072,- "actualItemsPerSec": 16607.728580673072,+ "attemptedItemsPerSec": 16414.805205046458,+ "actualItemsPerSec": 16414.805205046458,"rejected": 0,"ingest": {- "p50": 3059.862977,- "p95": 3608.4641619999998,- "p99": 3849.332037,- "ok": 20198,+ "p50": 3096.894328,+ "p95": 3656.6241760000003,+ "p99": 3898.193317,+ "ok": 19965,"errors": 0,"errRate": 0},@@ -265,22 +265,22 @@"longestMergeSec": 0},"passed": false,- "failReason": "achieved 166.08 req/sec is below 70% of target 325.00 req/sec (workers saturated, SUT past cliff)"+ "failReason": "achieved 164.15 req/sec is below 70% of target 325.00 req/sec (workers saturated, SUT past cliff)"},{"step": 6,"batchSize": 100,"requestRate": 212.5,- "attemptedItemsPerSec": 16618.477107899027,- "actualItemsPerSec": 16618.477107899027,+ "attemptedItemsPerSec": 16828.717117277138,+ "actualItemsPerSec": 16586.086182836163,"rejected": 0,"ingest": {- "p50": 3062.113422,- "p95": 3600.200766,- "p99": 3818.917135,- "ok": 20201,- "errors": 0,- "errRate": 0+ "p50": 3037.308098,+ "p95": 3575.279847,+ "p99": 3821.396914,+ "ok": 20166,+ "errors": 295,+ "errRate": 0.014417672645520747},"ch": {"reachable": false,@@ -294,5 +294,5 @@],"maxRequestRate": 212.5},- "maxSustainableItemsPerSec": 16618.477107899027+ "maxSustainableItemsPerSec": 16586.086182836163}diff --git a/benchmarks/results-throughput/ccx23-sqlite-metrics-throughput.json b/benchmarks/results-throughput/ccx23-sqlite-metrics-throughput.jsondeleted file mode 100644index 4f57e136..00000000--- a/benchmarks/results-throughput/ccx23-sqlite-metrics-throughput.json+++ /dev/null@@ -1,298 +0,0 @@-{- "tier": "ccx23",- "mode": "sqlite",- "signal": "metrics",- "scenario": "throughput",- "startedAt": "2026-05-23T01:50:50Z",- "endedAt": "2026-05-23T02:13:42Z",- "phase1": {- "kind": "batch-size-ramp",- "fixedRequestRate": 5,- "steps": [- {- "step": 1,- "batchSize": 256,- "requestRate": 5,- "attemptedItemsPerSec": 1281.9406024625573,- "actualItemsPerSec": 1281.9406024625573,- "rejected": 0,- "ingest": {- "p50": 14.373469,- "p95": 15.760814000000002,- "p99": 16.37836,- "ok": 601,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- },- {- "step": 2,- "batchSize": 1024,- "requestRate": 5,- "attemptedItemsPerSec": 3864.5588295930834,- "actualItemsPerSec": 3864.5588295930834,- "rejected": 0,- "ingest": {- "p50": 45.965535,- "p95": 48.173039,- "p99": 49.588989,- "ok": 453,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- },- {- "step": 3,- "batchSize": 4096,- "requestRate": 5,- "attemptedItemsPerSec": 15581.808620775046,- "actualItemsPerSec": 15581.808620775046,- "rejected": 0,- "ingest": {- "p50": 175.249369,- "p95": 180.299663,- "p99": 184.002258,- "ok": 457,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- },- {- "step": 4,- "batchSize": 8192,- "requestRate": 5,- "attemptedItemsPerSec": 28671.931492830765,- "actualItemsPerSec": 1449.3503831540827,- "rejected": 0,- "ingest": {- "p50": 13844.611633,- "p95": 27205.358106,- "p99": 28735.672296,- "ok": 23,- "errors": 432,- "errRate": 0.9494505494505494- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": false,- "failReason": "combined error rate 94.95% (http 94.95% + rejected 0.00%) \u003e 5.00% threshold"- }- ],- "maxBatchSize": 4096- },- "phase2": {- "kind": "request-rate-ramp",- "fixedBatchSize": 4096,- "steps": [- {- "step": 1,- "batchSize": 4096,- "requestRate": 1,- "attemptedItemsPerSec": 3843.921334473858,- "actualItemsPerSec": 472.61327882875304,- "rejected": 0,- "ingest": {- "p50": 22460.429516,- "p95": 28208.119936,- "p99": 28208.119936,- "ok": 15,- "errors": 107,- "errRate": 0.8770491803278688- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": false,- "failReason": "combined error rate 87.70% (http 87.70% + rejected 0.00%) \u003e 5.00% threshold"- }- ]- },- "phase3": {- "kind": "small-batch-rate-ramp",- "fixedBatchSize": 100,- "steps": [- {- "step": 1,- "batchSize": 100,- "requestRate": 10,- "attemptedItemsPerSec": 1001.6159708675704,- "actualItemsPerSec": 1001.6159708675704,- "rejected": 0,- "ingest": {- "p50": 5.272304999999999,- "p95": 10.194234,- "p99": 10.921793000000001,- "ok": 1202,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- },- {- "step": 2,- "batchSize": 100,- "requestRate": 100,- "attemptedItemsPerSec": 9751.652384343159,- "actualItemsPerSec": 9751.652384343159,- "rejected": 0,- "ingest": {- "p50": 4.901661,- "p95": 10.576791,- "p99": 13.988710999999999,- "ok": 11703,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- },- {- "step": 3,- "batchSize": 100,- "requestRate": 1000,- "attemptedItemsPerSec": 16520.394051587584,- "actualItemsPerSec": 16520.394051587584,- "rejected": 0,- "ingest": {- "p50": 3064.221612,- "p95": 3609.252122,- "p99": 3857.97566,- "ok": 20087,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": false,- "failReason": "achieved 165.20 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"- },- {- "step": 4,- "batchSize": 100,- "requestRate": 550,- "attemptedItemsPerSec": 16724.753642510736,- "actualItemsPerSec": 16724.753642510736,- "rejected": 0,- "ingest": {- "p50": 3043.820314,- "p95": 3575.048052,- "p99": 3829.841293,- "ok": 20325,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": false,- "failReason": "achieved 167.25 req/sec is below 70% of target 550.00 req/sec (workers saturated, SUT past cliff)"- },- {- "step": 5,- "batchSize": 100,- "requestRate": 325,- "attemptedItemsPerSec": 16603.33808393728,- "actualItemsPerSec": 16603.33808393728,- "rejected": 0,- "ingest": {- "p50": 3062.7498370000003,- "p95": 3606.210013,- "p99": 3865.9489059999996,- "ok": 20170,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": false,- "failReason": "achieved 166.03 req/sec is below 70% of target 325.00 req/sec (workers saturated, SUT past cliff)"- },- {- "step": 6,- "batchSize": 100,- "requestRate": 212.5,- "attemptedItemsPerSec": 16659.19663732722,- "actualItemsPerSec": 16659.19663732722,- "rejected": 0,- "ingest": {- "p50": 3048.84016,- "p95": 3600.212748,- "p99": 3833.650021,- "ok": 20254,- "errors": 0,- "errRate": 0- },- "ch": {- "reachable": false,- "uptimeSec": 0,- "partsCount": 0,- "activeMerges": 0,- "longestMergeSec": 0- },- "passed": true- }- ],- "maxRequestRate": 212.5- },- "maxSustainableItemsPerSec": 16659.19663732722-}diff --git a/benchmarks/results-throughput/chart-batch-efficiency-metrics.png b/benchmarks/results-throughput/chart-batch-efficiency-metrics.pngindex 76c20d86..6d7eff61 100644Binary files a/benchmarks/results-throughput/chart-batch-efficiency-metrics.png and b/benchmarks/results-throughput/chart-batch-efficiency-metrics.png differdiff --git a/benchmarks/results-throughput/chart-cliff-metrics.png b/benchmarks/results-throughput/chart-cliff-metrics.pngindex 292eb380..c8ed1f9e 100644Binary files a/benchmarks/results-throughput/chart-cliff-metrics.png and b/benchmarks/results-throughput/chart-cliff-metrics.png differdiff --git a/benchmarks/results-throughput/chart-metrics.png b/benchmarks/results-throughput/chart-metrics.pngindex 11e984ae..e7213acd 100644Binary files a/benchmarks/results-throughput/chart-metrics.png and b/benchmarks/results-throughput/chart-metrics.png differdiff --git a/benchmarks/results-throughput/chart-pareto-metrics.png b/benchmarks/results-throughput/chart-pareto-metrics.pngindex dfc882f1..5ded534a 100644Binary files a/benchmarks/results-throughput/chart-pareto-metrics.png and b/benchmarks/results-throughput/chart-pareto-metrics.png differdiff --git a/benchmarks/results-throughput/chart-phase1-batch-metrics.png b/benchmarks/results-throughput/chart-phase1-batch-metrics.pngindex 0517e48c..f6e2ccb4 100644Binary files a/benchmarks/results-throughput/chart-phase1-batch-metrics.png and b/benchmarks/results-throughput/chart-phase1-batch-metrics.png differdiff --git a/benchmarks/results-throughput/chart-phase2-rate-metrics.png b/benchmarks/results-throughput/chart-phase2-rate-metrics.pngindex 3c502448..4dccd184 100644Binary files a/benchmarks/results-throughput/chart-phase2-rate-metrics.png and b/benchmarks/results-throughput/chart-phase2-rate-metrics.png differdiff --git a/benchmarks/results-throughput/chart-phase3-rate-metrics.png b/benchmarks/results-throughput/chart-phase3-rate-metrics.pngindex d7a1f5f1..6f803190 100644Binary files a/benchmarks/results-throughput/chart-phase3-rate-metrics.png and b/benchmarks/results-throughput/chart-phase3-rate-metrics.png differdiff --git a/benchmarks/results-throughput/chart-signal-mix.png b/benchmarks/results-throughput/chart-signal-mix.pngindex 9cdf5f02..2e8c6c78 100644Binary files a/benchmarks/results-throughput/chart-signal-mix.png and b/benchmarks/results-throughput/chart-signal-mix.png differdiff --git a/benchmarks/results-throughput/chart-tier-scaling-metrics.png b/benchmarks/results-throughput/chart-tier-scaling-metrics.pngindex 59d85b3c..8b85e0ee 100644Binary files a/benchmarks/results-throughput/chart-tier-scaling-metrics.png and b/benchmarks/results-throughput/chart-tier-scaling-metrics.png differdiff --git a/benchmarks/results-throughput/summary.md b/benchmarks/results-throughput/summary.mdindex f180753c..9582a643 100644--- a/benchmarks/results-throughput/summary.md+++ b/benchmarks/results-throughput/summary.md@@ -2,7 +2,7 @@> See [../charts.md](../charts.md) for a guide to reading these charts.-Runs analyzed: 2+Runs analyzed: 1## Throughput scenario@@ -12,7 +12,6 @@ Failure threshold: combined HTTP error rate + OTLP rejected items > 5% of attemp| Tier | Mode | Max data points/sec | P1 max batch | P2 max req/sec @ batch | P3 max req/sec @ batch=100 ||------|------|---------|--------------|-------------------------|-------------------------------|-| ccx13 | sqlite | 16,618 | 4,096 | — | 212.5 |-| ccx23 | sqlite | 16,659 | 4,096 | — | 212.5 |+| ccx13 | sqlite | 16,586 | 4,096 | — | 212.5 |Generated by `benchmarks/scripts/chart.py`.