The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
blog: SQLite vs DuckDB benchmark + ingest admission gate
jstojiljkovic41d4e91c7594
diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe-60s-diagnostic.json b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe-60s-diagnostic.jsonnew file mode 100644index 00000000..055f0a28--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe-60s-diagnostic.json@@ -0,0 +1,133 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "logs",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T18:19:46Z",+ "endedAt": "2026-07-21T18:54:09Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs"+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs"+ }+ ],+ "readPath": "/api/logs",+ "readThresholdMs": 60000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1622016,+ "ingestSecondsElapsed": 29.849776256,+ "digestSeconds": 5.007748546,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 326.083069,+ "ok": true+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 74.746933,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 2.219858,+ "ok": true+ }+ ],+ "medianLatencyMs": 74.746933,+ "readLatencyMs": 74.746933,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10641408,+ "ingestSecondsElapsed": 169.171949097,+ "digestSeconds": 5.004557737,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 1895.5242220000002,+ "ok": true+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 81.393024,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 2.2702590000000002,+ "ok": true+ }+ ],+ "medianLatencyMs": 81.393024,+ "readLatencyMs": 81.393024,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 100581376,+ "ingestSecondsElapsed": 1752.447262896,+ "digestSeconds": 5.004060214,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 61010.336232999995,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=0bc48ced-c8b4-437e-8331-6441a2008ed6\": context deadline exceeded"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 3162.198998,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 4.405048,+ "ok": true+ }+ ],+ "medianLatencyMs": 3162.198998,+ "readLatencyMs": 3162.198998,+ "readOk": false,+ "passed": false,+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=0bc48ced-c8b4-437e-8331-6441a2008ed6\": context deadline exceeded (latency 61010ms)"+ }+ ],+ "maxFillLevelPassed": 10000000+ },+ "maxFillLevelPassed": 10000000,+ "bytesOnDisk": 13731495936,+ "storedRows": 100581376,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe.json b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe.jsonnew file mode 100644index 00000000..db2ad1a7--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-read-probe.json@@ -0,0 +1,133 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "logs",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T15:53:25Z",+ "endedAt": "2026-07-21T16:26:00Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs"+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs"+ }+ ],+ "readPath": "/api/logs",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1695744,+ "ingestSecondsElapsed": 31.215866858,+ "digestSeconds": 5.006626853,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 308.118162,+ "ok": true+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 27.046025,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 2.01207,+ "ok": true+ }+ ],+ "medianLatencyMs": 27.046025,+ "readLatencyMs": 27.046025,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10649600,+ "ingestSecondsElapsed": 160.098593092,+ "digestSeconds": 5.00439974,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 2862.714559,+ "ok": true+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 85.442927,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 2.495565,+ "ok": true+ }+ ],+ "medianLatencyMs": 85.442927,+ "readLatencyMs": 85.442927,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 100712448,+ "ingestSecondsElapsed": 1705.894526407,+ "digestSeconds": 5.006668129,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 6005.2135499999995,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=0412c6e7-5f61-41f4-9244-50ece71a3b7f\": context deadline exceeded"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 3525.108695,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 2.431163,+ "ok": true+ }+ ],+ "medianLatencyMs": 3525.108695,+ "readLatencyMs": 3525.108695,+ "readOk": false,+ "passed": false,+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=0412c6e7-5f61-41f4-9244-50ece71a3b7f\": context deadline exceeded (latency 6005ms)"+ }+ ],+ "maxFillLevelPassed": 10000000+ },+ "maxFillLevelPassed": 10000000,+ "bytesOnDisk": 9341763584,+ "storedRows": 100712448,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-logs-throughput.json b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-throughput.jsonnew file mode 100644index 00000000..28f3b408--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-logs-throughput.json@@ -0,0 +1,589 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "logs",+ "scenario": "throughput",+ "startedAt": "2026-07-21T13:33:00Z",+ "endedAt": "2026-07-21T14:09:35Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1282.0147335200115,+ "actualItemsPerSec": 1282.0147335200115,+ "rejected": 0,+ "ingest": {+ "p50": 17.26207,+ "p95": 20.646595,+ "p99": 24.239521,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 12288,+ "walSizeBytes": 72089224,+ "memoryUsedBytes": 91822080+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3890.1021650290145,+ "actualItemsPerSec": 3890.1021650290145,+ "rejected": 0,+ "ingest": {+ "p50": 41.299574,+ "p95": 47.441962,+ "p99": 61.803219,+ "ok": 456,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 52178944,+ "walSizeBytes": 33989045,+ "memoryUsedBytes": 97779712+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 15484.182970154025,+ "actualItemsPerSec": 15484.182970154025,+ "rejected": 0,+ "ingest": {+ "p50": 102.681886,+ "p95": 125.63351300000001,+ "p99": 992.965565,+ "ok": 454,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 207630336,+ "walSizeBytes": 128247991,+ "memoryUsedBytes": 367263744+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 30905.12202348657,+ "actualItemsPerSec": 30905.12202348657,+ "rejected": 0,+ "ingest": {+ "p50": 131.14354,+ "p95": 911.335455,+ "p99": 1269.347028,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 567816192,+ "walSizeBytes": 42114520,+ "memoryUsedBytes": 622592000+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 60925.38489691948,+ "actualItemsPerSec": 60925.38489691948,+ "rejected": 0,+ "ingest": {+ "p50": 1652.951507,+ "p95": 2388.499593,+ "p99": 2624.5160250000004,+ "ok": 454,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1242312704,+ "walSizeBytes": 99541833,+ "memoryUsedBytes": 1365245952+ },+ "passed": true+ }+ ],+ "maxBatchSize": 16384+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 16384,+ "steps": [+ {+ "step": 1,+ "batchSize": 16384,+ "requestRate": 1,+ "attemptedItemsPerSec": 16520.43807786408,+ "actualItemsPerSec": 16520.43807786408,+ "rejected": 0,+ "ingest": {+ "p50": 278.388168,+ "p95": 305.20088699999997,+ "p99": 1170.5415050000001,+ "ok": 121,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1396453376,+ "walSizeBytes": 245020518,+ "memoryUsedBytes": 1695809536+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 75225.16045662755,+ "actualItemsPerSec": 75225.16045662755,+ "rejected": 0,+ "ingest": {+ "p50": 2957.7459870000002,+ "p95": 4785.44721,+ "p99": 5096.376844,+ "ok": 572,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 2271227904,+ "walSizeBytes": 160792262,+ "memoryUsedBytes": 2467823616+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 16384,+ "requestRate": 10,+ "attemptedItemsPerSec": 143256.6261608713,+ "actualItemsPerSec": 71497.6044799239,+ "rejected": 0,+ "ingest": {+ "p50": 5662.247816,+ "p95": 6646.563777,+ "p99": 6880.767888,+ "ok": 547,+ "errors": 549,+ "errRate": 0.5009124087591241+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 3105632256,+ "walSizeBytes": 130175658,+ "memoryUsedBytes": 3268935680+ },+ "passed": false,+ "failReason": "combined error rate 50.09% (http 50.09% + rejected 0.00%) > 5.00% threshold"+ },+ {+ "step": 4,+ "batchSize": 16384,+ "requestRate": 7.5,+ "attemptedItemsPerSec": 93284.77559106264,+ "actualItemsPerSec": 61884.57062352403,+ "rejected": 0,+ "ingest": {+ "p50": 5617.176396999999,+ "p95": 6592.590075,+ "p99": 6779.297003,+ "ok": 473,+ "errors": 240,+ "errRate": 0.33660589060308554+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 3822071808,+ "walSizeBytes": 53603346,+ "memoryUsedBytes": 3742892032+ },+ "passed": false,+ "failReason": "combined error rate 33.66% (http 33.66% + rejected 0.00%) > 5.00% threshold"+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 6.25,+ "attemptedItemsPerSec": 79448.70719746505,+ "actualItemsPerSec": 64421.40238215504,+ "rejected": 0,+ "ingest": {+ "p50": 5551.942363,+ "p95": 6472.040691,+ "p99": 6688.987714,+ "ok": 493,+ "errors": 115,+ "errRate": 0.18914473684210525+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4543492096,+ "walSizeBytes": 160795453,+ "memoryUsedBytes": 3877896192+ },+ "passed": false,+ "failReason": "combined error rate 18.91% (http 18.91% + rejected 0.00%) > 5.00% threshold"+ },+ {+ "step": 6,+ "batchSize": 16384,+ "requestRate": 5.625,+ "attemptedItemsPerSec": 73095.08180662479,+ "actualItemsPerSec": 64726.41412214538,+ "rejected": 0,+ "ingest": {+ "p50": 5422.60898,+ "p95": 6287.980087,+ "p99": 6655.953776,+ "ok": 495,+ "errors": 64,+ "errRate": 0.11449016100178891+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5319438336,+ "walSizeBytes": 22969295,+ "memoryUsedBytes": 3709337600+ },+ "passed": false,+ "failReason": "combined error rate 11.45% (http 11.45% + rejected 0.00%) > 5.00% threshold"+ }+ ],+ "maxRequestRate": 5+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 1004.9267812440783,+ "actualItemsPerSec": 1004.9267812440783,+ "rejected": 0,+ "ingest": {+ "p50": 11.200266000000001,+ "p95": 15.275625,+ "p99": 21.741057,+ "ok": 1206,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5319438336,+ "walSizeBytes": 79714325,+ "memoryUsedBytes": 3777495040+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 100,+ "requestRate": 100,+ "attemptedItemsPerSec": 9754.40861507941,+ "actualItemsPerSec": 9754.40861507941,+ "rejected": 0,+ "ingest": {+ "p50": 7.9646040000000005,+ "p95": 13.466146,+ "p99": 501.01513900000003,+ "ok": 11706,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5419577344,+ "walSizeBytes": 118481377,+ "memoryUsedBytes": 3839098880+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 100,+ "requestRate": 1000,+ "attemptedItemsPerSec": 30093.297147245194,+ "actualItemsPerSec": 30093.297147245194,+ "rejected": 0,+ "ingest": {+ "p50": 1591.282434,+ "p95": 2655.65602,+ "p99": 2804.014969,+ "ok": 36615,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5768228864,+ "walSizeBytes": 48556398,+ "memoryUsedBytes": 3740532736+ },+ "passed": false,+ "failReason": "achieved 300.93 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 4,+ "batchSize": 100,+ "requestRate": 550,+ "attemptedItemsPerSec": 29465.46792248597,+ "actualItemsPerSec": 29433.3874878533,+ "rejected": 0,+ "ingest": {+ "p50": 1598.8326590000001,+ "p95": 2498.420931,+ "p99": 5001.769138000001,+ "ok": 35782,+ "errors": 39,+ "errRate": 0.0010887468244884285+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 6127890432,+ "walSizeBytes": 195554330,+ "memoryUsedBytes": 3449552896+ },+ "passed": false,+ "failReason": "achieved 294.33 req/sec is below 70% of target 550.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 5,+ "batchSize": 100,+ "requestRate": 325,+ "attemptedItemsPerSec": 30441.740603991988,+ "actualItemsPerSec": 30441.740603991988,+ "rejected": 0,+ "ingest": {+ "p50": 1571.473652,+ "p95": 2564.820214,+ "p99": 2755.983872,+ "ok": 37011,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 6420705280,+ "walSizeBytes": 143983013,+ "memoryUsedBytes": 3767795712+ },+ "passed": true+ },+ {+ "step": 6,+ "batchSize": 100,+ "requestRate": 437.5,+ "attemptedItemsPerSec": 30001.588816616724,+ "actualItemsPerSec": 29583.963806509822,+ "rejected": 0,+ "ingest": {+ "p50": 1548.747823,+ "p95": 2618.034174,+ "p99": 2686.060436,+ "ok": 35986,+ "errors": 508,+ "errRate": 0.01392009645421165+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 6989295616,+ "walSizeBytes": 44137546,+ "memoryUsedBytes": 2222981120+ },+ "passed": false,+ "failReason": "achieved 295.84 req/sec is below 70% of target 437.50 req/sec (workers saturated, SUT past cliff)"+ }+ ],+ "maxRequestRate": 325+ },+ "maxSustainableItemsPerSec": 75225.16045662755,+ "bytesOnDisk": 7033438208,+ "storedRows": 73713480,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe-60s-diagnostic.json b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe-60s-diagnostic.jsonnew file mode 100644index 00000000..03684a2d--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe-60s-diagnostic.json@@ -0,0 +1,165 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "metrics",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T17:11:18Z",+ "endedAt": "2026-07-21T18:13:13Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard"+ }+ ],+ "readPath": "/api/metrics/server",+ "readThresholdMs": 60000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 2596864,+ "ingestSecondsElapsed": 8.284285866,+ "digestSeconds": 5.007881861,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 140.18482400000002,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 138.57956199999998,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 220.36536800000002,+ "ok": true+ }+ ],+ "medianLatencyMs": 140.18482400000002,+ "readLatencyMs": 140.18482400000002,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 11550720,+ "ingestSecondsElapsed": 30.158111317,+ "digestSeconds": 5.007767715,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 335.28826200000003,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 381.869687,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 607.252369,+ "ok": true+ }+ ],+ "medianLatencyMs": 381.869687,+ "readLatencyMs": 381.869687,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 101474304,+ "ingestSecondsElapsed": 305.073646816,+ "digestSeconds": 5.005782441,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 2498.3773610000003,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 2986.210888,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 4761.32388,+ "ok": true+ }+ ],+ "medianLatencyMs": 2986.210888,+ "readLatencyMs": 2986.210888,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000000,+ "rowsIngested": 1001455616,+ "ingestSecondsElapsed": 3116.321125711,+ "digestSeconds": 5.004550418,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 61002.553677,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T18%3A10%3A10Z&projectId=5dd7a2f7-2701-4990-bf9b-f8617c4eee2a&toDate=2026-07-21T18%3A10%3A10Z\": context deadline exceeded"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 61001.405750000005,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-20T18%3A11%3A11Z&projectId=5dd7a2f7-2701-4990-bf9b-f8617c4eee2a&toDate=2026-07-21T18%3A11%3A11Z\": context deadline exceeded"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 61001.000224999996,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-20T18%3A12%3A12Z&projectId=5dd7a2f7-2701-4990-bf9b-f8617c4eee2a&toDate=2026-07-21T18%3A12%3A12Z\": context deadline exceeded"+ }+ ],+ "medianLatencyMs": 61001.405750000005,+ "readLatencyMs": 61001.405750000005,+ "readOk": false,+ "passed": false,+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T18%3A10%3A10Z&projectId=5dd7a2f7-2701-4990-bf9b-f8617c4eee2a&toDate=2026-07-21T18%3A10%3A10Z\": context deadline exceeded (latency 61003ms)"+ }+ ],+ "maxFillLevelPassed": 100000000+ },+ "maxFillLevelPassed": 100000000,+ "bytesOnDisk": 10795061248,+ "storedRows": 1001455616,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe.json b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe.jsonnew file mode 100644index 00000000..52d040e9--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-read-probe.json@@ -0,0 +1,165 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "metrics",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T14:47:29Z",+ "endedAt": "2026-07-21T15:46:57Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard"+ }+ ],+ "readPath": "/api/metrics/server",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 2580480,+ "ingestSecondsElapsed": 8.475577363,+ "digestSeconds": 5.007948499,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 146.0298,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 135.084231,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 213.30516899999998,+ "ok": true+ }+ ],+ "medianLatencyMs": 146.0298,+ "readLatencyMs": 146.0298,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 11550720,+ "ingestSecondsElapsed": 30.855640677,+ "digestSeconds": 5.007986705,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 343.63286500000004,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 381.574624,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 601.0998159999999,+ "ok": true+ }+ ],+ "medianLatencyMs": 381.574624,+ "readLatencyMs": 381.574624,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 101490688,+ "ingestSecondsElapsed": 305.46684237,+ "digestSeconds": 5.003962309,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 2467.547592,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 3001.6052050000003,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 4752.907417,+ "ok": true+ }+ ],+ "medianLatencyMs": 3001.6052050000003,+ "readLatencyMs": 3001.6052050000003,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000000,+ "rowsIngested": 1001472000,+ "ingestSecondsElapsed": 3132.254449848,+ "digestSeconds": 5.004906965,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 6005.238989,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T15%3A46%3A39Z&projectId=3d534138-1f31-4835-b08e-f841496cd3e9&toDate=2026-07-21T15%3A46%3A39Z\": context deadline exceeded"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 6000.54946,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-07-20T15%3A46%3A45Z&projectId=3d534138-1f31-4835-b08e-f841496cd3e9&toDate=2026-07-21T15%3A46%3A45Z\": context deadline exceeded"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 6005.2873389999995,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-07-20T15%3A46%3A51Z&projectId=3d534138-1f31-4835-b08e-f841496cd3e9&toDate=2026-07-21T15%3A46%3A51Z\": context deadline exceeded"+ }+ ],+ "medianLatencyMs": 6005.238989,+ "readLatencyMs": 6005.238989,+ "readOk": false,+ "passed": false,+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-07-20T15%3A46%3A39Z&projectId=3d534138-1f31-4835-b08e-f841496cd3e9&toDate=2026-07-21T15%3A46%3A39Z\": context deadline exceeded (latency 6005ms)"+ }+ ],+ "maxFillLevelPassed": 100000000+ },+ "maxFillLevelPassed": 100000000,+ "bytesOnDisk": 10823938048,+ "storedRows": 1001472000,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-throughput.json b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-throughput.jsonnew file mode 100644index 00000000..d9f1788c--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-metrics-throughput.json@@ -0,0 +1,618 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "metrics",+ "scenario": "throughput",+ "startedAt": "2026-07-21T12:48:00Z",+ "endedAt": "2026-07-21T13:26:20Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1282.0367777269973,+ "actualItemsPerSec": 1282.0367777269973,+ "rejected": 0,+ "ingest": {+ "p50": 8.388721,+ "p95": 11.190548,+ "p99": 12.992066999999999,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 12288,+ "walSizeBytes": 20806066,+ "memoryUsedBytes": 27492352+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3916.7726419306123,+ "actualItemsPerSec": 3916.7726419306123,+ "rejected": 0,+ "ingest": {+ "p50": 16.371666,+ "p95": 18.935693,+ "p99": 21.434116,+ "ok": 459,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 12288,+ "walSizeBytes": 84051387,+ "memoryUsedBytes": 105611264+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 15425.016712180306,+ "actualItemsPerSec": 15425.016712180306,+ "rejected": 0,+ "ingest": {+ "p50": 41.498647999999996,+ "p95": 47.420205,+ "p99": 54.80625,+ "ok": 452,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 21770240,+ "walSizeBytes": 76510334,+ "memoryUsedBytes": 119275520+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 30915.423404895144,+ "actualItemsPerSec": 30915.423404895144,+ "rejected": 0,+ "ingest": {+ "p50": 70.96242,+ "p95": 78.35168100000001,+ "p99": 253.431664,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 62926848,+ "walSizeBytes": 61641784,+ "memoryUsedBytes": 142082048+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 61822.383379369756,+ "actualItemsPerSec": 61822.383379369756,+ "rejected": 0,+ "ingest": {+ "p50": 99.663271,+ "p95": 122.064561,+ "p99": 542.562827,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 145764352,+ "walSizeBytes": 28618559,+ "memoryUsedBytes": 185073664+ },+ "passed": true+ }+ ],+ "maxBatchSize": 16384+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 16384,+ "steps": [+ {+ "step": 1,+ "batchSize": 16384,+ "requestRate": 1,+ "attemptedItemsPerSec": 16633.34190264782,+ "actualItemsPerSec": 16633.34190264782,+ "rejected": 0,+ "ingest": {+ "p50": 105.809615,+ "p95": 128.734228,+ "p99": 136.536756,+ "ok": 122,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 166735872,+ "walSizeBytes": 39625694,+ "memoryUsedBytes": 220463104+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 77912.30000410178,+ "actualItemsPerSec": 77912.30000410178,+ "rejected": 0,+ "ingest": {+ "p50": 99.736306,+ "p95": 124.617478,+ "p99": 566.485408,+ "ok": 571,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 270282752,+ "walSizeBytes": 8805716,+ "memoryUsedBytes": 287047680+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 16384,+ "requestRate": 10,+ "attemptedItemsPerSec": 143338.27451516208,+ "actualItemsPerSec": 143338.27451516208,+ "rejected": 0,+ "ingest": {+ "p50": 61.922116,+ "p95": 396.29122700000005,+ "p99": 660.553282,+ "ok": 1053,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 456929280,+ "walSizeBytes": 8805716,+ "memoryUsedBytes": 473694208+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 16384,+ "requestRate": 15,+ "attemptedItemsPerSec": 206136.23031869842,+ "actualItemsPerSec": 206136.23031869842,+ "rejected": 0,+ "ingest": {+ "p50": 61.575264999999995,+ "p95": 620.03202,+ "p99": 721.431008,+ "ok": 1511,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 705703936,+ "walSizeBytes": 244358405,+ "memoryUsedBytes": 1006108672+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 20,+ "attemptedItemsPerSec": 256340.29968471063,+ "actualItemsPerSec": 254242.38196134227,+ "rejected": 0,+ "ingest": {+ "p50": 3207.007778,+ "p95": 5075.152652,+ "p99": 5216.393919,+ "ok": 1939,+ "errors": 16,+ "errRate": 0.008184143222506393+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1058287616,+ "walSizeBytes": 127682774,+ "memoryUsedBytes": 1217134592+ },+ "passed": true+ },+ {+ "step": 6,+ "batchSize": 16384,+ "requestRate": 25,+ "attemptedItemsPerSec": 352077.7369994588,+ "actualItemsPerSec": 277942.3206521025,+ "rejected": 0,+ "ingest": {+ "p50": 5156.575400999999,+ "p95": 5284.039547,+ "p99": 5742.728115,+ "ok": 2122,+ "errors": 566,+ "errRate": 0.2105654761904762+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1431318528,+ "walSizeBytes": 156301325,+ "memoryUsedBytes": 1625292800+ },+ "passed": false,+ "failReason": "combined error rate 21.06% (http 21.06% + rejected 0.00%) > 5.00% threshold"+ },+ {+ "step": 7,+ "batchSize": 16384,+ "requestRate": 22.5,+ "attemptedItemsPerSec": 320468.7140275179,+ "actualItemsPerSec": 281310.5017291003,+ "rejected": 0,+ "ingest": {+ "p50": 5137.612766,+ "p95": 5270.028197000001,+ "p99": 5707.497281,+ "ok": 2148,+ "errors": 299,+ "errRate": 0.12219043727012668+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1804611584,+ "walSizeBytes": 239955551,+ "memoryUsedBytes": 2100297728+ },+ "passed": false,+ "failReason": "combined error rate 12.22% (http 12.22% + rejected 0.00%) > 5.00% threshold"+ }+ ],+ "maxRequestRate": 20+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 1009.0901821969892,+ "actualItemsPerSec": 1009.0901821969892,+ "rejected": 0,+ "ingest": {+ "p50": 6.715396,+ "p95": 8.796279,+ "p99": 31.050559,+ "ok": 1211,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1825058816,+ "walSizeBytes": 435610,+ "memoryUsedBytes": 1833172992+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 100,+ "requestRate": 100,+ "attemptedItemsPerSec": 9751.40723039604,+ "actualItemsPerSec": 9751.40723039604,+ "rejected": 0,+ "ingest": {+ "p50": 5.4039079999999995,+ "p95": 6.915218,+ "p99": 8.204802,+ "ok": 11702,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1825058816,+ "walSizeBytes": 159750440,+ "memoryUsedBytes": 2024275968+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 100,+ "requestRate": 1000,+ "attemptedItemsPerSec": 53259.34480514036,+ "actualItemsPerSec": 53259.34480514036,+ "rejected": 0,+ "ingest": {+ "p50": 934.397084,+ "p95": 996.504499,+ "p99": 1596.77871,+ "ok": 64411,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1906061312,+ "walSizeBytes": 12538779,+ "memoryUsedBytes": 1987575808+ },+ "passed": false,+ "failReason": "achieved 532.59 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 4,+ "batchSize": 100,+ "requestRate": 550,+ "attemptedItemsPerSec": 53090.9328358748,+ "actualItemsPerSec": 53090.9328358748,+ "rejected": 0,+ "ingest": {+ "p50": 947.972386,+ "p95": 991.64884,+ "p99": 1592.485309,+ "ok": 64208,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1966616576,+ "walSizeBytes": 118526270,+ "memoryUsedBytes": 2230845440+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 100,+ "requestRate": 775,+ "attemptedItemsPerSec": 53459.12621950558,+ "actualItemsPerSec": 53459.12621950558,+ "rejected": 0,+ "ingest": {+ "p50": 941.8437349999999,+ "p95": 975.135188,+ "p99": 1591.7919230000002,+ "ok": 64652,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 2027171840,+ "walSizeBytes": 230612977,+ "memoryUsedBytes": 2478309376+ },+ "passed": false,+ "failReason": "achieved 534.59 req/sec is below 70% of target 775.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 6,+ "batchSize": 100,+ "requestRate": 662.5,+ "attemptedItemsPerSec": 52746.649581601785,+ "actualItemsPerSec": 52746.649581601785,+ "rejected": 0,+ "ingest": {+ "p50": 938.609297,+ "p95": 980.475246,+ "p99": 1612.6566,+ "ok": 63780,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 2119708672,+ "walSizeBytes": 74851576,+ "memoryUsedBytes": 2406481920+ },+ "passed": true+ }+ ],+ "maxRequestRate": 662.5+ },+ "maxSustainableItemsPerSec": 254242.38196134227,+ "bytesOnDisk": 2194571264,+ "storedRows": 195695536,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe-60s-diagnostic.json b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe-60s-diagnostic.jsonnew file mode 100644index 00000000..59a0f504--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe-60s-diagnostic.json@@ -0,0 +1,127 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "spans",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T16:37:18Z",+ "endedAt": "2026-07-21T17:04:00Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces"+ }+ ],+ "readPath": "/api/endpoints/grouped",+ "readThresholdMs": 60000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 2211840,+ "ingestSecondsElapsed": 25.030675862,+ "digestSeconds": 5.005918072,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 205.227515,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 183.74937,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 2.813505,+ "ok": true+ }+ ],+ "medianLatencyMs": 183.74937,+ "readLatencyMs": 183.74937,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10838016,+ "ingestSecondsElapsed": 103.850431033,+ "digestSeconds": 5.005578797,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 845.7596739999999,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 759.485461,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 2.839823,+ "ok": true+ }+ ],+ "medianLatencyMs": 759.485461,+ "readLatencyMs": 759.485461,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 100630528,+ "ingestSecondsElapsed": 1251.663304011,+ "digestSeconds": 5.005885474,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 61010.333948,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=cc8e0fcf-03cb-4bad-b800-8913b1f85aa3\": context deadline exceeded"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 61000.414962999996,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=cc8e0fcf-03cb-4bad-b800-8913b1f85aa3\": context deadline exceeded"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 51954.515633,+ "ok": true+ }+ ],+ "medianLatencyMs": 61000.414962999996,+ "readLatencyMs": 61000.414962999996,+ "readOk": false,+ "passed": false,+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=cc8e0fcf-03cb-4bad-b800-8913b1f85aa3\": context deadline exceeded (latency 61010ms)"+ }+ ],+ "maxFillLevelPassed": 10000000+ },+ "maxFillLevelPassed": 10000000,+ "sutContainer": {}+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe.json b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe.jsonnew file mode 100644index 00000000..16eb488a--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-read-probe.json@@ -0,0 +1,128 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "spans",+ "scenario": "read-probe",+ "startedAt": "2026-07-21T14:23:14Z",+ "endedAt": "2026-07-21T14:40:18Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces"+ }+ ],+ "readPath": "/api/endpoints/grouped",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 8192,+ "fillRequestRate": 100,+ "steps": [+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1695744,+ "ingestSecondsElapsed": 16.074486227,+ "digestSeconds": 5.006851318,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 169.113337,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 139.026456,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 2.492602,+ "ok": true+ }+ ],+ "medianLatencyMs": 139.026456,+ "readLatencyMs": 139.026456,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10780672,+ "ingestSecondsElapsed": 83.963192231,+ "digestSeconds": 5.006837973,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 939.389513,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 902.3118890000001,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 3.0820079999999996,+ "ok": true+ }+ ],+ "medianLatencyMs": 902.3118890000001,+ "readLatencyMs": 902.3118890000001,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 100000000,+ "rowsIngested": 100646912,+ "ingestSecondsElapsed": 858.420307916,+ "digestSeconds": 5.005910713,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 6005.285989,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=d589ccbf-929d-47e7-8709-c4b57967efaa\": context deadline exceeded"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 6002.100358,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=d589ccbf-929d-47e7-8709-c4b57967efaa\": context deadline exceeded"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 6005.271328,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/exception-stack-traces?projectId=d589ccbf-929d-47e7-8709-c4b57967efaa\": context deadline exceeded"+ }+ ],+ "medianLatencyMs": 6005.271328,+ "readLatencyMs": 6005.271328,+ "readOk": false,+ "passed": false,+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=d589ccbf-929d-47e7-8709-c4b57967efaa\": context deadline exceeded (latency 6005ms)"+ }+ ],+ "maxFillLevelPassed": 10000000+ },+ "maxFillLevelPassed": 10000000,+ "sutContainer": {}+}diff --git a/benchmarks/blog/post-3-data/ccx13-duckdb-spans-throughput.json b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-throughput.jsonnew file mode 100644index 00000000..c40bfa28--- /dev/null+++ b/benchmarks/blog/post-3-data/ccx13-duckdb-spans-throughput.json@@ -0,0 +1,551 @@+{+ "tier": "ccx13",+ "mode": "duckdb",+ "signal": "spans",+ "scenario": "throughput",+ "startedAt": "2026-07-21T12:07:22Z",+ "endedAt": "2026-07-21T12:41:42Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1282.0259534142108,+ "actualItemsPerSec": 1282.0259534142108,+ "rejected": 0,+ "ingest": {+ "p50": 14.013178,+ "p95": 16.96497,+ "p99": 19.282500000000002,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 12288,+ "walSizeBytes": 47958770,+ "memoryUsedBytes": 63391744+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3864.8327211368464,+ "actualItemsPerSec": 3864.8327211368464,+ "rejected": 0,+ "ingest": {+ "p50": 27.599517,+ "p95": 31.053155,+ "p99": 36.705102000000004,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 12288,+ "walSizeBytes": 192159970,+ "memoryUsedBytes": 242436096+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 15415.069839229409,+ "actualItemsPerSec": 15415.069839229409,+ "rejected": 0,+ "ingest": {+ "p50": 79.253023,+ "p95": 89.611228,+ "p99": 667.014361,+ "ok": 455,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 164114432,+ "memoryUsedBytes": 165937152+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 31185.415455549006,+ "actualItemsPerSec": 31185.415455549006,+ "rejected": 0,+ "ingest": {+ "p50": 109.707679,+ "p95": 303.442473,+ "p99": 1034.8829250000001,+ "ok": 457,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 384315392,+ "walSizeBytes": 132306731,+ "memoryUsedBytes": 549715968+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 61868.83919864189,+ "actualItemsPerSec": 61868.83919864189,+ "rejected": 0,+ "ingest": {+ "p50": 153.437194,+ "p95": 1095.361743,+ "p99": 1326.792363,+ "ok": 454,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 879505408,+ "walSizeBytes": 106856836,+ "memoryUsedBytes": 1014497280+ },+ "passed": true+ }+ ],+ "maxBatchSize": 16384+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 16384,+ "steps": [+ {+ "step": 1,+ "batchSize": 16384,+ "requestRate": 1,+ "attemptedItemsPerSec": 16613.77616534193,+ "actualItemsPerSec": 16613.77616534193,+ "rejected": 0,+ "ingest": {+ "p50": 192.8409,+ "p95": 214.86640500000001,+ "p99": 308.191637,+ "ok": 122,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 989081600,+ "walSizeBytes": 208632672,+ "memoryUsedBytes": 1247281152+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 77244.81427079927,+ "actualItemsPerSec": 77244.81427079927,+ "rejected": 0,+ "ingest": {+ "p50": 151.988618,+ "p95": 1031.59392,+ "p99": 1312.071797,+ "ok": 571,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 1647325184,+ "memoryUsedBytes": 1649147904+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 16384,+ "requestRate": 10,+ "attemptedItemsPerSec": 138084.37152707088,+ "actualItemsPerSec": 107833.59325107424,+ "rejected": 0,+ "ingest": {+ "p50": 5371.09331,+ "p95": 6224.11976,+ "p99": 6456.579825000001,+ "ok": 827,+ "errors": 232,+ "errRate": 0.2190745986779981+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 2542022656,+ "walSizeBytes": 15267121,+ "memoryUsedBytes": 2566127616+ },+ "passed": false,+ "failReason": "combined error rate 21.91% (http 21.91% + rejected 0.00%) > 5.00% threshold"+ },+ {+ "step": 4,+ "batchSize": 16384,+ "requestRate": 7.5,+ "attemptedItemsPerSec": 95737.31828767364,+ "actualItemsPerSec": 95737.31828767364,+ "rejected": 0,+ "ingest": {+ "p50": 1167.7336220000002,+ "p95": 1838.536839,+ "p99": 2014.953619,+ "ok": 712,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 3312988160,+ "memoryUsedBytes": 3314810880+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 8.75,+ "attemptedItemsPerSec": 111024.5180973839,+ "actualItemsPerSec": 100208.7493155965,+ "rejected": 0,+ "ingest": {+ "p50": 5250.536638,+ "p95": 5967.252186,+ "p99": 6300.6231210000005,+ "ok": 769,+ "errors": 83,+ "errRate": 0.09741784037558686+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4139003904,+ "walSizeBytes": 15264471,+ "memoryUsedBytes": 3702259712+ },+ "passed": false,+ "failReason": "combined error rate 9.74% (http 9.74% + rejected 0.00%) > 5.00% threshold"+ }+ ],+ "maxRequestRate": 7.5+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 1007.3809777695797,+ "actualItemsPerSec": 1007.3809777695797,+ "rejected": 0,+ "ingest": {+ "p50": 9.489968999999999,+ "p95": 12.72621,+ "p99": 18.979889999999997,+ "ok": 1209,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4139003904,+ "walSizeBytes": 53169103,+ "memoryUsedBytes": 3749183488+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 100,+ "requestRate": 100,+ "attemptedItemsPerSec": 9754.603111536819,+ "actualItemsPerSec": 9754.603111536819,+ "rejected": 0,+ "ingest": {+ "p50": 7.615841,+ "p95": 10.681462,+ "p99": 26.597788,+ "ok": 11706,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4192743424,+ "walSizeBytes": 164160818,+ "memoryUsedBytes": 3890741248+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 100,+ "requestRate": 1000,+ "attemptedItemsPerSec": 36913.98112451267,+ "actualItemsPerSec": 36913.98112451267,+ "rejected": 0,+ "ingest": {+ "p50": 1295.6169630000002,+ "p95": 2293.325409,+ "p99": 2537.409309,+ "ok": 44772,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4514656256,+ "walSizeBytes": 31479949,+ "memoryUsedBytes": 3708551168+ },+ "passed": false,+ "failReason": "achieved 369.14 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 4,+ "batchSize": 100,+ "requestRate": 550,+ "attemptedItemsPerSec": 36555.4153674194,+ "actualItemsPerSec": 36555.4153674194,+ "rejected": 0,+ "ingest": {+ "p50": 1337.9296969999998,+ "p95": 2166.9471169999997,+ "p99": 2562.0238520000003,+ "ok": 44356,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 4783616000,+ "walSizeBytes": 141747760,+ "memoryUsedBytes": 3862429696+ },+ "passed": false,+ "failReason": "achieved 365.55 req/sec is below 70% of target 550.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 5,+ "batchSize": 100,+ "requestRate": 325,+ "attemptedItemsPerSec": 32769.56646378661,+ "actualItemsPerSec": 32769.56646378661,+ "rejected": 0,+ "ingest": {+ "p50": 6.744288999999999,+ "p95": 903.700262,+ "p99": 1278.921382,+ "ok": 39326,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5053886464,+ "walSizeBytes": 94589409,+ "memoryUsedBytes": 3805806592+ },+ "passed": true+ },+ {+ "step": 6,+ "batchSize": 100,+ "requestRate": 437.5,+ "attemptedItemsPerSec": 36576.94340054637,+ "actualItemsPerSec": 36576.94340054637,+ "rejected": 0,+ "ingest": {+ "p50": 1315.54054,+ "p95": 2160.389081,+ "p99": 2386.859315,+ "ok": 44375,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "sutIngest": {+ "reachable": true,+ "droppedRowsTotal": 0,+ "insertFailures": 0,+ "dbSizeBytes": 5321273344,+ "walSizeBytes": 205617910,+ "memoryUsedBytes": 3943956480+ },+ "passed": true+ }+ ],+ "maxRequestRate": 437.5+ },+ "maxSustainableItemsPerSec": 95737.31828767364,+ "bytesOnDisk": 5526900736,+ "storedRows": 81406272,+ "sutContainer": {+ "status": "running",+ "oomKilled": false,+ "exitCode": 0,+ "restartCount": 0+ }+}diff --git a/website/content/blog/sqlite-vs-duckdb.mdx b/website/content/blog/sqlite-vs-duckdb.mdxnew file mode 100644index 00000000..9b42fb4e--- /dev/null+++ b/website/content/blog/sqlite-vs-duckdb.mdx@@ -0,0 +1,159 @@+---+title: "SQLite vs DuckDB on the same $16 box: every cliff moved 100x"+date: "2026-07-21"+category: "engineering"+author: "jovan"+description: "Same $16.49/month server, same Traceway binary, two embedded databases. DuckDB writes 4x to 15x faster than SQLite, serves dashboards at 100x the row count, and stores a billion metric points in 10.8 GB. Full numbers, and the crash bug the benchmark found on the way."+---++## TL;DR++Same Hetzner CCX13 as [post 1](/blog/sqlite-observability-stack) and [post 2](/blog/break-sqlite-traceway): $16.49/month, 2 dedicated AMD vCPU, 8 GB RAM, 80 GB NVMe. Two builds of the same Traceway binary: the SQLite build post 2 measured, and the DuckDB build, where relational data stays in SQLite and telemetry moves to DuckDB's columnar engine. Same loadgen, same scenarios, same pass/fail rules.++The whole post in one table:++| | SQLite (post 2) | DuckDB (this post) | Change |+|---|---|---|---|+| Metrics writes | 61,712 pts/sec | 254,242 pts/sec | 4x |+| Spans writes | 30,508 spans/sec | 95,737 spans/sec | 3x |+| Logs writes | 4,877 rec/sec | 75,225 rec/sec | 15x |+| Metrics read cliff | 1M rows (3.85 s median) | 100M rows (3.0 s median) | 100x |+| Spans read cliff | 100k rows (2.27 s median) | 10M rows (902 ms median) | 100x |+| Logs read cliff | 100k rows (114 ms median) | 10M rows (85 ms median) | 100x |++A read cliff, throughout this series, is the largest table size at which real dashboard pages still load: the median of three real endpoint probes at or under 5 seconds, none timing out.++<mark>Each signal's read cliff sits exactly two decades further out on DuckDB than on SQLite, at equal or better latency, on identical hardware.</mark> The box also ingested a billion metric points in 52 minutes into 10.8 GB of disk and stayed up, something the SQLite build cannot get within two orders of magnitude of. Logs, the signal post 2 said to keep off SQLite entirely, is DuckDB's biggest win.++One catch before you trust any of it: my first DuckDB numbers were wrong, because the benchmark kept crashing my own backend. That bug, and the fix that made these numbers real, gets its own section.++## What's actually being compared++Post 2 ended promising a ClickHouse comparison, and that post is still coming. This one exists because a question kept getting in front of it: before reaching for a client-server OLAP database, how far does an embedded one go? Traceway ships a DuckDB telemetry backend as an opt-in build, same single binary, same deployment story as the SQLite build, different storage engine under the telemetry tables. If you self-host on one cheap box, these two builds are the actual decision in front of you, and nobody publishes numbers for that decision. So I ran post 2's entire methodology against the DuckDB build and put the results side by side.++Here's the plan. Methodology and its two new pieces first, then the bug that invalidated my first attempt, because half the final numbers changed after the fix. Then writes, all three signals against their SQLite baselines. Then reads, same shape. Then the billion-row run, what queries cost past the cliffs, the fleet math for both builds, and which build I would actually run.++## How I measured++The setup is post 2's, so I'll keep it short: a system under test (SUT) running the Traceway binary and a separate load generator on a private link, both in Hetzner's Nuremberg datacenter, OTLP over gzipped protobuf, uniform-random data in bounded ranges. Traceway built from commit [`14b4aa6e`](https://github.com/tracewayapp/traceway/commit/14b4aa6e175946d40383d938db22cfea8c23c695), Ubuntu 24.04, retention off during the bench.++Two scenarios per signal. The **throughput ramp**: batch-size ramp at fixed rate, then a request-rate ramp at the winning batch (this time with a finer rate ladder, 1 through 25 req/sec, because the interesting cliffs turned out to sit between post 2's coarser steps), then a small-batch high-rate ramp shaped like an SDK fleet. A step passes at under 5% errors and at least 70% of target rate achieved. The **read-probe**: fill the table to 1M, 10M, 100M, 1B, then 5B rows, and at each level load three real dashboard endpoints per signal, with a level passing when the median is at or under 5 s and nothing hits the 6 s timeout. The 5-second bar is generous on purpose and post 2's discussion of that still applies unchanged.++Three things differ from post 2's harness, all disclosed:++1. **The loadgen grew.** Post 2 generated load from a second CCX13, which SQLite's ceilings never stressed. On my first DuckDB spans run, that loadgen box died mid-ramp while the database sat at 0% errors, the traffic generator broke before the thing it was breaking. The loadgen is now a CCX23. The SUT, the box every number describes, is unchanged.+2. **The digestion gate.** DuckDB checkpoints its write-ahead log after ingest stops; probe during that and you measure a busy engine, not a query. The read-probe now polls the backend's deep-health endpoint until the database and WAL file sizes are stable, records the wait per level (5 seconds at every level in the final runs), then probes. SQLite reports no engine gauges and skips the wait, so its post 2 numbers were produced identically.+3. **Only accepted rows count.** The fill counts items confirmed by 2xx responses, not items attempted, which matters once the backend learns to refuse work. Fills still overshoot their targets slightly, so tables report actual row counts.++Caveats. These are single-shot runs, one throughput run and one read ladder per signal. Post 2 promised medians of three reps starting with this post, and I'm walking that back one more time: the fix-and-rerun cycle below consumed the budget, and I'd rather publish single-shot numbers with that admission than sit on them. The key cells reproduced within a few percent across the debugging runs (100M metrics reads landed within 4% across three runs on different days), but the protocol is single-shot. "Passing" still means three named endpoints on freshly written data on an otherwise idle box; read-under-ingest remains future work.++## The bug between me and the numbers++My first DuckDB throughput run posted 77,899 spans/sec, and then the box went silent, the run's JSON ends with `sutDied: true`. The first read-probe was stranger: logs failed its very first level, all three probes timing out at 3M rows, including a trace-ID lookup that matches at most one row. The same table on my laptop answered that lookup in 1 millisecond. A query does not slow six thousand times because of query cost.++The digestion gate made the failure legible: at the wedged levels it polled the health endpoint for ten straight minutes without one answer. Reproducing the fill locally with the backend's own logs showed the sequence. Under a sustained burst of fat log batches, ingest requests slow from milliseconds to 10, then 17 seconds, and then the process dies with no panic, which is the kernel's out-of-memory killer. Docker restarts it, DuckDB replays the write-ahead log on open, and on two slow cores with gigabytes of un-checkpointed WAL that replay runs for minutes, during which every query hangs. On my laptop the process died three times inside one 56-second fill.++The root cause was mine, not DuckDB's: the ingest path had no admission control. Every concurrent request was decompressed and decoded immediately, each holding megabytes of protobuf while it waited on the database. SQLite never surfaced this because its slower insert path errors early under pressure, post 2's logs wall at 5 req/sec was that error path working. DuckDB is fast enough to keep the front door open until memory runs out.++The fix is a gate at the door: a cap on concurrently processed ingest requests, acquired before the body is read, with a short wait and then a 503 plus Retry-After, which SDKs and OTLP collectors answer by retrying. It applies to every backend and shipped in the commit linked above. Against the same killer fill: 16.4M log rows accepted in 100 seconds, 6,188 requests refused cleanly, WAL held at 62 MB, zero restarts. The fill took 100 seconds instead of 56, and that gap is what surviving costs, part of the old speed was acknowledged data that was about to die with the process.++Two of the final numbers moved a lot once the box stopped dying mid-measurement: spans throughput rose 23% (the crash had been hiding the true ceiling between two coarse ramp steps) and logs throughput nearly doubled (the crash had been knocking the ramp down to a smaller batch size). Every number below is from the fixed backend.++## Writes: 4x, 3x, 15x++| Signal | SQLite | DuckDB | Winning shape (DuckDB) | First failing step |+|---|---|---|---|---|+| Metrics | 61,712/sec | **254,242/sec** | batch 16384 × 20 req/sec, p50 3.2 s | 22.5 req/sec, 12.2% errors |+| Spans | 30,508/sec | **95,737/sec** | batch 16384 × 7.5 req/sec, p50 1.2 s | 8.75 req/sec, 9.7% errors |+| Logs | 4,877/sec | **75,225/sec** | batch 16384 × 5 req/sec, p50 3.0 s | 5.625 req/sec, 11.4% errors |++Three things in this table deserve prose. First, the failure modes match now: every "first failing step" is clean 503 rejection from the admission gate, the container finished all three runs with zero restarts, which is the same graceful-cliff behavior SQLite showed in post 2, and which DuckDB only exhibits because of the fix above.++Second, logs. Post 2's starkest result was logs writing at 4,877/sec with a wall at 5 req/sec so sharp that 1.5x more requests took the error rate from 0% to 98%. The DuckDB build writes logs at 15 times that rate, and the wall became a slope of refusals. The signal with the lowest SQLite ceiling and the highest real-world volume is exactly the one the columnar engine helps most.++Third, the SDK-fleet shape, which post 2 could only measure on SQLite: thousands of small batches per second rather than a few fat ones. DuckDB sustains 53,091 metric points/sec, 36,577 spans/sec, and 30,442 log records/sec at batch 100. My first attempts never produced these numbers at all, the pre-fix backend always died before phase 3 ran.++## Reads: every cliff, two decades later++| Rows (actual) | Metrics | Spans | Logs |+|---|---|---|---|+| ~1M | 146 ms | 139 ms | 27 ms |+| ~10M | 382 ms | **902 ms** | **85 ms** |+| ~100M | **3.0 s, passes** | over 60 s, fails | fails unevenly: body search 3.5 s, trace-id 2 ms, severity filter over 60 s |+| ~1B | fill succeeds, queries over 60 s | not reached | not reached |++Each cell is the median of three real endpoint probes after fill and digestion; bold marks each signal's last passing level. For comparison at a glance: SQLite's last passing levels were 1M, 100k, and 100k. Same endpoints, same thresholds, one hundred times the rows.++Metrics reads scale almost politely: 146 ms at 1M, 382 ms at 10M, 3.0 s at 100M. A hundred times the rows costs twenty times the latency, the scan gets cheaper per row as the table grows. The level SQLite barely passed sits two decades below the level DuckDB passes with the same margin.++++Spans pay for their percentiles, same as they did on SQLite, just later. The two aggregation-heavy pages, P50/P95/P99 grouped by route and the stacked latency chart, cost 902 ms at 10M rows and blow past a full minute at 100M. Post 2 watched the same two pages die between 100k and 1M.++++Logs repeat post 2's most interesting pattern one decade up. At their passing levels they're the fastest dashboard of the three (85 ms median at 10M), and their failure at 100M is uneven in the same way SQLite's was at 1M: the trace-ID lookup still answers in 2 ms, the body search in 3.5 s, and only the severity filter, which matches and ranks a large slice of 100M rows, dies. The pages an on-call actually opens mid-incident, find this trace, search this error, remain usable at 100M rows; the full-table scan does not. Retention policy should still follow that split. The split just moved 100x.++++## A billion rows on an 80 GB disk++The top of the metrics ladder deserves its own numbers, because nothing in posts 1 or 2 gets near it. The fill accepted 1,001,472,000 metric points in 52 minutes, a sustained 320k points/sec with the gate shedding the excess, into **10.8 GB on disk**, about 10.8 bytes per point after columnar compression. The box stayed up, health checks answered throughout, and post-fill digestion took 5 seconds. On every pre-fix attempt this level ended with the container OOM-killed; on the fixed backend a billion rows is just a big table.++A big table you cannot look at: all three dashboard queries ran past 60 seconds at 1B, so the level fails, and 5B was never attempted since the ladder stops at the first failure. I left it that way on purpose. A 5B fill would prove the disk holds 54 GB of points, and I already know nothing could read them. The practical summary of the ladder's top: this box can store a billion metric points, and it can show you a hundred million of them.++## How slow is slow past the cliff++Post 2 could only say "it timed out" about anything past a cliff. This time I reran the entire ladder with a 60-second probe threshold to put real durations on the failing cells, and the durations barely exist: metrics at 1B, still running at 61 s. Spans aggregations at 100M, still running at 61 s. The logs severity filter at 100M, still running at 61 s while the body search finished in 3.2 s beside it.++So the cliffs are not slopes on either database. Queries go from 3 seconds to more than a minute across one decade of rows, the signature of an aggregation outgrowing the 4 GB memory budget and spilling. There is no slow-but-usable band above a cliff to plan around, which post 2 found on SQLite at one hundredth the scale, and which is why I keep publishing cliffs instead of curves.++The 60-second run also caught a smaller flaw for the fix list: a cancelled dashboard query does not promptly release its read-pool connection, so after the two spans aggregations timed out, a 3 ms exceptions query queued behind them for 52 seconds. It affected no passing number, and it goes on the same list the admission gate came from.++## What fits comfortably, on each build++Post 2's small-fleet shape, ten backends emitting 50 spans/sec, 200 log records/sec, and 10 metric points/sec each, held against both builds:++| | SQLite envelope used | DuckDB envelope used |+|---|---|---|+| Spans (500/sec) | 1.7% | 0.5% |+| Logs (2,000/sec) | **40%** | 2.7% |+| Metrics (100/sec) | 0.2% | 0.04% |++Writes were already a non-issue on SQLite for everything except logs, and on DuckDB they stop being a conversation entirely. Retention is still the real knob, with new units. At this fleet's rates, logs reach their last passing read level (10M rows) in about 85 minutes and the failing level in about 14 hours; spans reach 10M in 5.5 hours; metrics reach 100M in 11 days. On SQLite, post 2 measured the logs window at 50 seconds. An incident-sized working set, hours of spans and logs, days of metrics, fits on the right side of every DuckDB cliff with an ordinary retention job, which is the thing the SQLite build could not offer.++## What I deliberately didn't measure++- **Read under concurrent write.** Fill, digest, settle, probe. Stricter isolation than post 2, and still not what a dashboard experiences at 3 PM. Top of the future-work list.+- **Mixed-signal load.** Each signal had the box to itself; a real deployment writes all three at once and these ceilings do not simply add.+- **Result correctness.** I timed responses, I did not diff their contents.+- **Variance.** Single-shot, as admitted in the methodology, with the median-of-three protocol still owed.+- **Tuning.** Stock config: 4 GB DuckDB memory cap, 256 MB checkpoint threshold, no schema or query changes. The 1B read failure smells like it wants a memory-budget experiment, and that is a deliberate cliffhanger.+- **Durability over weeks.** A 52-minute billion-row fill says nothing about month three on the same disk.++## What surprised me++I expected the reads to win, that's what columnar storage is for. I did not expect all three cliffs to land exactly two decades up, a symmetry I would have dismissed as marketing if I hadn't produced it from the raw JSON myself. And I did not expect the database to be the easy part: every wrong-looking number in six days of benchmarking, a 1 ms lookup timing out, a ten-minute blackout, a dashboard "cliff" at 3M rows that wasn't real, traced to my own ingest path, not the engine. DuckDB, whenever my code got out of its way, was boring in the best way a database can be boring.++The other surprise was logs, again. Post 2's verdict was "keep logs off SQLite," and logs turned out to be the strongest single argument for the DuckDB build: 15x on writes, 100x on the read cliff, with the incident-workflow pages still answering at 100M rows.++## The verdict++Which build should you run on the cheap box? After these numbers, the DuckDB build, for almost everyone self-hosting on a single machine. Every signal writes faster than a small fleet emits, the dashboards survive row counts the SQLite build cannot approach, logs go from "keep them off this database" to the best result in the dataset, and a billion metric points cost 10.8 GB of disk.++The SQLite build keeps two honest advantages. It compiles anywhere Go does, while the DuckDB build needs CGO and glibc, so its container image is Debian rather than Alpine. And its storage engine does all its work inline, no deferred checkpoints, no digestion window, no 503s under burst, the trade being that its ceilings sit 4x to 100x lower. If your telemetry volume lives comfortably under post 2's numbers, the SQLite build remains the simplest thing that works. The moment logs matter or retention beyond an hour matters, the comparison stops being close.++And the honest asterisk on the whole table: these numbers exist because the benchmark first found a crash bug in my ingest path and I fixed it. If you run the DuckDB build, run a version with the admission gate. The engine was never the problem; the code in front of it was.++## Raw data + workflow++Everything in this post, runnable from the repo:++- **Raw JSON** lives in [`benchmarks/blog/post-3-data/`](https://github.com/tracewayapp/traceway/tree/main/benchmarks/blog/post-3-data): three throughput JSONs, three read-probe JSONs, and three 60-second diagnostic JSONs.+- **The workflow**: [`benchmark-hardware.yml`](https://github.com/tracewayapp/traceway/actions/workflows/benchmark-hardware.yml). Throughput run: [29828430297](https://github.com/tracewayapp/traceway/actions/runs/29828430297). Read-probe run: [29838394873](https://github.com/tracewayapp/traceway/actions/runs/29838394873). 60-second diagnostic: [29848976962](https://github.com/tracewayapp/traceway/actions/runs/29848976962). The pre-fix runs from the bug section: [29734966806](https://github.com/tracewayapp/traceway/actions/runs/29734966806) and [29815312404](https://github.com/tracewayapp/traceway/actions/runs/29815312404).+- **The fix**: commit [`14b4aa6e`](https://github.com/tracewayapp/traceway/commit/14b4aa6e175946d40383d938db22cfea8c23c695), the ingest admission gate.+- **Try Traceway**: [self-host in five minutes](https://tracewayapp.com/docs/install).++Questions, pushback, or "your number is wrong because X": `[email protected]`, or find me on [GitHub](https://github.com/jstojiljkovic).++**Next post:** the one post 2 already promised. Same $16.49/month box, ClickHouse with PostgreSQL in Docker, same loadgen, same signals, same ladder to a billion rows, now with the admission gate protecting every contender equally. Time to find out what the heavyweight buys over the embedded engines.diff --git a/website/public/blog/sqlite-vs-duckdb/chart-readprobe-logs.webp b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-logs.webpnew file mode 100644index 00000000..7b73c28cBinary files /dev/null and b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-logs.webp differdiff --git a/website/public/blog/sqlite-vs-duckdb/chart-readprobe-metrics.webp b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-metrics.webpnew file mode 100644index 00000000..756bd7aeBinary files /dev/null and b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-metrics.webp differdiff --git a/website/public/blog/sqlite-vs-duckdb/chart-readprobe-spans.webp b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-spans.webpnew file mode 100644index 00000000..6622b13bBinary files /dev/null and b/website/public/blog/sqlite-vs-duckdb/chart-readprobe-spans.webp differ