The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
post 2: break sqlite traceway + supporting bench infra (#189)
Jovan Stojiljkovicbb398902326f
diff --git a/.github/workflows/benchmark-hardware.yml b/.github/workflows/benchmark-hardware.ymlindex 26dbedaf..e7ac02d0 100644--- a/.github/workflows/benchmark-hardware.yml+++ b/.github/workflows/benchmark-hardware.yml@@ -38,6 +38,18 @@ on:required: truetype: booleandefault: false+ loadgen_tier:+ description: 'Override loadgen instance type (e.g. cpx11). Leave empty for the per-location default (cax11 in EU, cpx11 in US). Use cpx11 when CAX/ARM capacity is unavailable in the chosen EU datacenter.'+ required: false+ default: ''+ fill_batch_size:+ description: 'Read-probe fill phase batch size override (default 8192). Lower this (e.g. 1024) for SQLite to limit per-tick ingest overshoot — the loadgen drains counters every 200ms, so 8192×100req/s adds ~160k items per tick, blowing past low fill-level targets. Ignored unless scenario=read-probe.'+ required: false+ default: ''+ fill_request_rate:+ description: 'Read-probe fill phase request rate override (default 100 req/s). Combined with fill_batch_size sets total ingest rate. Lower together for SQLite to keep step-target overshoot small. Ignored unless scenario=read-probe.'+ required: false+ default: ''concurrency:group: benchmark-hardware@@ -88,6 +100,9 @@ jobs:HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}BENCH_LOCATION: ${{ github.event.inputs.location }}BENCH_SCENARIO: ${{ github.event.inputs.scenario }}+ LOADGEN_TIER: ${{ github.event.inputs.loadgen_tier }}+ BENCH_FILL_BATCH_SIZE: ${{ github.event.inputs.fill_batch_size }}+ BENCH_FILL_REQUEST_RATE: ${{ github.event.inputs.fill_request_rate }}# Managed-ClickHouse credentials. Used only when modes contains# managed-ch; empty for sqlite/pgch entries (harmless — sut-bootstrap# only validates them when MODE=managed-ch).diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-logs-read-probe.json b/benchmarks/blog/post-2-data/ccx13-sqlite-logs-read-probe.jsonnew file mode 100644index 00000000..7b6c9c8f--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-logs-read-probe.json@@ -0,0 +1,93 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "logs",+ "scenario": "read-probe",+ "startedAt": "2026-06-09T10:14:59Z",+ "endedAt": "2026-06-09T10:16:50Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs"+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs"+ }+ ],+ "readPath": "/api/logs",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 1024,+ "fillRequestRate": 25,+ "steps": [+ {+ "fillLevelTarget": 100000,+ "rowsIngested": 103424,+ "ingestSecondsElapsed": 7.37884842,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 113.801053,+ "ok": true+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 132.898146,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 101.51781199999999,+ "ok": true+ }+ ],+ "medianLatencyMs": 113.801053,+ "readLatencyMs": 113.801053,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1001472,+ "ingestSecondsElapsed": 70.127992178,+ "probes": [+ {+ "name": "logs-severity-error",+ "path": "/api/logs",+ "latencyMs": 6002.956789,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/logs?projectId=9f3faf2e-2ca7-4810-8176-f50e7f52c665\": context deadline exceeded"+ },+ {+ "name": "logs-body-search",+ "path": "/api/logs",+ "latencyMs": 4959.739051,+ "ok": true+ },+ {+ "name": "logs-trace-id",+ "path": "/api/logs",+ "latencyMs": 1702.958684,+ "ok": true+ }+ ],+ "medianLatencyMs": 4959.739051,+ "readLatencyMs": 4959.739051,+ "readOk": false,+ "passed": false,+ "failReason": "probe logs-severity-error failed: Post \"http://10.0.0.2/api/logs?projectId=9f3faf2e-2ca7-4810-8176-f50e7f52c665\": context deadline exceeded (latency 6003ms)"+ }+ ],+ "maxFillLevelPassed": 100000+ },+ "maxFillLevelPassed": 100000+}diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-logs-throughput.json b/benchmarks/blog/post-2-data/ccx13-sqlite-logs-throughput.jsonnew file mode 100644index 00000000..d1a78122--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-logs-throughput.json@@ -0,0 +1,275 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "logs",+ "scenario": "throughput",+ "startedAt": "2026-06-09T08:58:38Z",+ "endedAt": "2026-06-09T09:23:37Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1281.8105833517957,+ "actualItemsPerSec": 1281.8105833517957,+ "rejected": 0,+ "ingest": {+ "p50": 26.865206,+ "p95": 32.533356,+ "p99": 54.343305,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3862.815144303175,+ "actualItemsPerSec": 3862.815144303175,+ "rejected": 0,+ "ingest": {+ "p50": 97.534267,+ "p95": 162.871578,+ "p99": 376.23913799999997,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 12474.796948004725,+ "actualItemsPerSec": 9225.291341239697,+ "rejected": 0,+ "ingest": {+ "p50": 7439.978831,+ "p95": 24796.020858,+ "p99": 28513.927658,+ "ok": 335,+ "errors": 118,+ "errRate": 0.26048565121412803+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 26.05% (http 26.05% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxBatchSize": 1024+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 1024,+ "steps": [+ {+ "step": 1,+ "batchSize": 1024,+ "requestRate": 1,+ "attemptedItemsPerSec": 1040.0365962211288,+ "actualItemsPerSec": 1040.0365962211288,+ "rejected": 0,+ "ingest": {+ "p50": 101.45499099999999,+ "p95": 341.65785900000003,+ "p99": 632.502585,+ "ok": 122,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 4876.882380670827,+ "actualItemsPerSec": 4876.882380670827,+ "rejected": 0,+ "ingest": {+ "p50": 112.51915,+ "p95": 7718.946132,+ "p99": 14288.2334,+ "ok": 572,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 1024,+ "requestRate": 25,+ "attemptedItemsPerSec": 15710.384095091822,+ "actualItemsPerSec": 3925.889115461885,+ "rejected": 0,+ "ingest": {+ "p50": 11607.763025,+ "p95": 27959.518732,+ "p99": 29318.027183000002,+ "ok": 575,+ "errors": 1726,+ "errRate": 0.7501086484137331+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 75.01% (http 75.01% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 4,+ "batchSize": 1024,+ "requestRate": 15,+ "attemptedItemsPerSec": 12395.996213307182,+ "actualItemsPerSec": 1023.8983656366065,+ "rejected": 0,+ "ingest": {+ "p50": 13780.676479,+ "p95": 28483.394213,+ "p99": 29864.356925,+ "ok": 150,+ "errors": 1666,+ "errRate": 0.9174008810572687+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 91.74% (http 91.74% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 5,+ "batchSize": 1024,+ "requestRate": 10,+ "attemptedItemsPerSec": 8266.192643692513,+ "actualItemsPerSec": 348.12206839662934,+ "rejected": 0,+ "ingest": {+ "p50": 17408.916017,+ "p95": 29297.886866,+ "p99": 29410.928352,+ "ok": 51,+ "errors": 1160,+ "errRate": 0.9578860445912469+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 95.79% (http 95.79% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 6,+ "batchSize": 1024,+ "requestRate": 7.5,+ "attemptedItemsPerSec": 6195.186356460619,+ "actualItemsPerSec": 95.6258092507703,+ "rejected": 0,+ "ingest": {+ "p50": 13912.191435,+ "p95": 28444.071147,+ "p99": 28444.071147,+ "ok": 14,+ "errors": 893,+ "errRate": 0.9845644983461963+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 98.46% (http 98.46% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxRequestRate": 5+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 805.3162429442383,+ "actualItemsPerSec": 22.66618564578154,+ "rejected": 0,+ "ingest": {+ "p50": 17479.219193999998,+ "p95": 28658.365924,+ "p99": 29395.221698,+ "ok": 34,+ "errors": 1174,+ "errRate": 0.9718543046357616+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 97.19% (http 97.19% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ]+ },+ "maxSustainableItemsPerSec": 4876.882380670827+}diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-read-probe.json b/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-read-probe.jsonnew file mode 100644index 00000000..79aa663c--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-read-probe.json@@ -0,0 +1,124 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "metrics",+ "scenario": "read-probe",+ "startedAt": "2026-06-09T10:01:48Z",+ "endedAt": "2026-06-09T10:09:29Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard"+ }+ ],+ "readPath": "/api/metrics/server",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 1024,+ "fillRequestRate": 25,+ "steps": [+ {+ "fillLevelTarget": 100000,+ "rowsIngested": 103424,+ "ingestSecondsElapsed": 4.025633939,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 425.23123200000003,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 343.979527,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 443.048077,+ "ok": true+ }+ ],+ "medianLatencyMs": 425.23123200000003,+ "readLatencyMs": 425.23123200000003,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1003520,+ "ingestSecondsElapsed": 44.230255547,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 3743.352243,+ "ok": true+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 3849.445222,+ "ok": true+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 4938.900678999999,+ "ok": true+ }+ ],+ "medianLatencyMs": 3849.445222,+ "readLatencyMs": 3849.445222,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 10000000,+ "rowsIngested": 10000384,+ "ingestSecondsElapsed": 350.438980646,+ "probes": [+ {+ "name": "metrics-server",+ "path": "/api/metrics/server",+ "latencyMs": 6000.413476000001,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-06-08T10%3A09%3A11Z\u0026projectId=293a987a-c0fd-4b2b-9b70-783b17e1cb79\u0026toDate=2026-06-09T10%3A09%3A11Z\": context deadline exceeded"+ },+ {+ "name": "metrics-application",+ "path": "/api/metrics/application",+ "latencyMs": 6000.34941,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/metrics/application?fromDate=2026-06-08T10%3A09%3A17Z\u0026projectId=293a987a-c0fd-4b2b-9b70-783b17e1cb79\u0026toDate=2026-06-09T10%3A09%3A17Z\": context deadline exceeded"+ },+ {+ "name": "dashboard",+ "path": "/api/dashboard",+ "latencyMs": 6001.854048,+ "ok": false,+ "error": "Get \"http://10.0.0.2/api/dashboard?fromDate=2026-06-08T10%3A09%3A23Z\u0026projectId=293a987a-c0fd-4b2b-9b70-783b17e1cb79\u0026toDate=2026-06-09T10%3A09%3A23Z\": context deadline exceeded"+ }+ ],+ "medianLatencyMs": 6000.413476000001,+ "readLatencyMs": 6000.413476000001,+ "readOk": false,+ "passed": false,+ "failReason": "probe metrics-server failed: Get \"http://10.0.0.2/api/metrics/server?fromDate=2026-06-08T10%3A09%3A11Z\u0026projectId=293a987a-c0fd-4b2b-9b70-783b17e1cb79\u0026toDate=2026-06-09T10%3A09%3A11Z\": context deadline exceeded (latency 6000ms)"+ }+ ],+ "maxFillLevelPassed": 1000000+ },+ "maxFillLevelPassed": 1000000+}diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-throughput.json b/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-throughput.jsonnew file mode 100644index 00000000..a8fa50f0--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-metrics-throughput.json@@ -0,0 +1,371 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "metrics",+ "scenario": "throughput",+ "startedAt": "2026-06-09T08:22:43Z",+ "endedAt": "2026-06-09T08:52:44Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1282.014460668475,+ "actualItemsPerSec": 1282.014460668475,+ "rejected": 0,+ "ingest": {+ "p50": 10.651095,+ "p95": 12.958839,+ "p99": 15.768913999999999,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3890.0734046911844,+ "actualItemsPerSec": 3890.0734046911844,+ "rejected": 0,+ "ingest": {+ "p50": 31.777954999999995,+ "p95": 35.073718,+ "p99": 43.184578,+ "ok": 456,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 15454.648154032759,+ "actualItemsPerSec": 15454.648154032759,+ "rejected": 0,+ "ingest": {+ "p50": 107.526354,+ "p95": 163.073821,+ "p99": 577.1052559999999,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 30902.019019990206,+ "actualItemsPerSec": 30902.019019990206,+ "rejected": 0,+ "ingest": {+ "p50": 132.882231,+ "p95": 1050.9765049999999,+ "p99": 2192.1677609999997,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 5,+ "batchSize": 16384,+ "requestRate": 5,+ "attemptedItemsPerSec": 52938.74203365598,+ "actualItemsPerSec": 45957.80901822882,+ "rejected": 0,+ "ingest": {+ "p50": 3476.045195,+ "p95": 24838.586186,+ "p99": 27608.287499,+ "ok": 395,+ "errors": 60,+ "errRate": 0.13186813186813187+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 13.19% (http 13.19% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxBatchSize": 8192+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 8192,+ "steps": [+ {+ "step": 1,+ "batchSize": 8192,+ "requestRate": 1,+ "attemptedItemsPerSec": 8315.39698038745,+ "actualItemsPerSec": 8315.39698038745,+ "rejected": 0,+ "ingest": {+ "p50": 163.473908,+ "p95": 196.294058,+ "p99": 201.353089,+ "ok": 122,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 38920.26389972522,+ "actualItemsPerSec": 38920.26389972522,+ "rejected": 0,+ "ingest": {+ "p50": 124.799205,+ "p95": 593.362981,+ "p99": 3752.511282,+ "ok": 571,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 8192,+ "requestRate": 25,+ "attemptedItemsPerSec": 132148.32312213106,+ "actualItemsPerSec": 34566.06964310288,+ "rejected": 0,+ "ingest": {+ "p50": 8170.164094000001,+ "p95": 27652.94883,+ "p99": 29316.49986,+ "ok": 633,+ "errors": 1787,+ "errRate": 0.7384297520661157+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 73.84% (http 73.84% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 15,+ "attemptedItemsPerSec": 99162.63290466322,+ "actualItemsPerSec": 17473.59170126224,+ "rejected": 0,+ "ingest": {+ "p50": 10165.662945999999,+ "p95": 27582.143945,+ "p99": 29193.670879999998,+ "ok": 320,+ "errors": 1496,+ "errRate": 0.8237885462555066+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 82.38% (http 82.38% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 5,+ "batchSize": 8192,+ "requestRate": 10,+ "attemptedItemsPerSec": 67429.60943990525,+ "actualItemsPerSec": 58687.70301375734,+ "rejected": 0,+ "ingest": {+ "p50": 6251.887524,+ "p95": 24844.893581,+ "p99": 29098.505456000003,+ "ok": 1054,+ "errors": 157,+ "errRate": 0.129644921552436+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 12.96% (http 12.96% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 6,+ "batchSize": 8192,+ "requestRate": 7.5,+ "attemptedItemsPerSec": 61916.15013470516,+ "actualItemsPerSec": 61711.58135672706,+ "rejected": 0,+ "ingest": {+ "p50": 109.451668,+ "p95": 1811.3873350000001,+ "p99": 5041.599064999999,+ "ok": 905,+ "errors": 3,+ "errRate": 0.003303964757709251+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ }+ ],+ "maxRequestRate": 7.5+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 1006.6201698576238,+ "actualItemsPerSec": 1006.6201698576238,+ "rejected": 0,+ "ingest": {+ "p50": 5.335247000000001,+ "p95": 7.5556909999999995,+ "p99": 9.570946000000001,+ "ok": 1208,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 100,+ "requestRate": 100,+ "attemptedItemsPerSec": 9752.096384948332,+ "actualItemsPerSec": 9752.096384948332,+ "rejected": 0,+ "ingest": {+ "p50": 5.536316,+ "p95": 7.525412,+ "p99": 9.219549,+ "ok": 11703,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 100,+ "requestRate": 1000,+ "attemptedItemsPerSec": 38275.41389770552,+ "actualItemsPerSec": 30148.660738742234,+ "rejected": 0,+ "ingest": {+ "p50": 712.450016,+ "p95": 3364.7105279999996,+ "p99": 4528.25131,+ "ok": 1892,+ "errors": 510,+ "errRate": 0.21232306411323898+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 21.23% (http 21.23% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxRequestRate": 100+ },+ "maxSustainableItemsPerSec": 61711.58135672706+}diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-spans-read-probe.json b/benchmarks/blog/post-2-data/ccx13-sqlite-spans-read-probe.jsonnew file mode 100644index 00000000..1024b9cb--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-spans-read-probe.json@@ -0,0 +1,94 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "spans",+ "scenario": "read-probe",+ "startedAt": "2026-06-09T09:54:45Z",+ "endedAt": "2026-06-09T09:56:07Z",+ "readProbe": {+ "endpoints": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces"+ }+ ],+ "readPath": "/api/endpoints/grouped",+ "readThresholdMs": 5000,+ "settleSeconds": 10,+ "fillBatchSize": 1024,+ "fillRequestRate": 25,+ "steps": [+ {+ "fillLevelTarget": 100000,+ "rowsIngested": 103424,+ "ingestSecondsElapsed": 4.042245546,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 2270.673085,+ "ok": true+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 2286.140864,+ "ok": true+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 1.281663,+ "ok": true+ }+ ],+ "medianLatencyMs": 2270.673085,+ "readLatencyMs": 2270.673085,+ "readOk": true,+ "passed": true+ },+ {+ "fillLevelTarget": 1000000,+ "rowsIngested": 1005568,+ "ingestSecondsElapsed": 41.033156717,+ "probes": [+ {+ "name": "endpoints-grouped",+ "path": "/api/endpoints/grouped",+ "latencyMs": 6000.380988,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/grouped?projectId=090feab0-8f7b-473d-b042-83618aa141cb\": context deadline exceeded"+ },+ {+ "name": "endpoints-chart",+ "path": "/api/endpoints/chart",+ "latencyMs": 6003.66762,+ "ok": false,+ "error": "Post \"http://10.0.0.2/api/endpoints/chart?projectId=090feab0-8f7b-473d-b042-83618aa141cb\": context deadline exceeded"+ },+ {+ "name": "exception-stack-traces",+ "path": "/api/exception-stack-traces",+ "latencyMs": 15.630446999999998,+ "ok": true+ }+ ],+ "medianLatencyMs": 6000.380988,+ "readLatencyMs": 6000.380988,+ "readOk": false,+ "passed": false,+ "failReason": "probe endpoints-grouped failed: Post \"http://10.0.0.2/api/endpoints/grouped?projectId=090feab0-8f7b-473d-b042-83618aa141cb\": context deadline exceeded (latency 6000ms)"+ }+ ],+ "maxFillLevelPassed": 100000+ },+ "maxFillLevelPassed": 100000+}diff --git a/benchmarks/blog/post-2-data/ccx13-sqlite-spans-throughput.json b/benchmarks/blog/post-2-data/ccx13-sqlite-spans-throughput.jsonnew file mode 100644index 00000000..c40e0832--- /dev/null+++ b/benchmarks/blog/post-2-data/ccx13-sqlite-spans-throughput.json@@ -0,0 +1,372 @@+{+ "tier": "ccx13",+ "mode": "sqlite",+ "signal": "spans",+ "scenario": "throughput",+ "startedAt": "2026-06-09T07:47:16Z",+ "endedAt": "2026-06-09T08:17:16Z",+ "phase1": {+ "kind": "batch-size-ramp",+ "fixedRequestRate": 5,+ "steps": [+ {+ "step": 1,+ "batchSize": 256,+ "requestRate": 5,+ "attemptedItemsPerSec": 1281.9155915521958,+ "actualItemsPerSec": 1281.9155915521958,+ "rejected": 0,+ "ingest": {+ "p50": 17.131521,+ "p95": 18.971826,+ "p99": 23.706888,+ "ok": 601,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 1024,+ "requestRate": 5,+ "attemptedItemsPerSec": 3864.4809362344563,+ "actualItemsPerSec": 3864.4809362344563,+ "rejected": 0,+ "ingest": {+ "p50": 53.176734,+ "p95": 56.973969,+ "p99": 63.989968999999995,+ "ok": 453,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 15478.561680869701,+ "actualItemsPerSec": 15478.561680869701,+ "rejected": 0,+ "ingest": {+ "p50": 143.001605,+ "p95": 214.053628,+ "p99": 378.583769,+ "ok": 454,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 4,+ "batchSize": 8192,+ "requestRate": 5,+ "attemptedItemsPerSec": 27298.043495723847,+ "actualItemsPerSec": 25734.719418876226,+ "rejected": 0,+ "ingest": {+ "p50": 503.349597,+ "p95": 18527.423848000002,+ "p99": 28169.67943,+ "ok": 428,+ "errors": 26,+ "errRate": 0.05726872246696035+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 5.73% (http 5.73% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxBatchSize": 4096+ },+ "phase2": {+ "kind": "request-rate-ramp",+ "fixedBatchSize": 4096,+ "steps": [+ {+ "step": 1,+ "batchSize": 4096,+ "requestRate": 1,+ "attemptedItemsPerSec": 4157.322269069919,+ "actualItemsPerSec": 4157.322269069919,+ "rejected": 0,+ "ingest": {+ "p50": 186.598288,+ "p95": 231.03179500000002,+ "p99": 424.797007,+ "ok": 122,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 4096,+ "requestRate": 5,+ "attemptedItemsPerSec": 19471.728688316813,+ "actualItemsPerSec": 19471.728688316813,+ "rejected": 0,+ "ingest": {+ "p50": 115.06466,+ "p95": 159.72235600000002,+ "p99": 414.07030299999997,+ "ok": 571,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 4096,+ "requestRate": 25,+ "attemptedItemsPerSec": 64057.32454921227,+ "actualItemsPerSec": 16601.386754442054,+ "rejected": 0,+ "ingest": {+ "p50": 9189.775772,+ "p95": 26543.366202,+ "p99": 29351.827816,+ "ok": 608,+ "errors": 1738,+ "errRate": 0.7408354646206309+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 74.08% (http 74.08% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 4,+ "batchSize": 4096,+ "requestRate": 15,+ "attemptedItemsPerSec": 49748.068703138924,+ "actualItemsPerSec": 26819.03042972082,+ "rejected": 0,+ "ingest": {+ "p50": 9509.57405,+ "p95": 27128.938369,+ "p99": 29335.648576,+ "ok": 979,+ "errors": 837,+ "errRate": 0.46090308370044053+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 46.09% (http 46.09% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 5,+ "batchSize": 4096,+ "requestRate": 10,+ "attemptedItemsPerSec": 36715.47214411376,+ "actualItemsPerSec": 29372.377715291008,+ "rejected": 0,+ "ingest": {+ "p50": 10281.964751,+ "p95": 26324.430934,+ "p99": 29170.119928999997,+ "ok": 968,+ "errors": 242,+ "errRate": 0.2+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 20.00% (http 20.00% + rejected 0.00%) \u003e 5.00% threshold"+ },+ {+ "step": 6,+ "batchSize": 4096,+ "requestRate": 7.5,+ "attemptedItemsPerSec": 30507.965453189205,+ "actualItemsPerSec": 30507.965453189205,+ "rejected": 0,+ "ingest": {+ "p50": 98.190264,+ "p95": 207.712484,+ "p99": 460.340146,+ "ok": 894,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ }+ ],+ "maxRequestRate": 7.5+ },+ "phase3": {+ "kind": "small-batch-rate-ramp",+ "fixedBatchSize": 100,+ "steps": [+ {+ "step": 1,+ "batchSize": 100,+ "requestRate": 10,+ "attemptedItemsPerSec": 1006.5840412156434,+ "actualItemsPerSec": 1006.5840412156434,+ "rejected": 0,+ "ingest": {+ "p50": 7.897546999999999,+ "p95": 10.821043000000001,+ "p99": 13.174599,+ "ok": 1208,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 2,+ "batchSize": 100,+ "requestRate": 100,+ "attemptedItemsPerSec": 9753.760036962482,+ "actualItemsPerSec": 9753.760036962482,+ "rejected": 0,+ "ingest": {+ "p50": 7.378449,+ "p95": 10.611028,+ "p99": 13.136902000000001,+ "ok": 11705,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": true+ },+ {+ "step": 3,+ "batchSize": 100,+ "requestRate": 1000,+ "attemptedItemsPerSec": 26973.214539604716,+ "actualItemsPerSec": 26973.214539604716,+ "rejected": 0,+ "ingest": {+ "p50": 1314.711982,+ "p95": 5523.007444,+ "p99": 8384.693708,+ "ok": 32964,+ "errors": 0,+ "errRate": 0+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "achieved 269.73 req/sec is below 70% of target 1000.00 req/sec (workers saturated, SUT past cliff)"+ },+ {+ "step": 4,+ "batchSize": 100,+ "requestRate": 550,+ "attemptedItemsPerSec": 29211.416219376086,+ "actualItemsPerSec": 27267.032516353916,+ "rejected": 0,+ "ingest": {+ "p50": 1219.463868,+ "p95": 5071.119866,+ "p99": 7567.590748,+ "ok": 7166,+ "errors": 511,+ "errRate": 0.06656245929399505+ },+ "ch": {+ "reachable": false,+ "uptimeSec": 0,+ "partsCount": 0,+ "activeMerges": 0,+ "longestMergeSec": 0+ },+ "passed": false,+ "failReason": "combined error rate 6.66% (http 6.66% + rejected 0.00%) \u003e 5.00% threshold"+ }+ ],+ "maxRequestRate": 100+ },+ "maxSustainableItemsPerSec": 30507.965453189205+}diff --git a/benchmarks/loadgen/otlp_logs.go b/benchmarks/loadgen/otlp_logs.goindex 174c5761..bf760dbb 100644--- a/benchmarks/loadgen/otlp_logs.go+++ b/benchmarks/loadgen/otlp_logs.go@@ -23,6 +23,32 @@ var severityChoices = []struct {{logspb.SeverityNumber_SEVERITY_NUMBER_ERROR, "ERROR"},}+// logBodies is a deterministic pool of realistic-looking log lines so the+// /logs dashboard page renders publishable screenshots instead of 120 chars+// of random gibberish. Workload shape (uniform random selection across a+// fixed pool) is unchanged from the prior randomString path — only the+// per-record byte payload differs. See POSTS.md decision log 2026-06-08.+var logBodies = []string{+ "user logged in",+ "user logged out",+ "checkout completed for order 4821",+ "payment authorized via stripe (amount=42.99 usd)",+ "product 9123 added to cart",+ "product 9123 removed from cart",+ "search returned 18 results for query=\"running shoes\"",+ "cache miss for key=user:8821 — refilled from db (3.2ms)",+ "db pool reached 80% capacity (24/30 conns)",+ "slow query detected: SELECT * FROM orders WHERE ... (842ms)",+ "rate limit hit for ip=10.0.4.21 endpoint=/api/search",+ "request retried after 502 from upstream service=inventory",+ "connection timeout reaching downstream service=billing",+ "failed to deserialize webhook payload: unexpected token",+ "session expired for user_id=8821 — redirecting to /login",+ "feature flag \"new-checkout-flow\" evaluated to true",+ "background job \"daily-report\" finished in 12.4s",+ "OTLP exporter flushed batch=128 size=42.1KB",+}+type logsSender struct{}func (logsSender) Name() string { return "logs" }@@ -50,7 +76,7 @@ func (logsSender) BuildBody(rng *mathrand.Rand, batchSize int) ([]byte, error) {ObservedTimeUnixNano: uint64(ts),SeverityNumber: sev.number,SeverityText: sev.text,- Body: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: randomString(rng, 120)}},+ Body: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: logBodies[rng.Intn(len(logBodies))]}},Attributes: []*commonpb.KeyValue{strAttr("logger.name", "bench-loadgen"),strAttr("thread.name", "worker-1"),diff --git a/benchmarks/loadgen/otlp_metrics.go b/benchmarks/loadgen/otlp_metrics.goindex b9e0dfa4..108d3a1d 100644--- a/benchmarks/loadgen/otlp_metrics.go+++ b/benchmarks/loadgen/otlp_metrics.go@@ -15,9 +15,18 @@ import (// metrics in metric_registry on first sight; bench traffic with 10 stable// names exercises that path once per name per project and otherwise lets the// hot path stay hot.+//+// First 7 names match canonical constants in backend/app/models/metric_record.model.go+// (`MetricNameCpuUsage`, `MetricNameMemoryUsage`, etc.) so the dashboard cards+// at /metrics render real-looking values during ingest. Remaining 3 are+// application-style names that exist in the metric_points table for the+// read-probe's generic time-range queries but don't bind to a specific card.+// Workload shape (uniform random, 10 stable names) is unchanged from prior+// posts — see POSTS.md decision log 2026-06-08 for rationale.var metricNames = []string{- "bench.metric.cpu", "bench.metric.mem", "bench.metric.qps", "bench.metric.lat", "bench.metric.errs",- "bench.metric.disk", "bench.metric.net", "bench.metric.heap", "bench.metric.gc", "bench.metric.fd",+ "cpu.used_pcnt", "mem.used", "mem.total",+ "go.go_routines", "go.heap_objects", "go.num_gc", "go.gc_pause",+ "disk.used_pcnt", "net.bytes_tx", "app.requests_total",}type metricsSender struct{}diff --git a/benchmarks/loadgen/otlp_spans.go b/benchmarks/loadgen/otlp_spans.goindex 3819f94a..fd7308d2 100644--- a/benchmarks/loadgen/otlp_spans.go+++ b/benchmarks/loadgen/otlp_spans.go@@ -12,11 +12,25 @@ import ("google.golang.org/protobuf/proto")-var endpointPaths = []string{- "GET /api/users", "POST /api/users", "GET /api/orders", "POST /api/orders",- "GET /api/products/:id", "PUT /api/products/:id", "GET /api/search",- "POST /api/checkout", "GET /api/cart", "DELETE /api/cart/:id",- "GET /api/auth/me", "POST /api/auth/login", "GET /api/health",+// endpointPaths is a (method, path) pool. Keeping them paired so the synthetic+// traffic mix maps cleanly to real-world HTTP shapes (POST /users + GET /users+// etc.). The span Name is set to "{method} {path}" — what dashboards display —+// while http.method and http.route are set to the matched method and path+// individually. Earlier the loadgen hardcoded http.method="GET" and emitted+// Names like "GET /api/users", causing the /endpoints page to render+// "GET GET /api/users" (and "GET POST /api/users" for non-GET picks). Fixed+// 2026-06-08; see POSTS.md decision log.+var endpointPaths = []struct {+ method string+ path string+}{+ {"GET", "/api/users"}, {"POST", "/api/users"},+ {"GET", "/api/orders"}, {"POST", "/api/orders"},+ {"GET", "/api/products/:id"}, {"PUT", "/api/products/:id"},+ {"GET", "/api/search"}, {"POST", "/api/checkout"},+ {"GET", "/api/cart"}, {"DELETE", "/api/cart/:id"},+ {"GET", "/api/auth/me"}, {"POST", "/api/auth/login"},+ {"GET", "/api/health"},}type spansSender struct{}@@ -43,17 +57,18 @@ func (spansSender) BuildBody(rng *mathrand.Rand, batchSize int) ([]byte, error)duration := time.Duration(10+rng.Intn(990)) * time.MillisecondendNs := startNs + int64(duration)+ ep := endpointPaths[rng.Intn(len(endpointPaths))]spans[i] = &tracepb.Span{TraceId: traceId,SpanId: spanId,- Name: endpointPaths[rng.Intn(len(endpointPaths))],+ Name: ep.method + " " + ep.path,Kind: tracepb.Span_SPAN_KIND_SERVER,StartTimeUnixNano: uint64(startNs),EndTimeUnixNano: uint64(endNs),Attributes: []*commonpb.KeyValue{- strAttr("http.method", "GET"),+ strAttr("http.method", ep.method),intAttr("http.status_code", int64(pickStatus(rng))),- strAttr("http.route", endpointPaths[rng.Intn(len(endpointPaths))]),+ strAttr("http.route", ep.path),},Status: &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK},}diff --git a/benchmarks/loadgen/read_probe.go b/benchmarks/loadgen/read_probe.goindex 5cdcd823..69bcca54 100644--- a/benchmarks/loadgen/read_probe.go+++ b/benchmarks/loadgen/read_probe.go@@ -8,20 +8,40 @@ import ("io""net/http""net/url"+ "sort""time")+type endpointProbeResult struct {+ Name string `json:"name"`+ Path string `json:"path"`+ LatencyMs float64 `json:"latencyMs"`+ Ok bool `json:"ok"`+ Error string `json:"error,omitempty"`+}+type readProbeStep struct {- FillLevelTarget int64 `json:"fillLevelTarget"`- RowsIngested int64 `json:"rowsIngested"`- IngestSecondsElapsed float64 `json:"ingestSecondsElapsed"`- ReadLatencyMs float64 `json:"readLatencyMs"`- ReadOk bool `json:"readOk"`- Passed bool `json:"passed"`- FailReason string `json:"failReason,omitempty"`+ FillLevelTarget int64 `json:"fillLevelTarget"`+ RowsIngested int64 `json:"rowsIngested"`+ IngestSecondsElapsed float64 `json:"ingestSecondsElapsed"`+ Probes []endpointProbeResult `json:"probes"`+ MedianLatencyMs float64 `json:"medianLatencyMs"`+ // Back-compat: equals MedianLatencyMs and AND-of-all-probe-Ok respectively.+ // Existing chart.py renderers (and post-1 readers) consume these.+ ReadLatencyMs float64 `json:"readLatencyMs"`+ ReadOk bool `json:"readOk"`+ Passed bool `json:"passed"`+ FailReason string `json:"failReason,omitempty"`+}++type readProbeEndpointDescriptor struct {+ Name string `json:"name"`+ Path string `json:"path"`}type readProbeResult struct {+ Endpoints []readProbeEndpointDescriptor `json:"endpoints"`+ // Back-compat: first endpoint's path. Older chart.py renderers read this.ReadPath string `json:"readPath"`ReadThresholdMs int `json:"readThresholdMs"`SettleSeconds int `json:"settleSeconds"`@@ -31,24 +51,32 @@ type readProbeResult struct {MaxFillLevelPassed int64 `json:"maxFillLevelPassed"`}-// runReadProbe walks the configured fill levels. For each level it ingests-// rows until totalIngested >= target, settles for cfg.settleSeconds, then-// issues one read probe. If the probe times out, errors, or exceeds-// cfg.readThresholdMs, the step fails and the loop stops without ingesting-// further.// readProbeCheckpoint is invoked after every fill-level step so the caller// can persist a partial report — we never want to lose progress when the// process dies mid-run.type readProbeCheckpoint func(readProbeResult)+// runReadProbe walks the configured fill levels. For each level it ingests+// rows until totalIngested >= target, settles for cfg.settleSeconds, then+// issues one read probe **per configured endpoint** for the signal, records+// per-endpoint latencies, and uses the median to pass/fail against+// cfg.readThresholdMs. Any HTTP error on any probe fails the step.func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client, checkpoint readProbeCheckpoint) readProbeResult {+ endpoints := endpointSetForSignal(cfg.signal)+ descriptors := make([]readProbeEndpointDescriptor, len(endpoints))+ for i, ep := range endpoints {+ descriptors[i] = readProbeEndpointDescriptor{Name: ep.Name, Path: ep.Path}+ }res := readProbeResult{- ReadPath: readPathForSignal(cfg.signal),+ Endpoints: descriptors,ReadThresholdMs: cfg.readThresholdMs,SettleSeconds: int(cfg.settleSeconds.Seconds()),FillBatchSize: cfg.fillBatchSize,FillRequestRate: cfg.fillRequestRate,}+ if len(endpoints) > 0 {+ res.ReadPath = endpoints[0].Path+ }ing.SetBatchSize(cfg.fillBatchSize)ing.SetRequestRate(cfg.fillRequestRate)@@ -95,25 +123,51 @@ func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *lbreak}- latencyMs, readErr := probeRead(ctx, client, cfg)- step.ReadLatencyMs = latencyMs+ probeResults := make([]endpointProbeResult, 0, len(endpoints))+ allOk := true+ latencies := make([]float64, 0, len(endpoints))+ for _, ep := range endpoints {+ latency, err := probeReadEndpoint(ctx, client, cfg, ep)+ pr := endpointProbeResult{+ Name: ep.Name,+ Path: ep.Path,+ LatencyMs: latency,+ Ok: err == nil,+ }+ if err != nil {+ pr.Error = err.Error()+ allOk = false+ }+ probeResults = append(probeResults, pr)+ latencies = append(latencies, latency)+ }+ step.Probes = probeResults++ median := medianFloat(latencies)+ step.MedianLatencyMs = median+ step.ReadLatencyMs = median+ step.ReadOk = allOk+switch {- case readErr != nil:- step.ReadOk = false+ case !allOk:step.Passed = false- step.FailReason = fmt.Sprintf("read error: %v (latency %.0fms)", readErr, latencyMs)- case latencyMs > float64(cfg.readThresholdMs):- step.ReadOk = true+ // Find the first failed probe for the reason.+ for _, pr := range probeResults {+ if !pr.Ok {+ step.FailReason = fmt.Sprintf("probe %s failed: %s (latency %.0fms)", pr.Name, pr.Error, pr.LatencyMs)+ break+ }+ }+ case median > float64(cfg.readThresholdMs):step.Passed = false- step.FailReason = fmt.Sprintf("read latency %.0fms > %dms threshold", latencyMs, cfg.readThresholdMs)+ step.FailReason = fmt.Sprintf("median latency %.0fms > %dms threshold across %d probes", median, cfg.readThresholdMs, len(probeResults))default:- step.ReadOk = truestep.Passed = trueres.MaxFillLevelPassed = target}- fmt.Fprintf(stderrPrefix(), "read-probe target=%d read=%.0fms ok=%t passed=%t %s\n",- target, latencyMs, step.ReadOk, step.Passed, step.FailReason)+ fmt.Fprintf(stderrPrefix(), "read-probe target=%d median=%.0fms probes=%d allOk=%t passed=%t %s\n",+ target, median, len(probeResults), allOk, step.Passed, step.FailReason)res.Steps = append(res.Steps, step)if checkpoint != nil {checkpoint(res)@@ -144,22 +198,58 @@ func pollFillProgress(ctx context.Context, ing *ingester, totalIngested *int64,}}-func readPathForSignal(signal string) string {+func medianFloat(values []float64) float64 {+ if len(values) == 0 {+ return 0+ }+ sorted := make([]float64, len(values))+ copy(sorted, values)+ sort.Float64s(sorted)+ mid := len(sorted) / 2+ if len(sorted)%2 == 1 {+ return sorted[mid]+ }+ return (sorted[mid-1] + sorted[mid]) / 2+}++// readProbeEndpoint describes one HTTP probe target.+type readProbeEndpoint struct {+ Name string+ Path string+ Build func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error)+}++// endpointSetForSignal returns the 3 endpoints probed per fill-level for a+// given signal. Locked 2026-06-08 (see POSTS.md decision log). Order matters:+// the first entry's path becomes the back-compat readProbeResult.ReadPath.+func endpointSetForSignal(signal string) []readProbeEndpoint {switch signal {case "spans":- return "/api/endpoints/grouped"+ return []readProbeEndpoint{+ {Name: "endpoints-grouped", Path: "/api/endpoints/grouped", Build: buildPostJSON("/api/endpoints/grouped", spansGroupedBody)},+ {Name: "endpoints-chart", Path: "/api/endpoints/chart", Build: buildPostJSON("/api/endpoints/chart", spansChartBody)},+ {Name: "exception-stack-traces", Path: "/api/exception-stack-traces", Build: buildPostJSON("/api/exception-stack-traces", spansExceptionsBody)},+ }case "metrics":- return "/api/metrics/application"+ return []readProbeEndpoint{+ {Name: "metrics-server", Path: "/api/metrics/server", Build: buildGetQuery("/api/metrics/server", metricsTimeRangeQuery)},+ {Name: "metrics-application", Path: "/api/metrics/application", Build: buildGetQuery("/api/metrics/application", metricsTimeRangeQuery)},+ {Name: "dashboard", Path: "/api/dashboard", Build: buildGetQuery("/api/dashboard", metricsTimeRangeQuery)},+ }case "logs":- return "/api/logs"+ return []readProbeEndpoint{+ {Name: "logs-severity-error", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsSeverityErrorBody)},+ {Name: "logs-body-search", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsBodySearchBody)},+ {Name: "logs-trace-id", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsTraceIdBody)},+ }}- return ""+ return nil}-// probeRead issues one HTTP read against the signal-specific endpoint and-// returns wall-clock latency in ms plus any error. The request is hard-capped-// at threshold + 1s so a hanging SUT can't deadlock the loop.-func probeRead(ctx context.Context, client *http.Client, cfg config) (float64, error) {+// probeReadEndpoint issues one HTTP request for the given endpoint and returns+// wall-clock latency in ms plus any error. Hard-capped at threshold + 1s so a+// hanging SUT can't deadlock the loop.+func probeReadEndpoint(ctx context.Context, client *http.Client, cfg config, ep readProbeEndpoint) (float64, error) {now := time.Now().UTC()fromDate := now.Add(-24 * time.Hour).Format(time.RFC3339)toDate := now.Format(time.RFC3339)@@ -167,56 +257,9 @@ func probeRead(ctx context.Context, client *http.Client, cfg config) (float64, eprobeCtx, cancel := context.WithTimeout(ctx, time.Duration(cfg.readThresholdMs+1000)*time.Millisecond)defer cancel()- var req *http.Request- var err error- switch cfg.signal {- case "spans":- body, _ := json.Marshal(map[string]any{- "fromDate": fromDate,- "toDate": toDate,- "orderBy": "count",- "sortDirection": "desc",- "pagination": map[string]int{"page": 1, "pageSize": 50},- "search": "",- })- u, _ := url.Parse(cfg.target + "/api/endpoints/grouped")- q := u.Query()- q.Set("projectId", cfg.projectId)- u.RawQuery = q.Encode()- req, err = http.NewRequestWithContext(probeCtx, http.MethodPost, u.String(), bytes.NewReader(body))- if err != nil {- return 0, err- }- req.Header.Set("Content-Type", "application/json")- case "metrics":- u, _ := url.Parse(cfg.target + "/api/metrics/application")- q := u.Query()- q.Set("projectId", cfg.projectId)- q.Set("fromDate", fromDate)- q.Set("toDate", toDate)- u.RawQuery = q.Encode()- req, err = http.NewRequestWithContext(probeCtx, http.MethodGet, u.String(), nil)- if err != nil {- return 0, err- }- case "logs":- body, _ := json.Marshal(map[string]any{- "fromDate": fromDate,- "toDate": toDate,- "orderBy": "timestamp desc",- "pagination": map[string]int{"page": 1, "pageSize": 50},- })- u, _ := url.Parse(cfg.target + "/api/logs")- q := u.Query()- q.Set("projectId", cfg.projectId)- u.RawQuery = q.Encode()- req, err = http.NewRequestWithContext(probeCtx, http.MethodPost, u.String(), bytes.NewReader(body))- if err != nil {- return 0, err- }- req.Header.Set("Content-Type", "application/json")- default:- return 0, fmt.Errorf("no read endpoint configured for signal %q", cfg.signal)+ req, err := ep.Build(cfg, fromDate, toDate, probeCtx)+ if err != nil {+ return 0, err}req.Header.Set("Authorization", "Bearer "+cfg.jwt)@@ -233,3 +276,124 @@ func probeRead(ctx context.Context, client *http.Client, cfg config) (float64, e}return elapsed, nil}++// buildPostJSON returns a request-builder that POSTs a JSON body. The body+// callback receives the date range so it can populate fromDate/toDate.+func buildPostJSON(path string, body func(fromDate, toDate string) map[string]any) func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {+ return func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {+ payload, err := json.Marshal(body(fromDate, toDate))+ if err != nil {+ return nil, err+ }+ u, err := url.Parse(cfg.target + path)+ if err != nil {+ return nil, err+ }+ q := u.Query()+ q.Set("projectId", cfg.projectId)+ u.RawQuery = q.Encode()+ req, err := http.NewRequestWithContext(probeCtx, http.MethodPost, u.String(), bytes.NewReader(payload))+ if err != nil {+ return nil, err+ }+ req.Header.Set("Content-Type", "application/json")+ return req, nil+ }+}++// buildGetQuery returns a request-builder that GETs with extra query params.+func buildGetQuery(path string, query func(fromDate, toDate string) map[string]string) func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {+ return func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {+ u, err := url.Parse(cfg.target + path)+ if err != nil {+ return nil, err+ }+ q := u.Query()+ q.Set("projectId", cfg.projectId)+ for k, v := range query(fromDate, toDate) {+ q.Set(k, v)+ }+ u.RawQuery = q.Encode()+ return http.NewRequestWithContext(probeCtx, http.MethodGet, u.String(), nil)+ }+}++// --- Per-endpoint body / query builders ------------------------------------++func spansGroupedBody(fromDate, toDate string) map[string]any {+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "orderBy": "count",+ "sortDirection": "desc",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ "search": "",+ }+}++func spansChartBody(fromDate, toDate string) map[string]any {+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "metricType": "p95",+ "intervalMinutes": 5,+ }+}++func spansExceptionsBody(fromDate, toDate string) map[string]any {+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "orderBy": "count",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ "search": "",+ "searchType": "message",+ "includeArchived": false,+ }+}++func metricsTimeRangeQuery(fromDate, toDate string) map[string]string {+ return map[string]string{+ "fromDate": fromDate,+ "toDate": toDate,+ }+}++func logsSeverityErrorBody(fromDate, toDate string) map[string]any {+ // MinSeverity 17 = OTel SeverityNumber for ERROR.+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "orderBy": "timestamp",+ "sortDirection": "desc",+ "minSeverity": 17,+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ }+}++func logsBodySearchBody(fromDate, toDate string) map[string]any {+ // "error" is short, common, and inside the 24h unscoped-body-search+ // window allowed by /api/logs.+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "orderBy": "timestamp",+ "sortDirection": "desc",+ "search": "error",+ "searchType": "body",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ }+}++func logsTraceIdBody(fromDate, toDate string) map[string]any {+ // Fixed dummy trace_id — the query exercises the trace_id index path;+ // result count doesn't matter for latency measurement.+ return map[string]any{+ "fromDate": fromDate,+ "toDate": toDate,+ "orderBy": "timestamp",+ "sortDirection": "desc",+ "traceId": "00000000000000000000000000000001",+ "pagination": map[string]int{"page": 1, "pageSize": 50},+ }+}diff --git a/benchmarks/scripts/chart.py b/benchmarks/scripts/chart.pyindex 2f87ff8e..28dc886d 100755--- a/benchmarks/scripts/chart.py+++ b/benchmarks/scripts/chart.py@@ -13,6 +13,8 @@ each render function filters to its own scenario.See benchmarks/charts.md for a reader's guide to each output."""+from __future__ import annotations+import jsonimport sysfrom pathlib import Path@@ -978,6 +980,75 @@ def render_readprobe(runs: list[dict], signal: str, out: Path) -> None:plt.close(fig)+def render_readprobe_per_endpoint(runs: list[dict], signal: str, out: Path) -> None:+ """Latency vs rows ingested, **one line per probed endpoint**, per (tier, mode).+ Shows which specific query slows down first as the table grows. Replaces+ the single-line render_readprobe view when the JSON has per-endpoint+ probe data (T2+). Older JSON without `step["probes"]` falls back to the+ median-only view in render_readprobe."""+ sig_runs = [r for r in iter_runs(runs, signal, "read-probe") if _readprobe_steps(r)]+ sig_runs = [r for r in sig_runs if any(s.get("probes") for s in _readprobe_steps(r))]+ if not sig_runs:+ return++ fig, ax = plt.subplots(figsize=(12, 7))+ plotted = False+ threshold_ms = 5000+ # Deterministic palette per endpoint name. Falls back to grey if more than 6.+ endpoint_palette = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"]+ endpoint_color: dict[str, str] = {}++ for run in sig_runs:+ rp = run.get("readProbe") or {}+ threshold_ms = rp.get("readThresholdMs", threshold_ms)+ steps = _readprobe_steps(run)+ if not steps:+ continue++ # Collect per-endpoint (rowsIngested, latencyMs, ok) series.+ by_endpoint: dict[str, list[tuple[float, float, bool]]] = {}+ for s in steps:+ rows = s.get("rowsIngested", 0)+ for p in (s.get("probes") or []):+ name = p.get("name") or p.get("path") or "unknown"+ by_endpoint.setdefault(name, []).append((rows, p.get("latencyMs", 0), bool(p.get("ok", False))))++ run_lbl = run_label(run)+ for name, series in by_endpoint.items():+ if name not in endpoint_color:+ endpoint_color[name] = endpoint_palette[len(endpoint_color) % len(endpoint_palette)]+ color = endpoint_color[name]+ xs = [pt[0] for pt in series]+ ys = [pt[1] for pt in series]+ oks = [pt[2] for pt in series]+ ax.plot(xs, ys, linestyle="-", color=color, alpha=0.85,+ label=f"{run_lbl} · {name}")+ for x, y, ok in zip(xs, ys, oks):+ ax.plot(x, y, marker=("o" if ok else "x"), color=color,+ markersize=8, markeredgewidth=2 if not ok else 1, linestyle="")+ plotted = True++ if not plotted:+ plt.close(fig)+ return++ ax.axhline(threshold_ms, linestyle="--", color="#cc3333", alpha=0.7,+ label=f"threshold ({threshold_ms}ms)")+ ax.set_xlabel(f"{ROW_LABEL[signal]} ingested (log)")+ ax.set_ylabel("Read latency (ms, log)")+ ax.set_xscale("log")+ ax.set_yscale("log")+ ax.set_title(+ f"Per-endpoint read latency vs table size — {signal}\n"+ "○ = HTTP ok, × = HTTP error. Step pass uses median across endpoints."+ )+ ax.grid(True, which="both", linestyle=":", alpha=0.4)+ ax.legend(loc="upper left", fontsize=8, ncol=1)+ fig.tight_layout()+ fig.savefig(out, dpi=130)+ plt.close(fig)++def render_readprobe_ingest_time(runs: list[dict], signal: str, out: Path) -> None:sig_runs = [r for r in iter_runs(runs, signal, "read-probe") if _readprobe_steps(r)]if not sig_runs:@@ -1313,6 +1384,7 @@ def main() -> int:# Read-probe charts (no-op if no read-probe runs for this signal)render_readprobe_headline(runs, signal, results_dir / f"chart-readprobe-headline-{signal}.png")render_readprobe(runs, signal, results_dir / f"chart-readprobe-{signal}.png")+ render_readprobe_per_endpoint(runs, signal, results_dir / f"chart-readprobe-per-endpoint-{signal}.png")render_readprobe_ingest_time(runs, signal, results_dir / f"chart-readprobe-ingest-time-{signal}.png")render_readprobe_tier_scaling(runs, signal, results_dir / f"chart-readprobe-tier-scaling-{signal}.png")render_readprobe_cliff(runs, signal, results_dir / f"chart-readprobe-cliff-{signal}.png")diff --git a/benchmarks/scripts/run-matrix-entry.sh b/benchmarks/scripts/run-matrix-entry.shindex 5fdb70da..faca66c1 100755--- a/benchmarks/scripts/run-matrix-entry.sh+++ b/benchmarks/scripts/run-matrix-entry.sh@@ -87,6 +87,22 @@ if [[ "${SMOKE}" == "smoke" ]]; thenfifi+# Optional read-probe fill-phase overrides. Defaults (8192 × 100 req/s) push+# ~160k items per 200ms drain tick, which blows past low fill-level targets on+# slower DBs (notably SQLite: a "100k" step actually overshoots to ~1M for+# spans and ~4M for logs, smearing the cliff measurement across decades of+# table size). Lowering both knobs together keeps per-step overshoot under the+# next fill-level boundary. Only meaningful when scenario=read-probe; loadgen+# silently ignores them otherwise.+if [[ "${SCENARIO}" == "read-probe" ]]; then+ if [[ -n "${BENCH_FILL_BATCH_SIZE:-}" ]]; then+ extra_args+=( --fill-batch-size "${BENCH_FILL_BATCH_SIZE}" )+ fi+ if [[ -n "${BENCH_FILL_REQUEST_RATE:-}" ]]; then+ extra_args+=( --fill-request-rate "${BENCH_FILL_REQUEST_RATE}" )+ fi+fi+# SQLite has no merge-idle equivalent — /health/deep returns chReachable=false# and waitForMergesIdle skips immediately. Compensate with a longer per-step# drain and a fixed inter-phase cooldown so the SUT can finish digestingdiff --git a/website/components/blog-article.tsx b/website/components/blog-article.tsxindex 3daffba8..9ce838b5 100644--- a/website/components/blog-article.tsx+++ b/website/components/blog-article.tsx@@ -49,7 +49,7 @@ export function BlogArticle({<div className="prose"><Eyebrow>{eyebrow}</Eyebrow><h1 className="mt-4 mb-3">{post.title}</h1>- <BlogByline date={formatDate(post.date)} />+ <BlogByline date={formatDate(post.date)} author={post.author} />{post.description && (<p className="blog-lead mb-12">{post.description}</p>diff --git a/website/components/blog-byline.tsx b/website/components/blog-byline.tsxindex a8134bd7..8c37ed02 100644--- a/website/components/blog-byline.tsx+++ b/website/components/blog-byline.tsx@@ -1,28 +1,44 @@import Image from "next/image";-const AUTHOR = {- name: "Dusan Stanojevic",- url: "https://github.com/dusanstanojeviccs",- avatar: "/images/dusan-stanojevic.jpg",+type Author = {+ name: string;+ url: string;+ avatar: string;};-export function BlogByline({ date }: { date: string }) {+const AUTHORS: Record<string, Author> = {+ dusan: {+ name: "Dusan Stanojevic",+ url: "https://www.linkedin.com/in/dusanstanojeviccs",+ avatar: "/images/dusan-stanojevic.jpg",+ },+ jovan: {+ name: "Jovan Stojiljkovic",+ url: "https://github.com/jstojiljkovic",+ avatar: "/images/jovan-stojiljkovic.jpg",+ },+};++const DEFAULT_AUTHOR_KEY = "dusan";++export function BlogByline({ date, author }: { date: string; author?: string }) {+ const author_record = AUTHORS[author ?? DEFAULT_AUTHOR_KEY] ?? AUTHORS[DEFAULT_AUTHOR_KEY];return (<div className="blog-byline"><a- href={AUTHOR.url}+ href={author_record.url}target="_blank"rel="noopener noreferrer"className="blog-author"><Image- src={AUTHOR.avatar}- alt={AUTHOR.name}+ src={author_record.avatar}+ alt={author_record.name}width={32}height={32}className="blog-author-avatar"/>- {AUTHOR.name}+ {author_record.name}</a>{date && (<>diff --git a/website/content/blog/break-sqlite-traceway.mdx b/website/content/blog/break-sqlite-traceway.mdxnew file mode 100644index 00000000..9cd7e9fc--- /dev/null+++ b/website/content/blog/break-sqlite-traceway.mdx@@ -0,0 +1,224 @@+---+title: "I tried to break SQLite. The reads broke first."+date: "2026-06-09"+category: "engineering"+author: "jovan"+description: "Post 1 only measured metrics writes. This time I ran spans, metrics, and logs against the same $16.49/month CCX13 box, and probed real dashboard queries at 100k, 1M, and 10M rows."+---++## TL;DR++Same Hetzner CCX13 box as [post 1](/blog/sqlite-observability-stack) ($16.49/month, 2 dedicated AMD vCPU, 8 GB RAM, 80 GB NVMe). Same single binary, SQLite mode. This time I pushed all three signals (spans, metrics, and logs) through it instead of one, and measured reads on top of writes.++Write throughput (one 30-minute run per signal, out-of-the-box defaults, full methodology below):++- **Metrics: 61,712 points/sec** sustained, p50 ingest 109 ms, p95 1.8 s.+- **Spans: 30,508 spans/sec** sustained, p50 98 ms, p95 208 ms.+- **Logs: 4,877 records/sec** sustained, p50 113 ms, p95 7.7 s.++Read latency (fill the table to a target row count, then load real dashboard pages against it; a fill level passes when the median probe is at or under 5 s and no probe times out):++- **Metrics**: passes at 1M rows (median 3.85 s, barely). At 10M every probe hits the timeout.+- **Spans**: passes at 100k rows (median 2.27 s). Fails at 1M.+- **Logs**: passes at 100k rows (median 114 ms). Fails at 1M.++<mark>The single binary on the cheap box ran spans, metrics, and logs side by side, and the dashboard was usable up to roughly 100k-1M rows depending on signal.</mark> Above that there was no slow-but-working middle ground: the probes either came back under the threshold or hit the timeout.++## What post 1 didn't measure++[Post 1](/blog/sqlite-observability-stack) made a claim that's easy to mistake for a stunt: a $16/month box can run your entire observability stack on SQLite. The benchmark behind it measured exactly one thing, metrics writes, and the number held. That was deliberate: pick the simplest signal, prove the floor, ship. But "you can run your stack on $16/month" only lands if it's the whole stack, not one tab of it. So this post is me trying to break that claim before someone else does.++Two things were missing:++1. **Spans and logs writes.** OTLP, the OpenTelemetry wire protocol, defines three signals: traces, metrics, and logs. An observability stack benchmark that only does one is half the picture.+2. **Anything read-side.** A database that ingests fast but kills the dashboard at 200k rows isn't something you'd ship.++This post fills both gaps on the same hardware, same DB mode, same load generator. The only thing that grew between the two posts is the harness: the load generator plus the scripts that orchestrate the runs.++Here's the plan. Methodology first, because half of it is new and the numbers mean nothing without the pass/fail rules. Then one section per signal: metrics first, since post 1 gives it a baseline, then spans, then logs. Each covers its write ceiling and its read ceiling. After that: the three cliffs compared, why the 5-second threshold is generous, what this means for a real fleet, and what I left out.++## How I measured++The setup is two machines: the **system under test** (SUT), which runs the Traceway binary, and a separate **load generator** (loadgen), which sends it traffic. Keeping them apart means the SUT is never CPU-starved by sharing a box with traffic generation.++**Hardware:** same spec as post 1. Both machines are Hetzner Cloud CCX13s (2 dedicated AMD vCPU, 8 GB RAM, 80 GB NVMe local disk) in the same Nuremberg datacenter, $16.49/month at the time of publication, connected over a private 10 Gbit link inside the same Hetzner project.++**Software:** Traceway commit [`2395238e`](https://github.com/tracewayapp/traceway/commit/2395238ee805f93d439d440ea6e8a9aac2b14bc9), branch `perf/sqlite`, image built on the SUT from that tree. SQLite via `modernc.org/sqlite v1.18.1` (pure Go). Loadgen built from the same SHA. Ubuntu 24.04. Retention TTLs disabled during the bench so cleanup doesn't compete with ingest for I/O.++**Workload:** OTLP/HTTP, gzipped protobuf. Three signals, each run separately with a fresh VM pair, so cross-signal interference is zero by construction. The data is uniform-random within bounded ranges (13 endpoints, 10 metric names, 18 log body strings, 5 severities), enough variety to fill the dashboard and flat enough to isolate raw DB behavior from traffic-shape noise.++Each signal gets two scenarios: a **throughput ramp** (how fast can the box write) and a **read-probe** (does the dashboard still load once the table is full).++### The throughput ramp++Every ingest request carries a batch of items, whether metric points, spans, or log records. That gives the loadgen two knobs: the **batch size** (items per request) and the **request rate** (requests per second). Target items/sec is simply batch size × request rate.++The ramp explores those knobs in three phases:++1. **Phase 1, batch-size ramp.** Hold the request rate at 5 req/sec and keep growing the batch until the box fails a step. This finds the largest batch the SUT can swallow.+2. **Phase 2, request-rate ramp.** Take the winning batch from phase 1 and ramp the request rate. This is where the headline numbers come from.+3. **Phase 3, small-batch rate ramp.** Fix the batch at 100 items and crank the request rate instead: the "many small clients" shape rather than "a few fat ones".++A step passes when both hold:++1. Error rate (HTTP failures + OTLP `PartialSuccess` rejected items) is ≤ 5%.+2. Achieved items/sec is ≥ 70% of the target. This catches the soft cliff: the rate limiter happily *sent* 100 req/sec but the workers only *managed* 30. That's a failure, not a pass.++The headline per signal is the highest achieved items/sec across all passing steps from any phase (the `maxSustainableItemsPerSec` field in the raw JSON). It's measured from real OK responses, not the rate-limiter target. Ingest latency, wherever it appears in this post, is the time from the loadgen sending a request to receiving the OK response.++### The read-probe++This scenario answers the question post 1 punted on: you've ingested all this data, can you actually look at it?++Fill the table to a target row count (100k, then 1M, then 10M, then 100M), stopping at the first level that fails. At each level: fill, let the SUT settle for 10 seconds, then issue three real dashboard queries against the populated table, each with a 6-second timeout. A fill level passes when the median latency across the three probes is ≤ 5 s **and** all three come back before the timeout. The second condition matters: one hung query fails the level even if the other two are instant, because during an incident you don't get to pick which page you need.++The probed endpoints per signal:++- **Metrics**: `/api/metrics/server`, `/api/metrics/application`, `/api/dashboard`.+- **Spans**: `/api/endpoints/grouped` (P50/P95/P99 aggregates), `/api/endpoints/chart` (stacked latency over time), `/api/exception-stack-traces` (grouped exception list).+- **Logs**: three filter shapes on `/api/logs`, a `severity=ERROR` filter, a free-text body search, and a trace-ID lookup.++`maxFillLevelPassed` is the largest row count that passed. And if 5 seconds sounds like a long time to wait for a dashboard, it is, and there's a section on exactly that after the results.++### Caveats++These sit inside the methodology for a reason. Read them before quoting numbers from this post:++- **Single-shot.** Posts 1 and 2 are one run per configuration on purpose, so they stay directly comparable. From post 3 onward, headlines are the median of three reps with the range shown as an error bar. Until then, these numbers are *not* a published median; each is a single 30-minute sustained run.+- **What "queryable" means here.** A fill level "passing" means *one specific check across three named endpoints* came back under the thresholds on freshly-written data after a settle period. I did not measure: correctness of the result, behavior under concurrent ingest while probing, behavior on backdated data, or any other dashboard page. Each of those is a future post.+- **Harness change vs post 1.** The throughput-ramp output schema changed between post 1 and post 2, and the headline is computed from slightly different ramp steps. Post 1's 58k metric-points/sec and this post's 61.7k are not directly comparable; both are honest "single-shot maximum sustained" numbers, but the ramp shape that produced them differs. Any comparison to post 1 in this post is therefore qualitative, not digit-for-digit.+- **Loadgen body shape.** Log records carry a body sampled from a fixed pool of 18 realistic strings averaging ~30-60 bytes (the kind of thing you'd see in a real app log). An earlier baseline used a 120-byte random string instead, and the smaller realistic bodies are part of why the logs throughput number is what it is.+- **The exception-stack-traces probe is currently an empty query** (no specific exception hash filter). It reads the grouped exception list, which on a synthetic loadgen with all-OK spans is a tiny response. A more representative probe lands in post 3.++## Metrics: 61,712 points/sec, dashboard holds at 1M rows++Metrics first, because it's the one signal with a baseline: post 1 measured exactly this, on the same box, in the same DB mode.++The ramp settled at batch=8192, 7.5 req/sec in phase 2: **61,712 points/sec sustained**, error rate 0.33%, p50 ingest 109 ms, p95 1.8 s. Phase 1 found the batch ceiling at 16384, which failed with a 13% error rate. Past a point, a bigger batch just takes longer to write than the request budget allows, and the SUT starts rejecting.++That's slightly ahead of post 1's 58k headline, but as flagged in the caveats, the harness changed between posts and the two numbers shouldn't be compared digit-for-digit. The qualitative read is the same as post 1: SQLite handles metrics ingest comfortably at this scale, with hundreds of milliseconds of ingest-latency headroom.++No `/metrics` screenshot this time. The loadgen emits `rng.Float64() * 100` per point, so the cards render real shapes but nonsense values. Post 1 already showed the UI reading real values; the read-probe chart below is the visual for this section.++On the read side, metrics held longer than either other signal. **1M rows passed**, but barely: median 3.85 s across the three probes (`/api/metrics/server` 3.74 s, `/api/metrics/application` 3.85 s, `/api/dashboard` 4.94 s). At 10M rows every probe hit the 6 s timeout. And 10M is closer than it sounds: at the headline write rate, it's under three minutes of sustained ingest.++++The cliff in one chart: comfortable at 100k, on the edge at 1M, gone at 10M. All three probes track together, with `/api/dashboard` running hottest at every step.++## Spans: 30,508 spans/sec, dashboard usable to 100k++Metrics was the known quantity. Spans are the first new signal, and they're heavier: each span carries a route, timing, status, and a bag of attributes that the dashboard later has to aggregate into percentiles.++The ramp settled at batch=4096, 7.5 req/sec: **30,508 spans/sec sustained**, error rate 0%, p50 ingest 98 ms, p95 208 ms. Phase 1 found the batch ceiling at 8192 (5.7% error rate); phase 3, the many-small-clients shape, failed at batch=100, 550 req/sec with a 6.6% error rate.++Here's the `/endpoints` page after a few minutes of ingest:++++This is the dashboard reading the same spans the loadgen just wrote. 31k+ calls per route, p50 around 500 ms, p95 around 950 ms, p99 990 ms; those are the loadgen's synthetic ranges, not Traceway's overhead. The chart legend tracks 13 `(method, path)` tuples; one of them rendered as `Other` because the page shows the top 6 by impact and groups the rest.++Drilling into one route, `POST /api/users`, gives the per-endpoint trace table:++++504 ms average, p50 505 ms, p95 951 ms, p99 990 ms, 26,749 rpm sustained, 2.08% error rate. Each row in the table is one span: `http.method=POST`, `http.route=/api/users`, version 1.0.0, server `bench-loadgen`. This is the page that pays off the "I can pull a slow request out of a million" promise: filter to a route, sort by duration, there's your tail.++On the read side, the spans dashboard stayed usable up to **100k rows**, median 2.27 s. `/api/endpoints/grouped` and `/api/endpoints/chart` both sat around 2.3 s. The third probe, `/api/exception-stack-traces`, came back in ~1 ms, which is the empty-query artifact from the caveats, so ignore it. At 1M rows both real probes timed out. The cliff sits somewhere between 100k and 1M; I didn't bisect it. At the full 30k spans/sec ingest rate, 100k rows is about three seconds of traffic and 1M is a little over half a minute, so at peak you blow past the usable read window almost as fast as you fill it. Traceway on SQLite will let you read recent spans. It's not the database for "show me last quarter's p99 across every route."++## Logs: 4,877 records/sec, dashboard fast at 100k++Logs are the highest-volume signal in most real systems, and going in, the one I expected to hurt the most.++The ramp settled at batch=1024, 5 req/sec: **4,877 records/sec sustained**, error rate 0%, p50 ingest 113 ms, p95 7.7 s. The phase 2 cliff between 5 and 7.5 req/sec is the sharpest in the whole dataset: bumping the request rate by 1.5× takes the error rate from 0% to 98% almost immediately. Past 5 req/sec the box stops accepting batches at all.++`/logs` after a few minutes of ingest:++++INFO/WARN/ERROR mixed, real-shape body lines from the 18-string pool (`"product 9123 removed from cart"`, `"slow query detected: SELECT * FROM orders WHERE ... (842ms)"`, `"payment authorized via stripe (amount=42.99 usd)"`, etc.), service column populated, trace IDs truncated for the list view. Click into one row and the structured context shows up:++++Trace ID, span ID, scope, `code.namespace`, `logger.name`, resource attributes, `retry.count`: every OTel attribute the loadgen attached is one click from the request that produced the log.++The read side is where logs surprised me. **100k rows passed at a 114 ms median**, with all three query shapes (severity filter, free-text body search, trace-ID lookup) coming back in 102-133 ms. That's an order of magnitude faster than the spans dashboard at the same fill.++At 1M rows the cliff hit, and it hit unevenly: the trace-ID lookup came back in 1.7 s, the body search in 5 s, and the severity filter timed out. The median was just under the 5 s bar, but the severity filter hitting the timeout fails the level anyway: a median can look fine while one of the pages is dead.++++The uneven failure has a simple shape. The trace-ID probe matches at most one row per query, so it stays fast as the table grows. The body search and severity filter both return many rows, and their cost scales with the result set, not the lookup.++## Where it breaks++Three signals, each with a write ceiling and a read ceiling:++| Signal | Write cliff (single-shot, sustained) | Read cliff (largest fill level passed) | What happens above it |+|---|---|---|---|+| Metrics | ~62k points/sec | 1M rows, median 3.85 s (narrow headroom) | At 10M every probe hits the timeout |+| Spans | ~30k spans/sec | 100k rows, median 2.27 s | At 1M both representative probes time out |+| Logs | ~5k records/sec, sharp HTTP-5xx cliff above 5 req/sec | 100k rows, median 114 ms | At 1M the trace-ID lookup is still fast (1.7 s) but the severity filter times out |++The shapes are different. Metrics has the highest ingest ceiling and the highest read ceiling, but the read margin at 1M is thin. Spans ingest at half the metrics rate and their reads give out at 100k. Logs ingest the slowest and also stop at 100k, but the read story at 100k is the most comfortable of the three, and the failure at 1M is uneven: some queries stay fast while others time out hard.++The uneven logs failure matters for retention policy. The page that paginates raw rows ("show me the last 50 ERRORs") stays usable longer than the page that scans the whole table to find a string. SQLite indexes do their job; full table scans do not. Plan retention so the page your on-call actually opens during an incident is on the indexed side of the cliff.++## 5 seconds is generous++I promised in the methodology to come back to the threshold. Your users expect 500 ms. A page that takes 5 seconds to load is broken. Users hit refresh and blame the network. So why is the read-probe threshold 5 s?++Because the threshold isn't a UX target, it's a "the database is still responding" floor. It separates "this query is slow" from "this query died." The series uses the same threshold across every database so the numbers compare; it does not claim that 5 s is what your on-call experiences.++So re-read the cliffs with a 500 ms threshold instead:++- **Logs at 100k still passes**: every probe came back inside 135 ms.+- **Spans at 100k fails**: both representative probes are above 2 s.+- **Metrics at 100k passes** at 343-443 ms across the three probes, but **1M fails** at 3.7-4.9 s.++Hold the dashboards to 500 ms and the comfortable ceiling shrinks: ~100k rows on logs and metrics, not even 100k on spans. The 1M+ headline numbers are "still loading" territory, not the floor of good UX. I kept the 5 s threshold so the series stays comparable. If you're a small-fleet operator deciding when to switch databases, halve the row counts in the headlines first.++## What fits comfortably++The cliffs say where it breaks. The other half of the question is what fits underneath them. Take a reasonable small-fleet shape (10 backends, each emitting 50 spans/sec, 200 log records/sec, and 10 metric points/sec) and hold it against the write ceilings:++- **Spans:** 10 × 50 = 500 spans/sec → 1.7% of the 30k/sec envelope+- **Logs:** 10 × 200 = 2,000 records/sec → 40% of the 5k/sec envelope (the binding constraint)+- **Metrics:** 10 × 10 = 100 points/sec → 0.2% of the 62k/sec envelope (essentially free)++Every signal fits comfortably on the write side. The binding constraint is logs throughput, and even there you're at 40% of the cliff, with room for spikes.++The read side is tighter. At this fleet rate, logs hit their 100k read cliff in 50 seconds. Spans take just over 3 minutes. Metrics take under 3 hours to reach 1M. So retention, not throughput, is the real config knob for Traceway on SQLite. A few hours of retention, the bare minimum for an incident workflow, keeps every signal on the usable side of the cliff. Days of history for trend analysis pushes you off the cheap box entirely and into post 3's territory: ClickHouse on the same hardware.++## What I deliberately didn't measure++In the spirit of [post 1's "be specific"](https://tracewayapp.com/blog/sqlite-observability-stack), here is what this post does *not* claim:++- **Read under concurrent write.** Ingest stops, settle, then probe. That isolates DB read performance from ingest pressure, but it is also not what a production dashboard does at 3 PM on a Tuesday. The "read while writing" scenario comes later in the series, after the `read-under-load` harness lands.+- **Correctness of the returned data.** I measured response time, not row counts or content. A fast wrong answer would have passed the probe.+- **The full dashboard.** Three named endpoints per signal, all on Traceway's hot path. Settings, project management, member invites, and everything else off the hot path went unmeasured.+- **Variance.** One 30-minute sustained run per signal. From post 3 onward the headline is the median of three reps with the range shown; this post (and post 1) are explicitly single-shot, see the caveats above.+- **Sustained durability.** 30 minutes of sustained load tells you the box is not falling over inside those 30 minutes. It does not tell you what week three on the same disk looks like.+- **Tuning.** Out-of-the-box SQLite defaults plus the transactional-insert change from post 1 (wrapping each batch insert in a single transaction; baseline now, not an experiment). PRAGMA tuning, custom indexes, schema rewrites: none of that. Tuning is its own post.++If you need any of these, the answer is "not in this post." I will get to them.++## What surprised me++I went into this expecting SQLite to cliff hard on logs and was ready to write up "the cheap tier is only for metrics." Logs writes do cliff sharply at 5k/sec, sure. But the logs *dashboard* at 100k rows came back in 114 ms, faster than the spans dashboard at the same fill. That was the surprise of the run, and the reason this post isn't titled the way I expected it would be.++SQLite is doing more than people give it credit for at this scale. The $16.49/month box ingested all three signals and served real dashboard pages on top. Two of the three stayed usable through 100k rows. Metrics held through 1M. For small fleets, side projects, or a backend behind a mobile frontend, the single-binary configuration is real. You don't need a multi-component LGTM deployment (Loki, Grafana, Tempo, Mimir) to have a working observability story.++What the data also says, plainly: Traceway on SQLite isn't the configuration for the day you grow past a million spans an hour. The cliff is closer than the ingest number suggests, and the next post moves the same workload to ClickHouse on the same box to find out what that buys you.++## Raw data + workflow++Everything in this post, runnable from the repo:++- **Raw JSON** lives in [`benchmarks/blog/post-2-data/`](https://github.com/tracewayapp/traceway/tree/main/benchmarks/blog/post-2-data): three throughput JSONs and three read-probe JSONs.+- **The workflow that produced the numbers**: [`benchmark-hardware.yml`](https://github.com/tracewayapp/traceway/actions/workflows/benchmark-hardware.yml). Throughput run: [27191483023](https://github.com/tracewayapp/traceway/actions/runs/27191483023). Read-probe run: [27198043577](https://github.com/tracewayapp/traceway/actions/runs/27198043577).+- **Try Traceway**: [self-host in five minutes](https://tracewayapp.com/docs/install).++Questions, pushback, or "your number is wrong because X": `[email protected]`, or find me on [GitHub](https://github.com/jstojiljkovic).++**Next post:** same $16.49/month box, swap SQLite for ClickHouse (with a local PostgreSQL alongside for relational data). Same loadgen. Same three signals. Same read-probe. Find out what the heavier database actually buys you, and where it doesn't.diff --git a/website/lib/blog.ts b/website/lib/blog.tsindex 3b8fa209..ca4cdf35 100644--- a/website/lib/blog.ts+++ b/website/lib/blog.ts@@ -13,6 +13,7 @@ export type BlogPostMeta = {version?: string;category: BlogCategory;description?: string;+ author?: string;};export type BlogPost = BlogPostMeta & {@@ -31,7 +32,8 @@ function readPostFile(filename: string): BlogPost | null {data.category === "engineering" ? "engineering" : "release";const description =typeof data.description === "string" ? data.description : undefined;- return { slug, title, date, version, category, description, content };+ const author = typeof data.author === "string" ? data.author : undefined;+ return { slug, title, date, version, category, description, author, content };}export function getAllPosts(): BlogPostMeta[] {diff --git a/website/public/blog/break-sqlite-traceway/chart-readprobe-logs.webp b/website/public/blog/break-sqlite-traceway/chart-readprobe-logs.webpnew file mode 100644index 00000000..0e1ad4daBinary files /dev/null and b/website/public/blog/break-sqlite-traceway/chart-readprobe-logs.webp differdiff --git a/website/public/blog/break-sqlite-traceway/chart-readprobe-metrics.webp b/website/public/blog/break-sqlite-traceway/chart-readprobe-metrics.webpnew file mode 100644index 00000000..075f8d60Binary files /dev/null and b/website/public/blog/break-sqlite-traceway/chart-readprobe-metrics.webp differdiff --git a/website/public/blog/break-sqlite-traceway/img1-endpoints-overview.webp b/website/public/blog/break-sqlite-traceway/img1-endpoints-overview.webpnew file mode 100644index 00000000..59dd1862Binary files /dev/null and b/website/public/blog/break-sqlite-traceway/img1-endpoints-overview.webp differdiff --git a/website/public/blog/break-sqlite-traceway/img2-endpoint-detail.webp b/website/public/blog/break-sqlite-traceway/img2-endpoint-detail.webpnew file mode 100644index 00000000..2e9c439eBinary files /dev/null and b/website/public/blog/break-sqlite-traceway/img2-endpoint-detail.webp differdiff --git a/website/public/blog/break-sqlite-traceway/img3-logs-overview.webp b/website/public/blog/break-sqlite-traceway/img3-logs-overview.webpnew file mode 100644index 00000000..00431442Binary files /dev/null and b/website/public/blog/break-sqlite-traceway/img3-logs-overview.webp differdiff --git a/website/public/blog/break-sqlite-traceway/img4-logs-drilldown.webp b/website/public/blog/break-sqlite-traceway/img4-logs-drilldown.webpnew file mode 100644index 00000000..a08c97d2Binary files /dev/null and b/website/public/blog/break-sqlite-traceway/img4-logs-drilldown.webp differdiff --git a/website/public/images/jovan-stojiljkovic.jpg b/website/public/images/jovan-stojiljkovic.jpgnew file mode 100644index 00000000..c8c0d19eBinary files /dev/null and b/website/public/images/jovan-stojiljkovic.jpg differ