AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
bench(index-size): measure ZBS4+ZFM5 combined on the merged tree
xerj-orgef3e75ae4fa1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a323fa9ca..e6e1e6570 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -111,7 +111,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0family drops 200,826 → 13,774 B (−93.1 %), total durable −3.5 %, withevery other extension byte-identical; the win scaleswith term cardinality, which is exactly the shape of unique-id fields- on large deployments. ZFM4-and-older segments decode unchanged.+ on large deployments. Measured again on a tree carrying ZBS4 as well+ (`results/result-zbs4-zfm5-balanced.json` vs+ `result-zbs4-only-control-balanced.json`, same day): 4,176,306 →+ 3,989,254 B — the two stages together take the pre-stage baseline+ 5,398,478 → 3,989,254 B (**−26.1 %**), with the four staged deltas+ (ZNV2, ZPS2, ZBS4, ZFM5) summing to within 12 B of the directly+ measured endpoint. ZFM4-and-older segments decode unchanged.- **Stored sections get typed-int columns, duplicate-column references anda merge-path zstd effort chooser (`ZBS4`)** — stage 1 of the index-sizediff --git a/benchmarks/index-size/results/result-zbs4-only-control-balanced.json b/benchmarks/index-size/results/result-zbs4-only-control-balanced.jsonnew file mode 100644index 000000000..09f8c15fe--- /dev/null+++ b/benchmarks/index-size/results/result-zbs4-only-control-balanced.json@@ -0,0 +1,157 @@+{+ "label": "zbs4-only-control",+ "compression_level": "balanced",+ "docs": 100000,+ "durable_bytes_excluding_wal": 4176306,+ "wal_bytes": 4117,+ "by_extension": {+ "post": 2364905,+ "ids": 613485,+ "seg": 449092,+ "dv": 355156,+ "meta": 200826,+ "fst": 105760,+ "json": 47365,+ "norms": 35670,+ "jsonl": 3256,+ "ftsan": 624,+ "xerj_master_key": 64,+ "key": 64,+ "sidx": 32,+ "lock": 7+ },+ "by_field_post_meta": {+ "body.post": 1749695,+ "doc_id.post": 256447,+ "doc_id.meta": 138727,+ "completion_tokens.post": 68679,+ "top_doc.post": 60673,+ "latency_ms.post": 40298,+ "intent.post": 39443,+ "tenant.post": 35963,+ "body.norms": 35194,+ "model.post": 34122,+ "panels.fst": 30847,+ "@timestamp.fst": 21536,+ "prompt_tokens.post": 19430,+ "cache_hit.post": 19102,+ "body.fst": 15254,+ "context_tokens.post": 15193,+ "context_tokens.fst": 13846,+ "cost_usd.post": 13802,+ "cost_usd.fst": 12360,+ "body.meta": 12073,+ "cost_usd.meta": 10271,+ "context_tokens.meta": 10169,+ "@timestamp.meta": 8704,+ "prompt_tokens.meta": 8014,+ "@timestamp.post": 7345,+ "latency_ms.meta": 6454,+ "prompt_tokens.fst": 4991,+ "completion_tokens.meta": 4177,+ "status.post": 4082,+ "latency_ms.fst": 3321,+ "completion_tokens.fst": 1097,+ "id.fst": 296,+ "name.fst": 293,+ "doc_id.fst": 279,+ "top_doc.fst": 275,+ "panels.meta": 251,+ "intent.fst": 213,+ "intent.meta": 206,+ "top_doc.meta": 201,+ "panels.post": 177,+ "id.meta": 149,+ "model.meta": 131,+ "section.fst": 122,+ "model.fst": 121,+ "tenant.meta": 117,+ "name.meta": 108,+ "section.meta": 97,+ "tenant.fst": 90,+ "id.post": 87,+ "group.meta": 83,+ "status.meta": 73,+ "cache_hit.meta": 73,+ "created_at.fst": 66,+ "updated_at.fst": 66,+ "group.fst": 61,+ "org_id.meta": 58,+ "updated_at.meta": 58,+ "created_at.meta": 58,+ "visibility.meta": 58,+ "version.meta": 58,+ "owner.meta": 58,+ "managed.meta": 58,+ "seed_revision.meta": 58,+ "filters_default.meta": 58,+ "kind.meta": 58,+ "cache_hit.fst": 54,+ "status.fst": 53,+ "kind.fst": 50,+ "name.post": 49,+ "org_id.fst": 48,+ "visibility.fst": 48,+ "owner.fst": 47,+ "managed.fst": 45,+ "filters_default.fst": 45,+ "time_default.meta": 42,+ "cloned_from.meta": 42,+ "etag.meta": 42,+ "deleted_at.meta": 42,+ "section.post": 41,+ "seed_revision.fst": 40,+ "version.fst": 40,+ "cloned_from.fst": 39,+ "time_default.fst": 39,+ "etag.fst": 39,+ "deleted_at.fst": 39,+ "group.post": 29,+ "id.norms": 29,+ "panels.norms": 27,+ "org_id.post": 26,+ "version.post": 26,+ "filters_default.post": 26,+ "managed.post": 26,+ "owner.post": 26,+ "created_at.post": 26,+ "visibility.post": 26,+ "updated_at.post": 26,+ "kind.post": 21,+ "seed_revision.post": 19,+ "tenant.norms": 14,+ "doc_id.norms": 14,+ "prompt_tokens.norms": 14,+ "top_doc.norms": 14,+ "@timestamp.norms": 14,+ "intent.norms": 14,+ "status.norms": 14,+ "model.norms": 14,+ "completion_tokens.norms": 14,+ "context_tokens.norms": 14,+ "latency_ms.norms": 14,+ "cache_hit.norms": 14,+ "cost_usd.norms": 14,+ "updated_at.norms": 14,+ "etag.norms": 14,+ "managed.norms": 14,+ "visibility.norms": 14,+ "cloned_from.norms": 14,+ "version.norms": 14,+ "filters_default.norms": 14,+ "kind.norms": 14,+ "org_id.norms": 14,+ "name.norms": 14,+ "section.norms": 14,+ "group.norms": 14,+ "seed_revision.norms": 14,+ "owner.norms": 14,+ "created_at.norms": 14,+ "deleted_at.norms": 14,+ "time_default.norms": 14,+ "etag.post": 0,+ "time_default.post": 0,+ "cloned_from.post": 0,+ "deleted_at.post": 0+ }+}\ No newline at end of filediff --git a/benchmarks/index-size/results/result-zbs4-zfm5-balanced.json b/benchmarks/index-size/results/result-zbs4-zfm5-balanced.jsonnew file mode 100644index 000000000..656328759--- /dev/null+++ b/benchmarks/index-size/results/result-zbs4-zfm5-balanced.json@@ -0,0 +1,157 @@+{+ "label": "zbs4-zfm5",+ "compression_level": "balanced",+ "docs": 100000,+ "durable_bytes_excluding_wal": 3989254,+ "wal_bytes": 4117,+ "by_extension": {+ "post": 2364905,+ "ids": 613485,+ "seg": 449092,+ "dv": 355156,+ "fst": 105760,+ "json": 47365,+ "norms": 35670,+ "meta": 13774,+ "jsonl": 3256,+ "ftsan": 624,+ "xerj_master_key": 64,+ "key": 64,+ "sidx": 32,+ "lock": 7+ },+ "by_field_post_meta": {+ "body.post": 1749695,+ "doc_id.post": 256447,+ "completion_tokens.post": 68679,+ "top_doc.post": 60673,+ "latency_ms.post": 40298,+ "intent.post": 39443,+ "tenant.post": 35963,+ "body.norms": 35194,+ "model.post": 34122,+ "panels.fst": 30847,+ "@timestamp.fst": 21536,+ "prompt_tokens.post": 19430,+ "cache_hit.post": 19102,+ "body.fst": 15254,+ "context_tokens.post": 15193,+ "context_tokens.fst": 13846,+ "cost_usd.post": 13802,+ "cost_usd.fst": 12360,+ "@timestamp.post": 7345,+ "prompt_tokens.fst": 4991,+ "status.post": 4082,+ "latency_ms.fst": 3321,+ "body.meta": 2600,+ "prompt_tokens.meta": 2067,+ "latency_ms.meta": 1938,+ "cost_usd.meta": 1847,+ "context_tokens.meta": 1616,+ "completion_tokens.meta": 1542,+ "completion_tokens.fst": 1097,+ "@timestamp.meta": 558,+ "id.fst": 296,+ "name.fst": 293,+ "doc_id.fst": 279,+ "top_doc.fst": 275,+ "doc_id.meta": 216,+ "intent.fst": 213,+ "panels.post": 177,+ "section.fst": 122,+ "model.fst": 121,+ "intent.meta": 101,+ "top_doc.meta": 99,+ "tenant.fst": 90,+ "id.post": 87,+ "model.meta": 77,+ "tenant.meta": 73,+ "id.meta": 68,+ "updated_at.fst": 66,+ "created_at.fst": 66,+ "section.meta": 63,+ "group.fst": 61,+ "panels.meta": 58,+ "cache_hit.meta": 57,+ "name.meta": 56,+ "cache_hit.fst": 54,+ "status.meta": 54,+ "status.fst": 53,+ "group.meta": 52,+ "kind.fst": 50,+ "name.post": 49,+ "visibility.fst": 48,+ "org_id.fst": 48,+ "owner.fst": 47,+ "filters_default.meta": 46,+ "managed.meta": 46,+ "owner.meta": 46,+ "version.meta": 46,+ "created_at.meta": 46,+ "updated_at.meta": 46,+ "visibility.meta": 46,+ "org_id.meta": 46,+ "kind.meta": 46,+ "seed_revision.meta": 46,+ "managed.fst": 45,+ "filters_default.fst": 45,+ "time_default.meta": 43,+ "etag.meta": 43,+ "cloned_from.meta": 43,+ "deleted_at.meta": 43,+ "section.post": 41,+ "seed_revision.fst": 40,+ "version.fst": 40,+ "etag.fst": 39,+ "deleted_at.fst": 39,+ "cloned_from.fst": 39,+ "time_default.fst": 39,+ "group.post": 29,+ "id.norms": 29,+ "panels.norms": 27,+ "created_at.post": 26,+ "updated_at.post": 26,+ "org_id.post": 26,+ "owner.post": 26,+ "visibility.post": 26,+ "version.post": 26,+ "filters_default.post": 26,+ "managed.post": 26,+ "kind.post": 21,+ "seed_revision.post": 19,+ "cache_hit.norms": 14,+ "latency_ms.norms": 14,+ "completion_tokens.norms": 14,+ "status.norms": 14,+ "model.norms": 14,+ "doc_id.norms": 14,+ "cost_usd.norms": 14,+ "tenant.norms": 14,+ "intent.norms": 14,+ "@timestamp.norms": 14,+ "context_tokens.norms": 14,+ "prompt_tokens.norms": 14,+ "top_doc.norms": 14,+ "time_default.norms": 14,+ "visibility.norms": 14,+ "version.norms": 14,+ "filters_default.norms": 14,+ "seed_revision.norms": 14,+ "group.norms": 14,+ "org_id.norms": 14,+ "name.norms": 14,+ "owner.norms": 14,+ "updated_at.norms": 14,+ "section.norms": 14,+ "created_at.norms": 14,+ "kind.norms": 14,+ "cloned_from.norms": 14,+ "managed.norms": 14,+ "deleted_at.norms": 14,+ "etag.norms": 14,+ "deleted_at.post": 0,+ "time_default.post": 0,+ "cloned_from.post": 0,+ "etag.post": 0+ }+}\ No newline at end of file