Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

bench(index-size): measure ZBS4+ZFM5 combined on the merged tree

xerj-orgef3e75ae4fa1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a323fa9ca..e6e1e6570 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -111,7 +111,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
family drops 200,826 → 13,774 B (−93.1 %), total durable −3.5 %, with
every other extension byte-identical; the win scales
with term cardinality, which is exactly the shape of unique-id fields
- on large deployments. ZFM4-and-older segments decode unchanged.
+ on large deployments. Measured again on a tree carrying ZBS4 as well
+ (`results/result-zbs4-zfm5-balanced.json` vs
+ `result-zbs4-only-control-balanced.json`, same day): 4,176,306 →
+ 3,989,254 B — the two stages together take the pre-stage baseline
+ 5,398,478 → 3,989,254 B (**−26.1 %**), with the four staged deltas
+ (ZNV2, ZPS2, ZBS4, ZFM5) summing to within 12 B of the directly
+ measured endpoint. ZFM4-and-older segments decode unchanged.
- **Stored sections get typed-int columns, duplicate-column references and
a merge-path zstd effort chooser (`ZBS4`)** — stage 1 of the index-size
diff --git a/benchmarks/index-size/results/result-zbs4-only-control-balanced.json b/benchmarks/index-size/results/result-zbs4-only-control-balanced.json
new file mode 100644
index 000000000..09f8c15fe
--- /dev/null
+++ b/benchmarks/index-size/results/result-zbs4-only-control-balanced.json
@@ -0,0 +1,157 @@
+{
+ "label": "zbs4-only-control",
+ "compression_level": "balanced",
+ "docs": 100000,
+ "durable_bytes_excluding_wal": 4176306,
+ "wal_bytes": 4117,
+ "by_extension": {
+ "post": 2364905,
+ "ids": 613485,
+ "seg": 449092,
+ "dv": 355156,
+ "meta": 200826,
+ "fst": 105760,
+ "json": 47365,
+ "norms": 35670,
+ "jsonl": 3256,
+ "ftsan": 624,
+ "xerj_master_key": 64,
+ "key": 64,
+ "sidx": 32,
+ "lock": 7
+ },
+ "by_field_post_meta": {
+ "body.post": 1749695,
+ "doc_id.post": 256447,
+ "doc_id.meta": 138727,
+ "completion_tokens.post": 68679,
+ "top_doc.post": 60673,
+ "latency_ms.post": 40298,
+ "intent.post": 39443,
+ "tenant.post": 35963,
+ "body.norms": 35194,
+ "model.post": 34122,
+ "panels.fst": 30847,
+ "@timestamp.fst": 21536,
+ "prompt_tokens.post": 19430,
+ "cache_hit.post": 19102,
+ "body.fst": 15254,
+ "context_tokens.post": 15193,
+ "context_tokens.fst": 13846,
+ "cost_usd.post": 13802,
+ "cost_usd.fst": 12360,
+ "body.meta": 12073,
+ "cost_usd.meta": 10271,
+ "context_tokens.meta": 10169,
+ "@timestamp.meta": 8704,
+ "prompt_tokens.meta": 8014,
+ "@timestamp.post": 7345,
+ "latency_ms.meta": 6454,
+ "prompt_tokens.fst": 4991,
+ "completion_tokens.meta": 4177,
+ "status.post": 4082,
+ "latency_ms.fst": 3321,
+ "completion_tokens.fst": 1097,
+ "id.fst": 296,
+ "name.fst": 293,
+ "doc_id.fst": 279,
+ "top_doc.fst": 275,
+ "panels.meta": 251,
+ "intent.fst": 213,
+ "intent.meta": 206,
+ "top_doc.meta": 201,
+ "panels.post": 177,
+ "id.meta": 149,
+ "model.meta": 131,
+ "section.fst": 122,
+ "model.fst": 121,
+ "tenant.meta": 117,
+ "name.meta": 108,
+ "section.meta": 97,
+ "tenant.fst": 90,
+ "id.post": 87,
+ "group.meta": 83,
+ "status.meta": 73,
+ "cache_hit.meta": 73,
+ "created_at.fst": 66,
+ "updated_at.fst": 66,
+ "group.fst": 61,
+ "org_id.meta": 58,
+ "updated_at.meta": 58,
+ "created_at.meta": 58,
+ "visibility.meta": 58,
+ "version.meta": 58,
+ "owner.meta": 58,
+ "managed.meta": 58,
+ "seed_revision.meta": 58,
+ "filters_default.meta": 58,
+ "kind.meta": 58,
+ "cache_hit.fst": 54,
+ "status.fst": 53,
+ "kind.fst": 50,
+ "name.post": 49,
+ "org_id.fst": 48,
+ "visibility.fst": 48,
+ "owner.fst": 47,
+ "managed.fst": 45,
+ "filters_default.fst": 45,
+ "time_default.meta": 42,
+ "cloned_from.meta": 42,
+ "etag.meta": 42,
+ "deleted_at.meta": 42,
+ "section.post": 41,
+ "seed_revision.fst": 40,
+ "version.fst": 40,
+ "cloned_from.fst": 39,
+ "time_default.fst": 39,
+ "etag.fst": 39,
+ "deleted_at.fst": 39,
+ "group.post": 29,
+ "id.norms": 29,
+ "panels.norms": 27,
+ "org_id.post": 26,
+ "version.post": 26,
+ "filters_default.post": 26,
+ "managed.post": 26,
+ "owner.post": 26,
+ "created_at.post": 26,
+ "visibility.post": 26,
+ "updated_at.post": 26,
+ "kind.post": 21,
+ "seed_revision.post": 19,
+ "tenant.norms": 14,
+ "doc_id.norms": 14,
+ "prompt_tokens.norms": 14,
+ "top_doc.norms": 14,
+ "@timestamp.norms": 14,
+ "intent.norms": 14,
+ "status.norms": 14,
+ "model.norms": 14,
+ "completion_tokens.norms": 14,
+ "context_tokens.norms": 14,
+ "latency_ms.norms": 14,
+ "cache_hit.norms": 14,
+ "cost_usd.norms": 14,
+ "updated_at.norms": 14,
+ "etag.norms": 14,
+ "managed.norms": 14,
+ "visibility.norms": 14,
+ "cloned_from.norms": 14,
+ "version.norms": 14,
+ "filters_default.norms": 14,
+ "kind.norms": 14,
+ "org_id.norms": 14,
+ "name.norms": 14,
+ "section.norms": 14,
+ "group.norms": 14,
+ "seed_revision.norms": 14,
+ "owner.norms": 14,
+ "created_at.norms": 14,
+ "deleted_at.norms": 14,
+ "time_default.norms": 14,
+ "etag.post": 0,
+ "time_default.post": 0,
+ "cloned_from.post": 0,
+ "deleted_at.post": 0
+ }
+}
\ No newline at end of file
diff --git a/benchmarks/index-size/results/result-zbs4-zfm5-balanced.json b/benchmarks/index-size/results/result-zbs4-zfm5-balanced.json
new file mode 100644
index 000000000..656328759
--- /dev/null
+++ b/benchmarks/index-size/results/result-zbs4-zfm5-balanced.json
@@ -0,0 +1,157 @@
+{
+ "label": "zbs4-zfm5",
+ "compression_level": "balanced",
+ "docs": 100000,
+ "durable_bytes_excluding_wal": 3989254,
+ "wal_bytes": 4117,
+ "by_extension": {
+ "post": 2364905,
+ "ids": 613485,
+ "seg": 449092,
+ "dv": 355156,
+ "fst": 105760,
+ "json": 47365,
+ "norms": 35670,
+ "meta": 13774,
+ "jsonl": 3256,
+ "ftsan": 624,
+ "xerj_master_key": 64,
+ "key": 64,
+ "sidx": 32,
+ "lock": 7
+ },
+ "by_field_post_meta": {
+ "body.post": 1749695,
+ "doc_id.post": 256447,
+ "completion_tokens.post": 68679,
+ "top_doc.post": 60673,
+ "latency_ms.post": 40298,
+ "intent.post": 39443,
+ "tenant.post": 35963,
+ "body.norms": 35194,
+ "model.post": 34122,
+ "panels.fst": 30847,
+ "@timestamp.fst": 21536,
+ "prompt_tokens.post": 19430,
+ "cache_hit.post": 19102,
+ "body.fst": 15254,
+ "context_tokens.post": 15193,
+ "context_tokens.fst": 13846,
+ "cost_usd.post": 13802,
+ "cost_usd.fst": 12360,
+ "@timestamp.post": 7345,
+ "prompt_tokens.fst": 4991,
+ "status.post": 4082,
+ "latency_ms.fst": 3321,
+ "body.meta": 2600,
+ "prompt_tokens.meta": 2067,
+ "latency_ms.meta": 1938,
+ "cost_usd.meta": 1847,
+ "context_tokens.meta": 1616,
+ "completion_tokens.meta": 1542,
+ "completion_tokens.fst": 1097,
+ "@timestamp.meta": 558,
+ "id.fst": 296,
+ "name.fst": 293,
+ "doc_id.fst": 279,
+ "top_doc.fst": 275,
+ "doc_id.meta": 216,
+ "intent.fst": 213,
+ "panels.post": 177,
+ "section.fst": 122,
+ "model.fst": 121,
+ "intent.meta": 101,
+ "top_doc.meta": 99,
+ "tenant.fst": 90,
+ "id.post": 87,
+ "model.meta": 77,
+ "tenant.meta": 73,
+ "id.meta": 68,
+ "updated_at.fst": 66,
+ "created_at.fst": 66,
+ "section.meta": 63,
+ "group.fst": 61,
+ "panels.meta": 58,
+ "cache_hit.meta": 57,
+ "name.meta": 56,
+ "cache_hit.fst": 54,
+ "status.meta": 54,
+ "status.fst": 53,
+ "group.meta": 52,
+ "kind.fst": 50,
+ "name.post": 49,
+ "visibility.fst": 48,
+ "org_id.fst": 48,
+ "owner.fst": 47,
+ "filters_default.meta": 46,
+ "managed.meta": 46,
+ "owner.meta": 46,
+ "version.meta": 46,
+ "created_at.meta": 46,
+ "updated_at.meta": 46,
+ "visibility.meta": 46,
+ "org_id.meta": 46,
+ "kind.meta": 46,
+ "seed_revision.meta": 46,
+ "managed.fst": 45,
+ "filters_default.fst": 45,
+ "time_default.meta": 43,
+ "etag.meta": 43,
+ "cloned_from.meta": 43,
+ "deleted_at.meta": 43,
+ "section.post": 41,
+ "seed_revision.fst": 40,
+ "version.fst": 40,
+ "etag.fst": 39,
+ "deleted_at.fst": 39,
+ "cloned_from.fst": 39,
+ "time_default.fst": 39,
+ "group.post": 29,
+ "id.norms": 29,
+ "panels.norms": 27,
+ "created_at.post": 26,
+ "updated_at.post": 26,
+ "org_id.post": 26,
+ "owner.post": 26,
+ "visibility.post": 26,
+ "version.post": 26,
+ "filters_default.post": 26,
+ "managed.post": 26,
+ "kind.post": 21,
+ "seed_revision.post": 19,
+ "cache_hit.norms": 14,
+ "latency_ms.norms": 14,
+ "completion_tokens.norms": 14,
+ "status.norms": 14,
+ "model.norms": 14,
+ "doc_id.norms": 14,
+ "cost_usd.norms": 14,
+ "tenant.norms": 14,
+ "intent.norms": 14,
+ "@timestamp.norms": 14,
+ "context_tokens.norms": 14,
+ "prompt_tokens.norms": 14,
+ "top_doc.norms": 14,
+ "time_default.norms": 14,
+ "visibility.norms": 14,
+ "version.norms": 14,
+ "filters_default.norms": 14,
+ "seed_revision.norms": 14,
+ "group.norms": 14,
+ "org_id.norms": 14,
+ "name.norms": 14,
+ "owner.norms": 14,
+ "updated_at.norms": 14,
+ "section.norms": 14,
+ "created_at.norms": 14,
+ "kind.norms": 14,
+ "cloned_from.norms": 14,
+ "managed.norms": 14,
+ "deleted_at.norms": 14,
+ "etag.norms": 14,
+ "deleted_at.post": 0,
+ "time_default.post": 0,
+ "cloned_from.post": 0,
+ "etag.post": 0
+ }
+}
\ No newline at end of file