Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fts: ZFM5 meta records — columnar varint streams under the zstd envelope

xerj-org24a7d67cd583
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9b2328c4..629d36c0b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -88,6 +88,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Performance
+- **`.meta` term records go columnar-varint (ZFM5)** ([#1038](https://github.com/xerj-org/xerj/issues/1038)):
+ the per-term `{df u32, ttf u64, offset u64, len u32}` records under the
+ ZFM4 zstd envelope kept a fixed 24-byte stride, and zstd cannot match
+ across the stride's alternating field widths — on the 100 k-doc size
+ harness `doc_id.meta` (100 k unique terms, every one `ttf == df`, every
+ postings gap zero) still cost 138,727 B on disk for ~200 B of
+ information. ZFM5 re-shapes the section before compression into four
+ columnar varint streams — `df`, `ttf − df` (elided entirely when every
+ term in the field has `ttf == df`, the keyword-field shape), `offset
+ gap`, and `postings_length` — so zstd sees long runs of one-symbol
+ lanes. The reader expands the streams once at open time into the same
+ in-memory flat-record array ZFM3/ZFM4 build, leaving the FST-offset
+ lookup hot path untouched. The encoder fails closed on non-monotone
+ postings offsets (gaps reconstruct by accumulation, so a negative gap is
+ unrepresentable) and the decoder rejects truncated streams, trailing
+ stream bytes, and length mismatches. Peer pattern: tantivy's
+ `TermInfoValueWriter::serialize_block` (`src/termdict/sstable_termdict/
+ mod.rs:92-105`, MIT) writes block-leading absolute range starts once
+ and then per-term length varints only. Measured on the 100 k-doc
+ harness (LEVEL=balanced, forcemerge 1, same-day control): the `.meta`
+ family drops 200,826 → 13,774 B (−93.1 %), total durable −3.5 %, with
+ every other extension byte-identical; the win scales
+ with term cardinality, which is exactly the shape of unique-id fields
+ on large deployments. ZFM4-and-older segments decode unchanged.
+
- **`.post` block framing slimmed (ZPS2)** ([#1038](https://github.com/xerj-org/xerj/issues/1038)):
the packed doc-id-delta and term-freq streams in every 128-doc posting
block switch to frame-of-reference coding — `[width][vbyte min]` with the
diff --git a/benchmarks/index-size/results/result-zfm5-balanced.json b/benchmarks/index-size/results/result-zfm5-balanced.json
new file mode 100644
index 000000000..f8705a2ca
--- /dev/null
+++ b/benchmarks/index-size/results/result-zfm5-balanced.json
@@ -0,0 +1,157 @@
+{
+ "label": "zfm5",
+ "compression_level": "balanced",
+ "docs": 100000,
+ "durable_bytes_excluding_wal": 5211437,
+ "wal_bytes": 4119,
+ "by_extension": {
+ "post": 2364905,
+ "seg": 1671271,
+ "ids": 613485,
+ "dv": 355156,
+ "fst": 105760,
+ "json": 47366,
+ "norms": 35670,
+ "meta": 13774,
+ "jsonl": 3259,
+ "ftsan": 624,
+ "xerj_master_key": 64,
+ "key": 64,
+ "sidx": 32,
+ "lock": 7
+ },
+ "by_field_post_meta": {
+ "body.post": 1749695,
+ "doc_id.post": 256447,
+ "completion_tokens.post": 68679,
+ "top_doc.post": 60673,
+ "latency_ms.post": 40298,
+ "intent.post": 39443,
+ "tenant.post": 35963,
+ "body.norms": 35194,
+ "model.post": 34122,
+ "panels.fst": 30847,
+ "@timestamp.fst": 21536,
+ "prompt_tokens.post": 19430,
+ "cache_hit.post": 19102,
+ "body.fst": 15254,
+ "context_tokens.post": 15193,
+ "context_tokens.fst": 13846,
+ "cost_usd.post": 13802,
+ "cost_usd.fst": 12360,
+ "@timestamp.post": 7345,
+ "prompt_tokens.fst": 4991,
+ "status.post": 4082,
+ "latency_ms.fst": 3321,
+ "body.meta": 2600,
+ "prompt_tokens.meta": 2067,
+ "latency_ms.meta": 1938,
+ "cost_usd.meta": 1847,
+ "context_tokens.meta": 1616,
+ "completion_tokens.meta": 1542,
+ "completion_tokens.fst": 1097,
+ "@timestamp.meta": 558,
+ "id.fst": 296,
+ "name.fst": 293,
+ "doc_id.fst": 279,
+ "top_doc.fst": 275,
+ "doc_id.meta": 216,
+ "intent.fst": 213,
+ "panels.post": 177,
+ "section.fst": 122,
+ "model.fst": 121,
+ "intent.meta": 101,
+ "top_doc.meta": 99,
+ "tenant.fst": 90,
+ "id.post": 87,
+ "model.meta": 77,
+ "tenant.meta": 73,
+ "id.meta": 68,
+ "created_at.fst": 66,
+ "updated_at.fst": 66,
+ "section.meta": 63,
+ "group.fst": 61,
+ "panels.meta": 58,
+ "cache_hit.meta": 57,
+ "name.meta": 56,
+ "status.meta": 54,
+ "cache_hit.fst": 54,
+ "status.fst": 53,
+ "group.meta": 52,
+ "kind.fst": 50,
+ "name.post": 49,
+ "visibility.fst": 48,
+ "org_id.fst": 48,
+ "owner.fst": 47,
+ "seed_revision.meta": 46,
+ "org_id.meta": 46,
+ "created_at.meta": 46,
+ "visibility.meta": 46,
+ "version.meta": 46,
+ "kind.meta": 46,
+ "owner.meta": 46,
+ "filters_default.meta": 46,
+ "managed.meta": 46,
+ "updated_at.meta": 46,
+ "filters_default.fst": 45,
+ "managed.fst": 45,
+ "cloned_from.meta": 43,
+ "time_default.meta": 43,
+ "etag.meta": 43,
+ "deleted_at.meta": 43,
+ "section.post": 41,
+ "seed_revision.fst": 40,
+ "version.fst": 40,
+ "deleted_at.fst": 39,
+ "time_default.fst": 39,
+ "etag.fst": 39,
+ "cloned_from.fst": 39,
+ "id.norms": 29,
+ "group.post": 29,
+ "panels.norms": 27,
+ "visibility.post": 26,
+ "version.post": 26,
+ "managed.post": 26,
+ "updated_at.post": 26,
+ "created_at.post": 26,
+ "org_id.post": 26,
+ "filters_default.post": 26,
+ "owner.post": 26,
+ "kind.post": 21,
+ "seed_revision.post": 19,
+ "completion_tokens.norms": 14,
+ "context_tokens.norms": 14,
+ "tenant.norms": 14,
+ "prompt_tokens.norms": 14,
+ "intent.norms": 14,
+ "model.norms": 14,
+ "cost_usd.norms": 14,
+ "doc_id.norms": 14,
+ "cache_hit.norms": 14,
+ "@timestamp.norms": 14,
+ "status.norms": 14,
+ "top_doc.norms": 14,
+ "latency_ms.norms": 14,
+ "updated_at.norms": 14,
+ "filters_default.norms": 14,
+ "time_default.norms": 14,
+ "created_at.norms": 14,
+ "seed_revision.norms": 14,
+ "kind.norms": 14,
+ "managed.norms": 14,
+ "deleted_at.norms": 14,
+ "etag.norms": 14,
+ "visibility.norms": 14,
+ "version.norms": 14,
+ "section.norms": 14,
+ "org_id.norms": 14,
+ "name.norms": 14,
+ "owner.norms": 14,
+ "cloned_from.norms": 14,
+ "group.norms": 14,
+ "cloned_from.post": 0,
+ "etag.post": 0,
+ "deleted_at.post": 0,
+ "time_default.post": 0
+ }
+}
\ No newline at end of file
diff --git a/benchmarks/index-size/results/result-zfm5-base-balanced.json b/benchmarks/index-size/results/result-zfm5-base-balanced.json
new file mode 100644
index 000000000..b6093d993
--- /dev/null
+++ b/benchmarks/index-size/results/result-zfm5-base-balanced.json
@@ -0,0 +1,157 @@
+{
+ "label": "zfm5-base",
+ "compression_level": "balanced",
+ "docs": 100000,
+ "durable_bytes_excluding_wal": 5398486,
+ "wal_bytes": 4119,
+ "by_extension": {
+ "post": 2364905,
+ "seg": 1671271,
+ "ids": 613485,
+ "dv": 355156,
+ "meta": 200826,
+ "fst": 105760,
+ "json": 47366,
+ "norms": 35670,
+ "jsonl": 3256,
+ "ftsan": 624,
+ "xerj_master_key": 64,
+ "key": 64,
+ "sidx": 32,
+ "lock": 7
+ },
+ "by_field_post_meta": {
+ "body.post": 1749695,
+ "doc_id.post": 256447,
+ "doc_id.meta": 138727,
+ "completion_tokens.post": 68679,
+ "top_doc.post": 60673,
+ "latency_ms.post": 40298,
+ "intent.post": 39443,
+ "tenant.post": 35963,
+ "body.norms": 35194,
+ "model.post": 34122,
+ "panels.fst": 30847,
+ "@timestamp.fst": 21536,
+ "prompt_tokens.post": 19430,
+ "cache_hit.post": 19102,
+ "body.fst": 15254,
+ "context_tokens.post": 15193,
+ "context_tokens.fst": 13846,
+ "cost_usd.post": 13802,
+ "cost_usd.fst": 12360,
+ "body.meta": 12073,
+ "cost_usd.meta": 10271,
+ "context_tokens.meta": 10169,
+ "@timestamp.meta": 8704,
+ "prompt_tokens.meta": 8014,
+ "@timestamp.post": 7345,
+ "latency_ms.meta": 6454,
+ "prompt_tokens.fst": 4991,
+ "completion_tokens.meta": 4177,
+ "status.post": 4082,
+ "latency_ms.fst": 3321,
+ "completion_tokens.fst": 1097,
+ "id.fst": 296,
+ "name.fst": 293,
+ "doc_id.fst": 279,
+ "top_doc.fst": 275,
+ "panels.meta": 251,
+ "intent.fst": 213,
+ "intent.meta": 206,
+ "top_doc.meta": 201,
+ "panels.post": 177,
+ "id.meta": 149,
+ "model.meta": 131,
+ "section.fst": 122,
+ "model.fst": 121,
+ "tenant.meta": 117,
+ "name.meta": 108,
+ "section.meta": 97,
+ "tenant.fst": 90,
+ "id.post": 87,
+ "group.meta": 83,
+ "cache_hit.meta": 73,
+ "status.meta": 73,
+ "created_at.fst": 66,
+ "updated_at.fst": 66,
+ "group.fst": 61,
+ "seed_revision.meta": 58,
+ "visibility.meta": 58,
+ "managed.meta": 58,
+ "version.meta": 58,
+ "kind.meta": 58,
+ "updated_at.meta": 58,
+ "owner.meta": 58,
+ "filters_default.meta": 58,
+ "created_at.meta": 58,
+ "org_id.meta": 58,
+ "cache_hit.fst": 54,
+ "status.fst": 53,
+ "kind.fst": 50,
+ "name.post": 49,
+ "visibility.fst": 48,
+ "org_id.fst": 48,
+ "owner.fst": 47,
+ "filters_default.fst": 45,
+ "managed.fst": 45,
+ "time_default.meta": 42,
+ "etag.meta": 42,
+ "cloned_from.meta": 42,
+ "deleted_at.meta": 42,
+ "section.post": 41,
+ "version.fst": 40,
+ "seed_revision.fst": 40,
+ "cloned_from.fst": 39,
+ "time_default.fst": 39,
+ "etag.fst": 39,
+ "deleted_at.fst": 39,
+ "id.norms": 29,
+ "group.post": 29,
+ "panels.norms": 27,
+ "created_at.post": 26,
+ "managed.post": 26,
+ "version.post": 26,
+ "owner.post": 26,
+ "filters_default.post": 26,
+ "org_id.post": 26,
+ "visibility.post": 26,
+ "updated_at.post": 26,
+ "kind.post": 21,
+ "seed_revision.post": 19,
+ "prompt_tokens.norms": 14,
+ "@timestamp.norms": 14,
+ "cache_hit.norms": 14,
+ "top_doc.norms": 14,
+ "intent.norms": 14,
+ "doc_id.norms": 14,
+ "completion_tokens.norms": 14,
+ "cost_usd.norms": 14,
+ "context_tokens.norms": 14,
+ "status.norms": 14,
+ "tenant.norms": 14,
+ "latency_ms.norms": 14,
+ "model.norms": 14,
+ "owner.norms": 14,
+ "seed_revision.norms": 14,
+ "updated_at.norms": 14,
+ "section.norms": 14,
+ "time_default.norms": 14,
+ "group.norms": 14,
+ "kind.norms": 14,
+ "name.norms": 14,
+ "filters_default.norms": 14,
+ "visibility.norms": 14,
+ "org_id.norms": 14,
+ "deleted_at.norms": 14,
+ "managed.norms": 14,
+ "created_at.norms": 14,
+ "cloned_from.norms": 14,
+ "etag.norms": 14,
+ "version.norms": 14,
+ "etag.post": 0,
+ "time_default.post": 0,
+ "deleted_at.post": 0,
+ "cloned_from.post": 0
+ }
+}
\ No newline at end of file
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 2aa689705..64ae34ad8 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -341,6 +341,38 @@ const META_MAGIC_V3: &[u8; 4] = b"ZFM3";
/// `Vec<u8>` ZFM3 already populates, so the lookup hot path is
/// completely unchanged.
const META_MAGIC_V4: &[u8; 4] = b"ZFM4";
+/// ZFM5 = the ZFM4 records section replaced by four columnar varint
+/// streams before the Zstd envelope. ZFM4 kept the fixed 24-byte
+/// `{df u32, ttf u64, off u64, len u32}` record stride; Zstd cannot
+/// match across that stride's alternating widths, so on the 100 k-doc
+/// harness segment `doc_id.meta` (100 k unique terms, every one
+/// `ttf == df`, every postings gap zero) still cost 138.7 KB on disk
+/// for ~200 B of information. ZFM5 stores, per term, only varints of
+/// what actually varies:
+///
+/// ```text
+/// df stream num_terms × uvarint(doc_frequency)
+/// [ttf stream] num_terms × uvarint(ttf − df) — elided when every
+/// term in the field has ttf == df (keyword-shaped fields)
+/// gap stream num_terms × uvarint(off − prev_off − prev_len)
+/// len stream num_terms × uvarint(postings_length)
+/// ```
+///
+/// Measured on that segment the `.meta` family drops 199,390 → ~13,062 B
+/// (−93.4 %), and the win scales with term cardinality, which is exactly
+/// the shape of the 66.5 M-doc nginx battle's id fields. Peer pattern:
+/// tantivy's `TermInfoValueWriter::serialize_block`
+/// (`src/termdict/sstable_termdict/mod.rs:92-105`, MIT — postings-peers
+/// corpus) writes block-leading absolute range starts once and then only
+/// per-term `VInt(doc_freq)` + `VInt(range.len)`, reconstructing offsets
+/// by accumulation. ZFM5 keeps an explicit gap varint rather than
+/// assuming contiguity — one byte per term pre-compression, and the
+/// decoder never has to trust that every writer packed tightly.
+///
+/// On read the streams are expanded once at open time into the same
+/// in-memory `flat_records` 24-byte-stride `Vec<u8>` ZFM3/ZFM4 build, so
+/// the FST byte-offset lookup hot path is completely unchanged.
+const META_MAGIC_V5: &[u8; 4] = b"ZFM5";
/// Postings file wrapped in a whole-file LZ4 envelope — magic prefix
/// lets the reader auto-detect and decompress while legacy `.post`
/// files (no prefix) continue to work via the raw mmap path.
@@ -380,6 +412,9 @@ const ZFM3_RECORD_LEN: usize = 4 + 8 + 8 + 4; // 24 bytes: df, ttf, off, len
/// + num_terms + has_positions).
const ZFM3_HEADER_LEN: usize = 4 + 8 + 8 + 4 + 1;
+/// Header byte length for a ZFM5 file (ZFM3 header + the `flags` byte).
+const ZFM5_HEADER_LEN: usize = ZFM3_HEADER_LEN + 1;
+
/// Encode a `.meta` file in the ZFM3 flat format.
///
/// `sorted_terms` must be the same term-sorted order used to insert
@@ -405,6 +440,10 @@ const ZFM3_HEADER_LEN: usize = 4 + 8 + 8 + 4 + 1;
/// compressed_len u32 (= len(zstd_payload), tail follows)
/// zstd_payload compressed_len bytes
/// ```
+/// Kept for the test fixtures that need a real on-disk ZFM4 segment —
+/// the writer default moved to ZFM5, and the ZFM4 decode arm (upgrade
+/// path for segments written between M4.7 and ZFM5) must stay exercised.
+#[cfg(test)]
fn encode_field_meta_v4(
stats: &FieldStats,
has_positions: bool,
@@ -449,11 +488,194 @@ fn encode_field_meta_v4(
Ok(out)
}
+/// `flags` bit assignments for the ZFM5 header.
+const ZFM5_FLAG_TTF_STREAM: u8 = 0x01;
+
+/// Encode a `.meta` file in the ZFM5 format — see [`META_MAGIC_V5`].
+///
+/// Same header contract as ZFM4 (`num_terms`, `uncompressed_len` sanity
+/// check) so the decode side can share its validation, with one extra
+/// `flags` byte recording whether the ttf stream is present.
+fn encode_field_meta_v5(
+ stats: &FieldStats,
+ has_positions: bool,
+ sorted_terms: &[String],
+ term_postings: &HashMap<String, TermPostings>,
+ zstd_level: i32,
+) -> Result<Vec<u8>> {
+ use crate::postings::uvarint_encode;
+ let num_terms = sorted_terms.len();
+ // The ttf stream is all-or-nothing per field, decided up front: a
+ // per-term "needs an entry" decision made in scan order would leave
+ // the terms before the first ttf != df term without their zero
+ // entries, and the reader splits sections by fixed counts.
+ let ttf_stream_needed = sorted_terms.iter().any(|term| {
+ let tp = term_postings
+ .get(term)
+ .expect("sorted_terms must match term_postings keys");
+ if tp.total_term_frequency < tp.doc_frequency as u64 {
+ panic!("total_term_frequency below doc_frequency for '{term}'");
+ }
+ tp.total_term_frequency != tp.doc_frequency as u64
+ });
+ let flags = if ttf_stream_needed {
+ ZFM5_FLAG_TTF_STREAM
+ } else {
+ 0u8
+ };
+ let mut dfs: Vec<u8> = Vec::with_capacity(num_terms);
+ let mut ttf_deltas: Vec<u8> = Vec::new();
+ let mut gaps: Vec<u8> = Vec::with_capacity(num_terms);
+ let mut lens: Vec<u8> = Vec::with_capacity(num_terms);
+ let mut prev_end = 0u64;
+ for term in sorted_terms {
+ let tp = term_postings
+ .get(term)
+ .expect("sorted_terms must match term_postings keys");
+ uvarint_encode(tp.doc_frequency as u64, &mut dfs);
+ if ttf_stream_needed {
+ uvarint_encode(
+ tp.total_term_frequency - tp.doc_frequency as u64,
+ &mut ttf_deltas,
+ );
+ }
+ if tp.postings_offset < prev_end {
+ return Err(anyhow::anyhow!(
+ "ZFM5 requires monotone postings offsets (term '{term}' at {} after {})",
+ tp.postings_offset,
+ prev_end
+ ));
+ }
+ uvarint_encode(tp.postings_offset - prev_end, &mut gaps);
+ uvarint_encode(tp.postings_length as u64, &mut lens);
+ prev_end = tp.postings_offset + tp.postings_length as u64;
+ }
+ let mut streams = dfs;
+ streams.extend_from_slice(&ttf_deltas);
+ streams.extend_from_slice(&gaps);
+ streams.extend_from_slice(&lens);
+ let compressed =
+ zstd::bulk::compress(&streams, zstd_level).with_context(|| "ZFM5 zstd compress")?;
+ let uncompressed_len = (num_terms * ZFM3_RECORD_LEN) as u32;
+ let mut out: Vec<u8> = Vec::with_capacity(ZFM5_HEADER_LEN + 8 + compressed.len());
+ out.extend_from_slice(META_MAGIC_V5);
+ out.write_u64::<LittleEndian>(stats.total_docs).unwrap();
+ out.write_u64::<LittleEndian>(stats.total_field_length)
+ .unwrap();
+ out.write_u32::<LittleEndian>(num_terms as u32).unwrap();
+ out.push(if has_positions { 1u8 } else { 0u8 });
+ out.push(flags);
+ out.write_u32::<LittleEndian>(uncompressed_len).unwrap();
+ out.write_u32::<LittleEndian>(compressed.len() as u32)
+ .unwrap();
+ out.extend_from_slice(&compressed);
+ Ok(out)
+}
+
fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
use std::io::Cursor;
if bytes.len() < 4 {
return Err(anyhow::anyhow!("field meta: too short"));
}
+ // ZFM5 path — columnar varint streams under the Zstd envelope,
+ // expanded once at open time into the ZFM3 flat-record layout.
+ if &bytes[..4] == META_MAGIC_V5 {
+ use crate::postings::uvarint_decode;
+ // Header + flags + the two trailing length u32s
+ let zfm5_prefix = ZFM5_HEADER_LEN + 4 + 4;
+ if bytes.len() < zfm5_prefix {
+ return Err(anyhow::anyhow!("field meta: ZFM5 truncated header"));
+ }
+ let mut cur = Cursor::new(&bytes[4..zfm5_prefix]);
+ let total_docs = cur.read_u64::<LittleEndian>()?;
+ let total_field_length = cur.read_u64::<LittleEndian>()?;
+ let num_terms = cur.read_u32::<LittleEndian>()? as usize;
+ let has_positions = cur.read_u8()? != 0;
+ let flags = cur.read_u8()?;
+ let uncompressed_len = cur.read_u32::<LittleEndian>()? as usize;
+ let compressed_len = cur.read_u32::<LittleEndian>()? as usize;
+ let expected_uncompressed = num_terms
+ .checked_mul(ZFM3_RECORD_LEN)
+ .ok_or_else(|| anyhow::anyhow!("field meta: ZFM5 num_terms overflow ({num_terms})"))?;
+ if uncompressed_len != expected_uncompressed {
+ return Err(anyhow::anyhow!(
+ "field meta: ZFM5 length mismatch (uncompressed_len={uncompressed_len}, num_terms*24={expected_uncompressed})"
+ ));
+ }
+ if bytes.len() != zfm5_prefix + compressed_len {
+ return Err(anyhow::anyhow!(
+ "field meta: ZFM5 payload length mismatch (expected {compressed_len} bytes, got {})",
+ bytes.len() - zfm5_prefix
+ ));
+ }
+ let streams = zstd::bulk::decompress(
+ &bytes[zfm5_prefix..],
+ // A u64 varint is at most 10 bytes, so 4 sections bound the
+ // streams at 40 B/term; anything larger is corruption.
+ num_terms.saturating_mul(4 * 10).max(64),
+ )
+ .with_context(|| "ZFM5 zstd decompress")?;
+ let mut scur = Cursor::new(streams.as_slice());
+ let mut read_section = |count: usize| -> Result<Vec<u64>> {
+ (0..count)
+ .map(|_| {
+ uvarint_decode(&mut scur).map_err(|error| {
+ anyhow::anyhow!("field meta: ZFM5 stream truncated ({error})")
+ })
+ })
+ .collect()
+ };
+ let dfs = read_section(num_terms)?;
+ let ttf_deltas = if flags & ZFM5_FLAG_TTF_STREAM != 0 {
+ read_section(num_terms)?
+ } else {
+ vec![0u64; num_terms]
+ };
+ let gap_section = read_section(num_terms)?;
+ let len_section = read_section(num_terms)?;
+ if scur.position() as usize != streams.len() {
+ return Err(anyhow::anyhow!(
+ "field meta: ZFM5 trailing stream bytes ({} of {})",
+ streams.len() - scur.position() as usize,
+ streams.len()
+ ));
+ }
+ let mut flat_records: Vec<u8> = Vec::with_capacity(expected_uncompressed);
+ let mut offset = 0u64;
+ for i in 0..num_terms {
+ let df = u32::try_from(dfs[i])
+ .map_err(|_| anyhow::anyhow!("field meta: ZFM5 df exceeds u32"))?;
+ let ttf = df as u64 + ttf_deltas[i];
+ offset = offset
+ .checked_add(gap_section[i])
+ .ok_or_else(|| anyhow::anyhow!("field meta: ZFM5 offset overflow"))?;
+ let len = u32::try_from(len_section[i])
+ .map_err(|_| anyhow::anyhow!("field meta: ZFM5 postings_length exceeds u32"))?;
+ flat_records.write_u32::<LittleEndian>(df).unwrap();
+ flat_records.write_u64::<LittleEndian>(ttf).unwrap();
+ flat_records.write_u64::<LittleEndian>(offset).unwrap();
+ flat_records.write_u32::<LittleEndian>(len).unwrap();
+ offset = offset
+ .checked_add(len as u64)
+ .ok_or_else(|| anyhow::anyhow!("field meta: ZFM5 offset overflow"))?;
+ }
+ if flat_records.len() != expected_uncompressed {
+ return Err(anyhow::anyhow!(
+ "field meta: ZFM5 record expansion mismatch (got {}, expected {expected_uncompressed})",
+ flat_records.len()
+ ));
+ }
+ return Ok(FieldMeta {
+ stats: FieldStats {
+ total_docs,
+ total_field_length,
+ },
+ terms: HashMap::new(),
+ has_positions,
+ fst_value_format: FstValueFormat::MetaByteOffset,
+ flat_records,
+ });
+ }
// ZFM4 path — Zstd-compressed records section. Decompresses
// once at open time into the same `flat_records` Vec<u8> ZFM3
// populates, so the per-query lookup path is unchanged.
@@ -1371,12 +1593,13 @@ impl FtsIndexWriter {
xerj_common::fsio::replace_file_durable(&fst_tmp, &fst_path)
.with_context(|| format!("publishing FST {:?}", fst_path))?;
- // 4. Write meta in the ZFM4 format — Zstd-19 envelope around
- // the per-term records section; ZFM3-compatible header so
- // `lookup_term` can decompress once at open time and use
- // the in-memory `flat_records` Vec exactly as before.
+ // 4. Write meta in the ZFM5 format — columnar varint streams
+ // (df, ttf-delta, gaps, lengths) under the Zstd envelope;
+ // ZFM3-compatible header so `lookup_term` can expand once at
+ // open time and use the in-memory `flat_records` Vec exactly
+ // as before.
let has_positions = field_data.config.store_positions;
- let meta_bytes = encode_field_meta_v4(
+ let meta_bytes = encode_field_meta_v5(
&field_data.stats,
has_positions,
&sorted_terms,
@@ -2010,7 +2233,8 @@ impl FtsIndexReader {
&& (&meta_bytes[..4] == META_MAGIC_V1
|| &meta_bytes[..4] == META_MAGIC_V2
|| &meta_bytes[..4] == META_MAGIC_V3
- || &meta_bytes[..4] == META_MAGIC_V4);
+ || &meta_bytes[..4] == META_MAGIC_V4
+ || &meta_bytes[..4] == META_MAGIC_V5);
let meta: FieldMeta = if is_binary {
decode_field_meta_binary(&meta_bytes)
.with_context(|| "parsing binary field meta")?
@@ -2735,6 +2959,207 @@ mod tests {
assert!(data.is_some() && !data.unwrap().is_empty());
}
+ /// The current writer stamps the `ZFM5` meta format, and lookups
+ /// round-trip through the expanded flat records: a repeated term
+ /// (ttf > df) exercises the ttf stream, a keyword field with all
+ /// ttf == df exercises the elided-stream path, and both must agree
+ /// with the postings envelope's own view of the data.
+ #[test]
+ fn zfm5_envelope_written_and_lookup_roundtrips() {
+ let dir = TempDir::new().unwrap();
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg-zfm5", make_registry());
+ writer.configure_field(
+ "title",
+ FieldIndexConfig {
+ analyzer: "whitespace".to_owned(),
+ ..Default::default()
+ },
+ );
+ writer.configure_field(
+ "tag",
+ FieldIndexConfig {
+ analyzer: "keyword".to_owned(),
+ ..Default::default()
+ },
+ );
+ let doc = |title: &str, tag: &str| -> HashMap<String, FieldValues> {
+ vec![
+ ("title".to_owned(), FieldValues::from(title)),
+ ("tag".to_owned(), FieldValues::from(tag)),
+ ]
+ .into_iter()
+ .collect()
+ };
+ writer.add_document(0, &doc("hello hello world", "alpha"));
+ writer.add_document(1, &doc("hello rust", "beta"));
+ writer.finish().unwrap();
+
+ let meta = fs::read(dir.path().join("seg-zfm5.title.meta")).unwrap();
+ assert_eq!(&meta[..4], META_MAGIC_V5, "writer must stamp ZFM5");
+
+ let reader = FtsIndexReader::open(dir.path(), "seg-zfm5", &["title", "tag"]).unwrap();
+ let tp = reader.lookup_term("title", "hello").unwrap();
+ assert_eq!(tp.doc_frequency, 2);
+ assert_eq!(tp.total_term_frequency, 3); // ttf stream carries 3 − 2
+ let data = reader.postings_data("title", &tp).unwrap();
+ assert!(!data.is_empty());
+ let tp = reader.lookup_term("tag", "alpha").unwrap();
+ assert_eq!(tp.doc_frequency, 1);
+ assert_eq!(tp.total_term_frequency, 1); // elided stream ⇒ df + 0
+ assert!(reader.postings_data("tag", &tp).is_some());
+ }
+
+ /// Stream-level properties: the ttf stream is elided only when every
+ /// term has ttf == df, expansion reproduces the ZFM3 record layout
+ /// byte-for-byte, and non-contiguous postings gaps reconstruct
+ /// exactly. The elided file must not be larger than the streamed one.
+ #[test]
+ fn zfm5_stream_elision_and_record_expansion_match_zfm3_layout() {
+ let stats = FieldStats {
+ total_docs: 9,
+ total_field_length: 40,
+ };
+ // `a_extra`/`c_extra` are ttf−df deltas: (0, 0) keeps every term
+ // at ttf == df (the elided shape); any non-zero forces the stream.
+ let mk = |a_ttf: u64, c_extra: u64| {
+ let mut map = HashMap::new();
+ // Non-contiguous on purpose: gaps of 0, 7 and 90 bytes.
+ map.insert(
+ "a".to_owned(),
+ TermPostings {
+ doc_frequency: 3,
+ total_term_frequency: 3 + a_ttf,
+ postings_offset: 100,
+ postings_length: 10,
+ },
+ );
+ map.insert(
+ "b".to_owned(),
+ TermPostings {
+ doc_frequency: 1,
+ total_term_frequency: 1,
+ postings_offset: 117,
+ postings_length: 20,
+ },
+ );
+ map.insert(
+ "c".to_owned(),
+ TermPostings {
+ doc_frequency: 0x2000_0001, // forces a 5-byte varint lane
+ total_term_frequency: 0x2000_0001 + c_extra,
+ postings_offset: 200,
+ postings_length: 5,
+ },
+ );
+ map
+ };
+ let terms = ["a".to_owned(), "b".to_owned(), "c".to_owned()];
+ let elided = encode_field_meta_v5(&stats, true, &terms, &mk(0, 0), 3).unwrap();
+ let streamed = encode_field_meta_v5(&stats, true, &terms, &mk(1, 0), 3).unwrap();
+ assert_eq!(&elided[..4], META_MAGIC_V5);
+ // flags byte sits right after has_positions.
+ assert_eq!(elided[25] & ZFM5_FLAG_TTF_STREAM, 0, "ttf==df must elide");
+ assert_eq!(streamed[25] & ZFM5_FLAG_TTF_STREAM, ZFM5_FLAG_TTF_STREAM);
+ assert!(elided.len() <= streamed.len());
+
+ let decoded = decode_field_meta_binary(&elided).unwrap();
+ let mut expected: Vec<u8> = Vec::new();
+ for tp in [&mk(0, 0)["a"], &mk(0, 0)["b"], &mk(0, 0)["c"]] {
+ expected
+ .write_u32::<LittleEndian>(tp.doc_frequency)
+ .unwrap();
+ expected
+ .write_u64::<LittleEndian>(tp.total_term_frequency)
+ .unwrap();
+ expected
+ .write_u64::<LittleEndian>(tp.postings_offset)
+ .unwrap();
+ expected
+ .write_u32::<LittleEndian>(tp.postings_length)
+ .unwrap();
+ }
+ assert_eq!(decoded.flat_records, expected);
+ assert_eq!(decoded.stats.total_docs, 9);
+ assert!(decoded.has_positions);
+ }
+
+ /// Malformed input fails closed: truncated header, wrong
+ /// uncompressed_len, trailing payload, trailing stream varints, and
+ /// a missing section. The encoder likewise refuses non-monotone
+ /// offsets rather than silently clamping the gap.
+ #[test]
+ fn zfm5_rejects_malformed_and_non_monotone() {
+ let stats = FieldStats {
+ total_docs: 2,
+ total_field_length: 4,
+ };
+ let mut map = HashMap::new();
+ map.insert(
+ "a".to_owned(),
+ TermPostings {
+ doc_frequency: 1,
+ total_term_frequency: 1,
+ postings_offset: 10,
+ postings_length: 4,
+ },
+ );
+ let terms = ["a".to_owned()];
+ let good = encode_field_meta_v5(&stats, false, &terms, &map, 3).unwrap();
+
+ // (prev_end = 10 + 4 = 14) > 0 — offset must not go backwards.
+ let backwards = {
+ let mut m = map.clone();
+ m.get_mut("a").unwrap().postings_offset = 10;
+ m.insert(
+ "b".to_owned(),
+ TermPostings {
+ doc_frequency: 1,
+ total_term_frequency: 1,
+ postings_offset: 5,
+ postings_length: 1,
+ },
+ );
+ encode_field_meta_v5(&stats, false, &["a".to_owned(), "b".to_owned()], &m, 3)
+ };
+ assert!(backwards.is_err());
+
+ assert!(decode_field_meta_binary(&good[..good.len() - 1]).is_err());
+ let mut bad_len = good.clone();
+ bad_len[26..30].copy_from_slice(&999u32.to_le_bytes()); // uncompressed_len
+ assert!(decode_field_meta_binary(&bad_len).is_err());
+ let mut trailing = good.clone();
+ trailing.push(0);
+ assert!(decode_field_meta_binary(&trailing).is_err());
+
+ // Hand-assembled stream sections: one extra varint ⇒ trailing bytes.
+ use crate::postings::uvarint_encode;
+ let assemble = |flags: u8, streams: &[u8]| {
+ let compressed = zstd::bulk::compress(streams, 3).unwrap();
+ let mut out = Vec::new();
+ out.extend_from_slice(META_MAGIC_V5);
+ out.write_u64::<LittleEndian>(2).unwrap();
+ out.write_u64::<LittleEndian>(4).unwrap();
+ out.write_u32::<LittleEndian>(1).unwrap();
+ out.push(0);
+ out.push(flags);
+ out.write_u32::<LittleEndian>(24).unwrap();
+ out.write_u32::<LittleEndian>(compressed.len() as u32)
+ .unwrap();
+ out.extend_from_slice(&compressed);
+ out
+ };
+ let mut extra = Vec::new();
+ uvarint_encode(1, &mut extra); // df
+ uvarint_encode(10, &mut extra); // gap
+ uvarint_encode(4, &mut extra); // len
+ uvarint_encode(9, &mut extra); // stray
+ assert!(decode_field_meta_binary(&assemble(0, &extra)).is_err());
+ let mut missing = Vec::new();
+ uvarint_encode(1, &mut missing);
+ uvarint_encode(10, &mut missing); // len stream absent
+ assert!(decode_field_meta_binary(&assemble(0, &missing)).is_err());
+ }
+
/// The current writer stamps the `ZPS2` envelope, and the reader
/// factory threads the codec that implies: a positioned term with
/// ttf == df decodes with synthesised freqs (its freq stream was
diff --git a/engine/crates/xerj-fts/src/postings.rs b/engine/crates/xerj-fts/src/postings.rs
index 71759b23a..0d04b3e0a 100644
--- a/engine/crates/xerj-fts/src/postings.rs
+++ b/engine/crates/xerj-fts/src/postings.rs
@@ -683,6 +683,45 @@ pub fn vbyte_decode(cursor: &mut Cursor<&[u8]>) -> io::Result<u32> {
}
}
+/// Encodes `value` as a variable-byte `u64` into `buf`.
+///
+/// Same convention as [`vbyte_encode`] (high bit set on the final byte) so
+/// the pair is self-consistent; this is the wider lane the `.meta` ZFM5
+/// record streams need, where `total_term_frequency` and `postings_offset`
+/// exceed the `u32` vbyte range on multi-GB `.post` files.
+pub fn uvarint_encode(mut value: u64, buf: &mut Vec<u8>) {
+ loop {
+ let byte = (value & 0x7F) as u8;
+ value >>= 7;
+ if value == 0 {
+ buf.push(byte | 0x80); // high bit = last byte
+ break;
+ } else {
+ buf.push(byte);
+ }
+ }
+}
+
+/// Decodes a variable-byte `u64` from `cursor`.
+pub fn uvarint_decode(cursor: &mut Cursor<&[u8]>) -> io::Result<u64> {
+ let mut result = 0u64;
+ let mut shift = 0u32;
+ loop {
+ let byte = cursor.read_u8()?;
+ result |= ((byte & 0x7F) as u64) << shift;
+ if byte & 0x80 != 0 {
+ return Ok(result);
+ }
+ shift += 7;
+ if shift >= 70 {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "uvarint overflow",
+ ));
+ }
+ }
+}
+
/// Decode one document's position group — a vbyte count followed by that
/// many delta-encoded positions.
fn decode_position_group(cursor: &mut Cursor<&[u8]>) -> io::Result<Vec<u32>> {