AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): wire field-encoding read path for /v1/indices/:name/encodings
xerj-org2f7b1a18feb6
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex ef1b8ac59..5cf45d156 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -1285,13 +1285,57 @@ impl ShardedFtsMemtable {}/// Aggregated smart-field-encoding map across all shards.+ ///+ /// Computed lazily on the READ side from the doc-values `keyword`+ /// columns already maintained per shard — the same bounded-read pattern+ /// used by `all_keyword_values_for_field` / the `keyword_counts` fold.+ /// The ingest hot path only appends raw column values (it no longer runs+ /// `FieldAnalyzer` — the ingest-time `collect_sample` was dropped during+ /// the M4 perf pass, which is why `analyzed_encodings` is always empty),+ /// so this adds ZERO per-doc cost: the analysis happens here, once, when+ /// a caller (the `/v1/indices/:name/encodings` stats endpoint) asks for+ /// it.+ ///+ /// For each field we merge every shard's non-`None` keyword values into a+ /// single sample vector (capped at `ENCODING_SAMPLE_CAP` to bound work on+ /// a large memtable), then — for each field carrying at least+ /// `ENCODING_MIN_SAMPLES` values — run `FieldAnalyzer` to pick its optimal+ /// encoding. Read-only: it never touches the columns or the ingest path.pub fn aggregated_field_encodings(&self) -> HashMap<String, FieldEncoding> {- let mut out: HashMap<String, FieldEncoding> = HashMap::new();+ /// Upper bound on values analyzed per field — keeps this read bounded+ /// on a multi-hundred-k-doc memtable while staying statistically ample.+ const ENCODING_SAMPLE_CAP: usize = 4096;+ /// Minimum values a field needs before we report an encoding for it;+ /// below this the cardinality tiers aren't statistically meaningful.+ const ENCODING_MIN_SAMPLES: usize = 16;++ // Merge each shard's keyword columns into one capped sample vector+ // per field, under the shard READ lock.+ let mut samples: HashMap<String, Vec<String>> = HashMap::new();for s in &self.shards {let g = s.read();- for (k, v) in g.analyzed_field_encodings().iter() {- out.entry(k.clone()).or_insert_with(|| v.clone());+ for (field, col) in g.doc_values.keyword.iter() {+ let acc = samples.entry(field.clone()).or_default();+ if acc.len() >= ENCODING_SAMPLE_CAP {+ continue;+ }+ for v in col.iter().flatten() {+ acc.push(v.clone());+ if acc.len() >= ENCODING_SAMPLE_CAP {+ break;+ }+ }+ }+ }++ let analyzer = FieldAnalyzer::default();+ let mut out: HashMap<String, FieldEncoding> = HashMap::new();+ for (field, vals) in samples {+ if vals.len() < ENCODING_MIN_SAMPLES {+ continue;}+ let refs: Vec<&str> = vals.iter().map(|s| s.as_str()).collect();+ out.insert(field.clone(), analyzer.analyze(&field, &refs));}out}diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 6a7f256ba..e120df04d 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -4295,7 +4295,6 @@ async fn test_registry_apply_settings() {/// field analyzer auto-detects encodings and produces meaningful compression/// ratios.#[tokio::test]-#[ignore = "collect_sample was removed from push_field during M4 perf opt — field_encodings not populated"]async fn test_smart_field_encoding_apache_logs() {let dir = TempDir::new().unwrap();let engine = make_engine(&dir);