Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(engine): restore columnar fast-path tests after #1329 (the rest of the blast radius)

xerj-orgf4a6f3e18cd6
diff --git a/engine/crates/xerj-engine/tests/fast_aggs_exists_and_deletes.rs b/engine/crates/xerj-engine/tests/fast_aggs_exists_and_deletes.rs
index 40eeb01d8..efef7ad44 100644
--- a/engine/crates/xerj-engine/tests/fast_aggs_exists_and_deletes.rs
+++ b/engine/crates/xerj-engine/tests/fast_aggs_exists_and_deletes.rs
@@ -20,7 +20,7 @@
use serde_json::json;
use tempfile::TempDir;
use xerj_common::config::Config;
-use xerj_common::types::Schema;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
use xerj_engine::aggs::fast_path_aggs_served;
use xerj_engine::{Engine, Index};
use xerj_query::parse_request;
@@ -41,6 +41,29 @@ fn make_engine(dir: &TempDir) -> Engine {
Engine::new(config).expect("engine::new")
}
+/// A schema that DECLARES the given string fields as `keyword`, mirroring the
+/// real autoindex catalog mapping (`catalog.rs::catalog_mapping` declares
+/// `format`/`path`/`status`/`file_key` as keyword).
+///
+/// These tests stand in for that catalog but used `Schema::empty()`, letting
+/// their string fields (`extension`, `text`, `group`) be dynamically mapped.
+/// Since #1329 a dynamically mapped string is a `text` parent with NO
+/// doc-values (only its `.keyword` sub-field carries them, matching ES), and a
+/// `text` field never gets a doc-values column regardless of the flag
+/// (`FieldConfig::stored_doc_values` excludes `Text`). The columnar fast path
+/// — the terms executor and the `KwExists` arm alike — reads that doc-values
+/// column, so an inferred stand-in makes every "must serve columnarly"
+/// assertion bail to brute. Declaring the stand-ins keyword restores the
+/// column the catalog's real keyword fields have. Numeric fields (`v`, `i`)
+/// are unaffected and stay inferred.
+fn kw_schema(fields: &[&str]) -> Schema {
+ let mut s = Schema::empty();
+ for f in fields {
+ s.fields.push(FieldConfig::new(*f, FieldType::Keyword));
+ }
+ s
+}
+
async fn run(idx: &std::sync::Arc<Index>, body: serde_json::Value) -> serde_json::Value {
let req = parse_request(&body).unwrap();
let res = idx.search(&req).await.unwrap();
@@ -94,7 +117,7 @@ async fn exists_in_top_filter_matches_brute() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("existsq", Schema::empty()).unwrap();
+ engine.create_index("existsq", kw_schema(&["extension", "text"])).unwrap();
let idx = engine.get_index("existsq").unwrap();
seed_exists_shapes(&idx).await;
@@ -137,7 +160,7 @@ async fn exists_on_numeric_field_matches_brute() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("existsn", Schema::empty()).unwrap();
+ engine.create_index("existsn", kw_schema(&["group"])).unwrap();
let idx = engine.get_index("existsn").unwrap();
for i in 0..12_000u32 {
let mut doc = json!({ "group": if i % 2 == 0 { "a" } else { "b" } });
@@ -213,7 +236,7 @@ async fn unmerged_ghosts_still_bail() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("ghosts", Schema::empty()).unwrap();
+ engine.create_index("ghosts", kw_schema(&["group"])).unwrap();
let idx = engine.get_index("ghosts").unwrap();
for i in 0..12_000u32 {
idx.index_document(
@@ -265,7 +288,7 @@ async fn merged_delete_history_re_qualifies_for_fast_path() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("readmit", Schema::empty()).unwrap();
+ engine.create_index("readmit", kw_schema(&["extension"])).unwrap();
let idx = engine.get_index("readmit").unwrap();
for i in 0..12_000u32 {
idx.index_document(
@@ -303,8 +326,21 @@ async fn merged_delete_history_re_qualifies_for_fast_path() {
let q = json!({
"query": {
"bool": {
+ // A one-element `terms` SET, not a singular `term`: on a
+ // declared keyword field a singular `term` is deliberately
+ // kept in the FTS scoring tree (index.rs
+ // `query_node_to_fts_projected`, so a mixed bool can add its
+ // BM25 score), which makes `query_needs_fts` true and routes a
+ // size:0 agg to the brute path — never the columnar fast path
+ // this test asserts. The real finalize-verify window
+ // (`sync_executor::windowed_value_counts`) filters with a
+ // `terms` set over a digest window, so `terms` is also the
+ // production-faithful shape. A one-element set matches exactly
+ // the same docs as `term`. (Before #1329 `extension` was an
+ // inferred `text` field, classified differently, so the
+ // singular `term` happened to take the fast path.)
"filter": [
- { "term": { "extension": "css" } },
+ { "terms": { "extension": ["css"] } },
{ "exists": { "field": "i" } }
]
}
@@ -349,7 +385,7 @@ async fn object_valued_memtable_doc_counts_as_exists() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("objmem", Schema::empty()).unwrap();
+ engine.create_index("objmem", kw_schema(&["extension", "text"])).unwrap();
let idx = engine.get_index("objmem").unwrap();
for i in 0..12_000u32 {
idx.index_document(
diff --git a/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs b/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs
index 322769d09..0516be76b 100644
--- a/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs
+++ b/engine/crates/xerj-engine/tests/verify_window_terms_perf.rs
@@ -25,7 +25,7 @@ use serde_json::json;
use std::time::Instant;
use tempfile::TempDir;
use xerj_common::config::Config;
-use xerj_common::types::Schema;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
use xerj_engine::aggs::fast_path_aggs_served;
use xerj_engine::{Engine, Index};
use xerj_query::parse_request;
@@ -47,6 +47,25 @@ fn make_engine(dir: &TempDir) -> Engine {
Engine::new(config).expect("engine::new")
}
+/// Declare the exact-match fields this verify-shape reads as `keyword`, the
+/// way the real crawl's data index and the autoindex catalog declare theirs
+/// (`ax_file` is a content digest, `ax_dataset` a dataset name, `cve_id` an
+/// identifier — all keyword, never analysed text). These tests used
+/// `Schema::empty()`, so since #1329 those fields became `text` parents with
+/// NO doc-values column; the columnar count-lane (terms on `ax_file`) and the
+/// `KwExists` arm both read that column, so the window fell to the brute path
+/// and the "served columnar" assertions failed. `text` is declared keyword
+/// too so the `exists` arm has a column to read — a declared keyword carries
+/// no `ignore_above` by default, so the ~1 KB record body is still stored
+/// whole (presence is all `exists` reads).
+fn kw_schema(fields: &[&str]) -> Schema {
+ let mut s = Schema::empty();
+ for f in fields {
+ s.fields.push(FieldConfig::new(*f, FieldType::Keyword));
+ }
+ s
+}
+
async fn run(idx: &std::sync::Arc<Index>, body: serde_json::Value) -> serde_json::Value {
let req = parse_request(&body).unwrap();
let res = idx.search(&req).await.unwrap();
@@ -131,7 +150,7 @@ async fn count_lane_full_window_is_served_columnar() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("cves28", Schema::empty()).unwrap();
+ engine.create_index("cves28", kw_schema(&["ax_file", "ax_dataset", "cve_id", "text"])).unwrap();
let idx = engine.get_index("cves28").unwrap();
let digests = seed(&idx).await;
@@ -180,7 +199,7 @@ async fn count_lane_with_resident_ghosts_is_served_and_exact() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("cves28g", Schema::empty()).unwrap();
+ engine.create_index("cves28g", kw_schema(&["ax_file", "ax_dataset", "cve_id", "text"])).unwrap();
let idx = engine.get_index("cves28g").unwrap();
let digests = seed(&idx).await;
@@ -272,7 +291,7 @@ async fn doc_lane_window_fetch_is_bounded() {
let _sequential = ONE_AT_A_TIME.lock().await;
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("catalog", Schema::empty()).unwrap();
+ engine.create_index("catalog", kw_schema(&["file_key", "run_id", "ax_path"])).unwrap();
let idx = engine.get_index("catalog").unwrap();
// The catalog holds ONE doc per (digest, alias) — seed the same digests
// plus a run_id that matches everything (the resume-rewrote-everything