Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): skip tombstone-only segments in agg corpus (#161)

Leonid Bugaev47027db75a06
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 5e2140df4..ecf495e38 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -15845,6 +15845,17 @@ impl Index {
.collect();
let snap_bg = self.store.snapshot();
for seg in &snap_bg.segments {
+ // A tombstone-only segment is an intentional delete journal:
+ // it has `doc_count == 0` and no Stored section to hydrate.
+ // Its deletions are already represented in the version map,
+ // while this pass only assembles source documents for the
+ // aggregation corpus. Trying to load it makes the valid
+ // absence of Stored look like an unreadable segment and turns
+ // every fallback aggregation after a persisted delete into a
+ // 500 response.
+ if seg.doc_count == 0 {
+ continue;
+ }
// Cooperative deadline: the full-corpus assembly is the
// most expensive path in the engine (decompress + parse
// EVERY stored doc). Stop pulling further segments once
diff --git a/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rs b/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rs
new file mode 100644
index 000000000..b5f0c7908
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rs
@@ -0,0 +1,115 @@
+//! Search regressions for delete-only (`doc_count == 0`) segments.
+//!
+//! A persisted delete has no stored documents. Its segment intentionally
+//! contains only the tombstone section, so query paths must not try to hydrate
+//! a `Stored` section from it.
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+async fn seed_delete_only_segment(engine: &Engine, name: &str) {
+ engine.create_index(name, Schema::empty()).unwrap();
+ let index = engine.get_index(name).unwrap();
+
+ index
+ .index_document(Some("live".into()), json!({"group": "kept", "amount": 10}))
+ .await
+ .unwrap();
+ index
+ .index_document(
+ Some("deleted".into()),
+ json!({"group": "kept", "amount": 999}),
+ )
+ .await
+ .unwrap();
+ index.flush().await.unwrap();
+
+ assert!(index.delete_document("deleted").await.unwrap());
+ index.flush().await.unwrap();
+
+ assert!(
+ index
+ .store_snapshot()
+ .segments
+ .iter()
+ .any(|segment| segment.doc_count == 0 && segment.has_tombstones),
+ "test setup must publish a tombstone-only segment"
+ );
+}
+
+async fn assert_filtered_aggregation_works(index: &Index) {
+ let unfiltered = parse_request(&json!({
+ "query": {"match_all": {}},
+ "track_total_hits": true
+ }))
+ .unwrap();
+ let result = index
+ .search(&unfiltered)
+ .await
+ .expect("unfiltered search must succeed");
+ assert_eq!(result.total.value, 1);
+ assert_eq!(result.hits.len(), 1);
+ assert_eq!(result.hits[0].id, "live");
+
+ // A filtered aggregation enters the JSON full-corpus fallback before the
+ // later doc-values fallback can answer. Before the fix that corpus walk
+ // attempted to hydrate the tombstone-only segment and returned
+ // `segment ... unreadable during corpus assembly`.
+ let request = parse_request(&json!({
+ "size": 0,
+ "track_total_hits": true,
+ "query": {"terms": {"group": ["kept"]}},
+ "aggs": {
+ "amount_min": {"min": {"field": "amount"}},
+ "amount_max": {"max": {"field": "amount"}}
+ }
+ }))
+ .unwrap();
+
+ let result = index.search(&request).await.expect("search must succeed");
+ assert_eq!(result.total.value, 1);
+ let aggs = result.aggs.expect("aggregations");
+ assert_eq!(aggs["amount_min"]["value"], json!(10.0));
+ assert_eq!(aggs["amount_max"]["value"], json!(10.0));
+}
+
+#[tokio::test]
+async fn filtered_aggregation_skips_live_tombstone_only_segment() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed_delete_only_segment(&engine, "live-delete").await;
+
+ let index = engine.get_index("live-delete").unwrap();
+ assert_filtered_aggregation_works(&index).await;
+}
+
+#[tokio::test]
+async fn filtered_aggregation_skips_reopened_tombstone_only_segment() {
+ let dir = TempDir::new().unwrap();
+ {
+ let engine = make_engine(&dir);
+ seed_delete_only_segment(&engine, "reopened-delete").await;
+ }
+
+ let reopened = make_engine(&dir);
+ let index = reopened.get_index("reopened-delete").unwrap();
+ assert!(
+ index
+ .store_snapshot()
+ .segments
+ .iter()
+ .any(|segment| segment.doc_count == 0 && segment.has_tombstones),
+ "restart must retain the tombstone-only segment"
+ );
+ assert_filtered_aggregation_works(&index).await;
+}