AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): skip tombstone-only segments in agg corpus (#161)
Leonid Bugaev47027db75a06
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 5e2140df4..ecf495e38 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -15845,6 +15845,17 @@ impl Index {.collect();let snap_bg = self.store.snapshot();for seg in &snap_bg.segments {+ // A tombstone-only segment is an intentional delete journal:+ // it has `doc_count == 0` and no Stored section to hydrate.+ // Its deletions are already represented in the version map,+ // while this pass only assembles source documents for the+ // aggregation corpus. Trying to load it makes the valid+ // absence of Stored look like an unreadable segment and turns+ // every fallback aggregation after a persisted delete into a+ // 500 response.+ if seg.doc_count == 0 {+ continue;+ }// Cooperative deadline: the full-corpus assembly is the// most expensive path in the engine (decompress + parse// EVERY stored doc). Stop pulling further segments oncediff --git a/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rs b/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rsnew file mode 100644index 000000000..b5f0c7908--- /dev/null+++ b/engine/crates/xerj-engine/tests/tombstone_only_segment_search.rs@@ -0,0 +1,115 @@+//! Search regressions for delete-only (`doc_count == 0`) segments.+//!+//! A persisted delete has no stored documents. Its segment intentionally+//! contains only the tombstone section, so query paths must not try to hydrate+//! a `Stored` section from it.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn seed_delete_only_segment(engine: &Engine, name: &str) {+ engine.create_index(name, Schema::empty()).unwrap();+ let index = engine.get_index(name).unwrap();++ index+ .index_document(Some("live".into()), json!({"group": "kept", "amount": 10}))+ .await+ .unwrap();+ index+ .index_document(+ Some("deleted".into()),+ json!({"group": "kept", "amount": 999}),+ )+ .await+ .unwrap();+ index.flush().await.unwrap();++ assert!(index.delete_document("deleted").await.unwrap());+ index.flush().await.unwrap();++ assert!(+ index+ .store_snapshot()+ .segments+ .iter()+ .any(|segment| segment.doc_count == 0 && segment.has_tombstones),+ "test setup must publish a tombstone-only segment"+ );+}++async fn assert_filtered_aggregation_works(index: &Index) {+ let unfiltered = parse_request(&json!({+ "query": {"match_all": {}},+ "track_total_hits": true+ }))+ .unwrap();+ let result = index+ .search(&unfiltered)+ .await+ .expect("unfiltered search must succeed");+ assert_eq!(result.total.value, 1);+ assert_eq!(result.hits.len(), 1);+ assert_eq!(result.hits[0].id, "live");++ // A filtered aggregation enters the JSON full-corpus fallback before the+ // later doc-values fallback can answer. Before the fix that corpus walk+ // attempted to hydrate the tombstone-only segment and returned+ // `segment ... unreadable during corpus assembly`.+ let request = parse_request(&json!({+ "size": 0,+ "track_total_hits": true,+ "query": {"terms": {"group": ["kept"]}},+ "aggs": {+ "amount_min": {"min": {"field": "amount"}},+ "amount_max": {"max": {"field": "amount"}}+ }+ }))+ .unwrap();++ let result = index.search(&request).await.expect("search must succeed");+ assert_eq!(result.total.value, 1);+ let aggs = result.aggs.expect("aggregations");+ assert_eq!(aggs["amount_min"]["value"], json!(10.0));+ assert_eq!(aggs["amount_max"]["value"], json!(10.0));+}++#[tokio::test]+async fn filtered_aggregation_skips_live_tombstone_only_segment() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed_delete_only_segment(&engine, "live-delete").await;++ let index = engine.get_index("live-delete").unwrap();+ assert_filtered_aggregation_works(&index).await;+}++#[tokio::test]+async fn filtered_aggregation_skips_reopened_tombstone_only_segment() {+ let dir = TempDir::new().unwrap();+ {+ let engine = make_engine(&dir);+ seed_delete_only_segment(&engine, "reopened-delete").await;+ }++ let reopened = make_engine(&dir);+ let index = reopened.get_index("reopened-delete").unwrap();+ assert!(+ index+ .store_snapshot()+ .segments+ .iter()+ .any(|segment| segment.doc_count == 0 && segment.has_tombstones),+ "restart must retain the tombstone-only segment"+ );+ assert_filtered_aggregation_works(&index).await;+}