AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api,fts,engine): search timeout actually fires — parse body/param timeout + cooperative FTS-expansion deadline (RC4 blocker 12)
xerj-orgcc89ecf14c3d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 5020e1fc3..b650fc25b 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -2030,6 +2030,15 @@ pub struct EsSearchBody {/// `_seq_no` at-or-below the snapshot taken at PIT open time.#[serde(default)]pub pit: Option<Value>,+ /// Request-level timeout in ES time-units (`"100ms"`, `"2s"`, bare+ /// number = millis). Parsed into `SearchRequest.timeout_ms` — the+ /// wall-clock deadline every cooperative check in the engine polls.+ /// RC4 blocker 12: this field did not exist, so `{"timeout": "1ms"}`+ /// was silently dropped by serde, `timeout_ms` stayed `None` (30 s+ /// engine default), and the deadline machinery was never armed+ /// (live: a 1 ms-timeout wildcard ran 3.7 s with `timed_out: false`).+ #[serde(default)]+ pub timeout: Option<String>,}impl Default for EsSearchBody {@@ -2064,6 +2073,7 @@ impl Default for EsSearchBody {version: None,slice: None,pit: None,+ timeout: None,}}}@@ -3679,6 +3689,38 @@ fn default_size() -> usize {}/// Build a `SearchRequest` from the ES body, forwarding all relevant options.+/// Parse an ES time-units string to milliseconds: `"500ms"`, `"30s"`,+/// `"2m"`, `"1h"`, `"1d"`, `"5micros"`, `"100nanos"`, bare number =+/// millis. Sub-millisecond values round up to 1 ms so `timeout=100nanos`+/// still arms a (minimal) deadline instead of none at all.+fn parse_time_to_millis(s: &str) -> Option<u64> {+ let s = s.trim();+ if s.is_empty() {+ return None;+ }+ if let Some(n) = s.strip_suffix("nanos") {+ let v: u64 = n.parse().ok()?;+ return Some((v / 1_000_000).max(1));+ }+ if let Some(n) = s.strip_suffix("micros") {+ let v: u64 = n.parse().ok()?;+ return Some((v / 1_000).max(1));+ }+ if let Some(n) = s.strip_suffix("ms") {+ let v: u64 = n.parse().ok()?;+ return Some(v.max(1));+ }+ let (digits, mul_ms) = match s.as_bytes().last()? {+ b's' => (&s[..s.len() - 1], 1_000u64),+ b'm' => (&s[..s.len() - 1], 60_000),+ b'h' => (&s[..s.len() - 1], 3_600_000),+ b'd' => (&s[..s.len() - 1], 86_400_000),+ _ => (s, 1), // bare number → milliseconds (ES convention)+ };+ let n: u64 = digits.parse().ok()?;+ Some(n.saturating_mul(mul_ms).max(1))+}+fn build_search_request(body: &EsSearchBody,aggs_value: Option<Value>,@@ -3709,6 +3751,14 @@ fn build_search_request(req.from = body.from;req.explain = body.explain;+ // Request-level timeout (RC4 blocker 12): arm the engine's+ // cooperative wall-clock deadline. Unparsable values are ignored+ // (engine default 30 s) rather than 400 — matching our lenient+ // handling of other ES time-unit params.+ if req.timeout_ms.is_none() {+ req.timeout_ms = body.timeout.as_deref().and_then(parse_time_to_millis);+ }+// Forward aggs.if req.aggs.is_none() {req.aggs = aggs_value;@@ -4339,6 +4389,12 @@ pub struct EsSearchQueryParams {pub search_type: Option<String>,pub scroll: Option<String>,pub track_total_hits: Option<String>,+ /// Request-level search timeout (`?timeout=100ms`) — same semantics+ /// as the body's `timeout` (body wins when both are set). RC4+ /// blocker 12: previously this param (and the body field) were+ /// silently dropped, so the engine's cooperative deadline was never+ /// armed and `timeout` could not fire.+ pub timeout: Option<String>,/// ES 7+ compat: when true, return `hits.total` as a bare integer/// instead of `{value, relation}`. Many ES YAML tests set this.pub rest_total_hits_as_int: Option<String>,@@ -4523,6 +4579,11 @@ pub async fn search(}}}+ // `?timeout=100ms` → promote to body (body wins when both set) so+ // build_search_request arms the cooperative deadline (RC4 blocker 12).+ if body.timeout.is_none() {+ body.timeout = params.timeout.clone();+ }// ── Scroll support ──────────────────────────────────────────────────────// If `?scroll=1m` is present, snapshot all matching hits into a scroll// context and return the first page + `_scroll_id`. Subsequent pages@@ -12924,6 +12985,7 @@ pub async fn search_with_scroll(version: body.version,slice: body.slice.clone(),pit: body.pit.clone(),+ timeout: body.timeout.clone(),};// page_size: what the caller requested (or default 10)let page_size = body.size;diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex e6433cfc7..2efeb2fa2 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -6961,6 +6961,11 @@ impl Index {// Try FTS path first if we have an FTS query.let mut fts_handled = false;+ // Set when the FTS searcher's cooperative deadline fired+ // mid-expansion (RC4 blocker 12) — its hits/total for this+ // segment are partial and the response must say+ // `timed_out: true`.+ let mut fts_deadline_tripped = false;let t_fts = std::time::Instant::now();let reader_opt = if needs_fts {FtsIndexReader::open(&fts_dir, &seg_id, &field_refs).ok()@@ -6970,8 +6975,17 @@ impl Index {dbg_fts_ms += t_fts.elapsed().as_millis() as u64;if let Some(reader) = reader_opt {let reader = Arc::new(reader);+ // Arm the cooperative deadline (RC4 blocker 12): the+ // searcher's multi-term expansions (prefix / wildcard /+ // fuzzy term-dictionary walks) are otherwise+ // uninterruptible units that the outer tokio timeout+ // can never preempt (`block_in_place` completes the+ // whole search on first poll — live: `timeout: 1ms`+ // wildcard ran 3.7 s / fuzzy 8.3 s with+ // `timed_out: false`).let searcher =- FtsSearcher::new(Arc::clone(&reader), Arc::clone(&self.registry));+ FtsSearcher::new(Arc::clone(&reader), Arc::clone(&self.registry))+ .with_deadline(Some(search_deadline));let fts_query = query_node_to_fts(query, &text_fields, &exact_fields);// Count-only fast path for single-term FTS queries:@@ -7051,9 +7065,14 @@ impl Index {materialisation_limit};if fts_has_field {- if let Ok((seg_hits, seg_total)) =- searcher.search_bounded(&fq, fts_cap, false)- {+ let bounded = searcher.search_bounded(&fq, fts_cap, false);+ // Record a deadline trip regardless of Ok/Err —+ // the expansion may have stopped early either+ // way (RC4 blocker 12).+ if searcher.deadline_tripped() {+ fts_deadline_tripped = true;+ }+ if let Ok((seg_hits, seg_total)) = bounded {// FTS is authoritative for this segment (field// present + searcher ran) — mark handled the same// way for count-only and size>0 so the fall-through@@ -7319,6 +7338,15 @@ impl Index {} // close fts_has_field}+ // Propagate an FTS expansion deadline trip into the+ // response-level flag (RC4 blocker 12): the segment's FTS+ // hits/total above are partial, so the response must carry+ // `timed_out: true` even if this was the last segment (no+ // later between-segment check would run).+ if fts_deadline_tripped {+ deadline_exceeded = true;+ }+// If FTS didn't handle it, fall through to stored-doc scan.// This covers MatchAll (no FTS query), term-level queries// that are routed through doc scan, AND segments missingdiff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex 919c3a266..37d96f771 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -1276,6 +1276,45 @@ impl FtsIndexReader {.map(|idx| loaded.norms[idx].1)}+ /// Stream every term of `field` through `f` in lexicographic order,+ /// stopping as soon as `f` returns `false`.+ ///+ /// Unlike [`Self::all_terms`] this never materialises the dictionary —+ /// multi-term expansion (prefix/wildcard/fuzzy) walks dictionaries with+ /// millions of entries, and building a `Vec<String>` first both doubles+ /// the walk cost and makes the whole enumeration an uninterruptible+ /// unit (RC4 blocker 12: search `timeout` could never fire during+ /// expansion). The callback's `bool` return is the cooperative+ /// cancellation hook.+ pub fn for_each_term<F: FnMut(&str) -> bool>(&self, field: &str, mut f: F) {+ let Some(loaded) = self.fields.get(field) else {+ return;+ };+ use fst::Streamer;+ match &loaded.fst {+ FstData::Mmap(m) => {+ let mut stream = m.stream();+ while let Some((key, _)) = stream.next() {+ if let Ok(s) = std::str::from_utf8(key) {+ if !f(s) {+ return;+ }+ }+ }+ }+ FstData::Owned(m) => {+ let mut stream = m.stream();+ while let Some((key, _)) = stream.next() {+ if let Ok(s) = std::str::from_utf8(key) {+ if !f(s) {+ return;+ }+ }+ }+ }+ }+ }+/// Enumerate all terms in a field (lexicographic order, for debugging/admin).pub fn all_terms(&self, field: &str) -> Vec<String> {let Some(loaded) = self.fields.get(field) else {diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rsindex 4cb2dcf39..c472b5381 100644--- a/engine/crates/xerj-fts/src/search.rs+++ b/engine/crates/xerj-fts/src/search.rs@@ -424,11 +424,57 @@ pub struct FtsSearcher {/// Kept for future use: query-time analysis (e.g., synonym expansion, normalization).#[allow(dead_code)]registry: Arc<AnalyzerRegistry>,+ /// Cooperative wall-clock deadline (RC4 blocker 12). Multi-term+ /// expansion (prefix/wildcard/fuzzy) walks the WHOLE term dictionary+ /// — O(unique terms) predicate probes, multi-second on wide+ /// vocabularies — and the engine runs searches to completion inside+ /// `block_in_place`, so the outer `tokio::time::timeout` can never+ /// interrupt it. When set, the expansion loops poll this every 1 024+ /// terms and stop early, latching [`Self::deadline_tripped`] so the+ /// caller returns partial results with `timed_out: true` (ES shard+ /// timeout semantics).+ deadline: Option<std::time::Instant>,+ /// Set when any cooperative deadline check fired — results from this+ /// searcher are partial. Atomic (not `Cell`) so the searcher stays+ /// `Sync`.+ deadline_tripped: std::sync::atomic::AtomicBool,}impl FtsSearcher {pub fn new(reader: Arc<FtsIndexReader>, registry: Arc<AnalyzerRegistry>) -> Self {- Self { reader, registry }+ Self {+ reader,+ registry,+ deadline: None,+ deadline_tripped: std::sync::atomic::AtomicBool::new(false),+ }+ }++ /// Builder: arm the cooperative deadline. `None` disables (default).+ pub fn with_deadline(mut self, deadline: Option<std::time::Instant>) -> Self {+ self.deadline = deadline;+ self+ }++ /// True when any expansion loop stopped early because the deadline+ /// passed — the hits/totals produced by this searcher are partial.+ pub fn deadline_tripped(&self) -> bool {+ self.deadline_tripped+ .load(std::sync::atomic::Ordering::Relaxed)+ }++ /// Poll the deadline (called every N iterations, not every item):+ /// true once the deadline has passed, latching `deadline_tripped`.+ #[inline]+ fn deadline_hit(&self) -> bool {+ match self.deadline {+ Some(d) if std::time::Instant::now() >= d => {+ self.deadline_tripped+ .store(true, std::sync::atomic::Ordering::Relaxed);+ true+ }+ _ => false,+ }}/// Execute a query and return up to `limit` scored hits, sorted by descending score.@@ -707,6 +753,11 @@ impl FtsSearcher {let mut score_map: std::collections::HashMap<u32, (f32, Option<QueryExplanation>)> =std::collections::HashMap::new();for exp in &expansions {+ // Cooperative deadline (RC4 blocker 12): each expansion runs a+ // full positional phrase query — poll per expansion.+ if self.deadline_hit() {+ break;+ }let mut terms: Vec<String> = head.to_vec();terms.push(exp.clone());let pq = PhraseQuery {@@ -750,13 +801,25 @@ impl FtsSearcher {}fn expand_prefix(&self, field: &str, prefix: &str, max: usize) -> Vec<String> {- // Use the reader's all_terms which does an FST stream scan- let all_terms = self.reader.all_terms(field);- all_terms- .into_iter()- .filter(|t| t.starts_with(prefix))- .take(max)- .collect()+ // Streaming FST scan with a cooperative deadline poll every 1 024+ // terms (RC4 blocker 12) — a wide dictionary made the old+ // materialise-then-filter walk a multi-second uninterruptible unit.+ let mut out: Vec<String> = Vec::new();+ let mut seen = 0usize;+ self.reader.for_each_term(field, |t| {+ seen += 1;+ if seen & 1023 == 0 && self.deadline_hit() {+ return false;+ }+ if t.starts_with(prefix) {+ out.push(t.to_owned());+ if out.len() >= max {+ return false;+ }+ }+ true+ });+ out}// ── Wildcard query ──────────────────────────────────────────────────────────@@ -777,20 +840,36 @@ impl FtsSearcher {/// the standard analyzer and ES does not lowercase the pattern, so an/// uppercase pattern correctly matches nothing.fn expand_wildcard(&self, field: &str, pattern: &str, case_insensitive: bool) -> Vec<String> {+ // Streaming FST scan + deadline poll every 1 024 terms (RC4+ // blocker 12): the per-term wildcard walk over a wide dictionary+ // was the reproduced 3.7 s uninterruptible unit.+ let mut out: Vec<String> = Vec::new();+ let mut seen = 0usize;if case_insensitive {let pat_lc = pattern.to_lowercase();- self.reader- .all_terms(field)- .into_iter()- .filter(|t| term_matches_wildcard(t, &pat_lc))- .collect()+ self.reader.for_each_term(field, |t| {+ seen += 1;+ if seen & 1023 == 0 && self.deadline_hit() {+ return false;+ }+ if term_matches_wildcard(t, &pat_lc) {+ out.push(t.to_owned());+ }+ true+ });} else {- self.reader- .all_terms(field)- .into_iter()- .filter(|t| wildcard_match(t, pattern))- .collect()+ self.reader.for_each_term(field, |t| {+ seen += 1;+ if seen & 1023 == 0 && self.deadline_hit() {+ return false;+ }+ if wildcard_match(t, pattern) {+ out.push(t.to_owned());+ }+ true+ });}+ out}// ── Fuzzy query ─────────────────────────────────────────────────────────────@@ -817,20 +896,36 @@ impl FtsSearcher {max_edits: usize,case_insensitive: bool,) -> Vec<String> {+ // Streaming FST scan + deadline poll every 1 024 terms (RC4+ // blocker 12): per-term edit distance is the most expensive+ // expansion predicate — the reproduced 8.3 s uninterruptible unit.+ let mut out: Vec<String> = Vec::new();+ let mut seen = 0usize;if case_insensitive {let q_lower = value.to_lowercase();- self.reader- .all_terms(field)- .into_iter()- .filter(|t| term_matches_fuzzy(t, &q_lower, max_edits))- .collect()+ self.reader.for_each_term(field, |t| {+ seen += 1;+ if seen & 1023 == 0 && self.deadline_hit() {+ return false;+ }+ if term_matches_fuzzy(t, &q_lower, max_edits) {+ out.push(t.to_owned());+ }+ true+ });} else {- self.reader- .all_terms(field)- .into_iter()- .filter(|t| levenshtein_distance(t, value) <= max_edits)- .collect()+ self.reader.for_each_term(field, |t| {+ seen += 1;+ if seen & 1023 == 0 && self.deadline_hit() {+ return false;+ }+ if levenshtein_distance(t, value) <= max_edits {+ out.push(t.to_owned());+ }+ true+ });}+ out}/// Run one term query per expanded term and merge their per-doc scores — the@@ -854,6 +949,12 @@ impl FtsSearcher {if constant_score {let mut docs: std::collections::HashSet<u32> = std::collections::HashSet::new();for term in terms {+ // Cooperative deadline (RC4 blocker 12): a pathological+ // pattern can expand to hundreds of thousands of terms,+ // each with its own postings walk — poll per term.+ if self.deadline_hit() {+ break;+ }let tp = match self.reader.lookup_term(field, term) {Some(tp) => tp,None => continue,@@ -881,6 +982,11 @@ impl FtsSearcher {let mut score_map: std::collections::HashMap<u32, (f32, Option<QueryExplanation>)> =std::collections::HashMap::new();for term in terms {+ // Cooperative deadline (RC4 blocker 12) — see the+ // constant_score arm above.+ if self.deadline_hit() {+ break;+ }let tq = TermQuery::boosted(field, term, boost);let term_hits = self.execute_term(&tq, explain)?;for hit in term_hits {@@ -1726,6 +1832,54 @@ mod tests {);}+ #[test]+ fn wildcard_expansion_respects_cooperative_deadline() {+ // RC4 blocker 12 regression: multi-term expansion must stop at the+ // cooperative deadline and latch `deadline_tripped` so the engine+ // returns partial results with `timed_out: true`. Pre-fix the+ // expansion walked the entire dictionary regardless of the request+ // timeout (live: `timeout: 1ms` wildcard ran 3.7 s, fuzzy 8.3 s,+ // both `timed_out: false`).+ let dir = TempDir::new().unwrap();+ let registry = Arc::new(AnalyzerRegistry::default());+ let mut writer = FtsIndexWriter::new(dir.path(), "seg0", Arc::clone(®istry));+ let cfg = FieldIndexConfig {+ analyzer: "whitespace".to_owned(),+ ..Default::default()+ };+ writer.configure_field("body", cfg);+ // 5 000 unique terms — enough that the every-1 024-terms poll fires.+ let text: String = (0..5000)+ .map(|i| format!("term{i:05}"))+ .collect::<Vec<_>>()+ .join(" ");+ writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());+ writer.finish().unwrap();+ let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());++ let mut wq = WildcardQuery::new("body", "term*");+ wq.case_insensitive = false;+ let q = Query::Wildcard(wq);++ // Unarmed searcher (default): full expansion, flag stays clear.+ let searcher = FtsSearcher::new(Arc::clone(&reader), Arc::clone(®istry));+ let hits = searcher.search(&q, 10, false).unwrap();+ assert_eq!(hits.len(), 1, "the one doc matches via full expansion");+ assert!(!searcher.deadline_tripped(), "no deadline → never tripped");++ // Deadline already in the past: expansion stops at the first poll,+ // the search still returns Ok (graceful partial, not an error), and+ // the trip is latched for the caller to surface as timed_out:true.+ let expired = FtsSearcher::new(reader, registry).with_deadline(Some(+ std::time::Instant::now() - std::time::Duration::from_millis(1),+ ));+ let _ = expired.search(&q, 10, false).unwrap();+ assert!(+ expired.deadline_tripped(),+ "expired deadline must latch deadline_tripped"+ );+ }+#[test]fn phrase_query_nonexistent_returns_empty() {let dir = TempDir::new().unwrap();