Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api,fts,engine): search timeout actually fires — parse body/param timeout + cooperative FTS-expansion deadline (RC4 blocker 12)

xerj-orgcc89ecf14c3d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 5020e1fc3..b650fc25b 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -2030,6 +2030,15 @@ pub struct EsSearchBody {
/// `_seq_no` at-or-below the snapshot taken at PIT open time.
#[serde(default)]
pub pit: Option<Value>,
+ /// Request-level timeout in ES time-units (`"100ms"`, `"2s"`, bare
+ /// number = millis). Parsed into `SearchRequest.timeout_ms` — the
+ /// wall-clock deadline every cooperative check in the engine polls.
+ /// RC4 blocker 12: this field did not exist, so `{"timeout": "1ms"}`
+ /// was silently dropped by serde, `timeout_ms` stayed `None` (30 s
+ /// engine default), and the deadline machinery was never armed
+ /// (live: a 1 ms-timeout wildcard ran 3.7 s with `timed_out: false`).
+ #[serde(default)]
+ pub timeout: Option<String>,
}
impl Default for EsSearchBody {
@@ -2064,6 +2073,7 @@ impl Default for EsSearchBody {
version: None,
slice: None,
pit: None,
+ timeout: None,
}
}
}
@@ -3679,6 +3689,38 @@ fn default_size() -> usize {
}
/// Build a `SearchRequest` from the ES body, forwarding all relevant options.
+/// Parse an ES time-units string to milliseconds: `"500ms"`, `"30s"`,
+/// `"2m"`, `"1h"`, `"1d"`, `"5micros"`, `"100nanos"`, bare number =
+/// millis. Sub-millisecond values round up to 1 ms so `timeout=100nanos`
+/// still arms a (minimal) deadline instead of none at all.
+fn parse_time_to_millis(s: &str) -> Option<u64> {
+ let s = s.trim();
+ if s.is_empty() {
+ return None;
+ }
+ if let Some(n) = s.strip_suffix("nanos") {
+ let v: u64 = n.parse().ok()?;
+ return Some((v / 1_000_000).max(1));
+ }
+ if let Some(n) = s.strip_suffix("micros") {
+ let v: u64 = n.parse().ok()?;
+ return Some((v / 1_000).max(1));
+ }
+ if let Some(n) = s.strip_suffix("ms") {
+ let v: u64 = n.parse().ok()?;
+ return Some(v.max(1));
+ }
+ let (digits, mul_ms) = match s.as_bytes().last()? {
+ b's' => (&s[..s.len() - 1], 1_000u64),
+ b'm' => (&s[..s.len() - 1], 60_000),
+ b'h' => (&s[..s.len() - 1], 3_600_000),
+ b'd' => (&s[..s.len() - 1], 86_400_000),
+ _ => (s, 1), // bare number → milliseconds (ES convention)
+ };
+ let n: u64 = digits.parse().ok()?;
+ Some(n.saturating_mul(mul_ms).max(1))
+}
+
fn build_search_request(
body: &EsSearchBody,
aggs_value: Option<Value>,
@@ -3709,6 +3751,14 @@ fn build_search_request(
req.from = body.from;
req.explain = body.explain;
+ // Request-level timeout (RC4 blocker 12): arm the engine's
+ // cooperative wall-clock deadline. Unparsable values are ignored
+ // (engine default 30 s) rather than 400 — matching our lenient
+ // handling of other ES time-unit params.
+ if req.timeout_ms.is_none() {
+ req.timeout_ms = body.timeout.as_deref().and_then(parse_time_to_millis);
+ }
+
// Forward aggs.
if req.aggs.is_none() {
req.aggs = aggs_value;
@@ -4339,6 +4389,12 @@ pub struct EsSearchQueryParams {
pub search_type: Option<String>,
pub scroll: Option<String>,
pub track_total_hits: Option<String>,
+ /// Request-level search timeout (`?timeout=100ms`) — same semantics
+ /// as the body's `timeout` (body wins when both are set). RC4
+ /// blocker 12: previously this param (and the body field) were
+ /// silently dropped, so the engine's cooperative deadline was never
+ /// armed and `timeout` could not fire.
+ pub timeout: Option<String>,
/// ES 7+ compat: when true, return `hits.total` as a bare integer
/// instead of `{value, relation}`. Many ES YAML tests set this.
pub rest_total_hits_as_int: Option<String>,
@@ -4523,6 +4579,11 @@ pub async fn search(
}
}
}
+ // `?timeout=100ms` → promote to body (body wins when both set) so
+ // build_search_request arms the cooperative deadline (RC4 blocker 12).
+ if body.timeout.is_none() {
+ body.timeout = params.timeout.clone();
+ }
// ── Scroll support ──────────────────────────────────────────────────────
// If `?scroll=1m` is present, snapshot all matching hits into a scroll
// context and return the first page + `_scroll_id`. Subsequent pages
@@ -12924,6 +12985,7 @@ pub async fn search_with_scroll(
version: body.version,
slice: body.slice.clone(),
pit: body.pit.clone(),
+ timeout: body.timeout.clone(),
};
// page_size: what the caller requested (or default 10)
let page_size = body.size;
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index e6433cfc7..2efeb2fa2 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -6961,6 +6961,11 @@ impl Index {
// Try FTS path first if we have an FTS query.
let mut fts_handled = false;
+ // Set when the FTS searcher's cooperative deadline fired
+ // mid-expansion (RC4 blocker 12) — its hits/total for this
+ // segment are partial and the response must say
+ // `timed_out: true`.
+ let mut fts_deadline_tripped = false;
let t_fts = std::time::Instant::now();
let reader_opt = if needs_fts {
FtsIndexReader::open(&fts_dir, &seg_id, &field_refs).ok()
@@ -6970,8 +6975,17 @@ impl Index {
dbg_fts_ms += t_fts.elapsed().as_millis() as u64;
if let Some(reader) = reader_opt {
let reader = Arc::new(reader);
+ // Arm the cooperative deadline (RC4 blocker 12): the
+ // searcher's multi-term expansions (prefix / wildcard /
+ // fuzzy term-dictionary walks) are otherwise
+ // uninterruptible units that the outer tokio timeout
+ // can never preempt (`block_in_place` completes the
+ // whole search on first poll — live: `timeout: 1ms`
+ // wildcard ran 3.7 s / fuzzy 8.3 s with
+ // `timed_out: false`).
let searcher =
- FtsSearcher::new(Arc::clone(&reader), Arc::clone(&self.registry));
+ FtsSearcher::new(Arc::clone(&reader), Arc::clone(&self.registry))
+ .with_deadline(Some(search_deadline));
let fts_query = query_node_to_fts(query, &text_fields, &exact_fields);
// Count-only fast path for single-term FTS queries:
@@ -7051,9 +7065,14 @@ impl Index {
materialisation_limit
};
if fts_has_field {
- if let Ok((seg_hits, seg_total)) =
- searcher.search_bounded(&fq, fts_cap, false)
- {
+ let bounded = searcher.search_bounded(&fq, fts_cap, false);
+ // Record a deadline trip regardless of Ok/Err —
+ // the expansion may have stopped early either
+ // way (RC4 blocker 12).
+ if searcher.deadline_tripped() {
+ fts_deadline_tripped = true;
+ }
+ if let Ok((seg_hits, seg_total)) = bounded {
// FTS is authoritative for this segment (field
// present + searcher ran) — mark handled the same
// way for count-only and size>0 so the fall-through
@@ -7319,6 +7338,15 @@ impl Index {
} // close fts_has_field
}
+ // Propagate an FTS expansion deadline trip into the
+ // response-level flag (RC4 blocker 12): the segment's FTS
+ // hits/total above are partial, so the response must carry
+ // `timed_out: true` even if this was the last segment (no
+ // later between-segment check would run).
+ if fts_deadline_tripped {
+ deadline_exceeded = true;
+ }
+
// If FTS didn't handle it, fall through to stored-doc scan.
// This covers MatchAll (no FTS query), term-level queries
// that are routed through doc scan, AND segments missing
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 919c3a266..37d96f771 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -1276,6 +1276,45 @@ impl FtsIndexReader {
.map(|idx| loaded.norms[idx].1)
}
+ /// Stream every term of `field` through `f` in lexicographic order,
+ /// stopping as soon as `f` returns `false`.
+ ///
+ /// Unlike [`Self::all_terms`] this never materialises the dictionary —
+ /// multi-term expansion (prefix/wildcard/fuzzy) walks dictionaries with
+ /// millions of entries, and building a `Vec<String>` first both doubles
+ /// the walk cost and makes the whole enumeration an uninterruptible
+ /// unit (RC4 blocker 12: search `timeout` could never fire during
+ /// expansion). The callback's `bool` return is the cooperative
+ /// cancellation hook.
+ pub fn for_each_term<F: FnMut(&str) -> bool>(&self, field: &str, mut f: F) {
+ let Some(loaded) = self.fields.get(field) else {
+ return;
+ };
+ use fst::Streamer;
+ match &loaded.fst {
+ FstData::Mmap(m) => {
+ let mut stream = m.stream();
+ while let Some((key, _)) = stream.next() {
+ if let Ok(s) = std::str::from_utf8(key) {
+ if !f(s) {
+ return;
+ }
+ }
+ }
+ }
+ FstData::Owned(m) => {
+ let mut stream = m.stream();
+ while let Some((key, _)) = stream.next() {
+ if let Ok(s) = std::str::from_utf8(key) {
+ if !f(s) {
+ return;
+ }
+ }
+ }
+ }
+ }
+ }
+
/// Enumerate all terms in a field (lexicographic order, for debugging/admin).
pub fn all_terms(&self, field: &str) -> Vec<String> {
let Some(loaded) = self.fields.get(field) else {
diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rs
index 4cb2dcf39..c472b5381 100644
--- a/engine/crates/xerj-fts/src/search.rs
+++ b/engine/crates/xerj-fts/src/search.rs
@@ -424,11 +424,57 @@ pub struct FtsSearcher {
/// Kept for future use: query-time analysis (e.g., synonym expansion, normalization).
#[allow(dead_code)]
registry: Arc<AnalyzerRegistry>,
+ /// Cooperative wall-clock deadline (RC4 blocker 12). Multi-term
+ /// expansion (prefix/wildcard/fuzzy) walks the WHOLE term dictionary
+ /// — O(unique terms) predicate probes, multi-second on wide
+ /// vocabularies — and the engine runs searches to completion inside
+ /// `block_in_place`, so the outer `tokio::time::timeout` can never
+ /// interrupt it. When set, the expansion loops poll this every 1 024
+ /// terms and stop early, latching [`Self::deadline_tripped`] so the
+ /// caller returns partial results with `timed_out: true` (ES shard
+ /// timeout semantics).
+ deadline: Option<std::time::Instant>,
+ /// Set when any cooperative deadline check fired — results from this
+ /// searcher are partial. Atomic (not `Cell`) so the searcher stays
+ /// `Sync`.
+ deadline_tripped: std::sync::atomic::AtomicBool,
}
impl FtsSearcher {
pub fn new(reader: Arc<FtsIndexReader>, registry: Arc<AnalyzerRegistry>) -> Self {
- Self { reader, registry }
+ Self {
+ reader,
+ registry,
+ deadline: None,
+ deadline_tripped: std::sync::atomic::AtomicBool::new(false),
+ }
+ }
+
+ /// Builder: arm the cooperative deadline. `None` disables (default).
+ pub fn with_deadline(mut self, deadline: Option<std::time::Instant>) -> Self {
+ self.deadline = deadline;
+ self
+ }
+
+ /// True when any expansion loop stopped early because the deadline
+ /// passed — the hits/totals produced by this searcher are partial.
+ pub fn deadline_tripped(&self) -> bool {
+ self.deadline_tripped
+ .load(std::sync::atomic::Ordering::Relaxed)
+ }
+
+ /// Poll the deadline (called every N iterations, not every item):
+ /// true once the deadline has passed, latching `deadline_tripped`.
+ #[inline]
+ fn deadline_hit(&self) -> bool {
+ match self.deadline {
+ Some(d) if std::time::Instant::now() >= d => {
+ self.deadline_tripped
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ true
+ }
+ _ => false,
+ }
}
/// Execute a query and return up to `limit` scored hits, sorted by descending score.
@@ -707,6 +753,11 @@ impl FtsSearcher {
let mut score_map: std::collections::HashMap<u32, (f32, Option<QueryExplanation>)> =
std::collections::HashMap::new();
for exp in &expansions {
+ // Cooperative deadline (RC4 blocker 12): each expansion runs a
+ // full positional phrase query — poll per expansion.
+ if self.deadline_hit() {
+ break;
+ }
let mut terms: Vec<String> = head.to_vec();
terms.push(exp.clone());
let pq = PhraseQuery {
@@ -750,13 +801,25 @@ impl FtsSearcher {
}
fn expand_prefix(&self, field: &str, prefix: &str, max: usize) -> Vec<String> {
- // Use the reader's all_terms which does an FST stream scan
- let all_terms = self.reader.all_terms(field);
- all_terms
- .into_iter()
- .filter(|t| t.starts_with(prefix))
- .take(max)
- .collect()
+ // Streaming FST scan with a cooperative deadline poll every 1 024
+ // terms (RC4 blocker 12) — a wide dictionary made the old
+ // materialise-then-filter walk a multi-second uninterruptible unit.
+ let mut out: Vec<String> = Vec::new();
+ let mut seen = 0usize;
+ self.reader.for_each_term(field, |t| {
+ seen += 1;
+ if seen & 1023 == 0 && self.deadline_hit() {
+ return false;
+ }
+ if t.starts_with(prefix) {
+ out.push(t.to_owned());
+ if out.len() >= max {
+ return false;
+ }
+ }
+ true
+ });
+ out
}
// ── Wildcard query ──────────────────────────────────────────────────────────
@@ -777,20 +840,36 @@ impl FtsSearcher {
/// the standard analyzer and ES does not lowercase the pattern, so an
/// uppercase pattern correctly matches nothing.
fn expand_wildcard(&self, field: &str, pattern: &str, case_insensitive: bool) -> Vec<String> {
+ // Streaming FST scan + deadline poll every 1 024 terms (RC4
+ // blocker 12): the per-term wildcard walk over a wide dictionary
+ // was the reproduced 3.7 s uninterruptible unit.
+ let mut out: Vec<String> = Vec::new();
+ let mut seen = 0usize;
if case_insensitive {
let pat_lc = pattern.to_lowercase();
- self.reader
- .all_terms(field)
- .into_iter()
- .filter(|t| term_matches_wildcard(t, &pat_lc))
- .collect()
+ self.reader.for_each_term(field, |t| {
+ seen += 1;
+ if seen & 1023 == 0 && self.deadline_hit() {
+ return false;
+ }
+ if term_matches_wildcard(t, &pat_lc) {
+ out.push(t.to_owned());
+ }
+ true
+ });
} else {
- self.reader
- .all_terms(field)
- .into_iter()
- .filter(|t| wildcard_match(t, pattern))
- .collect()
+ self.reader.for_each_term(field, |t| {
+ seen += 1;
+ if seen & 1023 == 0 && self.deadline_hit() {
+ return false;
+ }
+ if wildcard_match(t, pattern) {
+ out.push(t.to_owned());
+ }
+ true
+ });
}
+ out
}
// ── Fuzzy query ─────────────────────────────────────────────────────────────
@@ -817,20 +896,36 @@ impl FtsSearcher {
max_edits: usize,
case_insensitive: bool,
) -> Vec<String> {
+ // Streaming FST scan + deadline poll every 1 024 terms (RC4
+ // blocker 12): per-term edit distance is the most expensive
+ // expansion predicate — the reproduced 8.3 s uninterruptible unit.
+ let mut out: Vec<String> = Vec::new();
+ let mut seen = 0usize;
if case_insensitive {
let q_lower = value.to_lowercase();
- self.reader
- .all_terms(field)
- .into_iter()
- .filter(|t| term_matches_fuzzy(t, &q_lower, max_edits))
- .collect()
+ self.reader.for_each_term(field, |t| {
+ seen += 1;
+ if seen & 1023 == 0 && self.deadline_hit() {
+ return false;
+ }
+ if term_matches_fuzzy(t, &q_lower, max_edits) {
+ out.push(t.to_owned());
+ }
+ true
+ });
} else {
- self.reader
- .all_terms(field)
- .into_iter()
- .filter(|t| levenshtein_distance(t, value) <= max_edits)
- .collect()
+ self.reader.for_each_term(field, |t| {
+ seen += 1;
+ if seen & 1023 == 0 && self.deadline_hit() {
+ return false;
+ }
+ if levenshtein_distance(t, value) <= max_edits {
+ out.push(t.to_owned());
+ }
+ true
+ });
}
+ out
}
/// Run one term query per expanded term and merge their per-doc scores — the
@@ -854,6 +949,12 @@ impl FtsSearcher {
if constant_score {
let mut docs: std::collections::HashSet<u32> = std::collections::HashSet::new();
for term in terms {
+ // Cooperative deadline (RC4 blocker 12): a pathological
+ // pattern can expand to hundreds of thousands of terms,
+ // each with its own postings walk — poll per term.
+ if self.deadline_hit() {
+ break;
+ }
let tp = match self.reader.lookup_term(field, term) {
Some(tp) => tp,
None => continue,
@@ -881,6 +982,11 @@ impl FtsSearcher {
let mut score_map: std::collections::HashMap<u32, (f32, Option<QueryExplanation>)> =
std::collections::HashMap::new();
for term in terms {
+ // Cooperative deadline (RC4 blocker 12) — see the
+ // constant_score arm above.
+ if self.deadline_hit() {
+ break;
+ }
let tq = TermQuery::boosted(field, term, boost);
let term_hits = self.execute_term(&tq, explain)?;
for hit in term_hits {
@@ -1726,6 +1832,54 @@ mod tests {
);
}
+ #[test]
+ fn wildcard_expansion_respects_cooperative_deadline() {
+ // RC4 blocker 12 regression: multi-term expansion must stop at the
+ // cooperative deadline and latch `deadline_tripped` so the engine
+ // returns partial results with `timed_out: true`. Pre-fix the
+ // expansion walked the entire dictionary regardless of the request
+ // timeout (live: `timeout: 1ms` wildcard ran 3.7 s, fuzzy 8.3 s,
+ // both `timed_out: false`).
+ let dir = TempDir::new().unwrap();
+ let registry = Arc::new(AnalyzerRegistry::default());
+ let mut writer = FtsIndexWriter::new(dir.path(), "seg0", Arc::clone(&registry));
+ let cfg = FieldIndexConfig {
+ analyzer: "whitespace".to_owned(),
+ ..Default::default()
+ };
+ writer.configure_field("body", cfg);
+ // 5 000 unique terms — enough that the every-1 024-terms poll fires.
+ let text: String = (0..5000)
+ .map(|i| format!("term{i:05}"))
+ .collect::<Vec<_>>()
+ .join(" ");
+ writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());
+ writer.finish().unwrap();
+ let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());
+
+ let mut wq = WildcardQuery::new("body", "term*");
+ wq.case_insensitive = false;
+ let q = Query::Wildcard(wq);
+
+ // Unarmed searcher (default): full expansion, flag stays clear.
+ let searcher = FtsSearcher::new(Arc::clone(&reader), Arc::clone(&registry));
+ let hits = searcher.search(&q, 10, false).unwrap();
+ assert_eq!(hits.len(), 1, "the one doc matches via full expansion");
+ assert!(!searcher.deadline_tripped(), "no deadline → never tripped");
+
+ // Deadline already in the past: expansion stops at the first poll,
+ // the search still returns Ok (graceful partial, not an error), and
+ // the trip is latched for the caller to surface as timed_out:true.
+ let expired = FtsSearcher::new(reader, registry).with_deadline(Some(
+ std::time::Instant::now() - std::time::Duration::from_millis(1),
+ ));
+ let _ = expired.search(&q, 10, false).unwrap();
+ assert!(
+ expired.deadline_tripped(),
+ "expired deadline must latch deadline_tripped"
+ );
+ }
+
#[test]
fn phrase_query_nonexistent_returns_empty() {
let dir = TempDir::new().unwrap();