AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): enforce index.max_terms_count on terms queries (#2936)
Amartya Vikram Singh3bba497a55f4
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex b49c2b7b7..396e9aa13 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -6346,6 +6346,186 @@ fn rewrite_unqueryable_exists(q: &mut Value, state: &AppState, indices: &[Stringwalk(q, state, indices);}+/// ES's default for `index.max_terms_count` (`IndexSettings.MAX_TERMS_COUNT_SETTING`).+const DEFAULT_MAX_TERMS_COUNT: u64 = 65_536;++/// The first `terms` query whose value list is longer than+/// `index.max_terms_count`, with ES's message (`TermsQueryBuilder#doToQuery`).+///+/// ES counts the values each `terms` clause carries after a terms lookup has+/// been fetched, so this runs on the resolved body. The setting is per index+/// and ES checks it per shard: a request only fails outright when it exceeds+/// the limit of every target index, so the largest limit among the targets is+/// the one that decides. XERJ never read the setting, so an index configured+/// with a small limit accepted any number of terms, and the 65 536 default did+/// not bound a request either.+fn find_terms_query_over_limit(+ roots: &[Option<&Value>],+ state: &AppState,+ indices: &[&str],+) -> Option<String> {+ fn longest(v: &Value, out: &mut usize) {+ match v {+ Value::Object(obj) => {+ if let Some(Value::Object(terms)) = obj.get("terms") {+ for (field, values) in terms {+ if matches!(field.as_str(), "boost" | "_name") {+ continue;+ }+ if let Value::Array(values) = values {+ *out = (*out).max(values.len());+ }+ }+ }+ for child in obj.values() {+ longest(child, out);+ }+ }+ Value::Array(items) => {+ for item in items {+ longest(item, out);+ }+ }+ _ => {}+ }+ }+ let mut count = 0usize;+ for root in roots.iter().flatten() {+ longest(root, &mut count);+ }+ let limit = indices+ .iter()+ .map(|ix| {+ index_int_setting(state, ix, "max_terms_count").unwrap_or(DEFAULT_MAX_TERMS_COUNT)+ })+ .max()?;+ (count as u64 > limit).then(|| {+ format!(+ "The number of terms [{count}] used in the Terms Query request has exceeded the \+ allowed maximum of [{limit}]. This maximum can be set by changing the \+ [index.max_terms_count] index level setting."+ )+ })+}++#[cfg(test)]+mod max_terms_count_tests {+ use super::*;+ use axum::{body::Body, http::Request};+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ async fn send(app: &axum::Router, method: &str, uri: &str, body: Value) -> (StatusCode, Value) {+ let resp = app+ .clone()+ .oneshot(+ Request::builder()+ .method(method)+ .uri(uri)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = resp.status();+ let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap())+ }++ /// `small` allows two terms, `big` keeps the default.+ async fn app() -> axum::Router {+ let app = crate::router::build_es_compat_router(test_state());+ let mappings = json!({"properties": {"user": {"type": "keyword"},+ "followers": {"type": "keyword"}}});+ let (status, body) = send(+ &app,+ "PUT",+ "/small",+ json!({"settings": {"index.max_terms_count": 2}, "mappings": mappings}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = send(&app, "PUT", "/big", json!({ "mappings": mappings })).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ for (id, followers) in [+ ("u1", json!(["u2", "u3"])),+ ("u2", json!(["u1", "u3", "u4"])),+ ] {+ let doc = json!({"user": id, "followers": followers});+ let (status, body) =+ send(&app, "PUT", &format!("/small/_doc/{id}?refresh=true"), doc).await;+ assert!(status.is_success(), "{body}");+ }+ app+ }++ fn terms(values: Value) -> Value {+ json!({"query": {"terms": {"user": values}}})+ }++ /// FAIL-BEFORE: every request here answered 200; `index.max_terms_count`+ /// was never read.+ #[tokio::test]+ async fn terms_over_the_index_limit_is_a_400() {+ let app = app().await;+ let lookup = json!({"index": "small", "id": "u2", "path": "followers"});+ for body in [+ terms(json!(["u1", "u2", "u3"])),+ // The limit applies to the values a lookup fetched (3 followers).+ terms(lookup),+ // ... and to a terms query nested in a bool and in a filter agg.+ json!({"query": {"bool": {"filter": [{"terms": {"user": ["a", "b", "c"]}}]}}}),+ json!({"size": 0, "aggs": {"f": {"filter": {"terms": {"user": ["a", "b", "c"]}}}}}),+ ] {+ let (status, resp) = send(&app, "POST", "/small/_search", body.clone()).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body} -> {resp}");+ assert_eq!(+ resp["error"]["root_cause"][0]["reason"],+ "The number of terms [3] used in the Terms Query request has exceeded the \+ allowed maximum of [2]. This maximum can be set by changing the \+ [index.max_terms_count] index level setting.",+ "{body}"+ );+ }+ }++ /// At the limit, on an index with the default limit, and across a target+ /// whose limit is higher, the request still answers.+ #[tokio::test]+ async fn terms_within_a_target_limit_still_answer() {+ let app = app().await;+ let lookup = json!({"index": "small", "id": "u1", "path": "followers"});+ for (uri, body) in [+ ("/small/_search", terms(json!(["u1", "u2"]))),+ ("/small/_search", terms(lookup)),+ ("/big/_search", terms(json!(["u1", "u2", "u3"]))),+ ("/small,big/_search", terms(json!(["u1", "u2", "u3"]))),+ ] {+ let (status, resp) = send(&app, "POST", uri, body.clone()).await;+ assert_eq!(status, StatusCode::OK, "{uri} {body} -> {resp}");+ }+ let (_, resp) = send(&app, "POST", "/small/_search", terms(json!(["u1", "u2"]))).await;+ assert_eq!(resp["hits"]["total"]["value"], 2, "{resp}");+ }+}+async fn resolve_terms_lookups(q: &mut Value, state: &AppState) {// First pass: collect every (index, id, path) triple keyed by the// (terms_obj, field) pair we'll mutate after the fetch.@@ -15126,6 +15306,18 @@ async fn search_impl(if let Some(aggs) = body.aggregations.as_mut() {resolve_terms_lookups(aggs, &state).await;}+ // `index.max_terms_count`, counted on the resolved lookups above.+ if let Some(reason) = find_terms_query_over_limit(+ &[+ body.query.as_ref(),+ body.aggs.as_ref(),+ body.aggregations.as_ref(),+ ],+ &state,+ &index_names,+ ) {+ return search_shard_validation_error(&reason);+ }// Rewrite `match` queries that target a keyword field into `term`// queries. ES treats `match` on keyword fields as exact (the// keyword analyzer is a no-op), but our Match parser tokenizes thediff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txtindex 6180dc91a..3216203ae 100644--- a/engine/tests/es-compat-yaml/catch-lenient.txt+++ b/engine/tests/es-compat-yaml/catch-lenient.txt@@ -59,7 +59,6 @@ search/120_batch_reduce_size.yml · batched_reduce_size lower limitsearch/140_pre_filter_search_shards.yml · pre_filter_shard_size with invalid parametersearch/150_rewrite_on_coordinator.yml · Ensure that we fetch the document only oncesearch/160_exists_query.yml · Test exists query on _source field-search/170_terms_query.yml · Terms Query with No.of terms exceeding index.max_terms_count should FAILsearch/220_total_hits_object.yml · hits.total as an objectsearch/260_parameter_validation.yml · test negative size throws IAEsearch/260_parameter_validation.yml · test negative terminate_after throws IAE