AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(es-yaml): enforce `catch:` expectations, ratchet today's 151 gaps (#1583)
xerj-org867d8506fed9
diff --git a/engine/tests/es-compat-yaml/catch-lenient.txt b/engine/tests/es-compat-yaml/catch-lenient.txtnew file mode 100644index 000000000..080b5f5d2--- /dev/null+++ b/engine/tests/es-compat-yaml/catch-lenient.txt@@ -0,0 +1,161 @@+# Cases whose `catch:` expectation XERJ does not meet yet (see `mod catch_lenient`+# in src/main.rs). Each one asks for an error that XERJ does not return: it+# answers 2xx, a different status, or an error whose body does not match.+#+# Before the runner checked `catch:` at all, every one of these passed+# vacuously. They are listed so the check can be enforced for every OTHER case+# today. This is a ratchet: fix the gap, delete its line. Never add a line to+# make a new failure go away; a new failure is a regression.+#+# Format: <path under yaml/> · <test name>+aggregations/avg_bucket.yml · bad path+aggregations/avg_metric.yml · Aggregating wrong datatype test+aggregations/bucket_script.yml · bad script+aggregations/bucket_script.yml · invalid parent aggregation+aggregations/bucket_script.yml · invalid path+aggregations/bucket_script.yml · top level fails+aggregations/bucket_selector.yml · bad script+aggregations/bucket_selector.yml · invalid parent aggregation+aggregations/bucket_selector.yml · invalid path+aggregations/bucket_selector.yml · top level fails+aggregations/bucket_sort.yml · invalid parent aggregation+aggregations/bucket_sort.yml · invalid path fails+aggregations/bucket_sort.yml · no action fails+aggregations/bucket_sort.yml · top level fails+aggregations/cardinality_metric.yml · invalid execution hint throws+aggregations/cardinality_metric.yml · Invalid Precision test+aggregations/composite.yml · Composite aggregation and array size+aggregations/composite.yml · Composite aggregation with invalid format+aggregations/composite.yml · Composite aggregation with lossy format+aggregations/composite.yml · Duplicate sources+aggregations/composite.yml · Invalid Composite aggregation+aggregations/composite.yml · missing_order with missing_bucket = false+aggregations/composite.yml · missing_order without missing_bucket+aggregations/composite.yml · Missing source+aggregations/cumulative_sum.yml · bad path+aggregations/cumulative_sum.yml · invalid parent aggregation+aggregations/derivative.yml · bad path+aggregations/derivative.yml · invalid parent aggregation+aggregations/derivative.yml · Not under date_histo+aggregations/extended_stats_metric.yml · Bad sigma test+aggregations/filters_bucket.yml · Bad params+aggregations/filters_bucket.yml · null meta+aggregations/ip_prefix.yml · Invalid IPv4 prefix+aggregations/ip_prefix.yml · Invalid IPv6 prefix+aggregations/matrix_stats_multi_value_field.yml · With script+aggregations/matrix_stats_multi_value_field.yml · With script params+aggregations/matrix_stats_single_value_field.yml · With script+aggregations/matrix_stats_single_value_field.yml · With script params+aggregations/max_buckets.yml · Max bucket+aggregations/max_bucket.yml · bad path+aggregations/max_metric.yml · Aggregating wrong datatype test+aggregations/median_absolute_deviation_metric.yml · bad arguments+aggregations/median_absolute_deviation_metric.yml · with execution hint+aggregations/min_bucket.yml · bad path+aggregations/min_metric.yml · Aggregating wrong datatype test+aggregations/moving_fn.yml · bad path+aggregations/moving_fn.yml · Bad window+aggregations/moving_fn.yml · invalid parent aggregation+aggregations/moving_fn.yml · Not under date_histo+aggregations/nested.yml · Illegal reverse nested aggregation to a child nested object+aggregations/percentile_ranks_hdr_metric.yml · invalid params+aggregations/percentile_ranks_tdigest_metric.yml · invalid params+aggregations/percentile_ranks_tdigest_metric.yml · with execution hint+aggregations/percentiles_bucket.yml · bad path+aggregations/percentiles_hdr_metric.yml · Invalid params test+aggregations/percentiles_tdigest_metric.yml · Execution hint+aggregations/percentiles_tdigest_metric.yml · Invalid params test+aggregations/pipeline.yml · Max pipeline on percentiles hdr agg without specifying percent+aggregations/pipeline.yml · Max pipeline on percentiles tdigest agg without specifying percent+aggregations/pipeline.yml · Max pipeline on percentiles with incorrect name+aggregations/pipeline.yml · Max pipeline on terms agg+aggregations/pipeline.yml · Max pipeline through terms agg+aggregations/random_sampler.yml · Test random_sampler aggregation with poor settings+aggregations/rare_terms.yml · IP test+aggregations/scripted_metric_allow_list.yml · allowed inline scripts do not affect allowed stored scripts+aggregations/scripted_metric_allow_list.yml · allowed stored scripts do not affect allowed inline scripts+aggregations/scripted_metric_allow_list.yml · disallow all for empty allow lists+aggregations/scripted_metric_allow_list.yml · inline combine_script must be allowed+aggregations/scripted_metric_allow_list.yml · inline init_script must be allowed+aggregations/scripted_metric_allow_list.yml · inline map_script must be allowed+aggregations/scripted_metric_allow_list.yml · inline reduce_script must be allowed+aggregations/scripted_metric_allow_list.yml · stored combine_script must be allowed+aggregations/scripted_metric_allow_list.yml · stored init_script must be allowed+aggregations/scripted_metric_allow_list.yml · stored map_script must be allowed+aggregations/scripted_metric_allow_list.yml · stored reduce_script must be allowed+aggregations/serial_diff.yml · invalid parent aggregation+aggregations/sig_terms.yml · IP test+aggregations/sig_terms.yml · Misspelled fields get "did you mean"+aggregations/stats_metric_fail_formatting.yml · fail formatting+aggregations/sum_bucket.yml · bad path+aggregations/sum_metric.yml · Aggregating wrong datatype test+aggregations/terms.yml · IP test+aggregations/terms.yml · No field or script+aggregations/time_series.yml · auto_date_histogram aggregation with time_series aggregation+aggregations/time_series.yml · Composite aggregation with nested time series aggregation failure+aggregations/time_series.yml · Global aggregation with nested time series aggregation failure+aggregations/time_series.yml · rare_terms aggregation with time_series aggregation+aggregations/time_series.yml · Sampler aggregation with nested time series aggregation failure+aggregations/time_series.yml · variable_width_histogram aggregation with time_series aggregation+aggregations/top_hits.yml · having+aggregations/weighted_avg.yml · Multi weight field not allowed+bulk/10_basic.yml · empty action+bulk/60_deprecated.yml · Deprecated parameters should fail in Bulk query+indices/indices.create/10_basic.yml · Create index with invalid mappings+indices/indices.create/10_basic.yml · Create index without soft deletes+indices/indices.create/10_basic.yml · Create lookup index with two shards+indices/indices.create/10_basic.yml · Poorly formatted request returns bad_request+indices/indices.create/10_basic.yml · Throw exception for unsupported value type+indices/indices.get/10_basic.yml · Should return an exception when querying invalid indices+indices/indices.put_alias/10_basic.yml · Can't create alias with invalid characters+indices/indices.put_alias/10_basic.yml · Can't create alias with the same name as an index+indices/indices.put_alias/10_basic.yml · Throw exception for unknown token+indices/indices.put_mapping/10_basic.yml · Create index with invalid mappings+indices/indices.put_template/10_basic.yml · Put index template without index_patterns+indices/indices.put_template/10_basic.yml · Put template create+scroll/10_basic.yml · Scroll cannot used the request cache+scroll/10_basic.yml · Scroll with size 0+scroll/11_clear.yml · Body params with array param override query string+scroll/11_clear.yml · Body params with string param scroll id override query string+scroll/11_clear.yml · Clear scroll+scroll/12_slices.yml · Sliced scroll with invalid arguments+scroll/20_keep_alive.yml · Max keep alive+search/110_field_collapsing.yml · field collapsing and scroll+search/110_field_collapsing.yml · field collapsing and search_after with invalid sort+search/110_field_collapsing.yml · Test collapse with inner_hits and missing name fails+search/115_multiple_field_collapsing.yml · two levels fields collapsing+search/120_batch_reduce_size.yml · batched_reduce_size lower limit+search/140_pre_filter_search_shards.yml · pre_filter_shard_size with invalid parameter+search/150_rewrite_on_coordinator.yml · Ensure that we fetch the document only once+search/160_exists_query.yml · Test exists query on _source field+search/170_terms_query.yml · Terms Query with No.of terms exceeding index.max_terms_count should FAIL+search/220_total_hits_object.yml · hits.total as an object+search/260_parameter_validation.yml · test negative size throws IAE+search/260_parameter_validation.yml · test negative terminate_after throws IAE+search/30_limits.yml · Docvalues_fields size limit+search/30_limits.yml · Prefix length limit+search/30_limits.yml · Regexp length limit+search/30_limits.yml · Request window limits without scroll+search/30_limits.yml · Request window limits with scroll+search/30_limits.yml · Request with negative from value in body+search/30_limits.yml · Request with negative from value url parameter+search/30_limits.yml · Rescore window limits+search/320_disallow_queries.yml · Test disallow expensive queries+search/330_fetch_fields.yml · error for flattened includes whole path+search/330_fetch_fields.yml · error includes field name+search/330_fetch_fields.yml · error includes glob pattern+search/330_fetch_fields.yml · Test date formatting+search/350_point_in_time.yml · delete invalid PIT returns error+search/350_point_in_time.yml · msearch+search/390_lookup_fields.yml · Fails to query or aggregate on lookup fields+search/40_indices_boost.yml · Indices boost for nonexistent index/alias+search/40_indices_boost.yml · Remote expressions not supported in indices boost+search/500_date_range.yml · test bad dates in range - future+search/500_date_range.yml · test bad dates in range - past+search/550_ignore_above_invalid.yml · flattened ignore_above mapping setting negative value+search/550_ignore_above_invalid.yml · ignore_above index setting negative value+search/550_ignore_above_invalid.yml · keyword ignore_above mapping setting negative value+search/550_ignore_above_invalid.yml · wildcard ignore_above mapping setting negative value+search/90_search_after.yml · Format sort values+vectors/search.vectors/46_knn_search_bbq_ivf.yml · Test bad quantization parameters+vectors/search.vectors/search.vectors/46_knn_search_bbq_ivf.yml · Test bad quantization parametersdiff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rsindex fb3a1084a..ab8de5148 100644--- a/engine/tests/es-compat-yaml/src/main.rs+++ b/engine/tests/es-compat-yaml/src/main.rs@@ -41,6 +41,8 @@ struct Stats {passed: usize,failed: usize,skipped: usize,+ /// Every case run, by its `catch_lenient::key`.+ ran: std::collections::BTreeSet<String>,}fn main() {@@ -71,6 +73,7 @@ fn main() {passed: 0,failed: 0,skipped: 0,+ ran: Default::default(),};for path in &files {@@ -89,6 +92,19 @@ fn main() {format!("{}", total.skipped).yellow(),total.passed + total.failed + total.skipped,);+ let lenient = catch_lenient::hits();+ if lenient > 0 {+ println!(+ "{} cases ran catch-lenient (an expected error XERJ does not raise yet; see catch-lenient.txt)",+ format!("{lenient}").yellow()+ );+ }+ for k in catch_lenient::stale(&total.ran) {+ println!(+ "{} stale catch-lenient entry, delete it: {k}",+ "NOTE".yellow()+ );+ }// CI's "0 failures required" gate consumes this exit code — without it a// regressing PR shows a green conformance check (bit us on PR #23, which// went 1360→1345 passing with the job still SUCCESS).@@ -219,6 +235,9 @@ fn run_file(for (name, steps) in &tests {cleanup_indices(client, base_url);+ let key = catch_lenient::key(path, name);+ stats.ran.insert(key.clone());+ catch_lenient::enter(key);let mut setup_ok = true;let mut setup_response: Value = Value::Null;@@ -774,8 +793,14 @@ fn execute_do(resp_json = Value::Bool(status.is_success());}- if catch.is_some() {- // Expected error — don't fail on non-2xx+ if let Some(c) = &catch {+ let code = status.as_u16();+ if let Some(why) = catch_mismatch(c, code, &resp_text) {+ let msg = format!("catch: {c} — {method} {path} → {code}: {why}");+ if !catch_lenient::tolerate(&msg) {+ return Err(msg);+ }+ }*last_response = resp_json;return Ok(());}@@ -807,6 +832,121 @@ fn execute_do(Ok(())}+/// The ratchet for `catch:` expectations XERJ does not meet yet.+///+/// Until this existed the runner ignored `catch:` entirely: an expected+/// error step passed whatever the server answered, so a case asserting+/// "this request must be rejected" passed against a server that accepted it+/// with a 200. Checking every catch at once turned 151 of those vacuous+/// passes into failures, across areas owned by different people, so the+/// cases failing on the day the check landed are listed in+/// `catch-lenient.txt` (one `<path under yaml/> · <test name>` per line).+/// A listed case keeps its old behaviour — the mismatch is tolerated and+/// the case runs on — and is counted in the summary as `catch-lenient`;+/// every other case must now get the error it asks for. Fixing a gap means+/// deleting its line; entries that no longer mismatch are reported as stale.+mod catch_lenient {+ use std::cell::RefCell;+ use std::collections::BTreeSet;+ use std::sync::OnceLock;++ thread_local! {+ static CURRENT: RefCell<String> = const { RefCell::new(String::new()) };+ static HITS: RefCell<BTreeSet<String>> = const { RefCell::new(BTreeSet::new()) };+ }++ fn list() -> &'static BTreeSet<String> {+ static LIST: OnceLock<BTreeSet<String>> = OnceLock::new();+ LIST.get_or_init(|| {+ let path = concat!(env!("CARGO_MANIFEST_DIR"), "/catch-lenient.txt");+ std::fs::read_to_string(path)+ .unwrap_or_default()+ .lines()+ .map(str::trim)+ .filter(|l| !l.is_empty() && !l.starts_with('#'))+ .map(String::from)+ .collect()+ })+ }++ /// The key a case is listed under: its path below the `yaml/` root,+ /// so the list is independent of where the runner is invoked from.+ pub fn key(path: &std::path::Path, name: &str) -> String {+ let p = path.to_string_lossy().replace('\\', "/");+ let rel = p.rsplit_once("yaml/").map(|(_, r)| r).unwrap_or(&p);+ format!("{rel} · {name}")+ }++ pub fn enter(key: String) {+ CURRENT.with(|c| *c.borrow_mut() = key);+ }++ /// Should this catch mismatch be tolerated for the case now running?+ pub fn tolerate(msg: &str) -> bool {+ let key = CURRENT.with(|c| c.borrow().clone());+ if list().contains(&key) {+ if std::env::var("XERJ_DEBUG_CATCH").is_ok() {+ eprintln!(" catch-lenient {key}: {msg}");+ }+ HITS.with(|h| h.borrow_mut().insert(key));+ true+ } else {+ false+ }+ }++ /// Cases tolerated so far.+ pub fn hits() -> usize {+ HITS.with(|h| h.borrow().len())+ }++ /// Listed cases among `ran` that never needed tolerating.+ pub fn stale(ran: &BTreeSet<String>) -> Vec<String> {+ HITS.with(|h| {+ let h = h.borrow();+ list()+ .iter()+ .filter(|k| ran.contains(*k) && !h.contains(*k))+ .cloned()+ .collect()+ })+ }+}++/// Does a response satisfy a YAML `catch:` expectation? `None` when it does,+/// otherwise the reason it does not. Semantics follow the ES REST test spec:+/// a named catch pins the status code, `/regex/` requires an error status+/// whose body matches, and `param` is a client-side validation error the+/// runner cannot observe (accepted as-is).+fn catch_mismatch(catch: &str, code: u16, body: &str) -> Option<String> {+ let want: Option<&[u16]> = match catch {+ "bad_request" => Some(&[400]),+ "unauthorized" => Some(&[401]),+ "forbidden" => Some(&[403]),+ "missing" => Some(&[404]),+ "request_timeout" => Some(&[408]),+ "conflict" => Some(&[409]),+ "unavailable" => Some(&[503]),+ "request" => None,+ "param" => return None,+ _ => None,+ };+ if let Some(codes) = want {+ return (!codes.contains(&code)).then(|| format!("expected status {codes:?}"));+ }+ if code < 400 {+ return Some("expected an error status".into());+ }+ if let Some(pat) = catch.strip_prefix('/').and_then(|s| s.strip_suffix('/')) {+ return match regex::Regex::new(&format!("(?s){}", pat.trim())) {+ Ok(re) if re.is_match(body) => None,+ Ok(_) => Some("error body does not match".into()),+ Err(e) => Some(format!("bad catch regex: {e}")),+ };+ }+ None+}+// ES date-math path segments start with `<` and contain `/` or `|` which// collide with URL path semantics. Percent-encode them so the server sees// one path segment. Also handle `+` for index name addition syntax.