Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

rebase fixup: union with #1076's xerj_plan — thirteen tools

xerj-orgd3ff514fe3e5
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index d0e30594a..d28070ab0 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -822,6 +822,8 @@ mod sample_query_tests {
date_min: None,
date_max: None,
date_evidence: vec![],
+ num_min: None,
+ num_max: None,
}
}
diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rs
index 95c6ab3bf..a797106f3 100644
--- a/engine/crates/xerj-autoindex/src/infer/mod.rs
+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs
@@ -330,6 +330,19 @@ pub struct FieldSpec {
pub date_max: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub date_evidence: Vec<String>,
+ /// Sampled integer range of a `long` field (#1055). `FieldAcc` has held
+ /// `int_min`/`int_max` since the beginning; the spec never exposed them.
+ /// `#[serde(default)]` because every catalog and state file written before
+ /// this field lacks it, and those must keep deserializing.
+ ///
+ /// Deliberately `None` on `double` fields: the accumulator only tracks the
+ /// i64-parseable subset, so on a fractional column these would be the
+ /// range of the integers *within* it — a wrong range stated as fact. An
+ /// absent range is honest; a partial one is not.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub num_min: Option<i64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub num_max: Option<i64>,
}
fn p95(samples: &[u32]) -> u32 {
@@ -450,6 +463,8 @@ pub fn infer_fields_with_policy(
date_min: acc.date_min.map(|d| dates::to_rfc3339_millis(&d)),
date_max: acc.date_max.map(|d| dates::to_rfc3339_millis(&d)),
date_evidence: Vec::new(),
+ num_min: None,
+ num_max: None,
};
let n = acc.n;
let th95 = |x: u64| x * 100 >= n * 95;
@@ -481,6 +496,13 @@ pub fn infer_fields_with_policy(
// numeric
if th95(acc.long_ok) && acc.long_ok > 0 {
spec.es_type = "long".into();
+ // #1055: the sampled integer range rides the spec wherever the
+ // elected type is `long` — including the epoch-candidate that
+ // stays `long` pending corroboration and the one a sibling date
+ // range later flips to `date` (where it lands beside the
+ // RFC3339 `date_min`/`date_max` the flip fills in).
+ spec.num_min = Some(acc.int_min);
+ spec.num_max = Some(acc.int_max);
// epoch candidate?
let (lo, hi) = (acc.int_min, acc.int_max);
let in_ms = lo >= dates::EPOCH_MS_MIN && hi <= dates::EPOCH_MS_MAX;
@@ -1012,4 +1034,101 @@ mod tests {
assert_eq!(specs[0].es_type, "keyword", "{specs:#?}");
assert_eq!(elected_default_analyzer(&specs, &fields), None);
}
+
+ // ── #1055: the sampled numeric range rides the spec ───────────────────
+
+ fn one_field_spec(acc: FieldAcc, records: u64) -> Vec<FieldSpec> {
+ let mut fields = HashMap::new();
+ fields.insert("f".to_string(), acc);
+ infer_fields(&fields, records, true)
+ }
+
+ /// A `long` field carries the integer range its sample actually showed —
+ /// `FieldAcc::int_min`/`int_max` existed from the start; the spec just
+ /// never exposed them. Negative bounds are part of the pin.
+ #[test]
+ fn a_long_field_carries_its_sampled_integer_range() {
+ let mut acc = FieldAcc::default();
+ for v in [3i64, 17, -2, 9] {
+ acc.add(&Value::Number(v.into()));
+ }
+ let specs = one_field_spec(acc, 4);
+ assert_eq!(specs[0].es_type, "long");
+ assert_eq!(specs[0].num_min, Some(-2));
+ assert_eq!(specs[0].num_max, Some(17));
+ }
+
+ /// The range must survive the catalog round trip: `dataset_doc` stores the
+ /// specs as `fields_json`, and every reader (the CLI map, the MCP tool)
+ /// parses that string back. Serialisation is therefore pinned literally.
+ #[test]
+ fn the_numeric_range_serialises_into_fields_json_and_back() {
+ let mut acc = FieldAcc::default();
+ for v in [3i64, 17, -2] {
+ acc.add(&Value::Number(v.into()));
+ }
+ let specs = one_field_spec(acc, 3);
+ let raw = serde_json::to_string(&specs[0]).unwrap();
+ assert!(raw.contains(r#""num_min":-2"#), "{raw}");
+ assert!(raw.contains(r#""num_max":17"#), "{raw}");
+ let back: FieldSpec = serde_json::from_str(&raw).unwrap();
+ assert_eq!((back.num_min, back.num_max), (Some(-2), Some(17)));
+ }
+
+ /// Every catalog and state file written before #1055 lacks the keys
+ /// entirely; `#[serde(default)]` is what keeps those deserializing, so a
+ /// pre-range `fields_json` must come back as `None` — never as an error
+ /// and never as a fabricated `Some(0)`.
+ #[test]
+ fn a_spec_written_before_numeric_ranges_still_deserialises() {
+ let raw = r#"{"name":"n","es_type":"long","cardinality_est":3,
+ "cardinality_overflow":false,"null_ratio":0.0,"avg_len":0.0,
+ "coverage":1.0,"examples":[]}"#;
+ let spec: FieldSpec = serde_json::from_str(raw).expect("old fields_json parses");
+ assert_eq!(spec.num_min, None);
+ assert_eq!(spec.num_max, None);
+ assert_eq!(spec.es_type, "long");
+ }
+
+ /// A fractional column elects `double`, and the accumulator tracks only
+ /// its i64-parseable subset — so the spec must state NO range there
+ /// rather than the integers' range over a `double` field.
+ #[test]
+ fn a_double_field_states_no_range_rather_than_the_integer_subset() {
+ let mut acc = FieldAcc::default();
+ for v in ["1.5", "2.25", "0.125"] {
+ acc.add(&Value::String(v.into()));
+ }
+ let specs = one_field_spec(acc, 3);
+ assert_eq!(specs[0].es_type, "double");
+ assert_eq!(specs[0].num_min, None, "the int range would be a lie here");
+ assert_eq!(specs[0].num_max, None);
+ }
+
+ /// Non-numeric elects carry no range: the keys are absent from the JSON,
+ /// not null-padded.
+ #[test]
+ fn keyword_and_date_fields_omit_the_numeric_keys_entirely() {
+ let mut acc = FieldAcc::default();
+ for v in ["INFO", "WARN", "ERROR", "INFO"] {
+ acc.add(&Value::String(v.into()));
+ }
+ let specs = one_field_spec(acc, 4);
+ assert_eq!(specs[0].es_type, "keyword");
+ let raw = serde_json::to_string(&specs[0]).unwrap();
+ assert!(!raw.contains("num_min"), "{raw}");
+ assert!(!raw.contains("num_max"), "{raw}");
+
+ // A string-date field: the range it carries is `date_min`/`date_max`
+ // (pre-existing), never the numeric pair.
+ let mut acc = FieldAcc::default();
+ for d in ["2026-03-01T00:00:00Z", "2026-03-02T00:00:00Z"] {
+ acc.add(&Value::String(d.into()));
+ }
+ let specs = one_field_spec(acc, 2);
+ assert_eq!(specs[0].es_type, "date");
+ assert!(specs[0].date_min.is_some());
+ assert!(specs[0].date_max.is_some());
+ assert_eq!((specs[0].num_min, specs[0].num_max), (None, None));
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index d8450b43c..14693abaf 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3089,6 +3089,8 @@ fn pipeline_keyword_spec(name: &str) -> infer::FieldSpec {
date_min: None,
date_max: None,
date_evidence: Vec::new(),
+ num_min: None,
+ num_max: None,
}
}
diff --git a/engine/crates/xerj-autoindex/src/reconcile_plan.rs b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
index df121499c..6036462a9 100644
--- a/engine/crates/xerj-autoindex/src/reconcile_plan.rs
+++ b/engine/crates/xerj-autoindex/src/reconcile_plan.rs
@@ -671,6 +671,8 @@ mod tests {
date_min: None,
date_max: None,
date_evidence: vec![],
+ num_min: None,
+ num_max: None,
}
}
diff --git a/engine/crates/xerj-mcp/Cargo.toml b/engine/crates/xerj-mcp/Cargo.toml
index 18575f4ea..17108a871 100644
--- a/engine/crates/xerj-mcp/Cargo.toml
+++ b/engine/crates/xerj-mcp/Cargo.toml
@@ -4,7 +4,7 @@ version.workspace = true
edition.workspace = true
license.workspace = true
authors.workspace = true
-description = "Model Context Protocol (MCP) stdio server exposing the 11 canonical XERJ agent operations (search, memory, second-brain graph, reference-code) as thin proxies to an ES-compatible XERJ endpoint"
+description = "Model Context Protocol (MCP) stdio server exposing the 12 canonical XERJ agent operations (search, memory, second-brain graph, field map, reference-code) as thin proxies to an ES-compatible XERJ endpoint"
# Library first: the `xerj mcp` subcommand on the main binary calls
# `xerj_mcp::run`, so a user with the installed `xerj` binary already has the
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 0fb5e2d64..4b580f71b 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -9,12 +9,12 @@
//!
//! It speaks the MCP **stdio transport**: newline-delimited
//! JSON-RPC 2.0 messages on stdin/stdout. It exposes XERJ to any MCP-capable
-//! agent host (Claude Desktop, IDE agents, custom orchestrators) as twelve
+//! agent host (Claude Desktop, IDE agents, custom orchestrators) as thirteen
//! tools that map 1:1 onto XERJ's real, verified REST surface. Every tool is
//! a *thin proxy*: it constructs exactly the request the running engine
//! already accepts and forwards it to a configurable base URL.
//!
-//! ## The twelve canonical agent operations
+//! ## The thirteen canonical agent operations
//!
//! | MCP tool | XERJ endpoint | Real capability |
//! |------------------------|---------------------------------------|-----------------|
@@ -29,6 +29,7 @@
//! | `xerj_brain_unlink` | `DELETE /_graph/{brain}/link/{id}` | retire a link (soft-invalidate; never deletes) |
//! | `xerj_brain_overview` | `GET /_graph/{brain}/overview` | whole-brain orientation: counts, hubs, detectors, timeline |
//! | `xerj_plan` | `POST /_ask` | natural-language prompt in, VALIDATED query DSL out (deterministic planner, not an LLM) |
+//! | `xerj_map` | `POST /autoindex-catalog/_search` | per-index field map from the autoindex catalog (#1055) |
//!
//! ## Honesty notes (must match the engine, never oversell)
//!
@@ -86,6 +87,7 @@
//! words), with overlapping file:line passages deduped before budgeting and
//! over-budget hits reduced to locators.
+mod map;
mod token_budget;
use std::time::Duration;
@@ -113,10 +115,10 @@ USAGE:
xerj mcp [OPTIONS] (or the standalone binary: xerj-mcp [OPTIONS])
Speaks MCP over stdio (newline-delimited JSON-RPC 2.0 on stdin/stdout) and
-proxies twelve tools — xerj_search, xerj_semantic_search, xerj_vector_search,
+proxies thirteen tools — xerj_search, xerj_semantic_search, xerj_vector_search,
xerj_hybrid_search, xerj_memory_store, xerj_memory_recall, xerj_brain_ego,
xerj_brain_link, xerj_brain_unlink, xerj_brain_overview, xerj_code_search,
-xerj_plan — to a
+xerj_map, xerj_plan — to a
XERJ node that
is ALREADY RUNNING. This command does not start a node; start one first with
`xerj --data-dir ./data`.
@@ -414,6 +416,9 @@ fn initialize_result(msg: &Value) -> Value {
kNN over a dense_vector field, xerj_hybrid_search to fuse \
lexical + vector results (rrf|linear), and xerj_memory_store / \
xerj_memory_recall for durable agent memory recalled by meaning. \
+ Call xerj_map FIRST on an autoindexed node: it returns each \
+ index's field map (names, types, cardinality, coverage, \
+ examples, date/numeric ranges) so your DSL uses real fields. \
xerj_search and xerj_hybrid_search take an optional `rerank` \
argument: a second stage that re-judges the top hits with an \
external relevance model. It only works if the node's operator \
@@ -552,7 +557,7 @@ fn max_tokens_arg_schema() -> Value {
})
}
-/// The twelve tool specifications advertised via `tools/list`. Input schemas
+/// The thirteen tool specifications advertised via `tools/list`. Input schemas
/// are plain JSON Schema; every property maps onto a field the engine accepts.
///
/// Public because this is the single source of truth for the published
@@ -883,6 +888,51 @@ pub fn tool_specs() -> Value {
"required": ["brain"]
}
},
+ {
+ "name": "xerj_map",
+ "description": concat!(
+ "Per-index FIELD MAP from this node's autoindex catalog — call it ",
+ "BEFORE writing query DSL, so you stop guessing field names. For each ",
+ "catalogued index: record count, time field and time range, semantic ",
+ "body field, and every field the profiler inferred — name, es_type ",
+ "(keyword|text|long|double|boolean|date|semantic_text), date encoding, ",
+ "semantic tag, cardinality estimate, coverage, up to 5 examples, and ",
+ "the sampled date AND numeric min-max (numeric ranges appear on `long` ",
+ "fields only; fractional `double` ranges are not tracked). Reads the ",
+ "catalog index `autoindex-catalog` that `xerj autoindex <folder>` ",
+ "writes; never-autoindexed indexes are not listed. Honesty: every fact ",
+ "is inferred from a bounded sample taken at autoindex time — ",
+ "cardinality is an estimate (with `cardinality_overflow:true` it means ",
+ "AT LEAST that many), and min/max/coverage describe the sample, not ",
+ "the whole index. Up to 100 indexes per call (`indexes_omitted` ",
+ "reports more)."
+ ),
+ "inputSchema": {
+ "type": "object",
+ "properties": {
+ "index": {
+ "type": "string",
+ "description": "Index name or `*`-glob (e.g. `ax-logs`, `ax-*`). Default: every \
+ catalogued index. An exact name filters engine-side; a glob filters \
+ client-side over the catalog's index names."
+ },
+ "max_bytes": {
+ "type": "integer",
+ "minimum": map::MIN_MAX_BYTES,
+ "description": format!(
+ "Per-index byte budget for one rendered index entry (default {}). \
+ An entry NEVER exceeds it: the trim ladder first cuts per-field \
+ examples to one, then drops examples, then per-field notes, then \
+ the lowest-coverage fields — and the entry's `trimmed` and \
+ `fields_omitted` say exactly what went. The budget is per index; \
+ a multi-index response carries one such entry per index. Minimum {}.",
+ map::DEFAULT_MAX_BYTES,
+ map::MIN_MAX_BYTES,
+ )
+ }
+ }
+ }
+ },
{
"name": "xerj_code_search",
"description": concat!(
@@ -1001,6 +1051,10 @@ async fn call_tool(ctx: &Ctx, msg: &Value) -> Value {
"xerj_brain_overview" => build_brain_overview(&args),
// #1056: prompt in, validated DSL out — a plain engine proxy.
"xerj_plan" => build_plan(&args),
+ // #1055: the catalog read is a proxy-with-post-processing, like
+ // code_search — it renders the per-index field map itself, so it
+ // returns a tool result directly instead of a BuiltRequest.
+ "xerj_map" => return map::run(ctx, &args).await,
"xerj_code_search" => return run_code_search(ctx, &args).await,
other => return tool_text(format!("unknown tool: {other}"), true),
};
@@ -2404,7 +2458,7 @@ mod tests {
}
#[test]
- fn tools_list_has_all_twelve() {
+ fn tools_list_has_all_thirteen() {
let specs = tool_specs();
let names: Vec<&str> = specs
.as_array()
@@ -2412,7 +2466,7 @@ mod tests {
.iter()
.map(|t| t["name"].as_str().unwrap())
.collect();
- assert_eq!(names.len(), 12);
+ assert_eq!(names.len(), 13);
for n in [
"xerj_search",
"xerj_semantic_search",
@@ -2424,6 +2478,8 @@ mod tests {
"xerj_brain_link",
"xerj_brain_unlink",
"xerj_brain_overview",
+ // #1055: the per-index field map from the autoindex catalog.
+ "xerj_map",
// #977: reference-coding retrieval, served by the same pipeline
// as `xerj code`.
"xerj_code_search",
@@ -2441,6 +2497,43 @@ mod tests {
}
}
+ /// #1055: the field map is an orientation tool an agent calls before it
+ /// writes DSL — its description must say what to reach for, what the
+ /// facts ARE (sampled, not exhaustive), and how the byte cap trims.
+ #[test]
+ fn the_map_tool_description_carries_its_honesty_strings() {
+ let specs = tool_specs();
+ let tool = specs
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|t| t["name"] == "xerj_map")
+ .expect("xerj_map is served");
+ let d = tool["description"].as_str().expect("described");
+ for needle in [
+ "BEFORE writing query DSL",
+ "bounded sample",
+ "not tracked",
+ "indexes_omitted",
+ "at autoindex time",
+ ] {
+ assert!(d.contains(needle), "description lacks `{needle}`: {d}");
+ }
+ let mb = &tool["inputSchema"]["properties"]["max_bytes"];
+ assert_eq!(mb["minimum"], map::MIN_MAX_BYTES);
+ assert!(
+ mb["description"]
+ .as_str()
+ .unwrap()
+ .contains("NEVER exceeds"),
+ "the cap must be stated as never-exceeded: {mb}"
+ );
+ assert!(
+ tool["inputSchema"]["required"].is_null(),
+ "every xerj_map argument is optional"
+ );
+ }
+
#[test]
fn every_brain_tool_carries_both_honesty_strings() {
let specs = tool_specs();
diff --git a/engine/crates/xerj-mcp/src/map.rs b/engine/crates/xerj-mcp/src/map.rs
new file mode 100644
index 000000000..91160104d
--- /dev/null
+++ b/engine/crates/xerj-mcp/src/map.rs
@@ -0,0 +1,846 @@
+//! `xerj_map` — the per-index field map from the autoindex catalog (#1055).
+//!
+//! The catalog index (`autoindex-catalog`) holds one `doc_kind: dataset`
+//! document per dataset `xerj autoindex` indexed, and inside each of those,
+//! `fields_json` carries the `FieldSpec` list the profiler inferred: name,
+//! es_type, cardinality estimate, coverage, examples, and — since #1055 — the
+//! sampled date/numeric min-max. The CLI renders that as `xerj autoindex
+//! map`; an MCP agent had no way to reach it, which is why agents guessed
+//! field names and ate unknown-field 400s on the DSL they then sent.
+//!
+//! This module is the READ side only and re-derives nothing: every fact in
+//! the response comes from the catalog document verbatim. The catalog index
+//! name is repeated here (a string constant in `xerj-autoindex/src/catalog.rs`)
+//! because depending on that crate would drag the tree-sitter/SQLite/AWS
+//! stack into this thin proxy; the name is a frozen on-disk contract there
+//! (hashed into `index_identity`), which is what makes the copy safe — it
+//! cannot be renamed in place.
+//!
+//! ## The per-index byte cap
+//!
+//! A response renders at most [`DEFAULT_MAX_BYTES`] of JSON per index entry.
+//! The trim ladder, applied per entry until it fits and reported in the
+//! entry's `trimmed` array (a stage that changes nothing is not reported):
+//!
+//! 1. per-field `examples` cut to the first example;
+//! 2. `examples` dropped entirely;
+//! 3. per-field `notes` dropped;
+//! 4. fields dropped from the end of the list — it is sorted coverage
+//! descending, so the lowest-coverage fields go first — each counted in
+//! `fields_omitted`. A single field that alone exceeds the cap is dropped
+//! whole, which is what guarantees the ladder terminates.
+
+use serde_json::{json, Map, Value};
+
+use crate::{tool_text, Ctx};
+
+/// The catalog index, written by `xerj autoindex`. See the module docs for
+/// why this is a copy and not a dependency.
+pub const CATALOG_INDEX: &str = "autoindex-catalog";
+
+/// Default per-index byte budget for the rendered entry (#1055 acceptance:
+/// "default response ≤ 4 KB per index").
+pub const DEFAULT_MAX_BYTES: usize = 4096;
+/// Floor for a caller-supplied budget: below this not even one field entry
+/// plus the index header renders, and the ladder would drop everything.
+pub const MIN_MAX_BYTES: usize = 1024;
+/// Datasets fetched (and rendered) per call. Each catalog doc carries the
+/// whole `fields_json`, so this bounds the transfer; datasets past it are
+/// reported in `indexes_omitted`.
+pub const MAX_INDEXES: usize = 100;
+/// Examples kept per field. The profiler caps at 3 today; this cap is the
+/// published contract if that ever grows.
+pub const MAX_EXAMPLES: usize = 5;
+
+/// Parsed, validated `xerj_map` arguments.
+#[derive(Debug, Clone)]
+pub(crate) struct MapArgs {
+ /// Index name or `*`-glob; `None` = every catalogued index.
+ pub index: Option<String>,
+ /// Per-index byte budget for the rendered entry.
+ pub max_bytes: usize,
+}
+
+/// Parse and validate the tool arguments. Present-but-mistyped is an error,
+/// never a silent drop (the `opt_typed` policy the rest of this crate holds).
+pub(crate) fn parse_args(args: &Value) -> Result<MapArgs, String> {
+ let index = match args.get("index") {
+ None | Some(Value::Null) => None,
+ Some(Value::String(s)) => {
+ let t = s.trim();
+ (!t.is_empty()).then(|| t.to_string())
+ }
+ Some(_) => return Err("`index` must be a string (index name or glob)".into()),
+ };
+ let max_bytes = match args.get("max_bytes") {
+ None | Some(Value::Null) => DEFAULT_MAX_BYTES,
+ Some(v) => v
+ .as_u64()
+ .and_then(|n| usize::try_from(n).ok())
+ .filter(|n| *n >= MIN_MAX_BYTES)
+ .ok_or_else(|| {
+ format!(
+ "`max_bytes` must be an integer >= {MIN_MAX_BYTES} (default \
+ {DEFAULT_MAX_BYTES})"
+ )
+ })?,
+ };
+ Ok(MapArgs { index, max_bytes })
+}
+
+/// The `_search` body for the catalog read. A pattern without a `*` becomes
+/// an engine-side `term` on `index_name` (exact); a glob filters client-side
+/// — one less engine query behaviour for this proxy to assume, and index
+/// counts are small.
+pub(crate) fn catalog_body(args: &MapArgs) -> Value {
+ let mut must = vec![json!({ "term": { "doc_kind": "dataset" } })];
+ if let Some(name) = args.index.as_deref().filter(|p| !p.contains('*')) {
+ must.push(json!({ "term": { "index_name": name } }));
+ }
+ json!({
+ "query": { "bool": { "must": must } },
+ "size": MAX_INDEXES,
+ "sort": [{ "record_count": "desc" }],
+ // The dataset doc also carries sample_queries_json (by far its
+ // largest field); project it out rather than transfer it to drop it.
+ "_source": { "includes": [
+ "index_name", "slug", "record_count", "time_field", "time_min",
+ "time_max", "semantic_field", "fields_json", "notes",
+ ] },
+ })
+}
+
+/// `*`-glob match — the only wildcard the tool documents. The first and last
+/// segments are anchored at the ends of `s`; inner segments may sit
+/// anywhere, in order. (So `ax-*` matches `ax-logs` but not `tax-logs`.)
+fn glob_match(pattern: &str, s: &str) -> bool {
+ match pattern.split_once('*') {
+ None => pattern == s,
+ Some((head, tail)) => {
+ let Some(rest) = s.strip_prefix(head) else {
+ return false;
+ };
+ if tail.is_empty() {
+ return true; // trailing star: everything left matches
+ }
+ (0..=rest.len())
+ .filter(|i| rest.is_char_boundary(*i))
+ .any(|i| glob_match(tail, &rest[i..]))
+ }
+ }
+}
+
+/// Normalise one raw `FieldSpec` JSON object into the field entry the tool
+/// returns: the FieldSpec keys an agent acts on, examples capped, absent
+/// keys omitted. Unknown keys in the catalog's `fields_json` pass through —
+/// this is a projection, not a re-derivation.
+fn project_field(f: &Value) -> Value {
+ let Some(obj) = f.as_object() else {
+ return f.clone();
+ };
+ let mut out = Map::new();
+ for (k, v) in obj {
+ match k.as_str() {
+ // `null_ratio` is 1−coverage and `avg_len` drove the semantic
+ // election; neither changes a query an agent writes, and
+ // `date_evidence` is restated more compactly by `date_enc`.
+ "null_ratio" | "avg_len" | "date_evidence" => {}
+ "examples" => {
+ let capped: Vec<Value> = v
+ .as_array()
+ .map(|a| a.iter().take(MAX_EXAMPLES).cloned().collect())
+ .unwrap_or_default();
+ if !capped.is_empty() {
+ out.insert(k.clone(), Value::Array(capped));
+ }
+ }
+ // Only the overflow is information: an absent key already means
+ // "counted, not capped", so `false` is noise on every field.
+ "cardinality_overflow" => {
+ if v.as_bool().unwrap_or(false) {
+ out.insert(k.clone(), v.clone());
+ }
+ }
+ "coverage" => {
+ // 3 decimals: byte-lean, and beyond the sample's precision
+ // anyway.
+ if let Some(c) = v.as_f64() {
+ out.insert(k.clone(), json!((c * 1000.0).round() / 1000.0));
+ }
+ }
+ _ => {
+ if !(v.is_null() || v.as_array().is_some_and(Vec::is_empty)) {
+ out.insert(k.clone(), v.clone());
+ }
+ }
+ }
+ }
+ Value::Object(out)
+}
+
+/// Build the full index entry from the dataset doc and the (possibly
+/// trimmed) field list.
+fn assemble_entry(
+ doc: &Value,
+ kept: &[Value],
+ field_count: usize,
+ trimmed: &[&'static str],
+) -> Value {
+ let mut entry = Map::new();
+ entry.insert(
+ "index".into(),
+ doc.get("index_name").cloned().unwrap_or(Value::Null),
+ );
+ if let Some(n) = doc.get("record_count").filter(|v| v.is_number()) {
+ entry.insert("records".into(), n.clone());
+ }
+ for key in [
+ "time_field",
+ "time_min",
+ "time_max",
+ "semantic_field",
+ "notes",
+ ] {
+ if let Some(v) = doc
+ .get(key)
+ .filter(|v| !v.is_null() && !v.as_array().is_some_and(Vec::is_empty))
+ {
+ entry.insert(key.into(), v.clone());
+ }
+ }
+ entry.insert("field_count".into(), json!(field_count));
+ entry.insert("fields".into(), Value::Array(kept.to_vec()));
+ let omitted = field_count.saturating_sub(kept.len());
+ if omitted > 0 {
+ entry.insert("fields_omitted".into(), json!(omitted));
+ }
+ if !trimmed.is_empty() {
+ entry.insert(
+ "trimmed".into(),
+ json!(trimmed.iter().map(|s| s.to_string()).collect::<Vec<_>>()),
+ );
+ }
+ Value::Object(entry)
+}
+
+/// Render one index entry under the byte budget. `fields` arrive already
+/// projected and sorted (coverage desc, name asc) — the order the ladder's
+/// "drop from the end" relies on.
+fn render_entry(doc: &Value, mut kept: Vec<Value>, max_bytes: usize) -> Value {
+ let field_count = kept.len();
+ let mut trimmed: Vec<&'static str> = Vec::new();
+ let fits = |entry: &Value| {
+ serde_json::to_string(entry)
+ .map(|s| s.len())
+ .unwrap_or(usize::MAX)
+ <= max_bytes
+ };
+ let mut entry = assemble_entry(doc, &kept, field_count, &trimmed);
+ if fits(&entry) {
+ return entry;
+ }
+
+ // Stage 1: examples cut to the first. Recorded only when it actually
+ // cut something — an honest accounting, not a ritual label.
+ let cut = kept.iter_mut().filter_map(|f| f.as_object_mut()).any(|o| {
+ o.get("examples")
+ .and_then(Value::as_array)
+ .is_some_and(|ex| ex.len() > 1)
+ });
+ if cut {
+ for f in &mut kept {
+ if let Some(ex) = f.get("examples").and_then(Value::as_array) {
+ if ex.len() > 1 {
+ let first = ex[..1].to_vec();
+ if let Some(o) = f.as_object_mut() {
+ o.insert("examples".into(), Value::Array(first));
+ }
+ }
+ }
+ }
+ trimmed.push("examples-cut-to-1");
+ entry = assemble_entry(doc, &kept, field_count, &trimmed);
+ if fits(&entry) {
+ return entry;
+ }
+ }
+
+ // Stage 2: examples dropped entirely.
+ for f in &mut kept {
+ if let Some(o) = f.as_object_mut() {
+ o.remove("examples");
+ }
+ }
+ trimmed.push("examples-dropped");
+ entry = assemble_entry(doc, &kept, field_count, &trimmed);
+ if fits(&entry) {
+ return entry;
+ }
+
+ // Stage 3: per-field notes dropped.
+ for f in &mut kept {
+ if let Some(o) = f.as_object_mut() {
+ o.remove("notes");
+ }
+ }
+ trimmed.push("notes-dropped");
+ entry = assemble_entry(doc, &kept, field_count, &trimmed);
+ if fits(&entry) {
+ return entry;
+ }
+
+ // Stage 4: drop fields from the end (lowest coverage first). A field
+ // that alone exceeds the budget goes too — `kept` can reach empty, and
+ // an entry with zero fields (but a truthful `field_count`) still
+ // renders, which is what terminates this loop.
+ trimmed.push("fields-dropped");
+ while !kept.is_empty() {
+ kept.pop();
+ entry = assemble_entry(doc, &kept, field_count, &trimmed);
+ if fits(&entry) {
+ return entry;
+ }
+ }
+ entry
+}
+
+/// Render the whole tool response from a catalog `_search` response.
+/// Pure — every unit test below drives this with a hand-built catalog
+/// response, no node required.
+pub(crate) fn render_response(resp: &Value, args: &MapArgs) -> String {
+ let hits = resp
+ .pointer("/hits/hits")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ let total: u64 = resp
+ .pointer("/hits/total/value")
+ .and_then(Value::as_u64)
+ .unwrap_or(hits.len() as u64);
+
+ // Pattern filtering (glob only; exact names were filtered engine-side).
+ let docs: Vec<Value> = hits
+ .iter()
+ .filter_map(|h| h.get("_source").cloned())
+ .filter(|src| {
+ args.index.as_deref().is_none_or(|p| {
+ !p.contains('*')
+ || glob_match(
+ p,
+ src.get("index_name").and_then(Value::as_str).unwrap_or(""),
+ )
+ })
+ })
+ .collect();
+
+ let mut indexes: Vec<Value> = docs
+ .iter()
+ .map(|src| {
+ let mut fields: Vec<Value> = src
+ .get("fields_json")
+ .and_then(Value::as_str)
+ .and_then(|raw| serde_json::from_str::<Value>(raw).ok())
+ .and_then(|v| v.as_array().cloned())
+ .unwrap_or_default()
+ .iter()
+ .map(project_field)
+ .collect();
+ // Coverage desc, then name — the CLI map's order, and the order
+ // the trim ladder's "drop from the end" relies on.
+ fn cov_of(f: &Value) -> f64 {
+ f.get("coverage").and_then(Value::as_f64).unwrap_or(0.0)
+ }
+ fn name_of(f: &Value) -> &str {
+ f.get("name").and_then(Value::as_str).unwrap_or("")
+ }
+ fields.sort_by(|a, b| {
+ cov_of(b)
+ .partial_cmp(&cov_of(a))
+ .unwrap()
+ .then_with(|| name_of(a).cmp(name_of(b)))
+ });
+ render_entry(src, fields, args.max_bytes)
+ })
+ .collect();
+ // The engine already sorted by record_count desc; make the order total
+ // so tied counts (and cap-truncated result sets) render deterministically.
+ indexes.sort_by(|a, b| {
+ let rc = |v: &Value| v.get("records").and_then(Value::as_u64).unwrap_or(0);
+ rc(b).cmp(&rc(a)).then_with(|| {
+ a.get("index")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .cmp(b.get("index").and_then(Value::as_str).unwrap_or(""))
+ })
+ });
+
+ let mut out = Map::new();
+ out.insert("matched".into(), json!(indexes.len()));
+ if let Some(p) = &args.index {
+ out.insert("pattern".into(), json!(p));
+ }
+ out.insert("indexes".into(), Value::Array(indexes));
+ let omitted = total.saturating_sub(hits.len() as u64);
+ if omitted > 0 {
+ out.insert("indexes_omitted".into(), json!(omitted));
+ }
+ if docs.is_empty() {
+ out.insert(
+ "hint".into(),
+ json!(format!(
+ "no datasets in {CATALOG_INDEX} match — the catalog is written by `xerj \
+ autoindex <folder>`; never-autoindexed indexes are not listed (ask the \
+ node's _cat/indices for those)"
+ )),
+ );
+ }
+ Value::Object(out).to_string()
+}
+
+/// `xerj_map` dispatch: read the catalog, render the map. A missing catalog
+/// (the node never ran autoindex) is an error with the fix named — the
+/// response an agent can act on — while an empty-but-present catalog is a
+/// normal empty result, not an error.
+pub(crate) async fn run(ctx: &Ctx, args: &Value) -> Value {
+ let parsed = match parse_args(args) {
+ Ok(a) => a,
+ Err(msg) => return tool_text(format!("xerj_map: {msg}"), true),
+ };
+ let body = catalog_body(&parsed);
+ let url = format!("{}/{CATALOG_INDEX}/_search", ctx.base_url);
+ let mut req = ctx.client.post(&url).json(&body);
+ if let Some(auth) = &ctx.auth {
+ req = req.header("Authorization", auth);
+ }
+ match req.send().await {
+ Ok(resp) => {
+ let status = resp.status();
+ let text = resp.text().await.unwrap_or_default();
+ if status.is_success() {
+ match serde_json::from_str::<Value>(&text) {
+ Ok(v) => tool_text(render_response(&v, &parsed), false),
+ Err(e) => tool_text(
+ format!("xerj_map: catalog reply was not JSON ({e}): {text}"),
+ true,
+ ),
+ }
+ } else if status.as_u16() == 404 {
+ tool_text(
+ format!(
+ "xerj_map: no autoindex catalog on this node (index \
+ `{CATALOG_INDEX}` not found, HTTP 404) — run `xerj autoindex \
+ <folder>` first. Engine said: {text}"
+ ),
+ true,
+ )
+ } else {
+ tool_text(
+ format!("XERJ returned HTTP {status} from {CATALOG_INDEX}: {text}"),
+ true,
+ )
+ }
+ }
+ Err(e) => tool_text(format!("request to {url} failed: {e}"), true),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn dataset_doc(index: &str, records: u64, fields_json: &str) -> Value {
+ json!({
+ "_index": CATALOG_INDEX,
+ "_id": format!("ds:ax:{index}"),
+ "_source": {
+ "doc_kind": "dataset",
+ "slug": index,
+ "index_name": index,
+ "record_count": records,
+ "fields_json": fields_json,
+ }
+ })
+ }
+
+ fn catalog_response(hits: Vec<Value>) -> Value {
+ json!({ "took": 1, "hits": {
+ "total": { "value": hits.len(), "relation": "eq" }, "hits": hits
+ } })
+ }
+
+ fn args(index: Option<&str>) -> MapArgs {
+ MapArgs {
+ index: index.map(str::to_string),
+ max_bytes: DEFAULT_MAX_BYTES,
+ }
+ }
+
+ fn disconnected_ctx() -> crate::Ctx {
+ crate::Ctx {
+ client: reqwest::Client::new(),
+ base_url: "http://127.0.0.1:1".to_string(),
+ auth: None,
+ }
+ }
+
+ #[test]
+ fn glob_match_handles_the_documented_shapes() {
+ assert!(glob_match("ax-*", "ax-logs"));
+ assert!(glob_match("ax-*", "ax-"));
+ assert!(
+ !glob_match("ax-*", "tax-logs"),
+ "the star anchors the prefix"
+ );
+ assert!(glob_match("*", "anything"));
+ assert!(glob_match("ax-logs", "ax-logs"), "no star = exact");
+ assert!(!glob_match("ax-logs", "ax-logs2"));
+ assert!(glob_match("ax-*-2026", "ax-logs-2026"));
+ assert!(!glob_match("ax-*-2026", "ax-2026"));
+ }
+
+ #[test]
+ fn a_small_catalog_renders_every_field_untrimmed() {
+ let fields = r#"[
+ {"name":"ts","es_type":"date","date_enc":"rfc3339","cardinality_est":900,
+ "cardinality_overflow":false,"coverage":1.0,"null_ratio":0.0,"avg_len":24.0,
+ "examples":["2026-03-01T00:00:00.000Z"],"date_min":"2026-03-01T00:00:00.000Z",
+ "date_max":"2026-03-02T00:00:00.000Z","date_evidence":["rfc3339: 900"]},
+ {"name":"status","es_type":"long","cardinality_est":4,"coverage":0.5,
+ "null_ratio":0.5,"avg_len":0.0,"examples":["200"],"num_min":200,"num_max":404}
+ ]"#;
+ let resp = catalog_response(vec![dataset_doc("ax-logs", 10, fields)]);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ assert_eq!(out["matched"], 1);
+ let idx = &out["indexes"][0];
+ assert_eq!(idx["index"], "ax-logs");
+ assert_eq!(idx["records"], 10);
+ assert_eq!(idx["field_count"], 2);
+ assert!(idx.get("fields_omitted").is_none(), "nothing was dropped");
+ assert!(idx.get("trimmed").is_none(), "nothing was trimmed: {idx}");
+ let ts = &idx["fields"][0];
+ assert_eq!(ts["name"], "ts");
+ assert_eq!(ts["es_type"], "date");
+ assert_eq!(ts["date_min"], "2026-03-01T00:00:00.000Z");
+ assert_eq!(
+ ts["num_min"],
+ Value::Null,
+ "a date field carries no numeric range"
+ );
+ let status = &idx["fields"][1];
+ assert_eq!(status["num_min"], 200);
+ assert_eq!(status["num_max"], 404);
+ assert_eq!(status["coverage"], 0.5);
+ // date_evidence / null_ratio / avg_len are projected out;
+ // cardinality_overflow only appears when it is TRUE.
+ for f in idx["fields"].as_array().unwrap() {
+ assert!(f.get("date_evidence").is_none(), "{f}");
+ assert!(f.get("null_ratio").is_none(), "{f}");
+ assert!(f.get("avg_len").is_none(), "{f}");
+ assert!(f.get("cardinality_overflow").is_none(), "{f}");
+ }
+ // ...and a capped field DOES carry the flag: cardinality_est is then
+ // "at least this many".
+ let capped = json!([{
+ "name": "trace_id", "es_type": "keyword", "cardinality_est": 8192,
+ "cardinality_overflow": true, "coverage": 1.0
+ }]);
+ let capped_json = serde_json::to_string(&capped).unwrap();
+ let resp = catalog_response(vec![dataset_doc("ax-traces", 99, &capped_json)]);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ assert_eq!(out["indexes"][0]["fields"][0]["cardinality_overflow"], true);
+ }
+
+ /// #1055's hard rule: every rendered index entry is <= the per-index byte
+ /// cap, and when the cap binds, the accounting says what was cut.
+ #[test]
+ fn a_wide_dataset_is_trimmed_to_the_cap_with_accounting() {
+ let mut fields = Vec::new();
+ for i in 0..300 {
+ fields.push(json!({
+ "name": format!("field_with_a_deliberately_long_name_{i:03}"),
+ "es_type": "keyword",
+ "cardinality_est": i,
+ "coverage": 1.0 - (i as f64) / 1000.0,
+ "examples": [format!("example-value-{i:03}a"), format!("example-value-{i:03}b")],
+ "notes": ["a note of ordinary length for the trim ladder to consider"],
+ }));
+ }
+ let fields_json = serde_json::to_string(&Value::Array(fields)).unwrap();
+ let resp = catalog_response(vec![dataset_doc("ax-wide", 5000, &fields_json)]);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ let idx = &out["indexes"][0];
+ let raw = serde_json::to_string(idx).unwrap();
+ assert!(
+ raw.len() <= DEFAULT_MAX_BYTES,
+ "entry is {} bytes",
+ raw.len()
+ );
+ assert_eq!(idx["field_count"], 300);
+ assert!(
+ idx["fields_omitted"].as_u64().unwrap() > 0,
+ "fields had to go"
+ );
+ // The whole ladder ran, in order — examples were still gone by the
+ // time fields started going, which is what the order pins.
+ let trimmed: Vec<&str> = idx["trimmed"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .flat_map(|v| v.as_str())
+ .collect();
+ assert_eq!(
+ trimmed,
+ [
+ "examples-cut-to-1",
+ "examples-dropped",
+ "notes-dropped",
+ "fields-dropped"
+ ],
+ "{idx}"
+ );
+ // Highest coverage survived: the ladder drops from the END.
+ let kept = idx["fields"].as_array().unwrap();
+ assert!(kept.len() >= 2, "ladder kept {} fields", kept.len());
+ let cov = |f: &Value| f["coverage"].as_f64().unwrap();
+ assert!(
+ (cov(&kept[0]) - 1.0).abs() < 1e-9,
+ "first kept field is the best covered"
+ );
+ assert!(cov(&kept[kept.len() - 1]) < cov(&kept[0]));
+ assert!(
+ kept.iter().all(|f| f.get("examples").is_none()),
+ "examples were dropped before fields were"
+ );
+ }
+
+ /// The ladder order itself: examples before notes, notes before fields.
+ /// A dataset whose fields all fit once examples and notes are gone must
+ /// keep every field, and a stage that cut nothing must not be reported.
+ #[test]
+ fn the_ladder_trims_examples_then_notes_then_fields() {
+ let with_payload = |examples: bool, notes: bool| {
+ let mut fields = Vec::new();
+ for i in 0..30 {
+ let mut f = json!({
+ "name": format!("f{i:02}"),
+ "es_type": "keyword",
+ "cardinality_est": i,
+ "coverage": 0.9,
+ });
+ if examples {
+ f["examples"] = json!([format!("ex-{i:02}")]);
+ }
+ if notes {
+ f["notes"] = json!(["note"]);
+ }
+ fields.push(f);
+ }
+ serde_json::to_string(&Value::Array(fields)).unwrap()
+ };
+ let resp_full =
+ catalog_response(vec![dataset_doc("ax-ladder", 9, &with_payload(true, true))]);
+ let resp_base = catalog_response(vec![dataset_doc(
+ "ax-ladder",
+ 9,
+ &with_payload(false, false),
+ )]);
+ let huge = MapArgs {
+ index: None,
+ max_bytes: 1_000_000,
+ };
+ let full_len = serde_json::to_string(
+ &serde_json::from_str::<Value>(&render_response(&resp_full, &huge)).unwrap()["indexes"]
+ [0],
+ )
+ .unwrap()
+ .len();
+ let base_len = serde_json::to_string(
+ &serde_json::from_str::<Value>(&render_response(&resp_base, &huge)).unwrap()["indexes"]
+ [0],
+ )
+ .unwrap()
+ .len();
+ // A budget the notes-trimmed entry fits but the examples-trimmed one
+ // cannot: examples cost more bytes than the slack we leave.
+ let budget = base_len + 300;
+ assert!(
+ full_len > budget,
+ "test premise: the full entry ({full_len}) must overshoot {budget}"
+ );
+ let tight = MapArgs {
+ index: None,
+ max_bytes: budget,
+ };
+ let out: Value = serde_json::from_str(&render_response(&resp_full, &tight)).unwrap();
+ let idx = &out["indexes"][0];
+ let raw = serde_json::to_string(idx).unwrap();
+ assert!(raw.len() <= budget, "{} > {budget}: {raw}", raw.len());
+ assert!(
+ idx.get("fields_omitted").is_none(),
+ "all 30 fields fit once notes went: {idx}"
+ );
+ let trimmed: Vec<&str> = idx["trimmed"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .flat_map(|v| v.as_str())
+ .collect();
+ assert_eq!(
+ trimmed,
+ ["examples-dropped", "notes-dropped"],
+ "examples were already 1 each, so the cut-to-1 stage must not be reported"
+ );
+ }
+
+ #[test]
+ fn a_single_field_that_alone_exceeds_the_budget_is_dropped_whole() {
+ let huge = json!([{
+ "name": format!("n{}", "x".repeat(3000)),
+ "es_type": "keyword",
+ "cardinality_est": 1,
+ "coverage": 1.0,
+ }]);
+ let fields_json = serde_json::to_string(&huge).unwrap();
+ let resp = catalog_response(vec![dataset_doc("ax-huge", 1, &fields_json)]);
+ let tight = MapArgs {
+ index: None,
+ max_bytes: MIN_MAX_BYTES,
+ };
+ let out: Value = serde_json::from_str(&render_response(&resp, &tight)).unwrap();
+ let idx = &out["indexes"][0];
+ let raw = serde_json::to_string(idx).unwrap();
+ assert!(raw.len() <= MIN_MAX_BYTES, "{raw}");
+ assert_eq!(idx["fields"].as_array().unwrap().len(), 0);
+ assert_eq!(idx["fields_omitted"], 1);
+ assert_eq!(idx["field_count"], 1, "the count still tells the truth");
+ }
+
+ #[test]
+ fn exact_names_filter_engine_side_and_globs_client_side() {
+ let f = r#"[{"name":"a","es_type":"keyword","cardinality_est":1,"coverage":1.0,"examples":["a"]}]"#;
+ let hits = vec![
+ dataset_doc("ax-logs", 10, f),
+ dataset_doc("ax-metrics", 20, f),
+ dataset_doc("other", 30, f),
+ ];
+ let resp = catalog_response(hits);
+
+ // Exact name: engine filters, so the renderer passes it through.
+ let body = catalog_body(&args(Some("ax-logs")));
+ assert_eq!(body["query"]["bool"]["must"].as_array().unwrap().len(), 2);
+
+ // Glob: no engine-side term; the client filter keeps only matches.
+ let body = catalog_body(&args(Some("ax-*")));
+ assert_eq!(body["query"]["bool"]["must"].as_array().unwrap().len(), 1);
+ let out: Value =
+ serde_json::from_str(&render_response(&resp, &args(Some("ax-*")))).unwrap();
+ assert_eq!(out["matched"], 2);
+ let names: Vec<&str> = out["indexes"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|i| i["index"].as_str().unwrap())
+ .collect();
+ assert_eq!(
+ names,
+ ["ax-metrics", "ax-logs"],
+ "records desc, deterministic"
+ );
+ assert_eq!(out["pattern"], "ax-*");
+
+ // No match is NOT an error: 0 matched plus the hint.
+ let out: Value =
+ serde_json::from_str(&render_response(&resp, &args(Some("zz-*")))).unwrap();
+ assert_eq!(out["matched"], 0);
+ assert!(out["hint"].as_str().unwrap().contains("xerj autoindex"));
+ }
+
+ #[test]
+ fn dataset_notes_ride_the_entry_and_survive_small_caps() {
+ let f = r#"[{"name":"a","es_type":"keyword","cardinality_est":1,"coverage":1.0}]"#;
+ let mut doc = dataset_doc("ax-notes", 7, f);
+ doc["_source"]["notes"] = json!(["dataset-level note from the run"]);
+ let resp = catalog_response(vec![doc]);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ assert_eq!(
+ out["indexes"][0]["notes"],
+ json!(["dataset-level note from the run"])
+ );
+ }
+
+ #[test]
+ fn indexes_past_the_fetch_cap_are_reported_not_silently_dropped() {
+ let f = r#"[{"name":"a","es_type":"keyword","cardinality_est":1,"coverage":1.0}]"#;
+ let mut resp = catalog_response(vec![dataset_doc("ax-a", 1, f)]);
+ resp["hits"]["total"]["value"] = json!(250);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ assert_eq!(out["matched"], 1);
+ assert_eq!(out["indexes_omitted"], 250 - 1);
+ }
+
+ #[test]
+ fn a_corrupt_fields_json_renders_an_empty_field_list_not_an_error() {
+ let resp = catalog_response(vec![dataset_doc("ax-bad", 5, "not json")]);
+ let out: Value = serde_json::from_str(&render_response(&resp, &args(None))).unwrap();
+ assert_eq!(out["matched"], 1);
+ assert_eq!(out["indexes"][0]["field_count"], 0);
+ assert_eq!(out["indexes"][0]["fields"].as_array().unwrap().len(), 0);
+ }
+
+ #[test]
+ fn argument_validation_refuses_mistyped_and_sub_floor_budgets() {
+ assert_eq!(parse_args(&json!({})).unwrap().max_bytes, DEFAULT_MAX_BYTES);
+ assert!(parse_args(&json!({ "index": null, "max_bytes": null })).is_ok());
+ assert_eq!(parse_args(&json!({ "index": " " })).unwrap().index, None);
+ assert_eq!(
+ parse_args(&json!({ "index": "ax-*" }))
+ .unwrap()
+ .index
+ .as_deref(),
+ Some("ax-*")
+ );
+ let e = parse_args(&json!({ "index": 5 })).unwrap_err();
+ assert!(e.contains("`index` must be a string"), "{e}");
+ let e = parse_args(&json!({ "max_bytes": 100 })).unwrap_err();
+ assert!(e.contains(">= 1024"), "{e}");
+ let e = parse_args(&json!({ "max_bytes": "4096" })).unwrap_err();
+ assert!(e.contains("`max_bytes` must be an integer"), "{e}");
+ assert_eq!(
+ parse_args(&json!({ "max_bytes": 2048 })).unwrap().max_bytes,
+ 2048
+ );
+ }
+
+ /// The dispatch seam: a bad argument is an isError tool result BEFORE any
+ /// request is made (the same shape as the `max_tokens` refusal test).
+ #[tokio::test]
+ async fn a_bad_argument_is_refused_at_dispatch() {
+ let ctx = disconnected_ctx();
+ for bad in [json!({ "max_bytes": 10 }), json!({ "index": 7 })] {
+ let msg = json!({ "params": { "name": "xerj_map", "arguments": bad } });
+ let res = crate::call_tool(&ctx, &msg).await;
+ assert_eq!(res["isError"], true, "{bad}: {res}");
+ assert!(
+ res["content"][0]["text"]
+ .as_str()
+ .unwrap()
+ .contains("xerj_map"),
+ "{res}"
+ );
+ }
+ }
+
+ /// Transport failure reaches the agent as isError with the URL named —
+ /// the same honesty as every other proxied tool. A silently empty map
+ /// here would be the worst failure mode: it reads as "no data".
+ #[tokio::test]
+ async fn an_unreachable_node_is_an_error_not_an_empty_map() {
+ let ctx = disconnected_ctx();
+ let msg = json!({ "params": { "name": "xerj_map", "arguments": {} } });
+ let res = crate::call_tool(&ctx, &msg).await;
+ assert_eq!(res["isError"], true);
+ let text = res["content"][0]["text"].as_str().unwrap();
+ assert!(text.contains("request to"), "{text}");
+ assert!(text.contains(CATALOG_INDEX), "{text}");
+ }
+}
diff --git a/landing/docs/agents/schemas/mcp-tools.json b/landing/docs/agents/schemas/mcp-tools.json
index d44c80a26..5d1a63a58 100644
--- a/landing/docs/agents/schemas/mcp-tools.json
+++ b/landing/docs/agents/schemas/mcp-tools.json
@@ -490,6 +490,24 @@
]
}
},
+ {
+ "name": "xerj_map",
+ "description": "Per-index FIELD MAP from this node's autoindex catalog — call it BEFORE writing query DSL, so you stop guessing field names. For each catalogued index: record count, time field and time range, semantic body field, and every field the profiler inferred — name, es_type (keyword|text|long|double|boolean|date|semantic_text), date encoding, semantic tag, cardinality estimate, coverage, up to 5 examples, and the sampled date AND numeric min-max (numeric ranges appear on `long` fields only; fractional `double` ranges are not tracked). Reads the catalog index `autoindex-catalog` that `xerj autoindex <folder>` writes; never-autoindexed indexes are not listed. Honesty: every fact is inferred from a bounded sample taken at autoindex time — cardinality is an estimate (with `cardinality_overflow:true` it means AT LEAST that many), and min/max/coverage describe the sample, not the whole index. Up to 100 indexes per call (`indexes_omitted` reports more).",
+ "inputSchema": {
+ "type": "object",
+ "properties": {
+ "index": {
+ "type": "string",
+ "description": "Index name or `*`-glob (e.g. `ax-logs`, `ax-*`). Default: every catalogued index. An exact name filters engine-side; a glob filters client-side over the catalog's index names."
+ },
+ "max_bytes": {
+ "type": "integer",
+ "minimum": 1024,
+ "description": "Per-index byte budget for one rendered index entry (default 4096). An entry NEVER exceeds it: the trim ladder first cuts per-field examples to one, then drops examples, then per-field notes, then the lowest-coverage fields — and the entry's `trimmed` and `fields_omitted` say exactly what went. The budget is per index; a multi-index response carries one such entry per index. Minimum 1024."
+ }
+ }
+ }
+ },
{
"name": "xerj_code_search",
"description": "Reference-coding retrieval over a corpus of peer projects: how did real codebases solve this problem? Returns the matching definition (not a byte window), the file:line to cite, and per-hit licence warnings — AGPL/SSPL/Elastic/BUSL/GPL/LGPL/MPL sources are approach-only. Refuses indices older than 30 days (override with stale_ok). A no-match is NOT an error: it says the corpus is wrong for the task. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule). Corpora: `xerj corpus list`.",