AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(map): explain autoindex and managed corpus namespace boundaries
Jarvis9d02f048dd57
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1bd033cfe..379e2e858 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- Empty autoindex maps now explain that managed corpus slugs and autoindex+ dataset/index names belong to separate discovery surfaces. CLI diagnostics,+ MCP empty responses and the `xerj_map` description point to corpus discovery+ without claiming a corpus exists. Missing catalogs remain errors; auth+ refusals and successful dataset maps retain their behavior (#2412).+- `xerj sessions import --format claude-code --input FILE --output NEW.jsonl`prepares an explicit Claude Code JSONL transcript for ordinary `autoindex`.It emits text-bearing user/assistant records through the existing nativediff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rsindex 709a0769e..b27febd81 100644--- a/engine/crates/xerj-autoindex/src/catalog.rs+++ b/engine/crates/xerj-autoindex/src/catalog.rs@@ -10,6 +10,9 @@ use serde_json::{json, Value};pub const CATALOG_INDEX: &str = xerj_common::AUTOINDEX_CATALOG_INDEX;+/// Scope guidance, not an existence probe or a dataset record.+pub const MAP_NAMESPACE_HINT: &str = "This map lists autoindex datasets, not the managed corpus registry. A managed corpus slug is not an autoindex dataset/index pattern. Use `xerj corpus list` for this client's configured corpora, then `xerj code <corpus> <query>` (MCP `xerj_code_search`); the node's `_cat/indices` lists indices. An empty map does not establish that managed corpus data is absent; no managed corpus existence was checked.";+/// The corpus-scope field the #737/#693 exclusion sweeps term-query (#755).////// It exists because `prefix` cannot be relied on to be a `keyword` on andiff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex fa70cc173..bd648d716 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -493,7 +493,7 @@ pub fn help_text_with(feedback: bool) -> String {\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20the JSON decision request on stdout and exits 4 instead,\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20exactly like an agent-driven run. See ESTIMATE + DECISION\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20GATE.\n\- \x20\x20\x20\x20--dataset <SLUG> (map, no folder) show a single catalog dataset\n\+ \x20\x20\x20\x20--dataset <SLUG> (map, no folder) show one autoindex dataset (not a managed corpus slug)\n\\x20\x20\x20\x20--offset <N> (map, no folder) skip the first N dataset detail sections\n\\x20\x20\x20\x20--limit <N> (map, no folder) how many dataset detail sections to print\n\\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20(default: all of them when there are few, otherwise\n\diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 367943c25..3c762aae4 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -12869,7 +12869,15 @@ fn run_map(cfg: MapCfg) -> Result<i32> {if let Some(s) = sort {body["sort"] = s;}- let v = es.search(catalog::CATALOG_INDEX, &body)?;+ let v = es+ .search_present(catalog::CATALOG_INDEX, &body)?+ .ok_or_else(|| {+ anyhow::anyhow!(+ "autoindex catalog {} is missing (HTTP 404). {}",+ catalog::CATALOG_INDEX,+ catalog::MAP_NAMESPACE_HINT+ )+ })?;Ok(v.pointer("/hits/hits").and_then(|h| h.as_array()).map(|arr| {@@ -12901,17 +12909,18 @@ fn run_map(cfg: MapCfg) -> Result<i32> {if datasets.is_empty() {if let Some(prefix) = cfg.prefix.as_deref() {eprintln!(- "no autoindex catalog found for --prefix {prefix} at {} (index {}) — run `xerj autoindex <folder> --prefix {prefix}` first",+ "no autoindex datasets match --prefix {prefix} at {} (catalog {})",cfg.url,catalog::CATALOG_INDEX);} else {eprintln!(- "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",+ "no autoindex datasets match at {} (catalog {})",cfg.url,catalog::CATALOG_INDEX);}+ eprintln!("{}", catalog::MAP_NAMESPACE_HINT);return Ok(1);}let mut run_must = vec![json!({"term": {"doc_kind": "run"}})];diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rsindex 26f5082f7..1e2b1734f 100644--- a/engine/crates/xerj-mcp/src/lib.rs+++ b/engine/crates/xerj-mcp/src/lib.rs@@ -1266,7 +1266,7 @@ pub fn tool_specs_with(include_hub: bool) -> Value {{"name": "xerj_map","description": format!(- "{}{NEVER_DELETE_INDEX}",+ "{} {}{NEVER_DELETE_INDEX}",concat!("Per-index FIELD MAP from this node's autoindex catalog — call it ","BEFORE writing query DSL, so you stop guessing field names. For each ",@@ -1285,7 +1285,7 @@ pub fn tool_specs_with(include_hub: bool) -> Value {"reports more). When more than 8 indexes match and you did not pass ","`index`, the answer is a SUMMARY page (index, records, field names) ","— pass `index` or `detail: true` with `offset`/`limit` for full fields."- )),+ ), xerj_autoindex::catalog::MAP_NAMESPACE_HINT),"inputSchema": {"type": "object","properties": {@@ -6162,6 +6162,8 @@ mod tests {"not tracked","indexes_omitted","at autoindex time",+ "managed corpus registry",+ "xerj corpus list",] {assert!(d.contains(needle), "description lacks `{needle}`: {d}");}diff --git a/engine/crates/xerj-mcp/src/map.rs b/engine/crates/xerj-mcp/src/map.rsindex 5f1927b55..4727bf68e 100644--- a/engine/crates/xerj-mcp/src/map.rs+++ b/engine/crates/xerj-mcp/src/map.rs@@ -469,6 +469,10 @@ pub(crate) fn render_response(resp: &Value, args: &MapArgs) -> String {out.insert("indexes_omitted".into(), json!(omitted));}if docs.is_empty() {+ out.insert(+ "namespace_hint".into(),+ json!(xerj_autoindex::catalog::MAP_NAMESPACE_HINT),+ );out.insert("hint".into(),json!(format!(@@ -855,8 +859,8 @@ pub(crate) async fn run(ctx: &Ctx, args: &Value) -> Value {tool_text(format!("xerj_map: no autoindex catalog on this node (index \- `{CATALOG_INDEX}` not found, HTTP 404) — run `xerj autoindex \- <folder>` first. Engine said: {text}"+ `{CATALOG_INDEX}` not found, HTTP 404). {} Engine said: {text}",+ xerj_autoindex::catalog::MAP_NAMESPACE_HINT),true,)@@ -932,6 +936,80 @@ mod tests {assert!(!glob_match("ax-*-2026", "ax-2026"));}+ #[test]+ fn empty_maps_explain_namespace_scope_without_inventing_corpus_existence() {+ let rendered: Value = serde_json::from_str(&render_response(+ &catalog_response(vec![]),+ &args(Some("managed-*")),+ ))+ .unwrap();+ assert_eq!(rendered["matched"], 0);+ assert_eq!(rendered["indexes"], json!([]));+ let hint = rendered["namespace_hint"].as_str().unwrap();+ assert!(hint.contains("managed corpus slug"));+ assert!(hint.contains("xerj corpus list"));+ assert!(hint.contains("no managed corpus existence was checked"));+ let positive: Value = serde_json::from_str(&render_response(+ &catalog_response(vec![dataset_doc("ax-code", 2, "[]")]),+ &args(None),+ ))+ .unwrap();+ assert_eq!(positive["matched"], 1);+ assert!(positive.get("namespace_hint").is_none());+ }++ #[tokio::test]+ async fn namespace_guidance_is_added_to_404_but_not_authorization_refusals() {+ use std::io::{Read, Write};+ for (status, expected_hint) in [(404, true), (403, false)] {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let server = std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ stream+ .set_read_timeout(Some(std::time::Duration::from_secs(10)))+ .unwrap();+ let mut request = Vec::new();+ let mut chunk = [0u8; 2048];+ loop {+ let n = stream.read(&mut chunk).unwrap();+ assert!(n > 0);+ request.extend_from_slice(&chunk[..n]);+ if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") {+ let headers = String::from_utf8_lossy(&request[..end]);+ let length: usize = headers+ .lines()+ .find_map(|line| {+ line.to_ascii_lowercase()+ .strip_prefix("content-length:")+ .map(|v| v.trim().parse().unwrap())+ })+ .unwrap_or(0);+ if request.len() >= end + 4 + length {+ break;+ }+ }+ }+ assert!(request.starts_with(b"POST /autoindex-catalog/_search "));+ let body = "{\"error\":{\"reason\":\"fixture refusal\"}}";+ write!(stream, "HTTP/1.1 {status} Refused\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();+ });+ let ctx = crate::Ctx {+ client: reqwest::Client::new(),+ base_url: url,+ auth: None,+ };+ let result =+ crate::call_tool(&ctx, &json!({"params":{"name":"xerj_map","arguments":{}}})).await;+ server.join().unwrap();+ assert_eq!(result["isError"], true);+ let text = result["content"][0]["text"].as_str().unwrap();+ assert!(text.contains(&status.to_string()), "{text}");+ assert_eq!(text.contains("managed corpus"), expected_hint, "{text}");+ assert!(text.contains("fixture refusal"), "{text}");+ }+ }+#[test]fn a_small_catalog_renders_every_field_untrimmed() {let fields = r#"[diff --git a/tools/catalog-map/README.md b/tools/catalog-map/README.mdnew file mode 100644index 000000000..0f5c93efe--- /dev/null+++ b/tools/catalog-map/README.md@@ -0,0 +1,56 @@+# Autoindex maps and managed corpora are separate views++`xerj autoindex map` and MCP `xerj_map` read `autoindex-catalog` on the+configured node. That catalog lists datasets written by `xerj autoindex`.+CLI `--dataset` is a catalog dataset slug; `--prefix` scopes autoindex index+names. MCP `index` is a catalog index name or glob, not a managed corpus slug.++`xerj corpus list` lists corpora configured for the client process. A selected+corpus is queried through `xerj code <corpus> <query>` or MCP+`xerj_code_search`. The client's registry can differ from the node's environment+and does not prove that a corpus is indexed/reachable. The node's `_cat/indices`+lists node index names, including indices outside autoindex. These views are+not interchangeable and a managed corpus need not have an autoindex catalog row.++An empty map therefore does not establish that managed corpus data is absent.+Empty MCP maps keep `matched: 0` and `indexes: []`, adding a top-level+`namespace_hint`. A missing catalog (HTTP 404) stays an error with scope+guidance. CLI empty/missing results keep exit 1; diagnostics go to stderr,+including with `--json`, leaving stdout unmodified. HTTP 401/403, transport and+other server failures remain errors without being relabelled as empty catalogs.+Positive dataset maps do not gain `namespace_hint` or synthetic dataset rows.++No managed-corpus registry or index is probed to generate this hint. No extra+network requests, automatic indexing, index selection, fallback search or+schema/mapping migration are introduced. A corpus and an autoindex dataset+may happen to share a name; name similarity does not establish ownership or+equivalence. An empty page can also follow pagination/filtering, so use the+returned pattern, offset/limit and omission information when diagnosing a map.++## Reproduce the transport checks++From the repository root, with Python 3 and a built binary:++```sh+python3 tools/catalog-map/smoke.py --binary engine/target/release/xerj+```++The script uses only a private loopback HTTP fixture and temporary client cwd.+It does not run a node, read corpus configuration, index data or contact an+existing server. Eight CLI/MCP cases cover empty results, missing catalogs,+403 refusals and populated datasets; CLI also verifies `--json` stdout.+Requests must remain reads of `/`, catalog `_search` or dataset `_mapping`.+The client/fixture processes stop and temporary files are removed on completion.+These fixtures do not establish live managed-corpus availability or relevance.++MCP unit tests also cover additive empty-result guidance, unchanged positive+maps and real loopback 404/403 tool-error classification. The description is+generated into the published MCP schema using `scripts/mcp-schema-check.sh`;+do not edit that JSON by hand.++Reference before implementation: baseline `9cf3f29c`,+`engine/crates/xerj-autoindex/src/lib.rs:12857` (CLI map),+`engine/crates/xerj-autoindex/src/esclient.rs:2452` (`search_present` distinguishes+HTTP 404 from other errors), `engine/crates/xerj-mcp/src/map.rs:360` (bounded+map rendering), and `tools/xerj-code/README.md:103` (client-side corpus listing).+Existing query, renderer and error helpers are preserved; no external code copied.diff --git a/tools/catalog-map/smoke.py b/tools/catalog-map/smoke.pynew file mode 100644index 000000000..a5e34b39a--- /dev/null+++ b/tools/catalog-map/smoke.py@@ -0,0 +1,117 @@+#!/usr/bin/env python3+"""Verify CLI/MCP catalog scope and error classification against read-only loopback fixtures."""+import argparse+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+import json+import os+from pathlib import Path+import subprocess+import tempfile+import threading+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--binary', type=Path, required=True)+ args = parser.parse_args()+ binary = args.binary.resolve(strict=True)+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')+ and k.lower() not in ('all_proxy', 'http_proxy', 'https_proxy', 'no_proxy')}+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='')+ mode = 'empty'+ requests = []+ dataset = {'doc_kind': 'dataset', 'slug': 'ax-code', 'index_name': 'ax-code',+ 'record_count': 2, 'fields_json': '[]'}++ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *_):+ pass++ def reply(self, status, body):+ payload = json.dumps(body).encode()+ self.send_response(status)+ self.send_header('Content-Type', 'application/json')+ self.send_header('Content-Length', str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def do_GET(self):+ requests.append(('GET', self.path))+ self.reply(200, {})++ def do_POST(self):+ requests.append(('POST', self.path))+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))+ if mode in ('missing', 'forbidden'):+ status = 404 if mode == 'missing' else 403+ kind = 'index_not_found_exception' if status == 404 else 'security_exception'+ self.reply(status, {'error': {'type': kind, 'reason': 'fixture refusal'}, 'status': status})+ return+ is_dataset = '"dataset"' in json.dumps(body.get('query', {}))+ hits = [{'_source': dataset}] if mode == 'positive' and is_dataset else []+ self.reply(200, {'hits': {'total': {'value': len(hits)}, 'hits': hits}})++ server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)+ worker = threading.Thread(target=server.serve_forever, daemon=True)+ worker.start()+ url = f'http://127.0.0.1:{server.server_port}'+ failures = []+ try:+ with tempfile.TemporaryDirectory(prefix='xerj-catalog-map-') as temp:+ for mode in ('empty', 'missing', 'forbidden', 'positive'):+ for transport in ('cli', 'mcp'):+ requests.clear()+ try:+ if transport == 'cli':+ result = subprocess.run([str(binary), 'autoindex', 'map', '--url', url],+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)+ assert result.returncode == (0 if mode == 'positive' else 1), result.stderr+ text = result.stdout + result.stderr+ json_result = subprocess.run([str(binary), 'autoindex', 'map', '--url', url, '--json'],+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)+ assert json_result.returncode == result.returncode, json_result.stderr+ if mode == 'positive':+ assert json.loads(json_result.stdout)['datasets'][0]['index_name'] == 'ax-code'+ else:+ assert not json_result.stdout.strip(), json_result.stdout+ else:+ messages = [+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {'protocolVersion': '2024-11-05', 'capabilities': {}, 'clientInfo': {'name': 'catalog-fixture', 'version': '1'}}},+ {'jsonrpc': '2.0', 'method': 'notifications/initialized'},+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {'name': 'xerj_map', 'arguments': {'index': '*'}}},+ ]+ result = subprocess.run([str(binary), 'mcp', '--url', url], input=''.join(json.dumps(m) + '\n' for m in messages),+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)+ assert result.returncode == 0, result.stderr+ reply = next(json.loads(line)['result'] for line in result.stdout.splitlines() if json.loads(line).get('id') == 2)+ assert reply.get('isError', False) == (mode in ('missing', 'forbidden')), reply+ text = reply['content'][0]['text']+ if mode == 'empty':+ mapped = json.loads(text)+ assert mapped['matched'] == 0 and mapped['indexes'] == [], mapped+ if mode in ('empty', 'missing'):+ assert 'managed corpus' in text, text+ assert 'xerj corpus list' in text, text+ assert 'not establish' in text, text+ elif mode == 'forbidden':+ assert '403' in text, text+ assert 'managed corpus' not in text, text+ else:+ assert 'ax-code' in text, text+ assert 'namespace_hint' not in text, text+ assert all(method in ('GET', 'POST') and (path == '/' or path.endswith('/_search') or path.endswith('/_mapping'))+ for method, path in requests), requests+ print(f'PASS {transport} {mode}: classification, scope and read-only requests', flush=True)+ except (AssertionError, KeyError, StopIteration) as error:+ failures.append(f'{transport} {mode}: {error}')+ if failures:+ raise AssertionError('\n'.join(failures))+ finally:+ server.shutdown()+ server.server_close()+ worker.join(timeout=5)+ return 0+++if __name__ == '__main__':+ raise SystemExit(main())