Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(map): explain autoindex and managed corpus namespace boundaries

Jarvis9d02f048dd57
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1bd033cfe..379e2e858 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- Empty autoindex maps now explain that managed corpus slugs and autoindex
+ dataset/index names belong to separate discovery surfaces. CLI diagnostics,
+ MCP empty responses and the `xerj_map` description point to corpus discovery
+ without claiming a corpus exists. Missing catalogs remain errors; auth
+ refusals and successful dataset maps retain their behavior (#2412).
+
- `xerj sessions import --format claude-code --input FILE --output NEW.jsonl`
prepares an explicit Claude Code JSONL transcript for ordinary `autoindex`.
It emits text-bearing user/assistant records through the existing native
diff --git a/engine/crates/xerj-autoindex/src/catalog.rs b/engine/crates/xerj-autoindex/src/catalog.rs
index 709a0769e..b27febd81 100644
--- a/engine/crates/xerj-autoindex/src/catalog.rs
+++ b/engine/crates/xerj-autoindex/src/catalog.rs
@@ -10,6 +10,9 @@ use serde_json::{json, Value};
pub const CATALOG_INDEX: &str = xerj_common::AUTOINDEX_CATALOG_INDEX;
+/// Scope guidance, not an existence probe or a dataset record.
+pub const MAP_NAMESPACE_HINT: &str = "This map lists autoindex datasets, not the managed corpus registry. A managed corpus slug is not an autoindex dataset/index pattern. Use `xerj corpus list` for this client's configured corpora, then `xerj code <corpus> <query>` (MCP `xerj_code_search`); the node's `_cat/indices` lists indices. An empty map does not establish that managed corpus data is absent; no managed corpus existence was checked.";
+
/// The corpus-scope field the #737/#693 exclusion sweeps term-query (#755).
///
/// It exists because `prefix` cannot be relied on to be a `keyword` on an
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index fa70cc173..bd648d716 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -493,7 +493,7 @@ pub fn help_text_with(feedback: bool) -> String {
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20the JSON decision request on stdout and exits 4 instead,\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20exactly like an agent-driven run. See ESTIMATE + DECISION\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20GATE.\n\
- \x20\x20\x20\x20--dataset <SLUG> (map, no folder) show a single catalog dataset\n\
+ \x20\x20\x20\x20--dataset <SLUG> (map, no folder) show one autoindex dataset (not a managed corpus slug)\n\
\x20\x20\x20\x20--offset <N> (map, no folder) skip the first N dataset detail sections\n\
\x20\x20\x20\x20--limit <N> (map, no folder) how many dataset detail sections to print\n\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20(default: all of them when there are few, otherwise\n\
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 367943c25..3c762aae4 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -12869,7 +12869,15 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
if let Some(s) = sort {
body["sort"] = s;
}
- let v = es.search(catalog::CATALOG_INDEX, &body)?;
+ let v = es
+ .search_present(catalog::CATALOG_INDEX, &body)?
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "autoindex catalog {} is missing (HTTP 404). {}",
+ catalog::CATALOG_INDEX,
+ catalog::MAP_NAMESPACE_HINT
+ )
+ })?;
Ok(v.pointer("/hits/hits")
.and_then(|h| h.as_array())
.map(|arr| {
@@ -12901,17 +12909,18 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
if datasets.is_empty() {
if let Some(prefix) = cfg.prefix.as_deref() {
eprintln!(
- "no autoindex catalog found for --prefix {prefix} at {} (index {}) — run `xerj autoindex <folder> --prefix {prefix}` first",
+ "no autoindex datasets match --prefix {prefix} at {} (catalog {})",
cfg.url,
catalog::CATALOG_INDEX
);
} else {
eprintln!(
- "no autoindex catalog found at {} (index {}) — run `xerj autoindex <folder>` first",
+ "no autoindex datasets match at {} (catalog {})",
cfg.url,
catalog::CATALOG_INDEX
);
}
+ eprintln!("{}", catalog::MAP_NAMESPACE_HINT);
return Ok(1);
}
let mut run_must = vec![json!({"term": {"doc_kind": "run"}})];
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 26f5082f7..1e2b1734f 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -1266,7 +1266,7 @@ pub fn tool_specs_with(include_hub: bool) -> Value {
{
"name": "xerj_map",
"description": format!(
- "{}{NEVER_DELETE_INDEX}",
+ "{} {}{NEVER_DELETE_INDEX}",
concat!(
"Per-index FIELD MAP from this node's autoindex catalog — call it ",
"BEFORE writing query DSL, so you stop guessing field names. For each ",
@@ -1285,7 +1285,7 @@ pub fn tool_specs_with(include_hub: bool) -> Value {
"reports more). When more than 8 indexes match and you did not pass ",
"`index`, the answer is a SUMMARY page (index, records, field names) ",
"— pass `index` or `detail: true` with `offset`/`limit` for full fields."
- )),
+ ), xerj_autoindex::catalog::MAP_NAMESPACE_HINT),
"inputSchema": {
"type": "object",
"properties": {
@@ -6162,6 +6162,8 @@ mod tests {
"not tracked",
"indexes_omitted",
"at autoindex time",
+ "managed corpus registry",
+ "xerj corpus list",
] {
assert!(d.contains(needle), "description lacks `{needle}`: {d}");
}
diff --git a/engine/crates/xerj-mcp/src/map.rs b/engine/crates/xerj-mcp/src/map.rs
index 5f1927b55..4727bf68e 100644
--- a/engine/crates/xerj-mcp/src/map.rs
+++ b/engine/crates/xerj-mcp/src/map.rs
@@ -469,6 +469,10 @@ pub(crate) fn render_response(resp: &Value, args: &MapArgs) -> String {
out.insert("indexes_omitted".into(), json!(omitted));
}
if docs.is_empty() {
+ out.insert(
+ "namespace_hint".into(),
+ json!(xerj_autoindex::catalog::MAP_NAMESPACE_HINT),
+ );
out.insert(
"hint".into(),
json!(format!(
@@ -855,8 +859,8 @@ pub(crate) async fn run(ctx: &Ctx, args: &Value) -> Value {
tool_text(
format!(
"xerj_map: no autoindex catalog on this node (index \
- `{CATALOG_INDEX}` not found, HTTP 404) — run `xerj autoindex \
- <folder>` first. Engine said: {text}"
+ `{CATALOG_INDEX}` not found, HTTP 404). {} Engine said: {text}",
+ xerj_autoindex::catalog::MAP_NAMESPACE_HINT
),
true,
)
@@ -932,6 +936,80 @@ mod tests {
assert!(!glob_match("ax-*-2026", "ax-2026"));
}
+ #[test]
+ fn empty_maps_explain_namespace_scope_without_inventing_corpus_existence() {
+ let rendered: Value = serde_json::from_str(&render_response(
+ &catalog_response(vec![]),
+ &args(Some("managed-*")),
+ ))
+ .unwrap();
+ assert_eq!(rendered["matched"], 0);
+ assert_eq!(rendered["indexes"], json!([]));
+ let hint = rendered["namespace_hint"].as_str().unwrap();
+ assert!(hint.contains("managed corpus slug"));
+ assert!(hint.contains("xerj corpus list"));
+ assert!(hint.contains("no managed corpus existence was checked"));
+ let positive: Value = serde_json::from_str(&render_response(
+ &catalog_response(vec![dataset_doc("ax-code", 2, "[]")]),
+ &args(None),
+ ))
+ .unwrap();
+ assert_eq!(positive["matched"], 1);
+ assert!(positive.get("namespace_hint").is_none());
+ }
+
+ #[tokio::test]
+ async fn namespace_guidance_is_added_to_404_but_not_authorization_refusals() {
+ use std::io::{Read, Write};
+ for (status, expected_hint) in [(404, true), (403, false)] {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let server = std::thread::spawn(move || {
+ let (mut stream, _) = listener.accept().unwrap();
+ stream
+ .set_read_timeout(Some(std::time::Duration::from_secs(10)))
+ .unwrap();
+ let mut request = Vec::new();
+ let mut chunk = [0u8; 2048];
+ loop {
+ let n = stream.read(&mut chunk).unwrap();
+ assert!(n > 0);
+ request.extend_from_slice(&chunk[..n]);
+ if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") {
+ let headers = String::from_utf8_lossy(&request[..end]);
+ let length: usize = headers
+ .lines()
+ .find_map(|line| {
+ line.to_ascii_lowercase()
+ .strip_prefix("content-length:")
+ .map(|v| v.trim().parse().unwrap())
+ })
+ .unwrap_or(0);
+ if request.len() >= end + 4 + length {
+ break;
+ }
+ }
+ }
+ assert!(request.starts_with(b"POST /autoindex-catalog/_search "));
+ let body = "{\"error\":{\"reason\":\"fixture refusal\"}}";
+ write!(stream, "HTTP/1.1 {status} Refused\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
+ });
+ let ctx = crate::Ctx {
+ client: reqwest::Client::new(),
+ base_url: url,
+ auth: None,
+ };
+ let result =
+ crate::call_tool(&ctx, &json!({"params":{"name":"xerj_map","arguments":{}}})).await;
+ server.join().unwrap();
+ assert_eq!(result["isError"], true);
+ let text = result["content"][0]["text"].as_str().unwrap();
+ assert!(text.contains(&status.to_string()), "{text}");
+ assert_eq!(text.contains("managed corpus"), expected_hint, "{text}");
+ assert!(text.contains("fixture refusal"), "{text}");
+ }
+ }
+
#[test]
fn a_small_catalog_renders_every_field_untrimmed() {
let fields = r#"[
diff --git a/tools/catalog-map/README.md b/tools/catalog-map/README.md
new file mode 100644
index 000000000..0f5c93efe
--- /dev/null
+++ b/tools/catalog-map/README.md
@@ -0,0 +1,56 @@
+# Autoindex maps and managed corpora are separate views
+
+`xerj autoindex map` and MCP `xerj_map` read `autoindex-catalog` on the
+configured node. That catalog lists datasets written by `xerj autoindex`.
+CLI `--dataset` is a catalog dataset slug; `--prefix` scopes autoindex index
+names. MCP `index` is a catalog index name or glob, not a managed corpus slug.
+
+`xerj corpus list` lists corpora configured for the client process. A selected
+corpus is queried through `xerj code <corpus> <query>` or MCP
+`xerj_code_search`. The client's registry can differ from the node's environment
+and does not prove that a corpus is indexed/reachable. The node's `_cat/indices`
+lists node index names, including indices outside autoindex. These views are
+not interchangeable and a managed corpus need not have an autoindex catalog row.
+
+An empty map therefore does not establish that managed corpus data is absent.
+Empty MCP maps keep `matched: 0` and `indexes: []`, adding a top-level
+`namespace_hint`. A missing catalog (HTTP 404) stays an error with scope
+guidance. CLI empty/missing results keep exit 1; diagnostics go to stderr,
+including with `--json`, leaving stdout unmodified. HTTP 401/403, transport and
+other server failures remain errors without being relabelled as empty catalogs.
+Positive dataset maps do not gain `namespace_hint` or synthetic dataset rows.
+
+No managed-corpus registry or index is probed to generate this hint. No extra
+network requests, automatic indexing, index selection, fallback search or
+schema/mapping migration are introduced. A corpus and an autoindex dataset
+may happen to share a name; name similarity does not establish ownership or
+equivalence. An empty page can also follow pagination/filtering, so use the
+returned pattern, offset/limit and omission information when diagnosing a map.
+
+## Reproduce the transport checks
+
+From the repository root, with Python 3 and a built binary:
+
+```sh
+python3 tools/catalog-map/smoke.py --binary engine/target/release/xerj
+```
+
+The script uses only a private loopback HTTP fixture and temporary client cwd.
+It does not run a node, read corpus configuration, index data or contact an
+existing server. Eight CLI/MCP cases cover empty results, missing catalogs,
+403 refusals and populated datasets; CLI also verifies `--json` stdout.
+Requests must remain reads of `/`, catalog `_search` or dataset `_mapping`.
+The client/fixture processes stop and temporary files are removed on completion.
+These fixtures do not establish live managed-corpus availability or relevance.
+
+MCP unit tests also cover additive empty-result guidance, unchanged positive
+maps and real loopback 404/403 tool-error classification. The description is
+generated into the published MCP schema using `scripts/mcp-schema-check.sh`;
+do not edit that JSON by hand.
+
+Reference before implementation: baseline `9cf3f29c`,
+`engine/crates/xerj-autoindex/src/lib.rs:12857` (CLI map),
+`engine/crates/xerj-autoindex/src/esclient.rs:2452` (`search_present` distinguishes
+HTTP 404 from other errors), `engine/crates/xerj-mcp/src/map.rs:360` (bounded
+map rendering), and `tools/xerj-code/README.md:103` (client-side corpus listing).
+Existing query, renderer and error helpers are preserved; no external code copied.
diff --git a/tools/catalog-map/smoke.py b/tools/catalog-map/smoke.py
new file mode 100644
index 000000000..a5e34b39a
--- /dev/null
+++ b/tools/catalog-map/smoke.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Verify CLI/MCP catalog scope and error classification against read-only loopback fixtures."""
+import argparse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import threading
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--binary', type=Path, required=True)
+ args = parser.parse_args()
+ binary = args.binary.resolve(strict=True)
+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')
+ and k.lower() not in ('all_proxy', 'http_proxy', 'https_proxy', 'no_proxy')}
+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_AUTH='')
+ mode = 'empty'
+ requests = []
+ dataset = {'doc_kind': 'dataset', 'slug': 'ax-code', 'index_name': 'ax-code',
+ 'record_count': 2, 'fields_json': '[]'}
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *_):
+ pass
+
+ def reply(self, status, body):
+ payload = json.dumps(body).encode()
+ self.send_response(status)
+ self.send_header('Content-Type', 'application/json')
+ self.send_header('Content-Length', str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def do_GET(self):
+ requests.append(('GET', self.path))
+ self.reply(200, {})
+
+ def do_POST(self):
+ requests.append(('POST', self.path))
+ body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
+ if mode in ('missing', 'forbidden'):
+ status = 404 if mode == 'missing' else 403
+ kind = 'index_not_found_exception' if status == 404 else 'security_exception'
+ self.reply(status, {'error': {'type': kind, 'reason': 'fixture refusal'}, 'status': status})
+ return
+ is_dataset = '"dataset"' in json.dumps(body.get('query', {}))
+ hits = [{'_source': dataset}] if mode == 'positive' and is_dataset else []
+ self.reply(200, {'hits': {'total': {'value': len(hits)}, 'hits': hits}})
+
+ server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
+ worker = threading.Thread(target=server.serve_forever, daemon=True)
+ worker.start()
+ url = f'http://127.0.0.1:{server.server_port}'
+ failures = []
+ try:
+ with tempfile.TemporaryDirectory(prefix='xerj-catalog-map-') as temp:
+ for mode in ('empty', 'missing', 'forbidden', 'positive'):
+ for transport in ('cli', 'mcp'):
+ requests.clear()
+ try:
+ if transport == 'cli':
+ result = subprocess.run([str(binary), 'autoindex', 'map', '--url', url],
+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)
+ assert result.returncode == (0 if mode == 'positive' else 1), result.stderr
+ text = result.stdout + result.stderr
+ json_result = subprocess.run([str(binary), 'autoindex', 'map', '--url', url, '--json'],
+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)
+ assert json_result.returncode == result.returncode, json_result.stderr
+ if mode == 'positive':
+ assert json.loads(json_result.stdout)['datasets'][0]['index_name'] == 'ax-code'
+ else:
+ assert not json_result.stdout.strip(), json_result.stdout
+ else:
+ messages = [
+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {'protocolVersion': '2024-11-05', 'capabilities': {}, 'clientInfo': {'name': 'catalog-fixture', 'version': '1'}}},
+ {'jsonrpc': '2.0', 'method': 'notifications/initialized'},
+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {'name': 'xerj_map', 'arguments': {'index': '*'}}},
+ ]
+ result = subprocess.run([str(binary), 'mcp', '--url', url], input=''.join(json.dumps(m) + '\n' for m in messages),
+ cwd=temp, env=env, capture_output=True, text=True, timeout=30)
+ assert result.returncode == 0, result.stderr
+ reply = next(json.loads(line)['result'] for line in result.stdout.splitlines() if json.loads(line).get('id') == 2)
+ assert reply.get('isError', False) == (mode in ('missing', 'forbidden')), reply
+ text = reply['content'][0]['text']
+ if mode == 'empty':
+ mapped = json.loads(text)
+ assert mapped['matched'] == 0 and mapped['indexes'] == [], mapped
+ if mode in ('empty', 'missing'):
+ assert 'managed corpus' in text, text
+ assert 'xerj corpus list' in text, text
+ assert 'not establish' in text, text
+ elif mode == 'forbidden':
+ assert '403' in text, text
+ assert 'managed corpus' not in text, text
+ else:
+ assert 'ax-code' in text, text
+ assert 'namespace_hint' not in text, text
+ assert all(method in ('GET', 'POST') and (path == '/' or path.endswith('/_search') or path.endswith('/_mapping'))
+ for method, path in requests), requests
+ print(f'PASS {transport} {mode}: classification, scope and read-only requests', flush=True)
+ except (AssertionError, KeyError, StopIteration) as error:
+ failures.append(f'{transport} {mode}: {error}')
+ if failures:
+ raise AssertionError('\n'.join(failures))
+ finally:
+ server.shutdown()
+ server.server_close()
+ worker.join(timeout=5)
+ return 0
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())