Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(corpus): refuse a corpus name its index prefix would rewrite (#1846) (#1919)

xerj-org6e4c1d730fe1
diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rs
index f01559e7e..e6ebd9b9c 100644
--- a/engine/crates/xerj-autoindex/src/xc.rs
+++ b/engine/crates/xerj-autoindex/src/xc.rs
@@ -2030,7 +2030,15 @@ fn run_corpus_list(args: &[String]) -> i32 {
.unwrap_or_default();
match live {
usize::MAX => println!("{name} (indexed {indexed}, prefix {prefix}) — server unreachable/ambiguous at {url}"),
- 0 => println!("{name} (indexed {indexed}, prefix {prefix}) — NOT loaded here (0 indices) — stale/other-server{use_note}"),
+ 0 => {
+ println!("{name} (indexed {indexed}, prefix {prefix}) — NOT loaded here (0 indices) — stale/other-server{use_note}");
+ // #1846: a name the index prefix rewrites (`curl_cffi`) was
+ // indexed under another spelling, so "0 indices" is not the
+ // whole story — say which name to move it to.
+ if let Err(why) = xccode::pathgate::valid_corpus_name(name) {
+ println!(" {why}");
+ }
+ }
n => {
loaded += 1;
println!("{name} (indexed {indexed}, prefix {prefix}) — loaded — {n} index(es){use_note}");
diff --git a/engine/crates/xerj-common/src/xccode/pathgate.rs b/engine/crates/xerj-common/src/xccode/pathgate.rs
index 5e8dec0f0..67a2b6d26 100644
--- a/engine/crates/xerj-common/src/xccode/pathgate.rs
+++ b/engine/crates/xerj-common/src/xccode/pathgate.rs
@@ -47,9 +47,56 @@ pub fn valid_corpus_name(name: &str) -> Result<(), String> {
"corpus name '{name}' is reserved — it is a subcommand of `xerj code`/`xerj corpus`"
));
}
+ // #1846: the name is also the index prefix (`xc-<name>-b<stamp>`), and
+ // `xerj autoindex` keeps only lower-case [a-z0-9] runs joined by '-' in a
+ // prefix. `curl_cffi` was registered, indexed as `xc-curl-cffi-…`, then
+ // verified and listed under `xc-curl_cffi-…`: records on the node, a corpus
+ // that answered nothing. Refuse up front and name the spelling that works.
+ let index_safe = |c: char| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-';
+ if !name.chars().all(index_safe) || name.starts_with('-') || name.ends_with('-') {
+ let suggested = index_safe_corpus_name(name);
+ let instead = if suggested.is_empty() {
+ "use a name made of a-z, 0-9 and '-'".to_string()
+ } else {
+ format!(
+ "use '{suggested}' instead (an existing corpus: rename corpora/{name} to \
+ corpora/{suggested} and set \"corpus\": \"{suggested}\" in its corpus.json)"
+ )
+ };
+ return Err(format!(
+ "bad corpus name '{name}': a corpus name is also its index prefix, which keeps only \
+ lower-case a-z, 0-9 and '-', so '{name}' would be indexed under xc-{suggested}-… \
+ and then looked up under xc-{name}-…, where nothing is — {instead}"
+ ));
+ }
Ok(())
}
+/// The spelling of `name` that survives as an index prefix unchanged: lower
+/// case, every run of other characters turned into one `-`, no `-` at either
+/// end. A run of only `-` is kept as written (the hub's `owner--repo`). Used to suggest a
+/// name in [`valid_corpus_name`]'s refusal; empty when nothing survives.
+pub fn index_safe_corpus_name(name: &str) -> String {
+ let mut out = String::new();
+ let mut run = String::new();
+ for c in name.chars() {
+ if c.is_ascii_alphanumeric() {
+ if !out.is_empty() && !run.is_empty() {
+ if run.chars().all(|r| r == '-') {
+ out.push_str(&run);
+ } else {
+ out.push('-');
+ }
+ }
+ run.clear();
+ out.push(c.to_ascii_lowercase());
+ } else {
+ run.push(c);
+ }
+ }
+ out
+}
+
/// Validate a repo name from an untrusted manifest: `^[A-Za-z0-9_][A-Za-z0-9._-]*$`.
///
/// The message says WHY because the rule looks arbitrary until the checkout
@@ -110,11 +157,44 @@ mod tests {
"'{reserved}' is a subcommand noun"
);
}
- for good in ["kv", "xerj-search", "battle-terse", "rust_text", "c++"] {
+ for good in ["kv", "xerj-search", "battle-terse", "owner--repo", "lz4"] {
assert!(valid_corpus_name(good).is_ok(), "{good:?} is a fine name");
}
}
+ /// #1846: `curl_cffi` was accepted, indexed as `xc-curl-cffi-…` and then
+ /// looked up as `xc-curl_cffi-…` — registered, indexed, unreachable. A name
+ /// the index prefix would rewrite is refused, and the refusal names the
+ /// spelling that works and how to rename an existing corpus.
+ #[test]
+ fn a_name_the_index_prefix_would_rewrite_is_refused_with_the_working_spelling() {
+ let e = valid_corpus_name("curl_cffi").unwrap_err();
+ assert!(e.contains("use 'curl-cffi' instead"), "{e}");
+ assert!(e.contains("xc-curl-cffi-"), "{e}");
+ assert!(e.contains("corpora/curl-cffi"), "{e}");
+ for (bad, fix) in [
+ ("rust_text", "rust-text"),
+ ("Curl-Cffi", "curl-cffi"),
+ ("socket.io", "socket-io"),
+ ("c++", "c"),
+ ("trail-", "trail"),
+ ("-lead", "lead"),
+ ("a__b", "a-b"),
+ ] {
+ let e = valid_corpus_name(bad).unwrap_err();
+ assert!(e.contains(&format!("use '{fix}' instead")), "{bad}: {e}");
+ assert_eq!(index_safe_corpus_name(bad), fix, "{bad}");
+ assert!(
+ valid_corpus_name(fix).is_ok(),
+ "the suggestion {fix} must pass"
+ );
+ }
+ assert!(valid_corpus_name("+++")
+ .unwrap_err()
+ .contains("a-z, 0-9 and '-'"));
+ assert_eq!(index_safe_corpus_name("owner--repo"), "owner--repo");
+ }
+
#[test]
fn unsafe_repo_names_are_rejected_with_the_path_escape_reason() {
for bad in [