Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): do not repeat the prefix in the dataset slug (#1694)

xerj-orgbe1e769ef12c
diff --git a/engine/crates/xerj-autoindex/src/dataset.rs b/engine/crates/xerj-autoindex/src/dataset.rs
index 69502b50d..7ee2e43cf 100644
--- a/engine/crates/xerj-autoindex/src/dataset.rs
+++ b/engine/crates/xerj-autoindex/src/dataset.rs
@@ -422,6 +422,43 @@ fn assign_slugs(clusters: &mut [Cluster], rels: &[String], used: &mut HashSet<St
}
}
+/// Drop a leading `{prefix}-` from each slug.
+///
+/// The index name is `{prefix}-{slug}`. A folder of root-level logs indexed
+/// with `--prefix logs` otherwise becomes `logs-logs-syslog`: the slug's
+/// family fallback is `logs` and a single-file collision appends the stem.
+/// Stripping that one repeated segment makes the index `logs-syslog`. A slug
+/// that does not start with the prefix is left alone, so the default prefix
+/// `ax` does not rename `logs-syslog`. An empty remainder is not produced.
+/// Collisions created by the strip are numbered `-2`, `-3`, in slug order.
+pub fn drop_repeated_prefix(prefix: &str, clusters: &mut [Cluster]) {
+ let prefix = sanitize_slug(prefix);
+ if prefix.is_empty() {
+ return;
+ }
+ let needle = format!("{prefix}-");
+ let mut order: Vec<usize> = (0..clusters.len()).collect();
+ order.sort_by_key(|&i| clusters[i].slug.clone());
+ let mut used: HashSet<String> = HashSet::new();
+ let mut next = vec![String::new(); clusters.len()];
+ for &i in &order {
+ let base = match clusters[i].slug.strip_prefix(&needle) {
+ Some(rest) if !rest.is_empty() => rest.to_string(),
+ _ => clusters[i].slug.clone(),
+ };
+ let mut slug = base.clone();
+ let mut k = 2u32;
+ while !used.insert(slug.clone()) {
+ slug = format!("{base}-{k}");
+ k += 1;
+ }
+ next[i] = slug;
+ }
+ for (i, slug) in next.into_iter().enumerate() {
+ clusters[i].slug = slug;
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -859,4 +896,42 @@ mod tests {
);
assert_eq!(before[0].slug, after[0].slug);
}
+
+ /// `--prefix logs` on a folder of root-level log files. The slug's family
+ /// fallback is `logs` and a one-file collision appends the stem, so the
+ /// index `{prefix}-{slug}` was `logs-logs-syslog`. The default prefix
+ /// `ax` does not contain that segment and must not rename the slug.
+ #[test]
+ fn a_logs_prefix_does_not_repeat_inside_a_root_syslog_slug() {
+ let rels = vec!["syslog".into(), "xferlog".into()];
+ let sketches = || {
+ vec![
+ data_sketch_n(0, Family::Logs, &["ts", "host", "program", "message"], 40),
+ data_sketch_n(
+ 1,
+ Family::Logs,
+ &["date", "host", "bytes", "path", "user"],
+ 40,
+ ),
+ ]
+ };
+ let mut clusters = cluster(sketches(), &rels, &root_scopes(2));
+ let mut raw: Vec<String> = clusters.iter().map(|c| c.slug.clone()).collect();
+ raw.sort();
+ assert_eq!(
+ raw,
+ ["logs-syslog", "logs-xferlog"],
+ "the family fallback plus the file stem is what stuttered"
+ );
+ drop_repeated_prefix("logs", &mut clusters);
+ let mut slugs: Vec<String> = clusters.iter().map(|c| c.slug.clone()).collect();
+ slugs.sort();
+ assert_eq!(slugs, ["syslog", "xferlog"]);
+
+ let mut untouched = cluster(sketches(), &rels, &root_scopes(2));
+ let before: Vec<String> = untouched.iter().map(|c| c.slug.clone()).collect();
+ drop_repeated_prefix("ax", &mut untouched);
+ let after: Vec<String> = untouched.iter().map(|c| c.slug.clone()).collect();
+ assert_eq!(before, after, "the default prefix must not rename slugs");
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 432205ff9..8f20470b8 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3658,6 +3658,9 @@ fn build_phase_a(
}
}
let mut clusters = dataset::cluster(sketches, &rels, &scopes);
+ // `{prefix}-{slug}` must not stutter (`logs-logs-syslog`) when the slug's
+ // first segment is the prefix the operator already chose.
+ dataset::drop_repeated_prefix(&cfg.prefix, &mut clusters);
// Demoted one-off config files (#173) were sampled as flattened records;
// phase B will index them as documents. Re-sample them through the same