Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): truncate the catalog diff on a char boundary (#367)

xerj-org4b2227c6ee5f
diff --git a/engine/crates/xerj-autoindex/src/generation_catalog.rs b/engine/crates/xerj-autoindex/src/generation_catalog.rs
index 6784c037a..b62ce9cfb 100644
--- a/engine/crates/xerj-autoindex/src/generation_catalog.rs
+++ b/engine/crates/xerj-autoindex/src/generation_catalog.rs
@@ -89,8 +89,10 @@ impl CatalogProjection {
fn describe_field_diff(expected: &Value, actual: &Value) -> String {
fn short(v: &Value) -> String {
let s = v.to_string();
- if s.len() > 120 {
- format!("{}…", &s[..120])
+ // Truncate on a CHAR boundary (values can hold non-ASCII, e.g. a CJK
+ // `fields_json`); byte slicing `&s[..120]` would panic mid-codepoint.
+ if let Some((idx, _)) = s.char_indices().nth(120) {
+ format!("{}…", &s[..idx])
} else {
s
}
@@ -786,5 +788,15 @@ mod tests {
err.contains("time_min") && err.contains("dropped"),
"a dropped key must be named: {err}"
);
+
+ // A long non-ASCII differing value must be reported, not panic the
+ // truncation (byte-slicing would split a multibyte codepoint).
+ let mut wide = projection.documents.clone();
+ wide.get_mut("ds:ax:reports").unwrap()["notes"] = json!(["名前".repeat(60)]);
+ let err = projection.validate_observed(&wide).unwrap_err().to_string();
+ assert!(
+ err.contains("notes"),
+ "a wide non-ASCII diff must be reported: {err}"
+ );
}
}