Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): stop slicing SQL statement heads inside a character

snmsoodanc257fbbfb6bb
diff --git a/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs b/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
index 278527196..6ad117ac9 100644
--- a/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
+++ b/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
@@ -315,7 +315,12 @@ fn trim_sql_noise(s: &str) -> &str {
fn starts_with_kw(s: &str, kw: &str) -> bool {
let s = trim_sql_noise(s);
- if s.len() < kw.len() || !s[..kw.len()].eq_ignore_ascii_case(kw) {
+ // `get` also rejects a prefix that ends inside a character, so a head
+ // opening with CJK text is simply not the ASCII keyword (#2547).
+ let Some(head) = s.get(..kw.len()) else {
+ return false;
+ };
+ if !head.eq_ignore_ascii_case(kw) {
return false;
}
s.as_bytes()
@@ -348,8 +353,11 @@ impl<'a> Words<'a> {
fn eat(&mut self, kw: &str) -> bool {
self.skip_ws();
let rest = &self.s[self.i..];
- if rest.len() >= kw.len()
- && rest[..kw.len()].eq_ignore_ascii_case(kw)
+ // Boundary-checked for the same reason as `starts_with_kw` (#2547).
+ let Some(head) = rest.get(..kw.len()) else {
+ return false;
+ };
+ if head.eq_ignore_ascii_case(kw)
&& rest
.as_bytes()
.get(kw.len())
@@ -748,4 +756,38 @@ mod tests {
sn.family
);
}
+
+ #[test]
+ fn a_multibyte_head_is_not_sliced_at_a_keyword_width() {
+ // The dump probes compare a fixed byte prefix of the statement head
+ // against an ASCII keyword. `insert` is six bytes and `copy` is four,
+ // so a head starting with a CJK character used to be sliced through
+ // the middle of that character (#2547).
+ for head in ["### 请先删除旧表", "### 删", "中文注释", "请先删除"] {
+ assert!(!starts_with_kw(head, "insert"), "{head}");
+ assert!(!starts_with_kw(head, "copy"), "{head}");
+ // `Words::eat` slices at the same widths for `create` / `table`.
+ assert!(match_create(head).is_none(), "{head}");
+ }
+ }
+
+ #[test]
+ fn a_cjk_headed_migration_is_indexed_instead_of_aborting_the_run() {
+ // #2547: the first line is a comment the SQL noise stripper does not
+ // recognise (`###`), so it is its own statement whose head is CJK and
+ // every keyword probe sliced into the middle of the character. The
+ // statements after it still have to be indexed.
+ let sql = "### 请先删除旧表;\n\
+ -- 中文字段整改:演示复现\n\
+ CREATE TABLE t (id int);\n";
+ let dir = tempfile::tempdir().unwrap();
+ let path = write(dir.path(), "V1__cjk_head.sql", sql);
+ let (stats, recs) = indexed(&path);
+ assert!(stats.refusal.is_none(), "{stats:?}");
+ let defs = recs
+ .iter()
+ .find_map(|r| r.fields.get("defs").and_then(|v| v.as_str()))
+ .unwrap_or("");
+ assert!(defs.contains("table t"), "{defs}");
+ }
}