Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Integrate the CJK SQL regression from #2586 with the UTF-8 fix

xerj-orgd54e53ad5ef7
diff --cc engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
index 6ad117ac9,1c9a250e9..919f8f92c
--- a/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
+++ b/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs
@@@ -758,36 -753,13 +758,68 @@@ mod tests
}
#[test]
- fn cjk_comment_head_does_not_panic_or_hang() {
- let sql = "-- 中文字段整改:演示复现\nALTER TABLE t ADD COLUMN c VARCHAR(64);\n";
+ fn a_multibyte_head_is_not_sliced_at_a_keyword_width() {
+ // The dump probes compare a fixed byte prefix of the statement head
+ // against an ASCII keyword. `insert` is six bytes and `copy` is four,
+ // so a head starting with a CJK character used to be sliced through
+ // the middle of that character (#2547).
+ for head in ["### 请先删除旧表", "### 删", "中文注释", "请先删除"] {
+ assert!(!starts_with_kw(head, "insert"), "{head}");
+ assert!(!starts_with_kw(head, "copy"), "{head}");
+ // `Words::eat` slices at the same widths for `create` / `table`.
+ assert!(match_create(head).is_none(), "{head}");
+ }
+ }
+
+ #[test]
+ fn a_cjk_headed_migration_is_indexed_instead_of_aborting_the_run() {
+ // #2547: the first line is a comment the SQL noise stripper does not
+ // recognise (`###`), so it is its own statement whose head is CJK and
+ // every keyword probe sliced into the middle of the character. The
+ // statements after it still have to be indexed.
+ let sql = "### 请先删除旧表;\n\
+ -- 中文字段整改:演示复现\n\
+ CREATE TABLE t (id int);\n";
let dir = tempfile::tempdir().unwrap();
- let path = write(dir.path(), "cjk.sql", sql);
+ let path = write(dir.path(), "V1__cjk_head.sql", sql);
let (stats, recs) = indexed(&path);
- assert!(stats.records > 0, "CJK comment file must be indexed safely");
- assert!(!recs.is_empty(), "records must not be empty");
assert!(stats.refusal.is_none(), "{stats:?}");
+ let defs = recs
+ .iter()
+ .find_map(|r| r.fields.get("defs").and_then(|v| v.as_str()))
+ .unwrap_or("");
+ assert!(defs.contains("table t"), "{defs}");
+ }
++
++ #[test]
++ fn cjk_comment_head_does_not_panic_or_hang() {
++ // Keep the recognized-comment compatibility control from #2586, and
++ // also require SQL symbol extraction for the formerly panicking heads.
++ // An ALTER-only file can take the prose route and miss both probes.
++ for (sql, expect_table) in [
++ (
++ "-- 中文字段整改:演示复现\nALTER TABLE t ADD COLUMN c VARCHAR(64);\n",
++ false,
++ ),
++ ("### 请先删除旧表;\nCREATE TABLE t (id int);\n", true),
++ ("中文注释;\nCREATE TABLE t (id int);\n", true),
++ ] {
++ let dir = tempfile::tempdir().unwrap();
++ let path = write(dir.path(), "cjk.sql", sql);
++ let (stats, recs) = indexed(&path);
++ assert!(stats.records > 0, "CJK comment file must be indexed safely");
++ assert!(!recs.is_empty(), "records must not be empty");
++ assert!(stats.refusal.is_none(), "{stats:?}");
++ if expect_table {
++ assert!(
++ recs.iter().any(|r| r
++ .fields
++ .get("defs")
++ .and_then(|v| v.as_str())
++ .is_some_and(|defs| defs.contains("table t"))),
++ "the regression must exercise SQL extraction, not prose fallback"
++ );
++ }
++ }
+ }
}