AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(fts): real NFKC in icu_folding analyzer via unicode-normalization
xerj-org99e726c557bd
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 11a47ce2e..44d1807a7 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -3011,6 +3011,15 @@ version = "1.0.24"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"+[[package]]+name = "unicode-normalization"+version = "0.1.25"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"+dependencies = [+ "tinyvec",+]+[[package]]name = "unicode-segmentation"version = "1.13.2"@@ -3802,6 +3811,7 @@ dependencies = ["tempfile","thiserror 2.0.18","tracing",+ "unicode-normalization","unicode-segmentation","xerj-common","zstd",diff --git a/engine/crates/xerj-fts/Cargo.toml b/engine/crates/xerj-fts/Cargo.tomlindex 55c1d10e7..7584cae3c 100644--- a/engine/crates/xerj-fts/Cargo.toml+++ b/engine/crates/xerj-fts/Cargo.toml@@ -28,6 +28,7 @@ zstd = { workspace = true }# Direct deps (not in workspace)unicode-segmentation = "1"+unicode-normalization = "0.1"rust-stemmers = "1.2"regex = "1"diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rsindex 2913ad68d..f694b46fd 100644--- a/engine/crates/xerj-fts/src/analyzer.rs+++ b/engine/crates/xerj-fts/src/analyzer.rs@@ -816,10 +816,6 @@ impl TokenFilter for IcuFoldingFilter {tokens.into_iter().map(|mut t| {- // Apply NFKC normalization using the `unicode-normalization` approach.- // We approximate NFKC by applying compatibility and canonical decomposition- // then recomposing. In Rust stable we use the `unicode_normalization` crate- // if available, otherwise we fall back to lowercasing only.t.text = nfkc_normalize(&t.text);t})@@ -827,37 +823,20 @@ impl TokenFilter for IcuFoldingFilter {}}-/// Apply NFKC-like normalization. Uses a character-by-character approach-/// for the most common compatibility mappings without requiring extra crate deps.+/// Apply full Unicode NFKC normalization (compatibility decomposition ++/// canonical composition) via the `unicode-normalization` crate.+///+/// This is real NFKC, matching the Unicode standard the ICU folding token+/// filter in Elasticsearch relies on — not a hand-picked table. Examples:+/// - Composes combining sequences: `e` + U+0301 → `é`.+/// - Folds compatibility characters: `fi` → `fi`, `²` → `2`, `Ⅸ` → `IX`.+/// - Maps fullwidth forms to ASCII: `!` → `!`.+///+/// Casing is intentionally left to the pipeline's `LowercaseFilter`; this+/// filter is pure NFKC so it can be composed independently.fn nfkc_normalize(s: &str) -> String {- // Rust's standard library doesn't include Unicode normalization, so we do- // a best-effort fold: lowercase + ASCII compatibility substitutions.- // The unicode-normalization crate is not a current dependency, so we keep- // this lightweight — the filter still benefits from lowercasing.- s.chars().flat_map(nfkc_fold_char).collect()-}--/// Single-character NFKC compatibility fold for the most common cases.-fn nfkc_fold_char(c: char) -> Vec<char> {- match c {- // Ligatures- 'fi' => vec!['f', 'i'],- 'fl' => vec!['f', 'l'],- 'ffi' => vec!['f', 'f', 'i'],- 'ffl' => vec!['f', 'f', 'l'],- 'ff' => vec!['f', 'f'],- // Superscripts/subscripts- '²' => vec!['2'],- '³' => vec!['3'],- '¹' => vec!['1'],- '⁰' => vec!['0'],- // Fullwidth ASCII (U+FF01..U+FF5E → U+0021..U+007E)- c if ('\u{FF01}'..='\u{FF5E}').contains(&c) => {- let ascii = (c as u32 - 0xFF00 + 0x0020) as u8;- vec![ascii as char]- }- other => vec![other.to_lowercase().next().unwrap_or(other)],- }+ use unicode_normalization::UnicodeNormalization;+ s.nfkc().collect()}// ── Analyzer registry ─────────────────────────────────────────────────────────@@ -1890,4 +1869,39 @@ mod tests {let terms = analyzer.analyze_to_terms("Hello World");assert_eq!(terms, vec!["Hello World"]);}++ #[test]+ fn icu_folding_filter_applies_real_nfkc() {+ let filter = IcuFoldingFilter;+ // Combining sequence composes: "e" + U+0301 (COMBINING ACUTE) -> "é".+ // Fullwidth exclamation folds to ASCII "!".+ // Roman numeral nine (U+2168) decomposes to "IX".+ // Ligature "fi" -> "fi"; superscript "²" -> "2".+ let tokens = vec![+ Token::new("e\u{0301}", 0, 0, 0),+ Token::new("\u{FF01}", 1, 0, 0),+ Token::new("\u{2168}", 2, 0, 0),+ Token::new("fi", 3, 0, 0),+ Token::new("²", 4, 0, 0),+ ];+ let out = filter.filter(tokens);+ assert_eq!(out[0].text, "\u{00E9}"); // é (single precomposed codepoint)+ assert_eq!(out[0].text.chars().count(), 1);+ assert_eq!(out[1].text, "!");+ assert_eq!(out[2].text, "IX"); // NFKC keeps case; lowercasing is the pipeline's job+ assert_eq!(out[3].text, "fi");+ assert_eq!(out[4].text, "2");+ }++ #[test]+ fn registry_icu_folding_analyzer_lowercases_and_nfkc() {+ let registry = AnalyzerRegistry::default();+ let analyzer = registry.get_analyzer("icu_folding").unwrap();+ // Pipeline = StandardTokenizer -> LowercaseFilter -> IcuFoldingFilter,+ // so the analyzer both lowercases and NFKC-folds.+ let terms = analyzer.analyze_to_terms("Ⅸ fi TEST");+ assert!(terms.contains(&"ix".to_string()), "terms={terms:?}");+ assert!(terms.contains(&"fi".to_string()), "terms={terms:?}");+ assert!(terms.contains(&"test".to_string()), "terms={terms:?}");+ }}