RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tokenizer): fold Cyrillic before stemming so Russian keywords match (#21433)
CCLUCKYf7de7669ad2b
diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cppindex 6248287282..530774d6de 100644--- a/internal/binding/cpp/rag_analyzer.cpp+++ b/internal/binding/cpp/rag_analyzer.cpp@@ -1889,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {for (unsigned i = 0; i < term_list.size(); ++i) {lowercase_string_buffer_.resize(term_list[i].size() + 1);char *lowercase_term = lowercase_string_buffer_.data();- ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, lowercase_string_buffer_.size());+ ToLowerUTF8(term_list[i].c_str(), term_list[i].size(), lowercase_term);std::string term_to_stem;if (use_lemmatizer_) {term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);@@ -2038,7 +2038,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>// Apply lowercase before lemmatization to match Python NLTK behaviorlowercase_string_buffer_.resize(term.size() + 1);char *lowercase_term = lowercase_string_buffer_.data();- ToLower(term.c_str(), term.size(), lowercase_term, lowercase_string_buffer_.size());+ ToLowerUTF8(term.c_str(), term.size(), lowercase_term);std::string term_to_stem;if (use_lemmatizer_) {term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);diff --git a/internal/binding/cpp/string_utils.h b/internal/binding/cpp/string_utils.hindex 8f03b9c4d2..c651071734 100644--- a/internal/binding/cpp/string_utils.h+++ b/internal/binding/cpp/string_utils.h@@ -113,6 +113,84 @@ inline std::string ToLowerString(std::string_view s)return result;}+// Simple (1:1) lower-case mapping for Cyrillic U+0400-U+04FF. Upper/lower+// pairs are contiguous inside the main blocks; U+0460-U+0481 and+// U+048A-U+04BF interleave them (upper first), while U+04C3-U+04CE interleave+// with the upper case on the odd code point. U+04C0/U+04C1 are the two+// exceptions Unicode defines outside that pattern.+[[nodiscard]] constexpr uint32_t ToLowerCyrillic(uint32_t cp) noexcept+{+ if (cp >= 0x0400 && cp <= 0x040F)+ return cp + 0x50;+ if (cp >= 0x0410 && cp <= 0x042F)+ return cp + 0x20;+ if (cp >= 0x0460 && cp <= 0x0481 && (cp & 1) == 0)+ return cp + 1;+ if (cp >= 0x048A && cp <= 0x04BF && (cp & 1) == 0)+ return cp + 1;+ if (cp == 0x04C0)+ return 0x04CF;+ if (cp == 0x04C1)+ return 0x04C2;+ if (cp >= 0x04C3 && cp <= 0x04CE && (cp & 1) == 1)+ return cp + 1;+ if (cp >= 0x04D0 && cp <= 0x04FF && (cp & 1) == 0)+ return cp + 1;+ return cp;+}++// ToLowerUTF8 folds ASCII A-Z and upper-case Cyrillic U+0400-U+04FF to lower+// case in place. Byte length is preserved so callers that map token offsets+// back to the source string keep their indices valid. The Snowball Russian+// stemmer only accepts lower-case input, so index-side tokens must be folded+// before stemming; ToLower above is deliberately ASCII-only and leaves+// Cyrillic untouched.+inline void ToLowerUTF8(char* data, size_t len)+{+ for (size_t i = 0; i < len;)+ {+ const unsigned char c = static_cast<unsigned char>(data[i]);+ if (c < 0x80)+ {+ if ('A' <= c && c <= 'Z')+ data[i] += 32;+ i += 1;+ continue;+ }+ // Cyrillic occupies the two-byte leads D0-D3. All foldings stay inside+ // the same block, so the encoded length never changes.+ if (c >= 0xD0 && c <= 0xD3 && i + 1 < len)+ {+ const unsigned char t = static_cast<unsigned char>(data[i + 1]);+ if (t >= 0x80 && t <= 0xBF)+ {+ const uint32_t cp = ((c - 0xD0u) << 6) + (t - 0x80u) + 0x400u;+ const uint32_t folded = ToLowerCyrillic(cp);+ if (folded != cp)+ {+ const uint32_t offset = folded - 0x400u;+ data[i] = static_cast<char>(0xD0u + (offset >> 6));+ data[i + 1] = static_cast<char>(0x80u + (offset & 0x3Fu));+ }+ i += 2;+ continue;+ }+ }+ // Other multi-byte sequences pass through byte for byte.+ i += 1;+ }+}++// Copies `len` bytes into `out`, folds them and terminates the result. Callers+// must provide len + 1 bytes: byte length is preserved, so that is the only+// space the fold needs.+inline void ToLowerUTF8(const char* data, size_t len, char* out)+{+ memcpy(out, data, len);+ ToLowerUTF8(out, len);+ out[len] = '\0';+}+inline bool IsUTF8Sep(const uint8_t c) { return c < 128 && !std::isalnum(c); }template <typename T>diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.goindex ade35fdc12..5e1fd263fc 100644--- a/internal/tokenizer/tokenizer_test.go+++ b/internal/tokenizer/tokenizer_test.go@@ -420,6 +420,45 @@ func TestTokenize_SlovakCzechFoldDiacriticsAndSkipStemming(t *testing.T) {}}+func TestTokenize_RussianFoldsCyrillicBeforeStemming(t *testing.T) {+ restore := saveEngineType()+ defer restore()+ SetEngineType("")++ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 1,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 5 * time.Second,+ }); err != nil {+ t.Fatalf("Failed to initialize pool: %v", err)+ }+ defer Close()++ // Index-side folding must be Unicode-aware: the Russian stemmer only+ // accepts lower-case input, so an upper-case word would otherwise reach+ // the suffix table unchanged while the query side folds it with+ // strings.ToLower.+ var want string+ for _, form := range []string{"ДОГОВОРАМИ", "Договора", "договорами", "договор"} {+ got, err := New("Russian").Tokenize(form)+ if err != nil {+ t.Fatalf("Tokenize(Russian, %q) unexpected error: %v", form, err)+ }+ if want == "" {+ want = got+ continue+ }+ if got != want {+ t.Errorf("Tokenize(Russian, %q) = %q, want the shared stem %q", form, got, want)+ }+ }+ if want != "договор" {+ t.Errorf("Russian stem for the договор forms = %q, want %q", want, "договор")+ }+}+func findEnglishDutchDifferentiator(t *testing.T) languageDifferentiator {t.Helper()