Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(tokenizer): fold Cyrillic before stemming so Russian keywords match (#21433)

CCLUCKYf7de7669ad2b
diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cpp
index 6248287282..530774d6de 100644
--- a/internal/binding/cpp/rag_analyzer.cpp
+++ b/internal/binding/cpp/rag_analyzer.cpp
@@ -1889,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
for (unsigned i = 0; i < term_list.size(); ++i) {
lowercase_string_buffer_.resize(term_list[i].size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
- ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, lowercase_string_buffer_.size());
+ ToLowerUTF8(term_list[i].c_str(), term_list[i].size(), lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
@@ -2038,7 +2038,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
// Apply lowercase before lemmatization to match Python NLTK behavior
lowercase_string_buffer_.resize(term.size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
- ToLower(term.c_str(), term.size(), lowercase_term, lowercase_string_buffer_.size());
+ ToLowerUTF8(term.c_str(), term.size(), lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
diff --git a/internal/binding/cpp/string_utils.h b/internal/binding/cpp/string_utils.h
index 8f03b9c4d2..c651071734 100644
--- a/internal/binding/cpp/string_utils.h
+++ b/internal/binding/cpp/string_utils.h
@@ -113,6 +113,84 @@ inline std::string ToLowerString(std::string_view s)
return result;
}
+// Simple (1:1) lower-case mapping for Cyrillic U+0400-U+04FF. Upper/lower
+// pairs are contiguous inside the main blocks; U+0460-U+0481 and
+// U+048A-U+04BF interleave them (upper first), while U+04C3-U+04CE interleave
+// with the upper case on the odd code point. U+04C0/U+04C1 are the two
+// exceptions Unicode defines outside that pattern.
+[[nodiscard]] constexpr uint32_t ToLowerCyrillic(uint32_t cp) noexcept
+{
+ if (cp >= 0x0400 && cp <= 0x040F)
+ return cp + 0x50;
+ if (cp >= 0x0410 && cp <= 0x042F)
+ return cp + 0x20;
+ if (cp >= 0x0460 && cp <= 0x0481 && (cp & 1) == 0)
+ return cp + 1;
+ if (cp >= 0x048A && cp <= 0x04BF && (cp & 1) == 0)
+ return cp + 1;
+ if (cp == 0x04C0)
+ return 0x04CF;
+ if (cp == 0x04C1)
+ return 0x04C2;
+ if (cp >= 0x04C3 && cp <= 0x04CE && (cp & 1) == 1)
+ return cp + 1;
+ if (cp >= 0x04D0 && cp <= 0x04FF && (cp & 1) == 0)
+ return cp + 1;
+ return cp;
+}
+
+// ToLowerUTF8 folds ASCII A-Z and upper-case Cyrillic U+0400-U+04FF to lower
+// case in place. Byte length is preserved so callers that map token offsets
+// back to the source string keep their indices valid. The Snowball Russian
+// stemmer only accepts lower-case input, so index-side tokens must be folded
+// before stemming; ToLower above is deliberately ASCII-only and leaves
+// Cyrillic untouched.
+inline void ToLowerUTF8(char* data, size_t len)
+{
+ for (size_t i = 0; i < len;)
+ {
+ const unsigned char c = static_cast<unsigned char>(data[i]);
+ if (c < 0x80)
+ {
+ if ('A' <= c && c <= 'Z')
+ data[i] += 32;
+ i += 1;
+ continue;
+ }
+ // Cyrillic occupies the two-byte leads D0-D3. All foldings stay inside
+ // the same block, so the encoded length never changes.
+ if (c >= 0xD0 && c <= 0xD3 && i + 1 < len)
+ {
+ const unsigned char t = static_cast<unsigned char>(data[i + 1]);
+ if (t >= 0x80 && t <= 0xBF)
+ {
+ const uint32_t cp = ((c - 0xD0u) << 6) + (t - 0x80u) + 0x400u;
+ const uint32_t folded = ToLowerCyrillic(cp);
+ if (folded != cp)
+ {
+ const uint32_t offset = folded - 0x400u;
+ data[i] = static_cast<char>(0xD0u + (offset >> 6));
+ data[i + 1] = static_cast<char>(0x80u + (offset & 0x3Fu));
+ }
+ i += 2;
+ continue;
+ }
+ }
+ // Other multi-byte sequences pass through byte for byte.
+ i += 1;
+ }
+}
+
+// Copies `len` bytes into `out`, folds them and terminates the result. Callers
+// must provide len + 1 bytes: byte length is preserved, so that is the only
+// space the fold needs.
+inline void ToLowerUTF8(const char* data, size_t len, char* out)
+{
+ memcpy(out, data, len);
+ ToLowerUTF8(out, len);
+ out[len] = '\0';
+}
+
inline bool IsUTF8Sep(const uint8_t c) { return c < 128 && !std::isalnum(c); }
template <typename T>
diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go
index ade35fdc12..5e1fd263fc 100644
--- a/internal/tokenizer/tokenizer_test.go
+++ b/internal/tokenizer/tokenizer_test.go
@@ -420,6 +420,45 @@ func TestTokenize_SlovakCzechFoldDiacriticsAndSkipStemming(t *testing.T) {
}
}
+func TestTokenize_RussianFoldsCyrillicBeforeStemming(t *testing.T) {
+ restore := saveEngineType()
+ defer restore()
+ SetEngineType("")
+
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 5 * time.Second,
+ }); err != nil {
+ t.Fatalf("Failed to initialize pool: %v", err)
+ }
+ defer Close()
+
+ // Index-side folding must be Unicode-aware: the Russian stemmer only
+ // accepts lower-case input, so an upper-case word would otherwise reach
+ // the suffix table unchanged while the query side folds it with
+ // strings.ToLower.
+ var want string
+ for _, form := range []string{"ДОГОВОРАМИ", "Договора", "договорами", "договор"} {
+ got, err := New("Russian").Tokenize(form)
+ if err != nil {
+ t.Fatalf("Tokenize(Russian, %q) unexpected error: %v", form, err)
+ }
+ if want == "" {
+ want = got
+ continue
+ }
+ if got != want {
+ t.Errorf("Tokenize(Russian, %q) = %q, want the shared stem %q", form, got, want)
+ }
+ }
+ if want != "договор" {
+ t.Errorf("Russian stem for the договор forms = %q, want %q", want, "договор")
+ }
+}
+
func findEnglishDutchDifferentiator(t *testing.T) languageDifferentiator {
t.Helper()