RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Add Slovak & Czech language dataset support (#19013)
Martin Civáň0ee758067297
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex 0069425da3..5d75ea462a 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -285,6 +285,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req RetripreparedReq.DocScope = docIDspreparedReq.DatasetIDs = append([]string(nil), datasets.kbIDs...)nlpReq := nlpRequestFromRetrieval(preparedReq, datasets.tenantIDs, topN, embeddingModel, preparedReq.ExcludeCompiled)+ nlpReq.Language = entity.KnowledgebasesLanguage(datasets.kbs)nlpReq.RerankModel = rerankModelif rankFeature != nil {nlpReq.RankFeature = &rankFeaturediff --git a/internal/agentic_rag/bm25_service.go b/internal/agentic_rag/bm25_service.goindex 787f97971c..30946b30fb 100644--- a/internal/agentic_rag/bm25_service.go+++ b/internal/agentic_rag/bm25_service.go@@ -21,8 +21,10 @@ import ("fmt""ragflow/internal/agent/runtime"+ "ragflow/internal/dao""ragflow/internal/engine"enginetypes "ragflow/internal/engine/types"+ "ragflow/internal/entity")// Bm25Adapter backs search_bm25_chunks with pure lexical full-text (BM25)@@ -45,10 +47,26 @@ type Bm25Adapter struct {}// QueryTokenizer converts a raw query string into a tokenized match-// expression over the tokenized fields. Implemented by the service/nlp-// QueryBuilder (the same tokenizer family the ingestion pipeline used).+// expression over the tokenized fields, using the dataset language ("" =+// English). Implemented by the service/nlp QueryBuilder (the same tokenizer+// family the ingestion pipeline used).type QueryTokenizer interface {- Question(txt string, tbl string, minMatch float64) (*enginetypes.MatchTextExpr, []string)+ Question(txt string, tbl string, minMatch float64, language string) (*enginetypes.MatchTextExpr, []string)+}++// resolveDatasetsLanguage returns the language the searched datasets were+// indexed with (see entity.KnowledgebasesLanguage), or "" when it cannot be+// read. It is a package-level variable so unit tests can stub it without a+// database.+var resolveDatasetsLanguage = func(ctx context.Context, datasetIDs []string) string {+ if dao.DB == nil || len(datasetIDs) == 0 {+ return ""+ }+ kbs, err := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, datasetIDs)+ if err != nil {+ return ""+ }+ return entity.KnowledgebasesLanguage(kbs)}// SetQueryBuilder attaches the query tokenizer. Called at boot, after@@ -98,6 +116,11 @@ func (b *Bm25Adapter) SearchBm25(ctx context.Context, req runtime.Bm25Request) (ectx, ecancel := engineCallContext(ctx)defer ecancel()+ language := ""+ if b.queryTokenizer != nil {+ language = resolveDatasetsLanguage(ctx, req.DatasetIDs)+ }+seen := map[string]struct{}{}var merged []runtime.RetrievalChunkfor _, q := range queries {@@ -105,7 +128,7 @@ func (b *Bm25Adapter) SearchBm25(ctx context.Context, req runtime.Bm25Request) (// pipeline used (see Bm25Adapter.queryTokenizer): raw query words// cannot match their stemmed/subword index tokens. Falls back to the// raw text match when no query tokenizer is attached.- matchExpr := b.buildMatchExpr(q, topN)+ matchExpr := b.buildMatchExpr(q, topN, language)res, err := b.docEngine.Search(ectx, &enginetypes.SearchRequest{IndexNames: []string{fmt.Sprintf("ragflow_%s", req.TenantID)},KbIDs: req.DatasetIDs,@@ -164,11 +187,11 @@ func floatFromMap(m map[string]interface{}, key string) float64 {// buildMatchExpr converts a raw query into the match expression SearchBm25// sends to the engine: tokenized via the attached QueryTokenizer when one is// registered, otherwise the historical raw-text match.-func (b *Bm25Adapter) buildMatchExpr(q string, topN int) interface{} {+func (b *Bm25Adapter) buildMatchExpr(q string, topN int, language string) interface{} {if b == nil || b.queryTokenizer == nil {return &enginetypes.MatchTextExpr{MatchingText: q, TopN: topN}}- expr, _ := b.queryTokenizer.Question(q, "", 0.0)+ expr, _ := b.queryTokenizer.Question(q, "", 0.0, language)if expr == nil {return &enginetypes.MatchTextExpr{MatchingText: q, TopN: topN}}diff --git a/internal/agentic_rag/bm25_test.go b/internal/agentic_rag/bm25_test.goindex 5cc12f5039..2ef0bb659b 100644--- a/internal/agentic_rag/bm25_test.go+++ b/internal/agentic_rag/bm25_test.go@@ -219,10 +219,14 @@ func (s *bm25StubService) SearchBm25(_ context.Context, req runtime.Bm25Request)}// fakeQueryTokenizer stands in for the service/nlp QueryBuilder: it "tokenizes"-// the raw query into the word forms the ingestion pipeline would have indexed.-type fakeQueryTokenizer struct{}+// the raw query into the word forms the ingestion pipeline would have indexed,+// and records the language it was asked to tokenize with.+type fakeQueryTokenizer struct {+ language string+}-func (fakeQueryTokenizer) Question(txt string, _ string, _ float64) (*enginetypes.MatchTextExpr, []string) {+func (f *fakeQueryTokenizer) Question(txt string, _ string, _ float64, language string) (*enginetypes.MatchTextExpr, []string) {+ f.language = languagereturn &enginetypes.MatchTextExpr{Fields: []string{"content_ltks^2"},MatchingText: "tokenized(" + txt + ")",@@ -237,7 +241,7 @@ func (fakeQueryTokenizer) Question(txt string, _ string, _ float64) (*enginetypefunc TestBm25Adapter_TokenizedMatchExpr(t *testing.T) {fe := &grepFakeEngine{}adapter := NewBm25Adapter(fe)- adapter.SetQueryBuilder(fakeQueryTokenizer{})+ adapter.SetQueryBuilder(&fakeQueryTokenizer{})if _, err := adapter.SearchBm25(context.Background(), runtime.Bm25Request{TenantID: "t",@@ -257,3 +261,33 @@ func TestBm25Adapter_TokenizedMatchExpr(t *testing.T) {t.Fatalf("TopN = %d, want caller topN 12 (tokenizer's 100 overridden)", mt.TopN)}}++// TestBm25Adapter_TokenizesWithDatasetLanguage: the query is tokenized with+// the language the searched datasets were indexed with, so a Slovak dataset's+// folded index tokens are matched by a folded query.+func TestBm25Adapter_TokenizesWithDatasetLanguage(t *testing.T) {+ orig := resolveDatasetsLanguage+ t.Cleanup(func() { resolveDatasetsLanguage = orig })+ var gotIDs []string+ resolveDatasetsLanguage = func(_ context.Context, ids []string) string {+ gotIDs = ids+ return "Slovak"+ }++ qt := &fakeQueryTokenizer{}+ adapter := NewBm25Adapter(&grepFakeEngine{})+ adapter.SetQueryBuilder(qt)+ if _, err := adapter.SearchBm25(context.Background(), runtime.Bm25Request{+ TenantID: "t",+ Queries: []string{"škola"},+ DatasetIDs: []string{"kb1"},+ }); err != nil {+ t.Fatalf("SearchBm25: %v", err)+ }+ if len(gotIDs) != 1 || gotIDs[0] != "kb1" {+ t.Fatalf("language resolved for %v, want [kb1]", gotIDs)+ }+ if qt.language != "Slovak" {+ t.Fatalf("Question language = %q, want Slovak", qt.language)+ }+}diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cppindex b606d41e2d..c3ee7330f3 100644--- a/internal/binding/cpp/rag_analyzer.cpp+++ b/internal/binding/cpp/rag_analyzer.cpp@@ -699,6 +699,8 @@ RAGAnalyzer::~RAGAnalyzer() {void RAGAnalyzer::InitStemmer(Language language) {stemmer_->Init(language);use_lemmatizer_ = (language == STEM_LANG_ENGLISH);+ use_stemmer_ = true;+ fold_diacritics_ = false;}void RAGAnalyzer::SetLanguage(const std::string &language) {@@ -707,6 +709,16 @@ void RAGAnalyzer::SetLanguage(const std::string &language) {lang_key.erase(lang_key.find_last_not_of(" \t") + 1);lang_key.erase(0, lang_key.find_first_not_of(" \t"));+ // Slovak and Czech have no Snowball stemmer. Instead their diacritics+ // are folded to ASCII before tokenization, and stemming/lemmatization+ // are disabled, matching Infinity's RAG analyzer.+ if (lang_key == "slovak" || lang_key == "czech") {+ fold_diacritics_ = true;+ use_stemmer_ = false;+ use_lemmatizer_ = false;+ return;+ }+Language stem_lang = STEM_LANG_UNKNOWN;for (const auto &pair : SNOWBALL_LANGUAGE_MAP) {if (pair.first == lang_key) {@@ -716,8 +728,7 @@ void RAGAnalyzer::SetLanguage(const std::string &language) {}if (stem_lang != STEM_LANG_UNKNOWN) {- stemmer_->Init(stem_lang);- use_lemmatizer_ = (stem_lang == STEM_LANG_ENGLISH);+ InitStemmer(stem_lang);}}@@ -887,6 +898,71 @@ std::string RAGAnalyzer::StrQ2B(const std::string &input) {return output;}+// ASCII base letter for each codepoint in Latin-1 Supplement (U+00C0–U+00FF)+// and Latin Extended-A (U+0100–U+017F), indexed by codepoint - 0xC0. Only+// letters whose NFD decomposition is exactly one ASCII letter plus combining+// marks are folded; '\0' marks a pass-through codepoint (Æ, Ð, ×, Ø, Þ, ß,+// Đ, Ħ, ı, IJ, ĸ, Ŀ, Ł, ʼn, Ŋ, Œ, Ŧ, ſ and their case pairs). Derived from+// Unicode NFD, the same table Infinity's RAG analyzer folds with, so an index+// built by either analyzer is searchable through the other.+static constexpr char DIACRITICS_FOLD_TABLE[0x180 - 0xC0] = {+ // clang-format off+ // U+00C0 À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï+ 'A', 'A', 'A', 'A', 'A', 'A', 0, 'C', 'E', 'E', 'E', 'E', 'I', 'I', 'I', 'I',+ // U+00D0 Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß+ 0, 'N', 'O', 'O', 'O', 'O', 'O', 0, 0, 'U', 'U', 'U', 'U', 'Y', 0, 0,+ // U+00E0 à á â ã ä å æ ç è é ê ë ì í î ï+ 'a', 'a', 'a', 'a', 'a', 'a', 0, 'c', 'e', 'e', 'e', 'e', 'i', 'i', 'i', 'i',+ // U+00F0 ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ+ 0, 'n', 'o', 'o', 'o', 'o', 'o', 0, 0, 'u', 'u', 'u', 'u', 'y', 0, 'y',+ // U+0100 Ā ā Ă ă Ą ą Ć ć Ĉ ĉ Ċ ċ Č č Ď ď+ 'A', 'a', 'A', 'a', 'A', 'a', 'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd',+ // U+0110 Đ đ Ē ē Ĕ ĕ Ė ė Ę ę Ě ě Ĝ ĝ Ğ ğ+ 0, 0, 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g',+ // U+0120 Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĩ ĩ Ī ī Ĭ ĭ Į į+ 'G', 'g', 'G', 'g', 'H', 'h', 0, 0, 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',+ // U+0130 İ ı IJ ij Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ+ 'I', 0, 0, 0, 'J', 'j', 'K', 'k', 0, 'L', 'l', 'L', 'l', 'L', 'l', 0,+ // U+0140 ŀ Ł ł Ń ń Ņ ņ Ň ň ʼn Ŋ ŋ Ō ō Ŏ ŏ+ 0, 0, 0, 'N', 'n', 'N', 'n', 'N', 'n', 0, 0, 0, 'O', 'o', 'O', 'o',+ // U+0150 Ő ő Œ œ Ŕ ŕ Ŗ ŗ Ř ř Ś ś Ŝ ŝ Ş ş+ 'O', 'o', 0, 0, 'R', 'r', 'R', 'r', 'R', 'r', 'S', 's', 'S', 's', 'S', 's',+ // U+0160 Š š Ţ ţ Ť ť Ŧ ŧ Ũ ũ Ū ū Ŭ ŭ Ů ů+ 'S', 's', 'T', 't', 'T', 't', 0, 0, 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u',+ // U+0170 Ű ű Ų ų Ŵ ŵ Ŷ ŷ Ÿ Ź ź Ż ż Ž ž ſ+ 'U', 'u', 'U', 'u', 'W', 'w', 'Y', 'y', 'Y', 'Z', 'z', 'Z', 'z', 'Z', 'z', 0,+ // clang-format on+};++std::string RAGAnalyzer::FoldDiacritics(const std::string &input) {+ std::string output;+ output.reserve(input.size());++ size_t i = 0;+ while (i < input.size()) {+ unsigned char c = input[i];+ // Only 2-byte UTF-8 sequences can encode U+00C0–U+017F.+ if ((c & 0xE0) == 0xC0 && i + 1 < input.size()) {+ uint32_t codepoint = ((c & 0x1F) << 6) | (static_cast<unsigned char>(input[i + 1]) & 0x3F);+ if (codepoint >= 0xC0 && codepoint < 0x180) {+ if (char folded = DIACRITICS_FOLD_TABLE[codepoint - 0xC0]; folded != '\0') {+ output += folded;+ i += 2;+ continue;+ }+ }+ output += input[i];+ output += input[i + 1];+ i += 2;+ continue;+ }+ output += input[i];+ i += 1;+ }++ return output;+}+int32_t RAGAnalyzer::Freq(const std::string_view key) const {int32_t v = trie_->Get(key);v = DecodeFreq(v);@@ -1433,7 +1509,11 @@ void RAGAnalyzer::EnglishNormalize(const std::vector<std::string> &tokens, std::term_to_stem = lowercase_term;}std::string stem_term;- stemmer_->Stem(term_to_stem, stem_term);+ if (use_stemmer_) {+ stemmer_->Stem(term_to_stem, stem_term);+ } else {+ stem_term = term_to_stem;+ }res.push_back(stem_term);} else {res.push_back(t);@@ -1775,8 +1855,17 @@ std::string PCRE2GlobalReplace(const std::string &text, const std::string &patte}std::string RAGAnalyzer::Tokenize(const std::string &line) {+ // Fold diacritics to ASCII before any other processing when the+ // language asks for it (Slovak, Czech).+ std::string folded_line;+ const std::string *input = &line;+ if (fold_diacritics_) {+ folded_line = FoldDiacritics(line);+ input = &folded_line;+ }+// Python-style simple tokenization: re.sub(r"\\W+", " ", line)- std::string processed_line = PCRE2GlobalReplace(line, R"#(\W+)#", " ");+ std::string processed_line = PCRE2GlobalReplace(*input, R"#(\W+)#", " ");std::string str1 = StrQ2B(processed_line);std::string strline;opencc_->convert(str1, strline);@@ -1806,7 +1895,11 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {term_to_stem = lowercase_term;}std::string stem_term;- stemmer_->Stem(term_to_stem, stem_term);+ if (use_stemmer_) {+ stemmer_->Stem(term_to_stem, stem_term);+ } else {+ stem_term = term_to_stem;+ }res.push_back(stem_term);}continue;@@ -1839,9 +1932,23 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {}std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) {+ // Fold diacritics to ASCII before any other processing when the+ // language asks for it (Slovak, Czech). Folding shrinks 2-byte UTF-8+ // letters to 1-byte ASCII, so byte offsets shift; like the StrQ2B and+ // OpenCC stages below, a position mapping recovers offsets on the+ // original input.+ std::string folded_line;+ std::vector<unsigned> fold_pos_mapping;+ const std::string *input = &line;+ if (fold_diacritics_) {+ folded_line = FoldDiacritics(line);+ BuildPositionMapping(line, folded_line, fold_pos_mapping);+ input = &folded_line;+ }+// Python-style simple tokenization: re.sub(r"\W+", " ", line)// Get processed line and position mapping from PCRE2GlobalReplace- auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(line, R"#(\W+)#", " ");+ auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(*input, R"#(\W+)#", " ");std::string str1 = StrQ2B(processed_line);std::string strline;@@ -1857,7 +1964,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>std::vector<unsigned> opencc_pos_mapping;BuildPositionMapping(str1, strline, opencc_pos_mapping);- // Combine all position mappings: strline -> str1 -> processed_line -> line+ // Combine all position mappings: strline -> str1 -> processed_line -> (folded_)linestd::vector<unsigned> final_pos_mapping;final_pos_mapping.resize(strline.size() + 1);@@ -1869,19 +1976,26 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>if (processed_pos < pcre2_pos_mapping.size()) {final_pos_mapping[i] = pcre2_pos_mapping[processed_pos].first;} else {- final_pos_mapping[i] = static_cast<unsigned>(line.size());+ final_pos_mapping[i] = static_cast<unsigned>(input->size());}} else {- final_pos_mapping[i] = static_cast<unsigned>(line.size());+ final_pos_mapping[i] = static_cast<unsigned>(input->size());}} else {- final_pos_mapping[i] = static_cast<unsigned>(line.size());+ final_pos_mapping[i] = static_cast<unsigned>(input->size());}}// Fill the last positionif (strline.size() < final_pos_mapping.size()) {- final_pos_mapping[strline.size()] = static_cast<unsigned>(line.size());+ final_pos_mapping[strline.size()] = static_cast<unsigned>(input->size());+ }++ // Remap folded-line offsets back to the original input.+ if (fold_diacritics_) {+ for (auto &pos : final_pos_mapping) {+ pos = pos < fold_pos_mapping.size() ? fold_pos_mapping[pos] : static_cast<unsigned>(line.size());+ }}// Use SplitByLang to separate by language@@ -1929,7 +2043,11 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>term_to_stem = lowercase_term;}std::string stem_term;- stemmer_->Stem(term_to_stem, stem_term);+ if (use_stemmer_) {+ stemmer_->Stem(term_to_stem, stem_term);+ } else {+ stem_term = term_to_stem;+ }tokens.push_back(stem_term);@@ -2258,7 +2376,11 @@ void RAGAnalyzer::EnglishNormalizeWithPosition(const std::vector<std::string> &tterm_to_stem = lowercase_term;}std::string stem_term;- stemmer_->Stem(term_to_stem, stem_term);+ if (use_stemmer_) {+ stemmer_->Stem(term_to_stem, stem_term);+ } else {+ stem_term = term_to_stem;+ }normalize_tokens.push_back(stem_term);normalize_positions.emplace_back(start_pos, end_pos);diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.hindex 95da708609..c2b7c31885 100644--- a/internal/binding/cpp/rag_analyzer.h+++ b/internal/binding/cpp/rag_analyzer.h@@ -102,6 +102,14 @@ private:static std::string StrQ2B(const std::string& input);+ // Fold Latin-1 Supplement / Latin Extended-A letters whose NFD+ // decomposition is one ASCII letter plus combining marks to that base+ // letter, preserving case (Š→S, ď→d). Letters without such a+ // decomposition (æ, œ, ß, ø, ł, …) pass through unchanged. Applied to+ // the whole input before tokenization when SetLanguage selected a+ // diacritics-folding language (Slovak, Czech).+ static std::string FoldDiacritics(const std::string& input);+static void BuildPositionMapping(const std::string& original, const std::string& converted,std::vector<unsigned>& pos_mapping);@@ -161,6 +169,10 @@ public:bool use_lemmatizer_{true};+ bool use_stemmer_{true}; // Disabled for languages without a Snowball stemmer (Slovak, Czech)++ bool fold_diacritics_{false}; // Fold Latin diacritics to ASCII before tokenization (Slovak, Czech)+bool fine_grained_{false};bool enable_position_{false};diff --git a/internal/binding/rag_analyzer.go b/internal/binding/rag_analyzer.goindex c248ceff14..7a83c2b180 100644--- a/internal/binding/rag_analyzer.go+++ b/internal/binding/rag_analyzer.go@@ -97,7 +97,8 @@ func (a *Analyzer) SetEnablePosition(enablePosition bool) {}// SetLanguage configures the Snowball stemmer for the given language (e.g. "English", "Dutch").-// Falls back to the English Porter stemmer for unmapped languages.+// Diacritic-folding languages ("Slovak", "Czech") disable stemming and+// lemmatization instead; other unmapped languages keep the previous stemmer.func (a *Analyzer) SetLanguage(language string) {if a.handle == nil {returndiff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 3693e56231..6034e9d49c 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -51,7 +51,7 @@ var jsonIterator = sonic.Config{var memoryMessageVectorFieldRE = regexp.MustCompile(`^q_\d+_vec$`)// CreateChunkStore creates an index-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {if baseName == "" {return fmt.Errorf("index name cannot be empty")}@@ -158,8 +158,8 @@ func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID strin// chunks straight back (the chunk APIs, the debug endpoints) sees them right// away. Ingestion must use InsertChunksNoRefresh instead - see that method for// why the wait is the wrong trade there.-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {- return e.insertChunks(ctx, chunks, baseName, datasetID, "wait_for")+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {+ return e.insertChunks(ctx, chunks, baseName, datasetID, language, "wait_for")}// InsertChunksNoRefresh inserts chunks without waiting for an index refresh:@@ -171,13 +171,16 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface// This is the ingestion path's contract, and it matches Python's: the Python// ingestion inserts chunks with refresh=False (rag/svr/task_executor_refactor/// chunk_service.py:386 and :423), while its API default stays "wait_for".-func (e *Engine) InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {- return e.insertChunks(ctx, chunks, baseName, datasetID, "")+func (e *Engine) InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {+ return e.insertChunks(ctx, chunks, baseName, datasetID, language, "")}// insertChunks is the shared bulk-insert body. An empty refresh omits the// parameter, which leaves the decision to the index (no forced refresh).-func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, refresh string) ([]string, error) {+// language is the dataset language, which only engines whose fulltext analyzer+// is fixed at index creation need; Elasticsearch analyses client-side and+// ignores it.+func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string, refresh string) ([]string, error) {common.Info("ElasticsearchConnection.InsertChunks called", zap.String("index_name", baseName), zap.Int("chunkCount", len(chunks)))if len(chunks) == 0 {diff --git a/internal/engine/elasticsearch/chunk_readback_integration_test.go b/internal/engine/elasticsearch/chunk_readback_integration_test.goindex dcef822925..e1669a89b4 100644--- a/internal/engine/elasticsearch/chunk_readback_integration_test.go+++ b/internal/engine/elasticsearch/chunk_readback_integration_test.go@@ -59,7 +59,7 @@ func TestInsertChunks_ReadBackSuffixedFields(t *testing.T) {"position_int": int(2),}- if _, err := engine.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, datasetID); err != nil {+ if _, err := engine.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, datasetID, ""); err != nil {t.Fatalf("InsertChunks: %v", err)}diff --git a/internal/engine/elasticsearch/chunk_readback_test.go b/internal/engine/elasticsearch/chunk_readback_test.goindex a7b3540ef9..8665b7507a 100644--- a/internal/engine/elasticsearch/chunk_readback_test.go+++ b/internal/engine/elasticsearch/chunk_readback_test.go@@ -91,7 +91,7 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {visible := map[string]interface{}{"id": "visible", "doc_id": "doc-1", "content_with_weight": "visible source"}nullAvailability := map[string]interface{}{"id": "null", "doc_id": "doc-1", "available_int": nil}enabled := map[string]interface{}{"id": "enabled", "doc_id": "doc-1", "available_int": 1}- if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk, visible, nullAvailability, enabled}, "ragflow_chunk_readback_test", "kb-1"); err != nil {+ if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk, visible, nullAvailability, enabled}, "ragflow_chunk_readback_test", "kb-1", ""); err != nil {t.Fatalf("InsertChunks: %v", err)}diff --git a/internal/engine/elasticsearch/chunk_refresh_test.go b/internal/engine/elasticsearch/chunk_refresh_test.goindex 523d3ff34b..5441ea4747 100644--- a/internal/engine/elasticsearch/chunk_refresh_test.go+++ b/internal/engine/elasticsearch/chunk_refresh_test.go@@ -55,7 +55,7 @@ func TestInsertChunksWaitsForRefresh(t *testing.T) {var refresh []stringengine := newBulkCaptureEngine(t, &refresh)- if _, err := engine.InsertChunks(t.Context(), oneChunk(), "ragflow_tenant", "kb-1"); err != nil {+ if _, err := engine.InsertChunks(t.Context(), oneChunk(), "ragflow_tenant", "kb-1", ""); err != nil {t.Fatalf("InsertChunks: %v", err)}if len(refresh) != 1 || refresh[0] != "wait_for" {@@ -71,7 +71,7 @@ func TestInsertChunksNoRefreshOmitsRefreshParam(t *testing.T) {var refresh []stringengine := newBulkCaptureEngine(t, &refresh)- if _, err := engine.InsertChunksNoRefresh(t.Context(), oneChunk(), "ragflow_tenant", "kb-1"); err != nil {+ if _, err := engine.InsertChunksNoRefresh(t.Context(), oneChunk(), "ragflow_tenant", "kb-1", ""); err != nil {t.Fatalf("InsertChunksNoRefresh: %v", err)}if len(refresh) != 1 {diff --git a/internal/engine/engine.go b/internal/engine/engine.goindex 74fb347eb8..cf7455801f 100644--- a/internal/engine/engine.go+++ b/internal/engine/engine.go@@ -41,8 +41,14 @@ const (// DocEngine document storage engine interfacetype DocEngine interface {// Chunk operations- CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error- InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)+ // CreateChunkStore creates the chunk store for a dataset. language is the+ // dataset language, needed by engines whose fulltext analyzer is fixed when+ // the index is created; the others ignore it.+ CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error+ // InsertChunks writes chunks, creating the chunk store on first write.+ // language is the dataset language, needed there for the same reason as in+ // CreateChunkStore; engines that analyse client-side ignore it.+ InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) errorDeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error)Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error)diff --git a/internal/engine/infinity/analyzer_test.go b/internal/engine/infinity/analyzer_test.gonew file mode 100644index 0000000000..af8a451343--- /dev/null+++ b/internal/engine/infinity/analyzer_test.go@@ -0,0 +1,69 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package infinity++import "testing"++func TestAnalyzerForLanguage(t *testing.T) {+ tests := []struct {+ name string+ analyzer string+ language string+ want string+ }{+ {"slovak suffixes the coarse rag analyzer", "rag-coarse", "Slovak", "rag-coarse-slovak"},+ {"slovak suffixes the fine rag analyzer", "rag-fine", "Slovak", "rag-fine-slovak"},+ {"czech is handled the same way", "rag-coarse", "czech", "rag-coarse-czech"},+ {"the bare rag analyzer takes a suffix too", "rag", "Slovak", "rag-slovak"},+ {"case and surrounding space do not matter", "rag-fine", " CZECH ", "rag-fine-czech"},++ // Languages the analyzer treats no differently must not move: their+ // datasets are already indexed under the default analyzer, and+ // switching a Snowball stemmer on would silently change tokenization.+ {"english is left alone", "rag-coarse", "English", "rag-coarse"},+ {"dutch is left alone", "rag-coarse", "Dutch", "rag-coarse"},+ {"an unset language is left alone", "rag-coarse", "", "rag-coarse"},++ // Keyword analyzers are language independent.+ {"whitespace is left alone", "whitespace-#", "Slovak", "whitespace-#"},+ {"rankfeatures is left alone", "rankfeatures", "Slovak", "rankfeatures"},+ {"a rag prefix is not a rag analyzer", "ragged", "Slovak", "ragged"},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := analyzerForLanguage(tt.analyzer, tt.language); got != tt.want {+ t.Errorf("analyzerForLanguage(%q, %q) = %q, want %q", tt.analyzer, tt.language, got, tt.want)+ }+ })+ }+}++// Queries and filters reference a fulltext index by name (convertMatchingField),+// so the language must change only the analyzer, never the index name.+func TestFulltextIndexNameIsLanguageIndependent(t *testing.T) {+ for _, tt := range []struct{ field, analyzer, ref string }{+ {"content", "rag-coarse", "content_ltks"},+ {"content", "rag-fine", "content_sm_ltks"},+ {"docnm", "rag-coarse", "docnm_kwd"},+ } {+ name := fulltextIndexName(tt.field, tt.analyzer)+ if want := tt.field + "@" + name; convertMatchingField(tt.ref) != want {+ t.Errorf("convertMatchingField(%q) = %q, want %q", tt.ref, convertMatchingField(tt.ref), want)+ }+ }+}diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex e9880a5d07..bf2c29cd1c 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -55,14 +55,15 @@ var pagerankAdjustLocks [pagerankAdjustLockCount]sync.Mutex// The full table name is built as "{baseName}_{datasetID}"// For skill index (datasetID="skill"), tableName is just baseName and uses skill_infinity_mapping.json// For memory message indexes (baseName prefixed "memory_"), uses message_infinity_mapping.json-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {+// language is the dataset language, which selects the fulltext analyzer; see analyzerForLanguage.+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")if err != nil {return fmt.Errorf("failed to get database: %w", err)}defer release()- return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID)+ return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID, language)}// memoryIndexPrefix marks per-memory message indexes (MemoryIndexName).@@ -88,7 +89,7 @@ func chunkMappingFileName(baseName, datasetID, configured string) string {}}-func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID string) error {+func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID, language string) error {vecSize := vectorSize// Determine table name and mapping file based on index type@@ -227,7 +228,9 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset}common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))- // Create full-text indexes for varchar fields with analyzers+ // Create full-text indexes for varchar fields with analyzers. CreateTable+ // above is ConflictTypeIgnore, so this may be an existing table: its indexes+ // are kept as they are, and with them the analyzer it was created with.for _, fieldName := range schema.Keys {fieldInfo := schema.Fields[fieldName]if fieldInfo.Type != "varchar" || fieldInfo.Analyzer == nil {@@ -247,13 +250,10 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset}for _, analyzer := range analyzers {- indexNameFt := fmt.Sprintf("ft_%s_%s",- regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(fieldName, "_"),- regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(analyzer, "_"),- )+ indexNameFt := fulltextIndexName(fieldName, analyzer)_, err = table.CreateIndex(indexNameFt,- infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzer}),+ infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzerForLanguage(analyzer, language)}),infinity.ConflictTypeIgnore,"",)@@ -332,7 +332,7 @@ func (e *Engine) ensureChunkDataColumn(table *infinity.Table, exists bool) error// Table name format: {baseName}_{datasetID}// Auto-create the table if it doesn't exist// Delete existing rows with matching IDs before insert-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {tableName := buildChunkTableName(baseName, datasetID)common.Info("InfinityConnection.InsertChunks called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))@@ -369,8 +369,9 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interfacereturn nil, fmt.Errorf("cannot infer vector size from chunks")}- // Create table- if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks)); err != nil {+ // Create table. This is the path the ingestion pipeline actually takes,+ // so the dataset language has to reach it here.+ if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks), language); err != nil {return nil, fmt.Errorf("failed to create table: %w", err)}diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.goindex edeb53036e..466d6fa83f 100644--- a/internal/engine/infinity/common.go+++ b/internal/engine/infinity/common.go@@ -23,6 +23,7 @@ import ("fmt""ragflow/internal/common""ragflow/internal/engine/types"+ "regexp""strings""unicode"@@ -573,3 +574,54 @@ func buildChunkTableName(baseName, datasetID string) string {func buildMetadataTableName(tenantID string) string {return fmt.Sprintf("ragflow_doc_meta_%s", tenantID)}++// analyzerLanguages are the dataset languages that need their own RAG+// analyzer. Infinity's analyzer folds their diacritics to ASCII and skips+// stemming, so a dataset in one of them has to be indexed and queried through+// "rag-<language>" instead of the default "rag-coarse"/"rag-fine" pair.+//+// They are the only two languages whose analyzer differs from the default:+// every other name either selects a Snowball stemmer (english, dutch, german,+// ... -- SNOWBALL_LANGUAGE_MAP in Infinity's rag_analyzer) or is a no-op+// (Chinese, Japanese, Korean, ... use dictionary segmentation, not stemming).+// Naming a stemmer here would switch it for every newly created dataset in+// that language, which is a behaviour change of its own.+var analyzerLanguages = map[string]struct{}{+ "czech": {},+ "slovak": {},+}++// ragAnalyzer is the analyzer family whose behaviour the dataset language+// changes. Fields indexed with "whitespace-#" or "rankfeatures" carry+// keywords, not prose.+const ragAnalyzer = "rag"++// analyzerForLanguage returns the analyzer to index a dataset in language+// with. Infinity analyzes a query with the analyzer of the fulltext index it+// matches against, so the language has to be baked into the index when it is+// created -- there is no per-query analyzer to pass. Chunk tables are+// per-dataset, which is what makes a per-dataset analyzer possible.+//+// Languages outside analyzerLanguages, and analyzers outside the rag family,+// are returned unchanged.+func analyzerForLanguage(analyzer, language string) string {+ key := strings.ToLower(strings.TrimSpace(language))+ if _, ok := analyzerLanguages[key]; !ok {+ return analyzer+ }+ if base, _, _ := strings.Cut(analyzer, "-"); base != ragAnalyzer {+ return analyzer+ }+ return analyzer + "-" + key+}++// indexNamePartRe matches everything that cannot appear in an index name.+var indexNamePartRe = regexp.MustCompile(`[^a-zA-Z0-9]`)++// fulltextIndexName is the name of the fulltext index on fieldName for the+// configured analyzer. Queries and filters name the index explicitly (see+// convertMatchingField), so the name is the same in every language; the+// dataset language only changes the analyzer behind it.+func fulltextIndexName(fieldName, analyzer string) string {+ return fmt.Sprintf("ft_%s_%s", indexNamePartRe.ReplaceAllString(fieldName, "_"), indexNamePartRe.ReplaceAllString(analyzer, "_"))+}diff --git a/internal/engine/infinity/language_integration_test.go b/internal/engine/infinity/language_integration_test.gonew file mode 100644index 0000000000..21db5fc9f0--- /dev/null+++ b/internal/engine/infinity/language_integration_test.go@@ -0,0 +1,107 @@+//go:build integration++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package infinity++import (+ "context"+ "fmt"+ "os"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/engine/types"+ "ragflow/internal/server/config"+)++// TestSlovakChunkStoreMatchesWithAndWithoutDiacritics runs against a real+// Infinity: a chunk store created with the Slovak dataset language gets the+// folding rag-*-slovak analyzers, so an accented word in a chunk matches a query+// typed with or without its diacritics.+func TestSlovakChunkStoreMatchesWithAndWithoutDiacritics(t *testing.T) {+ if err := common.InitLogger("error", common.FileOutput{}, "infinity_language_test"); err != nil {+ t.Fatalf("init logger: %v", err)+ }+ uri := os.Getenv("RAGFLOW_INFINITY_URI")+ if uri == "" {+ uri = "localhost:23817"+ }+ ctx := t.Context()+ eng, err := NewEngine(ctx, config.InfinityConfig{URI: uri, DBName: "ragflow_language_test", PostgresPort: 5432})+ if err != nil {+ t.Fatalf("connect to Infinity at %s: %v", uri, err)+ }+ t.Cleanup(func() { _ = eng.Close() })++ baseName := fmt.Sprintf("ragflow_language_test_%d", time.Now().UnixNano())+ search := func(datasetID, query string) int {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{datasetID},+ Limit: 10,+ SelectFields: []string{"id", "content_with_weight"},+ MatchExprs: []interface{}{&types.MatchTextExpr{+ Fields: []string{"content_ltks", "content_sm_ltks", "docnm_kwd"},+ MatchingText: query,+ TopN: 10,+ }},+ })+ if err != nil {+ t.Fatalf("Search(%s, %q): %v", datasetID, query, err)+ }+ return len(res.Chunks)+ }++ content := "Deti chodia do školy každý deň."+ for _, tt := range []struct {+ datasetID, language string+ unaccentedHits int+ }{+ {"kb_slovak", "slovak", 1},+ // The default analyzer does not fold, which is what makes the Slovak+ // match above the analyzer's doing.+ {"kb_default", "", 0},+ } {+ if err := eng.CreateChunkStore(ctx, baseName, tt.datasetID, 2, "naive", tt.language); err != nil {+ t.Fatalf("CreateChunkStore(%s): %v", tt.datasetID, err)+ }+ t.Cleanup(func() { _ = eng.DropChunkStore(context.Background(), baseName, tt.datasetID) })+ chunk := map[string]interface{}{+ "id": "chunk-1",+ "doc_id": "doc-1",+ "kb_id": tt.datasetID,+ "docnm_kwd": "škola.txt",+ "content_with_weight": content,+ "content_ltks": content,+ "content_sm_ltks": content,+ "available_int": 1,+ "q_2_vec": []float64{0.1, 0.2},+ }+ if _, err := eng.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, tt.datasetID, tt.language); err != nil {+ t.Fatalf("InsertChunks(%s): %v", tt.datasetID, err)+ }++ if got := search(tt.datasetID, "školy"); got != 1 {+ t.Errorf("%s: accented query returned %d chunks, want 1", tt.datasetID, got)+ }+ if got := search(tt.datasetID, "skoly"); got != tt.unaccentedHits {+ t.Errorf("%s: unaccented query returned %d chunks, want %d", tt.datasetID, got, tt.unaccentedHits)+ }+ }+}diff --git a/internal/engine/oceanbase/crud.go b/internal/engine/oceanbase/crud.goindex acd1ac5760..a09b0e401e 100644--- a/internal/engine/oceanbase/crud.go+++ b/internal/engine/oceanbase/crud.go@@ -26,7 +26,7 @@ import ()// InsertChunks writes chunks or memory messages with legacy REPLACE semantics.-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {if len(chunks) == 0 {return []string{}, nil}@@ -48,7 +48,7 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interfaceif vectorSize == 0 {return nil, fmt.Errorf("cannot infer vector size from documents")}- if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", ""); err != nil {return nil, err}} else if vectorSize > 0 {diff --git a/internal/engine/oceanbase/crud_test.go b/internal/engine/oceanbase/crud_test.goindex c972b6baa4..8adab1565a 100644--- a/internal/engine/oceanbase/crud_test.go+++ b/internal/engine/oceanbase/crud_test.go@@ -66,7 +66,7 @@ func TestInsertChunksFindsLaterVectorAndWaitsForSeekDBIndexRefresh(t *testing.T)"id": "memory-1_2", "message_id": "2", "memory_id": "memory-1","content": "hello", "content_embed": []float64{0.1, 0.2},},- }, tableName, "memory-1")+ }, tableName, "memory-1", "")if err != nil {t.Fatal(err)}@@ -104,7 +104,7 @@ func TestInsertChunksReturnsNormalizationErrorWithoutPanic(t *testing.T) {_, err = engine.InsertChunks(t.Context(), []map[string]interface{}{{"id": "memory-1_1", "message_id": "1", "q_2_vec": "invalid",- }}, tableName, "memory-1")+ }}, tableName, "memory-1", "")if err == nil {t.Fatal("InsertChunks() error = nil, want vector normalization error")}diff --git a/internal/engine/oceanbase/document.go b/internal/engine/oceanbase/document.goindex f46c1c7bca..a9ea37b4fc 100644--- a/internal/engine/oceanbase/document.go+++ b/internal/engine/oceanbase/document.go@@ -40,7 +40,7 @@ func (e *Engine) IndexDocument(ctx context.Context, indexName, docID string, doc}vectorSize := vectorDimension(document)if !ready {- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {return err}} else if vectorSize > 0 {@@ -91,7 +91,7 @@ func (e *Engine) BulkIndex(ctx context.Context, indexName string, docs []interfareturn nil, err}if !ready {- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {return nil, err}} else if vectorSize > 0 {diff --git a/internal/engine/oceanbase/integration_test.go b/internal/engine/oceanbase/integration_test.goindex d6ffb2b488..ee0cec7c6a 100644--- a/internal/engine/oceanbase/integration_test.go+++ b/internal/engine/oceanbase/integration_test.go@@ -58,7 +58,7 @@ func TestLegacyStorageRoundTrip(t *testing.T) {datasetID := "kb-" + suffixdefer cleanupChunkStore(t, engine, tableName, "")- if err := engine.CreateChunkStore(ctx, tableName, datasetID, 2, "naive"); err != nil {+ if err := engine.CreateChunkStore(ctx, tableName, datasetID, 2, "naive", ""); err != nil {t.Fatal(err)}if _, err := engine.InsertChunks(ctx, []map[string]interface{}{{@@ -66,7 +66,7 @@ func TestLegacyStorageRoundTrip(t *testing.T) {"content_ltks": "hello oceanbase", "important_kwd": []string{"hello"},"metadata": map[string]interface{}{"_group_id": "group-1", "custom": "json-value"},"custom_field": "kept-in-extra", "q_2_vec": []float64{0.25, 0.5},- }}, tableName, datasetID); err != nil {+ }}, tableName, datasetID, ""); err != nil {t.Fatal(err)}row, err := engine.GetChunk(ctx, tableName, "chunk-1", []string{datasetID})@@ -127,13 +127,13 @@ func TestLegacyStorageRoundTrip(t *testing.T) {memoryA := "memory-a-" + suffixmemoryB := "memory-b-" + suffixdefer cleanupChunkStore(t, engine, memoryTable, "")- if err := engine.CreateChunkStore(ctx, memoryTable, memoryA, 2, ""); err != nil {+ if err := engine.CreateChunkStore(ctx, memoryTable, memoryA, 2, "", ""); err != nil {t.Fatal(err)}if _, err := engine.InsertChunks(ctx, []map[string]interface{}{{"id": memoryA + "_1", "message_id": "1", "memory_id": memoryA, "content": "first", "content_embed": []float64{0.1, 0.2}},{"id": memoryB + "_1", "message_id": "1", "memory_id": memoryB, "content": "second", "content_embed": []float64{0.3, 0.4}},- }, memoryTable, memoryA); err != nil {+ }, memoryTable, memoryA, ""); err != nil {t.Fatal(err)}if err := engine.DropChunkStore(ctx, memoryTable, memoryA); err != nil {diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.goindex 27adf734c8..8f40d4bb5e 100644--- a/internal/engine/oceanbase/schema.go+++ b/internal/engine/oceanbase/schema.go@@ -159,7 +159,7 @@ var tokenizedFullTextFields = []string{"title_tks", "title_sm_tks", "important_t// CreateChunkStore creates or upgrades the legacy shared tenant table. The// dataset ID is a row-level discriminator for chunk and memory tables.-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {if err := validateIdentifier(baseName); err != nil {return err}diff --git a/internal/engine/serenedb/chunk.go b/internal/engine/serenedb/chunk.goindex 9f82255c28..fb836da590 100644--- a/internal/engine/serenedb/chunk.go+++ b/internal/engine/serenedb/chunk.go@@ -65,7 +65,7 @@ func chunkTableDDL(tableName string, vectorSize int) []string {// CreateChunkStore ensures the tenant chunk table and its indexes exist. All// datasets in the tenant share this table, so it is created once and is a// no-op for later datasets.-func (e *serenedbEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {+func (e *serenedbEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {tableName := chunkTableName(baseName)for _, stmt := range chunkTableDDL(tableName, vectorSize) {if err := e.exec(ctx, stmt); err != nil {@@ -227,7 +227,7 @@ func prepareChunkRow(chunk map[string]interface{}, defaultKbID string) ([]string// InsertChunks upserts chunks by id. Rows are grouped by identical column set// and inserted in one multi-row statement per group. Returns an empty slice on// success (ids are the caller's own).-func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {+func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {if len(chunks) == 0 {return []string{}, nil}@@ -243,7 +243,7 @@ func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]isize, _ = strconv.Atoi(m[1])}}- if err := e.CreateChunkStore(ctx, baseName, datasetID, size, ""); err != nil {+ if err := e.CreateChunkStore(ctx, baseName, datasetID, size, "", ""); err != nil {return nil, err}}diff --git a/internal/engine/serenedb/integration_test.go b/internal/engine/serenedb/integration_test.goindex 3bdfa35de8..3a0cd4934c 100644--- a/internal/engine/serenedb/integration_test.go+++ b/internal/engine/serenedb/integration_test.go@@ -107,7 +107,7 @@ func TestIntegrationChunkLifecycle(t *testing.T) {_ = e.DropChunkStore(ctx, base, kb)defer func() { _ = e.DropChunkStore(ctx, base, kb) }()- if err := e.CreateChunkStore(ctx, base, kb, 4, ""); err != nil {+ if err := e.CreateChunkStore(ctx, base, kb, 4, "", ""); err != nil {t.Fatalf("CreateChunkStore: %v", err)}if ok, _ := e.ChunkStoreExists(ctx, base, kb); !ok {@@ -119,7 +119,7 @@ func TestIntegrationChunkLifecycle(t *testing.T) {{"id": "b", "doc_id": "d1", "kb_id": kb, "content_ltks": "gamma delta", "content_with_weight": "gamma delta", "q_4_vec": []float64{0, 1, 0, 0}, "important_kwd": []interface{}{"gamma"}},{"id": "c", "doc_id": "d2", "kb_id": kb, "content_ltks": "alpha gamma", "content_with_weight": "alpha gamma", "q_4_vec": []float64{0.9, 0.1, 0, 0}, "important_kwd": []interface{}{"alpha", "gamma"}},}- if _, err := e.InsertChunks(ctx, chunks, base, kb); err != nil {+ if _, err := e.InsertChunks(ctx, chunks, base, kb, ""); err != nil {t.Fatalf("InsertChunks: %v", err)}diff --git a/internal/engine/vastbase/crud.go b/internal/engine/vastbase/crud.goindex 20689df643..a4c77f16b3 100644--- a/internal/engine/vastbase/crud.go+++ b/internal/engine/vastbase/crud.go@@ -31,7 +31,7 @@ import (// delete-then-insert upsert: legacy tables carry no PRIMARY KEY, so ON// CONFLICT is not an option. The delete and the grouped inserts share one// transaction, keeping a failed batch from leaving half-replaced rows.-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {if len(chunks) == 0 {return []string{}, nil}@@ -63,7 +63,7 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interfaceif !tablePresent && vectorSize == 0 {return nil, fmt.Errorf("cannot infer vector size from documents")}- if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", ""); err != nil {return nil, err}} else if vectorSize > 0 {diff --git a/internal/engine/vastbase/crud_test.go b/internal/engine/vastbase/crud_test.goindex 22c59d0b55..8e611ca9f6 100644--- a/internal/engine/vastbase/crud_test.go+++ b/internal/engine/vastbase/crud_test.go@@ -88,7 +88,7 @@ func TestInsertChunksRunsDeleteThenInsertInOneTransaction(t *testing.T) {inserted, err := engine.InsertChunks(context.Background(), []map[string]interface{}{{"id": "c1", "content_ltks": "hello", "q_3_vec": []interface{}{0.1, 0.2, 0.3},- }}, "t1", "kb-1")+ }}, "t1", "kb-1", "")if err != nil {t.Fatal(err)}@@ -119,7 +119,7 @@ func TestInsertChunksReturnsEncodeErrorForMalformedVector(t *testing.T) {_, err = engine.InsertChunks(context.Background(), []map[string]interface{}{{"id": "c1", "q_3_vec": "oops",- }}, "t1", "kb-1")+ }}, "t1", "kb-1", "")if err == nil || !strings.Contains(err.Error(), "encode q_3_vec") {t.Fatalf("err = %v", err)}@@ -161,7 +161,7 @@ func TestInsertChunksRequiresVectorSizeForNewStore(t *testing.T) {mock.ExpectQuery(regexp.QuoteMeta(tableCountQuery())).WithArgs("t1").WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))if _, err := engine.InsertChunks(context.Background(),- []map[string]interface{}{{"id": "c1", "content_ltks": "x"}}, "t1", "kb-1"); err == nil {+ []map[string]interface{}{{"id": "c1", "content_ltks": "x"}}, "t1", "kb-1", ""); err == nil {t.Fatal("creating a new store without an inferable vector dimension must fail")}if err := mock.ExpectationsWereMet(); err != nil {@@ -225,7 +225,7 @@ func TestInsertChunksVectorlessBatchOnTableWithFailedFullTextIndex(t *testing.T)if _, err := engine.InsertChunks(context.Background(), []map[string]interface{}{{"id": "c3", "kb_id": "kb-1", "content_ltks": "third"},- }, "t1", "kb-1"); err != nil {+ }, "t1", "kb-1", ""); err != nil {t.Fatal(err)}if err := mock.ExpectationsWereMet(); err != nil {diff --git a/internal/engine/vastbase/document.go b/internal/engine/vastbase/document.goindex d5c659c27a..e1c104bb46 100644--- a/internal/engine/vastbase/document.go+++ b/internal/engine/vastbase/document.go@@ -40,7 +40,7 @@ func (e *Engine) IndexDocument(ctx context.Context, indexName, docID string, doc}vectorSize := vectorDimension(document)if !ready {- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {return err}} else if vectorSize > 0 {@@ -94,7 +94,7 @@ func (e *Engine) BulkIndex(ctx context.Context, indexName string, docs []interfareturn nil, err}if !ready {- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {return nil, err}} else if vectorSize > 0 {diff --git a/internal/engine/vastbase/integration_test.go b/internal/engine/vastbase/integration_test.goindex 258fe7e601..f9e7b48fa2 100644--- a/internal/engine/vastbase/integration_test.go+++ b/internal/engine/vastbase/integration_test.go@@ -82,7 +82,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {defer engine.DropChunkStore(ctx, table, "")// --- Chunk store: DDL, dynamic columns, storage-format round trip ---- if err := engine.CreateChunkStore(ctx, table, dataset, 3, "naive"); err != nil {+ if err := engine.CreateChunkStore(ctx, table, dataset, 3, "naive", ""); err != nil {t.Fatal(err)}exists, err := engine.ChunkStoreExists(ctx, table, dataset)@@ -105,7 +105,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {"content_with_weight": "second chunk", "content_ltks": "second chunk","available_int": 1, "q_3_vec": []interface{}{0.9, 0.8, 0.7},},- }, table, dataset); err != nil {+ }, table, dataset, ""); err != nil {t.Fatal(err)}row, err := engine.GetChunk(ctx, table, "c1", []string{dataset})@@ -213,13 +213,13 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {memoryTable := "memory_vb_it_" + suffixmemoryID := "mem-" + suffixdefer engine.DropChunkStore(ctx, memoryTable, "")- if err := engine.CreateChunkStore(ctx, memoryTable, memoryID, 0, ""); err != nil {+ if err := engine.CreateChunkStore(ctx, memoryTable, memoryID, 0, "", ""); err != nil {t.Fatal(err)}if _, err := engine.InsertChunks(ctx, []map[string]interface{}{{"id": memoryID + "_1", "content": "remember the vastbase meeting", "status": true,"content_embed": []interface{}{0.25, 0.5}},- }, memoryTable, memoryID); err != nil {+ }, memoryTable, memoryID, ""); err != nil {t.Fatal(err)}memoryText, err := engine.Search(ctx, &types.SearchRequest{@@ -293,7 +293,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {// --- Drop deletes dataset rows but keeps the shared table ---remaining, err := engine.InsertChunks(ctx, []map[string]interface{}{{"id": "c3", "kb_id": dataset, "content_ltks": "third"},- }, table, dataset)+ }, table, dataset, "")if err != nil {t.Fatal(err)}diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.goindex 1f43aad026..1b5cf1678a 100644--- a/internal/engine/vastbase/schema.go+++ b/internal/engine/vastbase/schema.go@@ -178,7 +178,7 @@ var skillFullTextFields = []string{"name_tks", "tags_tks", "description_tks", "c// CreateChunkStore creates or upgrades the shared baseName table. The dataset// ID is a row-level discriminator for chunk and memory tables.-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {if err := validateIdentifier(baseName); err != nil {return err}diff --git a/internal/engine/vastbase/schema_test.go b/internal/engine/vastbase/schema_test.goindex 64257e22f3..7e24aa7e84 100644--- a/internal/engine/vastbase/schema_test.go+++ b/internal/engine/vastbase/schema_test.go@@ -95,7 +95,7 @@ func TestCreateMemoryStoreDDLGolden(t *testing.T) {}// vectorSize 0: no vector column or index is created.- if err := engine.CreateChunkStore(context.Background(), "memory_t1", "mem-1", 0, ""); err != nil {+ if err := engine.CreateChunkStore(context.Background(), "memory_t1", "mem-1", 0, "", ""); err != nil {t.Fatal(err)}if err := mock.ExpectationsWereMet(); err != nil {@@ -144,7 +144,7 @@ func TestCreateChunkStoreBModeProvisionsFullTextPerColumn(t *testing.T) {quoteIdent(indexName), quoteIdent(field)))}- if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, ""); err != nil {+ if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, "", ""); err != nil {t.Fatal(err)}if err := mock.ExpectationsWereMet(); err != nil {@@ -185,7 +185,7 @@ func TestCreateChunkStoreBModeDegradesWhenFullTextIndexFails(t *testing.T) {// vectorSize 0: no vector column or index is created; the failed// full-text DDL must not surface as an error.- if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, ""); err != nil {+ if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, "", ""); err != nil {t.Fatal(err)}if err := mock.ExpectationsWereMet(); err != nil {diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 63b101e6f3..381cc8b854 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -16,7 +16,10 @@package entity-import "time"+import (+ "strings"+ "time"+)// DatasetNameLimit is the maximum length for dataset nameconst DatasetNameLimit = 128@@ -147,6 +150,29 @@ func (kb *Knowledgebase) TableName() string {return "knowledgebase"}+// KnowledgebasesLanguage returns the language to retrieve with over kbs, which+// is only well defined when they all agree: one query is tokenized once, but+// each dataset was indexed under its own language. A mixed set therefore falls+// back to "" (English, no folding) rather than letting the first dataset in the+// slice decide how the others are searched.+func KnowledgebasesLanguage(kbs []*Knowledgebase) string {+ language := ""+ for i, kb := range kbs {+ current := ""+ if kb != nil && kb.Language != nil {+ current = *kb.Language+ }+ if i == 0 {+ language = current+ continue+ }+ if !strings.EqualFold(strings.TrimSpace(current), strings.TrimSpace(language)) {+ return ""+ }+ }+ return language+}+// ToMap converts Knowledgebase to a map for JSON responsefunc (kb *Knowledgebase) ToMap() map[string]interface{} {result := map[string]interface{}{diff --git a/internal/entity/dataset_language_test.go b/internal/entity/dataset_language_test.gonew file mode 100644index 0000000000..8b018c14ab--- /dev/null+++ b/internal/entity/dataset_language_test.go@@ -0,0 +1,54 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package entity++import "testing"++func kb(language *string) *Knowledgebase {+ return &Knowledgebase{Language: language}+}++func ptr(s string) *string { return &s }++func TestKnowledgebasesLanguage(t *testing.T) {+ tests := []struct {+ name string+ kbs []*Knowledgebase+ want string+ }{+ {"no datasets", nil, ""},+ {"single dataset", []*Knowledgebase{kb(ptr("Slovak"))}, "Slovak"},+ {"all agree", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr("Slovak"))}, "Slovak"},+ {"agree ignoring case and space", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr(" slovak "))}, "Slovak"},+ // A mixed set has no single answer: one query is tokenized once, but the+ // datasets were indexed under different rules. Fall back to English.+ {"mixed languages", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr("English"))}, ""},+ {"mixed with unset", []*Knowledgebase{kb(ptr("Slovak")), kb(nil)}, ""},+ {"unset first, set second", []*Knowledgebase{kb(nil), kb(ptr("Slovak"))}, ""},+ {"all unset", []*Knowledgebase{kb(nil), kb(nil)}, ""},+ {"nil element", []*Knowledgebase{kb(ptr("Slovak")), nil}, ""},+ // Order must not decide the result.+ {"mixed, reversed", []*Knowledgebase{kb(ptr("English")), kb(ptr("Slovak"))}, ""},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := KnowledgebasesLanguage(tt.kbs); got != tt.want {+ t.Errorf("KnowledgebasesLanguage() = %q, want %q", got, tt.want)+ }+ })+ }+}diff --git a/internal/handler/dify_retrieval_handler.go b/internal/handler/dify_retrieval_handler.goindex 2da98d9249..6e826ee0e0 100644--- a/internal/handler/dify_retrieval_handler.go+++ b/internal/handler/dify_retrieval_handler.go@@ -273,6 +273,7 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {KNNTopK: topK,SimilarityThreshold: scoreThreshold,EmbeddingModel: embModel,+ Language: entity.KnowledgebasesLanguage(kbs),}if rankFeature != nil {sr.RankFeature = &rankFeaturediff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 712c682b33..2a6c74b290 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -230,7 +230,11 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in// conf/infinity_mapping.json) and merge them into the upstream input// chunks. The component stays DB-independent and no longer routes through a// separate writer seam: its output is plain chunks.- compiled, err := productsToChunkDocs(out.Products)+ // Compiled rows share the dataset's chunk table with the parsed chunks and+ // are retrieved by the same queries, so they tokenize with the dataset+ // language the Tokenizer component uses.+ language := globals.GlobalOrInput(ctx, inputs, "lang", "English")+ compiled, err := productsToChunkDocs(out.Products, language)if err != nil {return nil, err}@@ -396,7 +400,8 @@ func kindOrVariant(p common.Product) string {// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is// not persisted: only engine columns are written, because Infinity rejects an// insert naming an unknown column.-func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {+func productsToChunkDocs(products []common.Product, language string) ([]schema.ChunkDoc, error) {+ tok := tokenizer.New(language)docs := make([]schema.ChunkDoc, 0, len(products))for _, p := range products {doc := schema.ChunkDoc{@@ -420,9 +425,9 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {indexText = d}}- if ltks, err := tokenizer.Tokenize(indexText); err == nil && ltks != "" {+ if ltks, err := tok.Tokenize(indexText); err == nil && ltks != "" {doc.ContentLtks = ltks- if sm, err := tokenizer.FineGrainedTokenize(ltks); err == nil && sm != "" {+ if sm, err := tok.FineGrainedTokenize(ltks); err == nil && sm != "" {doc.ContentSmLtks = sm}}@@ -488,7 +493,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {}}// Per-variant fine-grained columns (conf/infinity_mapping.json §45–97).- if err := applyVariantColumns(&doc, p); err != nil {+ if err := applyVariantColumns(&doc, p, tok); err != nil {return nil, err}docs = append(docs, doc)@@ -499,7 +504,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {// applyVariantColumns emits the compile-specific columns defined in// conf/infinity_mapping.json lines 45–77, driven by Product.Meta keys that// each variant's build site populates. Unknown/absent keys are skipped.-func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {+func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Tokenizer) error {kind := metaString(p.Meta, "kind")switch p.Variant {@@ -537,7 +542,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {if err := doc.SetExtraValue("title_kwd", v); err != nil {return err}- setTitleTokens(doc, v)+ setTitleTokens(doc, v, tok)}if v := metaString(p.Meta, "page_type"); v != "" {if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {@@ -706,11 +711,11 @@ func metaString(m map[string]any, key string) string {// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into// `docnm`, so the twin would make docnm map-order dependent.-func setTitleTokens(doc *schema.ChunkDoc, title string) {+func setTitleTokens(doc *schema.ChunkDoc, title string, tok tokenizer.Tokenizer) {if title == "" {return}- if tks, err := tokenizer.Tokenize(title); err == nil && tks != "" {+ if tks, err := tok.Tokenize(title); err == nil && tks != "" {doc.TitleTks = tks}}diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex c5ccf566a6..bc27a695a4 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -1463,7 +1463,7 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {Content: "Section body", ParentID: "page-id",Meta: map[string]any{"kind": "section", "slug": "overview", "page_slug": "entity/alpha", "section_level": 1, "source_chunk_ids": []string{"c1"}},}- docs, err := productsToChunkDocs([]common.Product{page, section})+ docs, err := productsToChunkDocs([]common.Product{page, section}, "")if err != nil {t.Fatalf("productsToChunkDocs: %v", err)}diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.goindex 1cb27a3fd3..a957f0dedc 100644--- a/internal/ingestion/component/knowledge_compiler/storage_test.go+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go@@ -24,7 +24,7 @@ func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {} {t.Run(test.kind, func(t *testing.T) {docs, err := productsToChunkDocs([]common.Product{{ID: "row", Kind: test.kind, Variant: test.variant, Content: "summary",- Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}})+ Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}}, "")if err != nil {t.Fatal(err)}diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex ab41cfb45b..fa11a31f02 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -904,7 +904,8 @@ func declaredTitleTokens(chunks []schema.ChunkDoc) []string {// Mirrors python tokenizer.py:130-185 and rag/nlp/__init__.py tokenize() /// tokenize_chunks().//-// language sets the Snowball stemmer language, matching Python's+// language selects the analyzer language (Snowball stemmer, or diacritics+// folding without stemming for Slovak/Czech), matching Python's// rag_tokenizer.tokenizer.set_language(language) call inside tokenize().func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string) error {tok := tokenizer.New(language)diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 3d128bf0d7..e7c2fc259e 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -85,7 +85,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {c := &Consumer{scheduler: scheduler,reader: engineReader{eng: engine.Get()},- writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService()},+ writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService(), resolveLanguage: datasetLanguageByID},factory: defaultDeduperFactory,contributions: newWikiContributionStore(engine.Get()),ttl: 2 * time.Minute,diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.goindex e0fae24d58..958bd744f8 100644--- a/internal/ingestion/knowledge_compile/fake_engine_test.go+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go@@ -55,10 +55,10 @@ func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interf// --- stubs: the reader/writer path under test never exercises these. ----func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _ string) ([]string, error) {+func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _, _ string) ([]string, error) {f.insertedChunks = append(f.insertedChunks, chunks...)return nil, nil}diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.goindex ad872543e1..6df368136f 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -107,7 +107,7 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, dataset"content_with_weight": string(payload),"available_int": 0,}- _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)+ _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID, "")return err}diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex 3637b06ad2..b5a76d220d 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -18,6 +18,7 @@ package knowledge_compileimport ("context""encoding/json"+ "errors""strings""testing")@@ -362,3 +363,26 @@ func TestProjectWikiGraphRowsSelfLoopSkipped(t *testing.T) {t.Fatalf("the self-referencing entity must still exist")}}++// TestProjectWikiGraphRowsResolvesDatasetLanguage pins that graph rows are+// tokenized with the language of the dataset they are written to.+func TestProjectWikiGraphRowsResolvesDatasetLanguage(t *testing.T) {+ var resolved []string+ w := engineWriter{resolveLanguage: func(_ context.Context, datasetID string) (string, error) {+ resolved = append(resolved, datasetID)+ return "slovak", nil+ }}+ if _, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{page("entity", "škola")}); err != nil {+ t.Fatalf("project: %v", err)+ }+ if len(resolved) != 1 || resolved[0] != kbForTest {+ t.Fatalf("language resolved for %v, want [%s]", resolved, kbForTest)+ }++ w.resolveLanguage = func(context.Context, string) (string, error) {+ return "", errors.New("dataset lookup failed")+ }+ if _, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{page("entity", "škola")}); err == nil {+ t.Fatal("projection must fail when the dataset language cannot be resolved")+ }+}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.goindex 6c6921008e..804bd8e2e3 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go@@ -22,6 +22,7 @@ import ("fmt""strings"+ "ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -36,6 +37,9 @@ const (type wikiMapVersionStore struct {engine engine.DocEngineresolveVectorSize func(context.Context) (int, error)+ // resolveLanguage returns the dataset language a missing Infinity chunk+ // table is created with; nil reads it from the dataset row.+ resolveLanguage func(ctx context.Context, datasetID string) (string, error)}func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error) {@@ -84,7 +88,7 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k"content_with_weight": string(state.Payload),"available_int": 0,}- _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID)+ _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID, "")return err}@@ -124,12 +128,34 @@ func (s *wikiMapVersionStore) ensureInfinityChunkStore(ctx context.Context, tenaif vectorSize <= 0 {return fmt.Errorf("resolve Wiki MAP vector size: got %d", vectorSize)}- if err := s.engine.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {+ // The table is the dataset's chunk table, and Infinity fixes its fulltext+ // analyzer here, so it has to be created with the dataset language.+ resolveLanguage := s.resolveLanguage+ if resolveLanguage == nil {+ resolveLanguage = datasetLanguageByID+ }+ language, err := resolveLanguage(ctx, datasetID)+ if err != nil {+ return fmt.Errorf("resolve Wiki MAP dataset language: %w", err)+ }+ if err := s.engine.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", language); err != nil {return fmt.Errorf("initialize Infinity chunk store for Wiki MAP: %w", err)}return nil}+// datasetLanguageByID reads the dataset's language, "" when it is unset.+func datasetLanguageByID(ctx context.Context, datasetID string) (string, error) {+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)+ if err != nil {+ return "", err+ }+ if kb.Language == nil {+ return "", nil+ }+ return *kb.Language, nil+}+func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID, datasetID string, keys []string) (map[string][]byte, error) {out := make(map[string][]byte, len(keys))if len(keys) == 0 {@@ -215,7 +241,7 @@ func (s *wikiMapVersionStore) PutWikiMapVersions(ctx context.Context, versions [if len(rows) == 0 {continue}- if _, err := s.engine.InsertChunks(ctx, rows, fmt.Sprintf("ragflow_%s", batch[0].TenantID), batch[0].DatasetID); err != nil {+ if _, err := s.engine.InsertChunks(ctx, rows, fmt.Sprintf("ragflow_%s", batch[0].TenantID), batch[0].DatasetID, ""); err != nil {return fmt.Errorf("save Wiki MAP versions: %w", err)}}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.goindex 5165608a41..ceec7df3ab 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go@@ -36,6 +36,7 @@ type wikiMapStoreEngine struct {chunkStoreChecks intcreatedStores intcreatedVectorSize int+ createdLanguage string}func (e *wikiMapStoreEngine) GetType() string {@@ -50,9 +51,10 @@ func (e *wikiMapStoreEngine) ChunkStoreExists(context.Context, string, string) (return e.chunkStoreExists, nil}-func (e *wikiMapStoreEngine) CreateChunkStore(_ context.Context, _, _ string, vectorSize int, _ string) error {+func (e *wikiMapStoreEngine) CreateChunkStore(_ context.Context, _, _ string, vectorSize int, _, language string) error {e.createdStores++e.createdVectorSize = vectorSize+ e.createdLanguage = languagee.chunkStoreExists = truereturn nil}@@ -126,7 +128,7 @@ func matchesStoredCompilationRow(row, filter map[string]interface{}) bool {return true}-func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {+func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName, datasetID, _ string) ([]string, error) {e.insertBase = baseNamee.insertDataset = datasetIDfor _, chunk := range chunks {@@ -239,6 +241,14 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, func(context.Context) (int, error) {return 768, nil})+ // The table is the dataset's chunk table, so it is created with the+ // dataset language, which fixes Infinity's fulltext analyzer.+ store.(*wikiMapVersionStore).resolveLanguage = func(_ context.Context, datasetID string) (string, error) {+ if datasetID != "kb-1" {+ t.Fatalf("language resolved for dataset %q", datasetID)+ }+ return "slovak", nil+ }version := kccommon.WikiMapVersion{Key: "version-a", TenantID: "tenant-1", DatasetID: "kb-1",DocumentID: "doc-1", ChunkID: "chunk-1", Payload: []byte(`{}`),@@ -247,8 +257,8 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {if err := store.PutWikiMapVersions(t.Context(), []kccommon.WikiMapVersion{version}); err != nil {t.Fatalf("PutWikiMapVersions() error = %v", err)}- if engine.createdStores != 1 || engine.createdVectorSize != 768 {- t.Fatalf("created store: count=%d vector_size=%d", engine.createdStores, engine.createdVectorSize)+ if engine.createdStores != 1 || engine.createdVectorSize != 768 || engine.createdLanguage != "slovak" {+ t.Fatalf("created store: count=%d vector_size=%d language=%q", engine.createdStores, engine.createdVectorSize, engine.createdLanguage)}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 577c99b00b..371d95a3a1 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -126,6 +126,23 @@ type StructureBucket struct {type engineWriter struct {eng engine.DocEnginecommitService pageCommitter+ // resolveLanguage returns the dataset language the dataset-level rows are+ // tokenized with; nil tokenizes them as English.+ resolveLanguage func(ctx context.Context, datasetID string) (string, error)+}++// datasetTokenizer returns a tokenizer in kb's language. Dataset-level rows+// share the dataset's chunk table with the parsed chunks and are retrieved by+// the same queries, so they tokenize like them.+func (w engineWriter) datasetTokenizer(ctx context.Context, kb string) (tokenizer.Tokenizer, error) {+ if w.resolveLanguage == nil {+ return tokenizer.New(""), nil+ }+ language, err := w.resolveLanguage(ctx, kb)+ if err != nil {+ return tokenizer.Tokenizer{}, fmt.Errorf("resolve dataset language: %w", err)+ }+ return tokenizer.New(language), nil}type pageCommitter interface {@@ -273,7 +290,7 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produczap.String("id", metaString(chunks[0], "id")),)}- _, err := eng.InsertChunks(ctx, chunks, baseName, kb)+ _, err := eng.InsertChunks(ctx, chunks, baseName, kb, "")return err})}@@ -483,6 +500,10 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinif err != nil {return err}+ tok, err := w.datasetTokenizer(ctx, kb)+ if err != nil {+ return err+ }rows := make([]map[string]interface{}, 0, len(buckets))for _, b := range buckets {if !structureBucketWritable(b) {@@ -529,7 +550,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinreturn err}payload := string(payloadBytes)- contentLTKS, contentSMLTKS := tokenizeWikiGraphContent("", kccommon.PayloadDescription(payloadMap, nil))+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(tok, "", kccommon.PayloadDescription(payloadMap, nil))row := map[string]interface{}{// Stable dataset-level id keyed on the template-scoped entity name or// relation identity plus the raw compile kind.@@ -617,7 +638,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin}batch := rows[start:end]jobs = append(jobs, func() error {- _, err := eng.InsertChunks(ctx, batch, baseName, kb)+ _, err := eng.InsertChunks(ctx, batch, baseName, kb, "")return err})}@@ -1359,17 +1380,17 @@ func wikiGraphBareKey(slug string) string {// tokenizeWikiGraphContent tokenizes Python's entity search text,// `slug + " " + description` (dataset_wiki_generator.py:750-773); the entity row// gets no title token column.-func tokenizeWikiGraphContent(slug, description string) (string, string) {+func tokenizeWikiGraphContent(tok tokenizer.Tokenizer, slug, description string) (string, string) {content := strings.TrimSpace(strings.Join([]string{slug, description}, " "))if content == "" {return "", ""}- contentLTKS, err := tokenizer.Tokenize(content)+ contentLTKS, err := tok.Tokenize(content)if err != nil || contentLTKS == "" {contentLTKS = content}- contentSMLTKS, err := tokenizer.FineGrainedTokenize(contentLTKS)+ contentSMLTKS, err := tok.FineGrainedTokenize(contentLTKS)if err != nil || contentSMLTKS == "" {contentSMLTKS = contentLTKS}@@ -1726,7 +1747,11 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string// dangling edge, e.g. the target was pruned/deleted) is skipped, as is a// self-loop (src == tgt) — the graph is rebuilt from a consistent read each// time, so cross-batch dangling edges cannot accumulate.-func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {+func (w engineWriter) projectWikiGraphRows(ctx context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {+ tok, err := w.datasetTokenizer(ctx, kb)+ if err != nil {+ return nil, err+ }// Sort pages deterministically by full slug before building the indexes so// the outcome is reproducible across reprojections. The engine's row order// is otherwise scroll-order dependent, which made bare-slug first-match@@ -1786,7 +1811,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,if err != nil {return nil, err}- contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Slug, p.Summary)+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(tok, p.Slug, p.Summary)rows = append(rows, map[string]interface{}{"id": entityID,"doc_id": kb,@@ -1898,7 +1923,7 @@ func (w engineWriter) insertWikiGraphChunks(ctx context.Context, tenant, kb stri}batch := rows[start:end]jobs = append(jobs, func() error {- _, err := eng.InsertChunks(ctx, batch, baseName, kb)+ _, err := eng.InsertChunks(ctx, batch, baseName, kb, "")return err})}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 4ed4e6353a..5538a5ac3d 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -305,7 +305,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik}}if len(rows) > 0 {- if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID); err != nil {+ if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID, ""); err != nil {return err}}diff --git a/internal/ingestion/task/pipeline_e2e_test.go b/internal/ingestion/task/pipeline_e2e_test.goindex 0225419ab3..90569a5492 100644--- a/internal/ingestion/task/pipeline_e2e_test.go+++ b/internal/ingestion/task/pipeline_e2e_test.go@@ -114,10 +114,10 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da_ = docEngine.DropChunkStore(ctx, baseName, datasetID)// Create chunk store (note: vec dimension = 2 because our test chunks have q_2_vec)- if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive"); err != nil {+ if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive", ""); err != nil {// If create failed, maybe it exists; try dropping and recreating_ = docEngine.DropChunkStore(ctx, baseName, datasetID)- if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive"); err != nil {+ if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive", ""); err != nil {_ = docEngine.Close()t.Fatalf("Could not create chunk store: %v", err)}@@ -256,7 +256,7 @@ func TestPipelineE2E_PipelineExecutor(t *testing.T) {svc.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {insertChunksCalled = truet.Logf("DocEngine InsertChunks called! baseName=%s datasetID=%s len(chunks)=%d", baseName, datasetID, len(chunks))- ids, err := docEngine.InsertChunks(ctx, chunks, baseName, datasetID)+ ids, err := docEngine.InsertChunks(ctx, chunks, baseName, datasetID, "")if err != nil {t.Logf("WARNING: InsertChunks err=%v", err)}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex f6534c323a..1b6b8caae2 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -106,7 +106,7 @@ func validateTaskContext(taskCtx *TaskContext) error {// an engine that implements it can write chunks without waiting for an index// refresh.type noRefreshChunkInserter interface {- InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)+ InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)}// insertChunksForIngestion writes non-final chunks through the engine's@@ -116,21 +116,24 @@ type noRefreshChunkInserter interface {// Refreshing every batch adds unnecessary latency. The writer uses the regular// inserter for its final batch, so all preceding batches become searchable// before the document task is acknowledged and its completion event is sent.-func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {+//+// The first write creates the chunk store, and on Infinity that fixes the+// fulltext analyzer, so the dataset language has to travel with it.+func insertChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {if bulk, ok := eng.(noRefreshChunkInserter); ok {- return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID)+ return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID, language)}- return eng.InsertChunks(ctx, chunks, baseName, datasetID)+ return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)}}// insertFinalChunksForIngestion waits for the index refresh before returning.// The final write makes every preceding no-refresh batch searchable before the// document task is acknowledged and its completion event is published.-func insertFinalChunksForIngestion(eng engine.DocEngine) InsertFunc {+func insertFinalChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {- return eng.InsertChunks(ctx, chunks, baseName, datasetID)+ return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)}}@@ -150,11 +153,11 @@ func NewPipelineExecutor(canvasID: canvasID,docBulkSize: docBulkSize,indexWriter: newChunkIndexWriter(- insertChunksForIngestion(engine.Get()),+ insertChunksForIngestion(engine.Get(), datasetLanguage(taskCtx)),fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),taskCtx.Doc.KbID,docBulkSize,- ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get())),+ ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get(), datasetLanguage(taskCtx))),deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)},@@ -1267,3 +1270,13 @@ func injectDebugChunkCap(inputs map[string]any) map[string]any {}return inputs}++// datasetLanguage is the language of the dataset being ingested, or "" when it+// is unset. It selects the fulltext analyzer on engines that fix it when the+// chunk store is created.+func datasetLanguage(taskCtx *TaskContext) string {+ if taskCtx == nil || taskCtx.KB.Language == nil {+ return ""+ }+ return *taskCtx.KB.Language+}diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex f4dc141a45..5e18382371 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -736,6 +736,7 @@ func (s *ChatPipelineService) AsyncChat(RerankModel: rerankModel,EmbeddingModel: embModel,Aggs: func() *bool { v := true; return &v }(),+ Language: entity.KnowledgebasesLanguage(kbs),}result, retErr := retrievalSvc.Retrieval(ctx, req)diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex a867a9af81..c0b45fb063 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -101,8 +101,8 @@ type ChunkService struct {storeChunkImageFunc func(string, string, []byte, string) errorremoveChunkImageFunc func(string, string) errormarkWikiDirtyFunc func(tenantID, datasetID, documentID string, chunkIDs []string)- tokenizeFunc func(string) (string, error)- fineGrainedTokenizeFunc func(string) (string, error)+ tokenizeFunc func(text, language string) (string, error)+ fineGrainedTokenizeFunc func(text, language string) (string, error)numTokensFunc func(string) int}@@ -416,6 +416,7 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.RetrievalRankFeature: &labels,EmbeddingModel: embeddingModel,Highlight: req.Highlight,+ Language: entity.KnowledgebasesLanguage(kbRecords),}// Call RetrievalService to perform retrieval@@ -851,7 +852,7 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,if queryBuilder == nil {queryBuilder = nlp.NewQueryBuilder()}- if matchText, _ := queryBuilder.Question(keywords, "", 0.3); matchText != nil {+ if matchText, _ := queryBuilder.Question(keywords, "", 0.3, datasetLanguageOf(kb)); matchText != nil {matchExprs = append(matchExprs, matchText)}}@@ -1094,6 +1095,7 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunkreturn err}targetTenantID := kb.TenantID+ language := datasetLanguageOf(kb)// Verify document belongs to datasetdocDAO := dao.NewDocumentDAO()@@ -1139,8 +1141,8 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk// Tokenize contentcontentStr := d["content_with_weight"].(string)- d["content_ltks"], _ = tokenizer.Tokenize(contentStr)- d["content_sm_ltks"], _ = tokenizer.FineGrainedTokenize(d["content_ltks"].(string))+ d["content_ltks"], _ = s.tokenize(contentStr, language)+ d["content_sm_ltks"], _ = s.fineGrainedTokenize(d["content_ltks"].(string), language)// Important keywords - convert []string to []interface{} for transformChunkFieldsif req.ImportantKwd != nil {@@ -1351,20 +1353,21 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequeschunkID := common.ChunkID(req.DocumentID, req.Content)indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)- contentLtks, err := s.tokenize(req.Content)+ language := datasetLanguageOf(kb)+ contentLtks, err := s.tokenize(req.Content, language)if err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content: %v", err)}}- contentSmLtks, err := s.fineGrainedTokenize(contentLtks)+ contentSmLtks, err := s.fineGrainedTokenize(contentLtks, language)if err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content fine-grained: %v", err)}}- importantTks, err := s.tokenize(strings.Join(req.ImportantKeywords, " "))+ importantTks, err := s.tokenize(strings.Join(req.ImportantKeywords, " "), language)if err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize important keywords: %v", err)}}questionKwd := filterTrimmedStrings(req.Questions)- questionTks, err := s.tokenize(strings.Join(req.Questions, "\n"))+ questionTks, err := s.tokenize(strings.Join(req.Questions, "\n"), language)if err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize questions: %v", err)}}@@ -1447,7 +1450,7 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesctx, cancel := context.WithTimeout(ctx, 600*time.Second)defer cancel()- if _, err = s.docEngine.InsertChunks(ctx, []map[string]interface{}{chunkData}, indexName, req.DatasetID); err != nil {+ if _, err = s.docEngine.InsertChunks(ctx, []map[string]interface{}{chunkData}, indexName, req.DatasetID, datasetLanguageOf(kb)); err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("insert chunk: %v", err)}}@@ -1632,18 +1635,20 @@ func filterTrimmedStrings(values []string) []string {return filtered}-func (s *ChunkService) tokenize(text string) (string, error) {+// tokenize tokenizes a manually written chunk field with the dataset language,+// so it is indexed the way the dataset's parsed chunks are and its queries fold.+func (s *ChunkService) tokenize(text, language string) (string, error) {if s.tokenizeFunc != nil {- return s.tokenizeFunc(text)+ return s.tokenizeFunc(text, language)}- return tokenizer.Tokenize(text)+ return tokenizer.New(language).Tokenize(text)}-func (s *ChunkService) fineGrainedTokenize(text string) (string, error) {+func (s *ChunkService) fineGrainedTokenize(text, language string) (string, error) {if s.fineGrainedTokenizeFunc != nil {- return s.fineGrainedTokenizeFunc(text)+ return s.fineGrainedTokenizeFunc(text, language)}- return tokenizer.FineGrainedTokenize(text)+ return tokenizer.New(language).FineGrainedTokenize(text)}func (s *ChunkService) numTokens(text string) int {@@ -1845,3 +1850,14 @@ func releaseChunkImageLock(key string) {delete(chunkImageLocks.locks, key)}}++// datasetLanguageOf is the dataset's language, or "" when unset. A query is+// folded the way its dataset was indexed, and on engines whose fulltext+// analyzer is fixed when the chunk store is created a write that may create it+// has to carry the language too.+func datasetLanguageOf(kb *entity.Knowledgebase) string {+ if kb == nil || kb.Language == nil {+ return ""+ }+ return *kb.Language+}diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex 7eebb0f24b..c19eed1212 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -1067,6 +1067,8 @@ func TestAddChunkSuccess(t *testing.T) {engine := &addChunkTestEngine{}var incrementTokenNum, incrementChunkNum int64+ language := "Slovak"+ var tokenizeLanguages []stringsvc := &ChunkService{docEngine: engine,kbDAO: dao.NewKnowledgebaseDAO(),@@ -1078,7 +1080,7 @@ func TestAddChunkSuccess(t *testing.T) {return permission.ErrPermissionDenied},getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {- return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil+ return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language}, nil},getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {driver := &stubEmbeddingDriver{@@ -1098,9 +1100,15 @@ func TestAddChunkSuccess(t *testing.T) {incrementChunkNum = chunkNumreturn nil},- tokenizeFunc: func(text string) (string, error) { return text, nil },- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },- numTokensFunc: func(text string) int { return len(text) },+ tokenizeFunc: func(text, lang string) (string, error) {+ tokenizeLanguages = append(tokenizeLanguages, lang)+ return text, nil+ },+ fineGrainedTokenizeFunc: func(text, lang string) (string, error) {+ tokenizeLanguages = append(tokenizeLanguages, lang)+ return text + "_fg", nil+ },+ numTokensFunc: func(text string) int { return len(text) },}resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{@@ -1142,6 +1150,19 @@ func TestAddChunkSuccess(t *testing.T) {if inserted["img_id"] != nil {t.Fatalf("did not expect image id in inserted chunk: %#v", inserted)}+ // A manual chunk is tokenized like the dataset's parsed chunks, and the+ // write that may create the chunk store carries the dataset language.+ if len(tokenizeLanguages) != 4 {+ t.Fatalf("tokenizer calls = %v, want 4", tokenizeLanguages)+ }+ for _, lang := range tokenizeLanguages {+ if lang != language {+ t.Fatalf("tokenized with language %q, want %q", lang, language)+ }+ }+ if engine.insertLanguage != language {+ t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)+ }vec, ok := inserted["q_2_vec"].([]float64)if !ok {t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])@@ -1190,8 +1211,8 @@ func TestAddChunkPersistsTagKwdOnlyWhenProvided(t *testing.T) {return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil},incrementChunkStatsFunc: func(_, _ string, _, _ int64, _ float64) error { return nil },- tokenizeFunc: func(text string) (string, error) { return text, nil },- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },numTokensFunc: func(text string) int { return len(text) },}return svc, engine@@ -1337,8 +1358,8 @@ func TestAddChunkImageAndTagFeatureValidation(t *testing.T) {getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil},- tokenizeFunc: func(text string) (string, error) { return text, nil },- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },numTokensFunc: func(text string) int { return len(text) },getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {driver := &stubEmbeddingDriver{@@ -1435,8 +1456,8 @@ func TestAddChunkIncrementsStatsAfterInsert(t *testing.T) {incrementCalls++return nil},- tokenizeFunc: func(text string) (string, error) { return text, nil },- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },numTokensFunc: func(text string) int { return len(text) },}@@ -1959,11 +1980,11 @@ type parseTestDocEngine struct {chunkStoreExists bool // if true, ChunkStoreExists returns true}-func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}@@ -2064,13 +2085,15 @@ type addChunkTestEngine struct {insertedChunks []map[string]interface{}insertIndex stringinsertDataset string+ insertLanguage stringinsertErr error}-func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {+func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {e.insertedChunks = chunkse.insertIndex = baseNamee.insertDataset = datasetID+ e.insertLanguage = languagereturn nil, e.insertErr}@@ -2377,10 +2400,10 @@ type switchChunksEngineMock struct {updateCalls []updateChunksCall}-func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string) error {+func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}func (m *switchChunksEngineMock) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, indexName string, datasetID string) error {diff --git a/internal/service/dataset/fake_doc_engine_test.go b/internal/service/dataset/fake_doc_engine_test.goindex 7fa80346a8..6c98f3de85 100644--- a/internal/service/dataset/fake_doc_engine_test.go+++ b/internal/service/dataset/fake_doc_engine_test.go@@ -11,10 +11,10 @@ import (// fakeChatDocEngine is a no-op DocEngine implementation for tests.type fakeChatDocEngine struct{}-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 4c8351921a..14e16883c5 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -309,6 +309,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.SearchRankFeature: &labels,EmbeddingModel: embeddingModel,Highlight: req.Highlight,+ Language: entity.KnowledgebasesLanguage(kbRecords),}retrievalReq.Filter = knowledgeCompilationFilter(req.IncludeCompiledChunks, req.KnowledgeCompilationScope)if len(docIDs) > 0 && (req.IncludeCompiledChunks == nil || *req.IncludeCompiledChunks) && req.KnowledgeCompilationScope != "document" {diff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.goindex 8ea9870d14..45bf0e066a 100644--- a/internal/service/dataset_artifact_keyword_integration_test.go+++ b/internal/service/dataset_artifact_keyword_integration_test.go@@ -66,8 +66,8 @@ func TestWikiKeywordFilter_Integration(t *testing.T) {row["kb_id"] = kbID}// Best effort: the store may already exist from a previous run.- _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive")- if _, err := de.InsertChunks(ctx, rows, idx, kbID); err != nil {+ _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive", "")+ if _, err := de.InsertChunks(ctx, rows, idx, kbID, ""); err != nil {t.Fatalf("seed wiki rows: %v", err)}t.Cleanup(func() {diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex cc881aeea3..d5f8f74c51 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -619,9 +619,18 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat// so a raw multi-token keyword (e.g. "桃园三结义") never equals a stored// token and silently matches nothing. Tokenize the query like the other// lexical callers (see navKeywordExpressions); the raw input stays on- // original_query for engines that analyze the query themselves.+ // original_query for engines that analyze the query themselves. The+ // entity rows are tokenized in the dataset language, so the query is too.+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)+ if err != nil {+ return nil, err+ }+ kbLanguage := ""+ if kb.Language != nil {+ kbLanguage = *kb.Language+ }originalQuery := keywords- if tokenized, err := tokenizer.Tokenize(keywords); err == nil && tokenized != "" {+ if tokenized, err := tokenizer.New(kbLanguage).Tokenize(keywords); err == nil && tokenized != "" {keywords = tokenized}matchExprs = []interface{}{&types.MatchTextExpr{diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 38e005727b..5cd86a8198 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -138,7 +138,7 @@ func (s *DocumentService) UpdateDocument(ctx context.Context, id string, req *Upif err != nil {return common.CodeServerError, err}- if err = s.updateDocumentNameOnly(ctx, document, kb.TenantID, *req.Name); err != nil {+ if err = s.updateDocumentNameOnly(ctx, document, kb, *req.Name); err != nil {return common.CodeServerError, err}return common.CodeSuccess, nildiff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 77521913ad..e313c7f8df 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -145,7 +145,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat}if present["name"] && req.Name != nil && (doc.Name == nil || *req.Name != *doc.Name) {- if err = s.updateDocumentNameOnly(ctx, doc, kb.TenantID, *req.Name); err != nil {+ if err = s.updateDocumentNameOnly(ctx, doc, kb, *req.Name); err != nil {return nil, common.CodeDataError, err}}@@ -453,7 +453,9 @@ func validateMetaFields(meta map[string]any) error {return nil}-func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entity.Document, tenantID, newName string) error {+// updateDocumentNameOnly renames a document and re-tokenizes its chunks' title+// fields with the dataset language, the way ingestion tokenized them.+func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, newName string) error {if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"name": newName}); err != nil {return errors.New("database error (Document rename)")}@@ -470,9 +472,14 @@ func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entitreturn nil}- titleTks, _ := tokenizer.Tokenize(newName)- titleSmTks, _ := tokenizer.FineGrainedTokenize(titleTks)- indexName := fmt.Sprintf("ragflow_%s", tenantID)+ language := ""+ if kb.Language != nil {+ language = *kb.Language+ }+ tok := tokenizer.New(language)+ titleTks, _ := tok.Tokenize(newName)+ titleSmTks, _ := tok.FineGrainedTokenize(titleTks)+ indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)return s.docEngine.UpdateChunks(ctx,map[string]interface{}{"doc_id": doc.ID},diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex d5f22a171f..80b8a7b618 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -151,10 +151,10 @@ func (f *fakeUploadStorage) Close() error { return nil }type fakeChatDocEngine struct{}-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {diff --git a/internal/service/memory.go b/internal/service/memory.goindex 7406012347..b76d02cc88 100644--- a/internal/service/memory.go+++ b/internal/service/memory.go@@ -1448,7 +1448,7 @@ func memoryMessageTextExpr(question string, similarityThreshold float64) *engineif queryBuilder == nil {queryBuilder = nlp.NewQueryBuilder()}- if built, _ := queryBuilder.Question(question, "messages", similarityThreshold); built != nil {+ if built, _ := queryBuilder.Question(question, "messages", similarityThreshold, ""); built != nil {matchText.MatchingText = built.MatchingTextmatchText.ExtraOptions = built.ExtraOptionsif matchText.ExtraOptions == nil {diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex b79593a3ca..ca7f51b6ff 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -357,7 +357,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Crreturn fmt.Errorf("check message index: %w", err)}if !exists {- if err := s.memories.docEngine.CreateChunkStore(ctx, indexName, mem.ID, vectorDim, ""); err != nil {+ if err := s.memories.docEngine.CreateChunkStore(ctx, indexName, mem.ID, vectorDim, "", ""); err != nil {return fmt.Errorf("create message index: %w", err)}}@@ -365,7 +365,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Crfor i, message := range messages {docs[i] = mapStringAny(message)}- if _, err := s.memories.docEngine.InsertChunks(ctx, docs, indexName, mem.ID); err != nil {+ if _, err := s.memories.docEngine.InsertChunks(ctx, docs, indexName, mem.ID, ""); err != nil {return fmt.Errorf("insert message into memory: %w", err)}diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex 04b51e18ee..a94349b4d4 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -856,7 +856,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"content_sm_ltks": contentSmLtks,"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),}- _, err = de.InsertChunks(ctx, []map[string]interface{}{row}, idx, in.KbID)+ _, err = de.InsertChunks(ctx, []map[string]interface{}{row}, idx, in.KbID, "")return err}@@ -910,7 +910,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"content_sm_ltks": clusterContentSmLtks,"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),}- _, err = de.InsertChunks(ctx, []map[string]interface{}{clusterRow}, idx, in.KbID)+ _, err = de.InsertChunks(ctx, []map[string]interface{}{clusterRow}, idx, in.KbID, "")if err != nil {return err}@@ -935,7 +935,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"content_sm_ltks": contentSmLtks,"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),}- _, err = de.InsertChunks(ctx, []map[string]interface{}{docRow}, idx, in.KbID)+ _, err = de.InsertChunks(ctx, []map[string]interface{}{docRow}, idx, in.KbID, "")return err}@@ -1308,7 +1308,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clusif vec := pickAnyVector(children, i2boolForTarget(spl.name, splitA)); len(vec) > 0 {row["q_"+fmt.Sprintf("%d", len(vec))+"_vec"] = f32ToF64Slice(vec)}- if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID); err != nil {+ if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID, ""); err != nil {return err}}diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex d484e54ced..fc17df3cdc 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -29,7 +29,7 @@ type memNavEngine struct {func newMemNavEngine() *memNavEngine { return &memNavEngine{} }-func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string) ([]string, error) {+func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string, _ string) ([]string, error) {ids := make([]string, 0, len(chunks))for _, c := range chunks {// A caller-provided row id is kept, exactly as the real engines do it@@ -134,7 +134,7 @@ func (m *memNavEngine) Close() error { return nil }func (m *memNavEngine) Ping(context.Context) error { return nil }func (m *memNavEngine) GetType() string { return "mem" }func (m *memNavEngine) SupportsPageRank() bool { return false }-func (m *memNavEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (m *memNavEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}func (m *memNavEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {@@ -503,7 +503,7 @@ func TestNavService_ListClusters_SearchReturnsPrunedForest(t *testing.T) {"content_with_weight": `{"type":"nav_doc","description":"孤儿概要"}`,},}- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {t.Fatal(err)}@@ -807,7 +807,7 @@ func TestNavService_Search_DocScope(t *testing.T) {row(nav.TypeNavCluster, "kb1", "covering d1+d2", []string{"d1", "d2"}, 0.4),row(nav.TypeNavCluster, "kb1", "covering d2 only", []string{"d2"}, 1),}- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {t.Fatal(err)}q := make([]float32, dim)@@ -905,7 +905,7 @@ func TestNavService_Search_DocScopeBeyondAnyPoolSize(t *testing.T) {"q_4_vec": vec(0.4),},)- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {t.Fatal(err)}q := make([]float32, dim)@@ -1103,7 +1103,7 @@ func TestNavService_UpsertDoc_PlacementBySimilarity(t *testing.T) {"doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,"q_1024_vec": vec,}- if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1", ""); err != nil {t.Fatal(err)}}@@ -1254,7 +1254,7 @@ func TestNavService_FindBestCluster_DeepestMatchWins(t *testing.T) {"doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,"q_1024_vec": vec,}- if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1", ""); err != nil {t.Fatal(err)}}@@ -1390,7 +1390,7 @@ func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {"doc_id": "d9", "title_kwd": "出师表全文内容与赏析","content_with_weight": `{"type":"nav_doc"}`,},- }, "", "kb1"); err != nil {+ }, "", "kb1", ""); err != nil {t.Fatal(err)}got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d9"})@@ -1409,7 +1409,7 @@ func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {"doc_id": rawID, "title_kwd": rawID,"content_with_weight": `{"type":"nav_doc"}`,},- }, "", "kb1"); err != nil {+ }, "", "kb1", ""); err != nil {t.Fatal(err)}got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{rawID})@@ -1505,7 +1505,7 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {"doc_count_int": 1,})}- if _, err := eng.InsertChunks(t.Context(), rows, idx, "kb1"); err != nil {+ if _, err := eng.InsertChunks(t.Context(), rows, idx, "kb1", ""); err != nil {t.Fatal(err)}if err := ns.maybeSplitCluster(t.Context(), "t1", "kb1", clusterName, ""); err != nil {diff --git a/internal/service/nlp/query_builder.go b/internal/service/nlp/query_builder.goindex 3b9fb59812..1a471241cc 100644--- a/internal/service/nlp/query_builder.go+++ b/internal/service/nlp/query_builder.go@@ -243,11 +243,15 @@ func (qb *QueryBuilder) NeedFineGrainedTokenize(tk string) bool {}// Question builds a full-text query expression based on input text.-// References Python FulltextQueryer.question method.-func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*types.MatchTextExpr, []string) {+// References Python FulltextQueryer.question method. language is the dataset+// language ("" = English) the query is tokenized with, so query tokens match+// index tokens.+func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64, language string) (*types.MatchTextExpr, []string) {// originalQuery stores the original input text for later use in query expression.originalQuery := txt+ tok := tokenizer.New(language)+// Add space between English and ChinesetxtWithSpaces := qb.AddSpaceBetweenEngZh(txt)@@ -278,7 +282,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typtxtFinal := qb.RmWWW(txtNoStopWords)// Tokenize using rag_tokenizer- tokenized, err := tokenizer.Tokenize(txtFinal)+ tokenized, err := tok.Tokenize(txtFinal)if err != nil {// If tokenizer fails, use simple splittokenized = txtFinal@@ -297,7 +301,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ// Calculate term weights using TermWeightDealer// Reference: rag/nlp/query.py L56// tws holds the term weight list for each token.- tws := qb.termWeight.Weights(tks, false)+ tws := qb.termWeight.Weights(tks, false, language)// Clean tokens and filter// Reference: rag/nlp/query.py L57-60@@ -435,7 +439,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ// Get term weights// termWeightList holds term weights for the current segment.- termWeightList := qb.termWeight.Weights([]string{segment}, true)+ termWeightList := qb.termWeight.Weights([]string{segment}, true, language)// Lookup synonyms// syns are synonyms for the current segment.@@ -463,7 +467,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ// sm holds fine‑grained tokens for the current term.var sm []stringif qb.NeedFineGrainedTokenize(term) {- fineGrained, err := tokenizer.FineGrainedTokenize(term)+ fineGrained, err := tok.FineGrainedTokenize(term)if err == nil && fineGrained != "" {sm = strings.Fields(fineGrained)}@@ -512,7 +516,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typif s == "" {continue}- fg, err := tokenizer.FineGrainedTokenize(s)+ fg, err := tok.FineGrainedTokenize(s)if err == nil && fg != "" {// Quote if contains spaceif strings.Contains(fg, " ") {@@ -566,7 +570,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ// Add proximity query if multiple tokensif len(termWeightList) > 1 {// tokenized is the tokenized version of the segment.- tokenized, _ := tokenizer.Tokenize(segment)+ tokenized, _ := tok.Tokenize(segment)if tokenized != "" {tmsStr += fmt.Sprintf(` ("%s"~2)^1.5`, tokenized)}@@ -579,7 +583,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typfor _, s := range syns {s = qb.SubSpecialChar(s)if s != "" {- tokenized, _ := tokenizer.Tokenize(s)+ tokenized, _ := tok.Tokenize(s)if tokenized != "" {synParts = append(synParts, fmt.Sprintf(`"%s"`, tokenized))}diff --git a/internal/service/nlp/query_builder_language_test.go b/internal/service/nlp/query_builder_language_test.gonew file mode 100644index 0000000000..a2648c4e99--- /dev/null+++ b/internal/service/nlp/query_builder_language_test.go@@ -0,0 +1,79 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++//go:build manual++package nlp++import (+ "strings"+ "testing"+ "time"++ "ragflow/internal/tokenizer"+)++// TestQueryBuilder_Question_FoldingLanguage verifies that a Slovak query is+// tokenized by the analyzer with diacritics folded to ASCII, so every emitted+// keyword and the query expression match the folded index tokens.+func TestQueryBuilder_Question_FoldingLanguage(t *testing.T) {+ // Folding happens inside the analyzer, so the real dictionaries must be+ // loaded (manual tier, like the analyzer-level tests in internal/tokenizer).+ if err := tokenizer.Init(&tokenizer.PoolConfig{MinSize: 1, MaxSize: 1, IdleTimeout: 5 * time.Second, AcquireTimeout: 5 * time.Second}); err != nil {+ t.Fatalf("Failed to initialize pool: %v", err)+ }+ defer tokenizer.Close()++ qb := NewQueryBuilder()+ hasDiacritics := func(s string) bool {+ for _, r := range s {+ if r > 127 {+ return true+ }+ }+ return false+ }++ expr, keywords := qb.Question("daňové priznanie pre živnostníkov", "test", 0.5, "Slovak")+ if expr == nil {+ t.Fatal("Question(Slovak) returned nil expr")+ }+ if hasDiacritics(expr.MatchingText) {+ t.Errorf("Question(Slovak) query contains diacritics: %q", expr.MatchingText)+ }+ for _, k := range keywords {+ if hasDiacritics(k) {+ t.Errorf("Question(Slovak) keyword contains diacritics: %q", k)+ }+ }+ if !strings.Contains(expr.MatchingText, "danove") {+ t.Errorf("Question(Slovak) query missing folded token 'danove': %q", expr.MatchingText)+ }++ // original_query must stay untouched for highlighting.+ if orig, _ := expr.ExtraOptions["original_query"].(string); orig != "daňové priznanie pre živnostníkov" {+ t.Errorf("Question(Slovak) original_query = %q, want the unfolded input", orig)+ }++ // Non-folding languages keep accents untouched.+ exprEn, _ := qb.Question("daňové priznanie pre živnostníkov", "test", 0.5, "")+ if exprEn == nil {+ t.Fatal("Question(English) returned nil expr")+ }+ if !hasDiacritics(exprEn.MatchingText) {+ t.Errorf("Question(English) unexpectedly folded diacritics: %q", exprEn.MatchingText)+ }+}diff --git a/internal/service/nlp/query_builder_test.go b/internal/service/nlp/query_builder_test.goindex 33233af3e1..233a2b7c4c 100644--- a/internal/service/nlp/query_builder_test.go+++ b/internal/service/nlp/query_builder_test.go@@ -296,7 +296,7 @@ func TestQueryBuilder_Question(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- expr, keywords := qb.Question(tt.txt, tt.tbl, tt.minMatch)+ expr, keywords := qb.Question(tt.txt, tt.tbl, tt.minMatch, "")if tt.expectNil && expr != nil {t.Errorf("Question(%q) expected nil expr, got %v", tt.txt, expr)}@@ -474,7 +474,7 @@ func TestQueryBuilder_Question_SynonymSingleQuote(t *testing.T) {qb.synonym.dictionary = map[string][]string{"cat": {"cat-o'-nine-tails", "'"},}- expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3, "")if expr == nil {t.Fatal("Question returned nil expr")}@@ -518,7 +518,7 @@ func TestQueryBuilder_Question_SynonymQuestionMark(t *testing.T) {qb.synonym.dictionary = map[string][]string{"cat": {"what?cat"},}- expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3, "")if expr == nil {t.Fatal("Question returned nil expr")}diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex bf26fa7375..abbbee19ca 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -65,10 +65,11 @@ func Rerank(cfield string,qb *QueryBuilder,rankFeature map[string]float64,+ language string,) (sim []float64, tsim []float64, vsim []float64, err error) {// If reranker model is provided and there are results, use model rerankingif rerankModel != nil && total > 0 {- return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature)+ return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)}// Otherwise, use fallback logic based on engine type@@ -84,7 +85,7 @@ func Rerank(}// For Elasticsearch: need to perform reranking and apply rank features- sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature)+ sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)return sim, tsim, vsim, nil}@@ -100,6 +101,7 @@ func RerankByModel(cfield string,qb *QueryBuilder,rankFeature map[string]float64,+ language string,) (sim []float64, tsim []float64, vsim []float64, err error) {if chunks == nil || len(chunks) == 0 {return []float64{}, []float64{}, []float64{}, nil@@ -112,7 +114,7 @@ func RerankByModel(// Extract keywords from querykeywords := []string{}if qb != nil {- _, keywords = qb.Question(query, "qa", 0.6)+ _, keywords = qb.Question(query, "qa", 0.6, language)}common.Info("RerankByModel keywords extracted", zap.Any("keywords", keywords))@@ -162,7 +164,7 @@ func RerankByModel(}// Calculate token similarity- tsim = TokenSimilarity(keywords, insTw, qb)+ tsim = TokenSimilarity(keywords, insTw, qb, language)// Get similarity scores from reranker modelrerankResponse, err := rerankModel.Rerank(ctx, models.RerankRequest{Query: query, Documents: docs}, &models.RerankConfig{}, nil)@@ -277,6 +279,7 @@ func RerankStandard(cfield string,qb *QueryBuilder,rankFeature map[string]float64,+ language string,) (sim []float64, tsim []float64, vsim []float64) {chunkCount := len(chunks)if chunkCount == 0 {@@ -287,7 +290,7 @@ func RerankStandard(// Compute keywords fresh from queryif qb != nil && len(keywords) == 0 {- _, keywords = qb.Question(query, "qa", 0.6)+ _, keywords = qb.Question(query, "qa", 0.6, language)}common.Info("RerankStandard keywords", zap.Any("keywords", keywords))@@ -332,7 +335,7 @@ func RerankStandard(}// Calculate hybrid similarity- sim, tsim, vsim = HybridSimilarity(questionVector, insEmbd, keywords, insTw, tkWeight, vtWeight, qb)+ sim, tsim, vsim = HybridSimilarity(questionVector, insEmbd, keywords, insTw, tkWeight, vtWeight, qb, language)// Apply rank feature scores (tag_score * 10 + pagerank)// Always apply pageranks, even when rankFeature is nil/empty@@ -376,6 +379,7 @@ func HybridSimilarity(btkss [][]string,tkWeight, vtWeight float64,qb *QueryBuilder,+ language string,) (sim []float64, tsim []float64, vsim []float64) {// Calculate vector similarities using cosine similarityvsim = make([]float64, len(bvecs))@@ -383,7 +387,7 @@ func HybridSimilarity(vsim[i] = cosineSimilarity(avec, bvec)}- tsim = TokenSimilarity(atks, btkss, qb)+ tsim = TokenSimilarity(atks, btkss, qb, language)// Check if all vector similarities are zeroallZero := true@@ -409,11 +413,11 @@ func HybridSimilarity(}// TokenSimilarity calculates token-based similarity-func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder) []float64 {- atksDict, atksKeyOrder := tokensToDict(atks, qb)+func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder, language string) []float64 {+ atksDict, atksKeyOrder := tokensToDict(atks, qb, language)btkssDicts := make([]map[string]float64, len(btkss))for i, btks := range btkss {- btkssDicts[i], _ = tokensToDict(btks, qb)+ btkssDicts[i], _ = tokensToDict(btks, qb, language)}similarities := make([]float64, len(btkssDicts))@@ -426,13 +430,13 @@ func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder) []float6// tokensToDict converts tokens to a weighted dictionary.// Also returns the insertion order of keys to match Python's dict insertion order.-func tokensToDict(tks []string, qb *QueryBuilder) (map[string]float64, []string) {+func tokensToDict(tks []string, qb *QueryBuilder, language string) (map[string]float64, []string) {d := make(map[string]float64)var keyOrder []stringif qb == nil || qb.termWeight == nil {return d, keyOrder}- wts := qb.termWeight.Weights(tks, false)+ wts := qb.termWeight.Weights(tks, false, language)for i, tw := range wts {t := tw.Term@@ -921,6 +925,7 @@ func RerankWithKNN(cfield string,qb *QueryBuilder,rankFeature map[string]float64,+ language string,) (sim []float64, tsim []float64, vsim []float64) {if len(ids) == 0 {return []float64{}, []float64{}, []float64{}@@ -947,7 +952,7 @@ func RerankWithKNN(// Extract keywords from querykeywords := []string{}if qb != nil {- _, keywords = qb.Question(query, "qa", 0.6)+ _, keywords = qb.Question(query, "qa", 0.6, language)}common.InfoCtx(ctx, "RerankWithKNN keywords", zap.Any("keywords", keywords))@@ -987,7 +992,7 @@ func RerankWithKNN(}// Calculate token similarity- tsim = TokenSimilarity(keywords, insTw, qb)+ tsim = TokenSimilarity(keywords, insTw, qb, language)common.InfoCtx(ctx, "RerankWithKNN tsim", zap.Float64s("tsim", tsim))// Build vector similarity from knnScores - matches Python's np.array([knn_scores.get(chunk_id, 0.0) for chunk_id in sres.ids])diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.goindex 769554fcb9..0a1edfead6 100644--- a/internal/service/nlp/reranker_question_tks_test.go+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -57,6 +57,7 @@ func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {"content_ltks",nil,nil,+ "",)if len(driver.documents) != 1 {diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex e1a3f45696..9170eb9189 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -73,6 +73,9 @@ type RetrievalRequest struct {// bridge a wording gap the query's own words cannot cross; the scoring pass// then uses the engine's kNN score directly instead of re-scoring tokens.VectorOnly bool+ // Language is the dataset language ("" = English); diacritic-folding+ // languages (Slovak, Czech) fold the query so its tokens match index tokens.+ Language string}// RetrievalResult result from retrieval search@@ -178,6 +181,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)AllowDenseFallback: req.AllowDenseFallback,VectorOnly: req.VectorOnly,Filter: req.Filter,+ Language: req.Language,}searchResult, err := s.Search(ctx, searchReq)if err != nil {@@ -477,6 +481,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval"content_ltks",qb,*req.RankFeature,+ req.Language,)}@@ -507,6 +512,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval"content_ltks",qb,*req.RankFeature,+ req.Language,)return sim, tsim, vsim, nil}@@ -563,6 +569,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval"content_ltks",qb,*req.RankFeature,+ req.Language,)return sim, tsim, vsim, nil}@@ -579,6 +586,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval"content_ltks",qb,*req.RankFeature,+ req.Language,)return sim, tsim, vsim, nil}@@ -605,6 +613,8 @@ type RetrievalSearchRequest struct {EmbeddingModel *models.EmbeddingModelVectorSimilarityWeight *float64AllowDenseFallback *bool+ // Language is the dataset language ("" = English).+ Language string}func buildInfinityFusionExpr(topn int, vectorSimilarityWeight *float64) *types.FusionExpr {@@ -750,7 +760,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// Non-empty question// Compute keywords via QueryBuilder- matchText, keywords := GetQueryBuilder().Question(req.Question, "", minMatch(req.VectorSimilarityWeight, 0.3))+ matchText, keywords := GetQueryBuilder().Question(req.Question, "", minMatch(req.VectorSimilarityWeight, 0.3), req.Language)for _, k := range keywords {kwds[k] = struct{}{}}@@ -853,7 +863,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// and lower vector similarity threshold (0.17 vs default 0.1-0.2).// This provides a second chance for queries that were too strict// on the first attempt.- matchText, _ := GetQueryBuilder().Question(req.Question, "qa", minMatch(req.VectorSimilarityWeight, 0.1))+ matchText, _ := GetQueryBuilder().Question(req.Question, "qa", minMatch(req.VectorSimilarityWeight, 0.1), req.Language)matchDense = cloneDenseExpr(denseTemplate)matchDense.ExtraOptions["similarity"] = 0.17if req.VectorOnly || matchText == nil {@@ -885,9 +895,10 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque}// Build kwds from keywords with fine-grained tokenization+ tok := tokenizer.New(req.Language)for _, k := range keywords {kwds[k] = struct{}{}- fgToken, _ := tokenizer.FineGrainedTokenize(k)+ fgToken, _ := tok.FineGrainedTokenize(k)for kk := range strings.FieldsSeq(fgToken) {if len(kk) < 2 {continuediff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 0421ef86f6..e13e8096b2 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -268,10 +268,10 @@ func (e *retrievalCountEngine) Close() error { return nil }func (e *retrievalCountEngine) Ping(context.Context) error { return nil }func (e *retrievalCountEngine) GetType() string { return "elasticsearch" }func (e *retrievalCountEngine) SupportsPageRank() bool { return false }-func (e *retrievalCountEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (e *retrievalCountEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (e *retrievalCountEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (e *retrievalCountEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}func (e *retrievalCountEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.goindex f99c4cdd91..156f50c044 100644--- a/internal/service/nlp/term_weight.go+++ b/internal/service/nlp/term_weight.go@@ -191,12 +191,12 @@ func loadDict(fnm string) map[string]int {return res}-// Pretoken preprocesses and tokenizes text+// Pretoken preprocesses and tokenizes text with the dataset language.// Reference: term_weight.py L92-114-func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {+func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool, lang string) []string {warnPunctPatternOnce()res := []string{}- tokenized, err := tokenizer.Tokenize(txt)+ tokenized, err := tokenizer.New(lang).Tokenize(txt)if err != nil {// Fallback to simple splittokenized = txt@@ -346,9 +346,10 @@ func (d *TermWeightDealer) Split(txt string) []string {return tks}-// Weights calculates weights for tokens+// Weights calculates weights for tokens with the dataset language ("" = English).// Reference: term_weight.py L163-246-func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {+func (d *TermWeightDealer) Weights(tks []string, preprocess bool, lang string) []TermWeight {+ tok := tokenizer.New(lang)// One dictionary lookup per distinct term. Both halves come from the same// entry, tks repeats terms (title x2, important x5, question x6), and the// consumers below are pure functions of the term, so caching the pair for the@@ -426,7 +427,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {}if s == 0 && len([]rune(t)) >= 4 {// Try fine-grained tokenization- fgTokens, _ := tokenizer.FineGrainedTokenize(t)+ fgTokens, _ := tok.FineGrainedTokenize(t)tokens := strings.Fields(fgTokens)// Filter: only keep tokens with length > 1@@ -473,7 +474,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {}if len([]rune(t)) >= 4 {// Use fine-grained tokenization- fgTokens, _ := tokenizer.FineGrainedTokenize(t)+ fgTokens, _ := tok.FineGrainedTokenize(t)tokens := strings.Fields(fgTokens)// Filter: only keep tokens with length > 1@@ -540,7 +541,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {} else {// With preprocessingfor _, tk := range tks {- tokens := d.Pretoken(tk, true, true)+ tokens := d.Pretoken(tk, true, true, lang)tt := d.TokenMerge(tokens)if len(tt) == 0 {continuediff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.goindex 6282726545..0ab97f4a2e 100644--- a/internal/service/nlp/term_weight_test.go+++ b/internal/service/nlp/term_weight_test.go@@ -76,7 +76,7 @@ func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) {func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {d := NewTermWeightDealer(t.TempDir())terms := []string{"was", "largest", "supplier", "equipment"}- weights := d.Weights(terms, false)+ weights := d.Weights(terms, false, "")if len(weights) != len(terms) {t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms))}@@ -87,7 +87,7 @@ func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {}d.df["equipment"] = 1_000_000- weights = d.Weights([]string{"supplier", "equipment"}, false)+ weights = d.Weights([]string{"supplier", "equipment"}, false, "")if weights[1].Weight >= weights[0].Weight {t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights)}@@ -206,7 +206,7 @@ func TestPretoken(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- result := d.Pretoken(tt.txt, tt.num, tt.stpwd)+ result := d.Pretoken(tt.txt, tt.num, tt.stpwd, "")// Just check it doesn't panic and returns a sliceif result == nil {t.Error("Pretoken returned nil")@@ -392,7 +392,7 @@ func TestWeights(t *testing.T) {t.Run("without preprocess", func(t *testing.T) {tks := []string{"hello", "world", "123"}- weights := d.Weights(tks, false)+ weights := d.Weights(tks, false, "")if len(weights) != len(tks) {t.Errorf("Expected %d weights, got %d", len(tks), len(weights))@@ -410,7 +410,7 @@ func TestWeights(t *testing.T) {t.Run("with preprocess", func(t *testing.T) {tks := []string{"hello world", "test"}- weights := d.Weights(tks, true)+ weights := d.Weights(tks, true, "")// Check it doesn't panic and returns resultsif weights == nil {@@ -419,7 +419,7 @@ func TestWeights(t *testing.T) {})t.Run("empty input", func(t *testing.T) {- weights := d.Weights([]string{}, false)+ weights := d.Weights([]string{}, false, "")if len(weights) != 0 {t.Errorf("Expected empty weights for empty input, got %d", len(weights))}@@ -432,7 +432,7 @@ func TestWeights(t *testing.T) {d2 := NewTermWeightDealer(tmpDir2)tks := []string{"toxicterm", "normal"}- weights := d2.Weights(tks, false)+ weights := d2.Weights(tks, false, "")if len(weights) != 2 {t.Fatalf("Expected 2 weights, got %d", len(weights))@@ -613,7 +613,7 @@ func TestWeightsNormalization(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- weights := d.Weights(tt.tks, false)+ weights := d.Weights(tt.tks, false, "")if len(weights) != len(tt.tks) {t.Fatalf("Expected %d weights, got %d", len(tt.tks), len(weights))@@ -665,7 +665,7 @@ func BenchmarkWeights(b *testing.B) {b.ResetTimer()for i := 0; i < b.N; i++ {- d.Weights(tks, false)+ d.Weights(tks, false, "")}}@@ -721,7 +721,7 @@ func TestIntegration(t *testing.T) {// Step 2: Pretokenvar allTokens []stringfor _, s := range splitted {- tokens := d.Pretoken(s, true, true)+ tokens := d.Pretoken(s, true, true, "")allTokens = append(allTokens, tokens...)}@@ -729,7 +729,7 @@ func TestIntegration(t *testing.T) {merged := d.TokenMerge(allTokens)// Step 4: Calculate weights- weights := d.Weights(merged, false)+ weights := d.Weights(merged, false, "")// Verify resultsif len(weights) == 0 && len(merged) > 0 {@@ -752,7 +752,7 @@ func TestWeightsEdgeCases(t *testing.T) {t.Run("numbers pattern", func(t *testing.T) {tks := []string{"123,45", "abc"}- weights := d.Weights(tks, false)+ weights := d.Weights(tks, false, "")if len(weights) != 2 {t.Fatalf("Expected 2 weights, got %d", len(weights))}@@ -761,7 +761,7 @@ func TestWeightsEdgeCases(t *testing.T) {t.Run("short letters pattern", func(t *testing.T) {tks := []string{"ab", "abc"}- weights := d.Weights(tks, false)+ weights := d.Weights(tks, false, "")if len(weights) != 2 {t.Fatalf("Expected 2 weights, got %d", len(weights))}@@ -769,7 +769,7 @@ func TestWeightsEdgeCases(t *testing.T) {t.Run("letter pattern with spaces", func(t *testing.T) {tks := []string{"hello world test"}- weights := d.Weights(tks, true)+ weights := d.Weights(tks, true, "")// Should not panicif weights == nil {t.Error("Weights returned nil for letter pattern")@@ -782,7 +782,7 @@ func TestPretokenWithNumbers(t *testing.T) {d := NewTermWeightDealer("")t.Run("num=false filters single digits", func(t *testing.T) {- result := d.Pretoken("5", false, true)+ result := d.Pretoken("5", false, true, "")// Single digit should be filtered when num=falseif slices.Contains(result, "5") {t.Error("Single digit should be filtered when num=false")@@ -790,7 +790,7 @@ func TestPretokenWithNumbers(t *testing.T) {})t.Run("num=true keeps single digits", func(t *testing.T) {- result := d.Pretoken("5 123", true, true)+ result := d.Pretoken("5 123", true, true, "")// Check at least something is returnedif len(result) == 0 {t.Log("Single digit may still be filtered by other rules")@@ -803,7 +803,7 @@ func TestPretokenStopWords(t *testing.T) {d := NewTermWeightDealer("")t.Run("stpwd=true removes stop words", func(t *testing.T) {- result := d.Pretoken("请问", true, true)+ result := d.Pretoken("请问", true, true, "")// "请问" is a stop wordfor _, r := range result {if r == "请问" {@@ -813,7 +813,7 @@ func TestPretokenStopWords(t *testing.T) {})t.Run("stpwd=false keeps stop words", func(t *testing.T) {- result := d.Pretoken("请问", true, false)+ result := d.Pretoken("请问", true, false, "")// With tokenizer, this might still filter it_ = result})@@ -906,7 +906,7 @@ func TestWeightsDependOnlyOnTheTerm(t *testing.T) {d := NewTermWeightDealer("")ratio := func(tks []string, i, j int) float64 {- weights := d.Weights(tks, false)+ weights := d.Weights(tks, false, "")if len(weights) != len(tks) {t.Fatalf("got %d weights for %d terms", len(weights), len(tks))}diff --git a/internal/service/skill_indexer.go b/internal/service/skill_indexer.goindex b17b941ae3..3cb484f03f 100644--- a/internal/service/skill_indexer.go+++ b/internal/service/skill_indexer.go@@ -931,7 +931,7 @@ func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tena// Use the doc engine's CreateChunkStore method with skill-specific mapping// The mapping file is loaded from conf/skill_es_mapping.json or conf/skill_infinity_mapping.json- err := docEngine.CreateChunkStore(ctx, indexName, "skill", dimension, "")+ err := docEngine.CreateChunkStore(ctx, indexName, "skill", dimension, "", "")if err != nil {common.Error("Failed to create skill index", err)return errdiff --git a/internal/service/tenant.go b/internal/service/tenant.goindex f6ff646e86..63d91506f7 100644--- a/internal/service/tenant.go+++ b/internal/service/tenant.go@@ -256,9 +256,16 @@ func (s *TenantService) CreateChunkStore(ctx context.Context, req *CreateDataset// Build table name prefix: ragflow_<tenant_id>tableName := fmt.Sprintf("ragflow_%s", kb.TenantID)+ // The dataset language picks the fulltext analyzer on engines that fix it at+ // index creation, so it has to travel with the table, not the query.+ language := ""+ if kb.Language != nil {+ language = *kb.Language+ }+// Call document engine to create table// Full table name will be built as "{tableName}_{kb_id}"- err = s.docEngine.CreateChunkStore(ctx, tableName, req.KBID, vecSize, req.ParserID)+ err = s.docEngine.CreateChunkStore(ctx, tableName, req.KBID, vecSize, req.ParserID, language)if err != nil {return nil, common.CodeServerError, fmt.Errorf("failed to create dataset: %w", err)}diff --git a/internal/service/test_helpers_test.go b/internal/service/test_helpers_test.goindex 632dbf6397..0e860d53d8 100644--- a/internal/service/test_helpers_test.go+++ b/internal/service/test_helpers_test.go@@ -94,10 +94,10 @@ func getAPIConversationForTest(t *testing.T, db *gorm.DB, id string) *entity.API// fakeChatDocEngine is a stub engine.DocEngine used by parent-package tests.type fakeChatDocEngine struct{}-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {return nil}-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {return nil, nil}func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.goindex 669353cde4..ade35fdc12 100644--- a/internal/tokenizer/tokenizer_test.go+++ b/internal/tokenizer/tokenizer_test.go@@ -362,6 +362,64 @@ func TestTokenize_DefaultLanguageResetsAnalyzerState(t *testing.T) {}}+func TestTokenize_SlovakCzechFoldDiacriticsAndSkipStemming(t *testing.T) {+ restore := saveEngineType()+ defer restore()+ SetEngineType("")++ if err := Init(&PoolConfig{+ DictPath: "",+ MinSize: 1,+ MaxSize: 1,+ IdleTimeout: 5 * time.Second,+ AcquireTimeout: 5 * time.Second,+ }); err != nil {+ t.Fatalf("Failed to initialize pool: %v", err)+ }+ defer Close()++ tests := []struct {+ lang string+ input string+ want string+ }{+ // The analyzer folds diacritics before splitting, so the split regex+ // keeps words whole, and stemming/lemmatization are off.+ {"Slovak", "škola daňové priznanie", "skola danove priznanie"},+ {"Czech", "příliš žluťoučký kůň", "prilis zlutoucky kun"},+ // No stemming for folding languages: the token survives unchanged.+ {"Slovak", "running", "running"},+ }+ for _, tt := range tests {+ got, err := New(tt.lang).Tokenize(tt.input)+ if err != nil {+ t.Fatalf("Tokenize(%s, %q) unexpected error: %v", tt.lang, tt.input, err)+ }+ if got != tt.want {+ t.Errorf("Tokenize(%s, %q) = %q, want %q", tt.lang, tt.input, got, tt.want)+ }+ }++ // English tokenization is unchanged and the pool resets sticky Slovak+ // state: the default-language path still stems.+ engGot, err := Tokenize("running")+ if err != nil {+ t.Fatalf("Tokenize(default, \"running\") unexpected error: %v", err)+ }+ if engGot == "running" {+ t.Errorf("Tokenize(default, \"running\") = %q, want a stemmed form (English stemming must stay enabled)", engGot)+ }++ // FineGrainedTokenize on folded tokens must not stem either.+ fg, err := New("Slovak").FineGrainedTokenize("danove priznanie")+ if err != nil {+ t.Fatalf("FineGrainedTokenize(Slovak) unexpected error: %v", err)+ }+ if fg != "danove priznanie" {+ t.Errorf("FineGrainedTokenize(Slovak, \"danove priznanie\") = %q, want unchanged tokens", fg)+ }+}+func findEnglishDutchDifferentiator(t *testing.T) languageDifferentiator {t.Helper()diff --git a/web/src/components/cross-language-form-field.tsx b/web/src/components/cross-language-form-field.tsxindex f2f9734bbb..2c087dcc12 100644--- a/web/src/components/cross-language-form-field.tsx+++ b/web/src/components/cross-language-form-field.tsx@@ -39,6 +39,8 @@ export const Languages = ['Arabic','Turkish','Dutch',+ 'Slovak',+ 'Czech',];export function useCrossLanguageOptions() {diff --git a/web/src/constants/common.ts b/web/src/constants/common.tsindex abb09bd79a..2bbe9f6904 100644--- a/web/src/constants/common.ts+++ b/web/src/constants/common.ts@@ -32,6 +32,8 @@ export const LanguageList = ['Arabic','Turkish','Dutch',+ 'Slovak',+ 'Czech',];export const LanguageMap = {English: 'English',@@ -51,6 +53,8 @@ export const LanguageMap = {Arabic: 'العربية',Turkish: 'Türkçe',Dutch: 'Nederlands',+ Slovak: 'Slovenčina',+ Czech: 'Čeština',};export enum LanguageAbbreviation {@@ -125,6 +129,8 @@ export const LanguageTranslationMap = {Arabic: 'ar',Turkish: 'tr',Dutch: 'nl',+ Slovak: 'sk',+ Czech: 'cs',};export enum FileMimeType {diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 6cfc36c667..b8a51658f4 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -46,6 +46,8 @@ export default {arabic: 'Arabic',turkish: 'Turkish',dutch: 'Dutch',+ slovak: 'Slovak',+ czech: 'Czech',language: 'Language',languageMessage: 'Please input your language!',languagePlaceholder: 'select your language',@@ -3909,6 +3911,8 @@ Important structured information may include: names, dates, locations, events, karabic: 'Arabic',turkish: 'Turkish',dutch: 'Dutch',+ slovak: 'Slovak',+ czech: 'Czech',},pagination: {total: 'Total {{total}}',