Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Add Slovak & Czech language dataset support (#19013)

Martin Civáň0ee758067297
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.go
index 0069425da3..5d75ea462a 100644
--- a/internal/agent/tool/retrieval_nlp.go
+++ b/internal/agent/tool/retrieval_nlp.go
@@ -285,6 +285,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retri
preparedReq.DocScope = docIDs
preparedReq.DatasetIDs = append([]string(nil), datasets.kbIDs...)
nlpReq := nlpRequestFromRetrieval(preparedReq, datasets.tenantIDs, topN, embeddingModel, preparedReq.ExcludeCompiled)
+ nlpReq.Language = entity.KnowledgebasesLanguage(datasets.kbs)
nlpReq.RerankModel = rerankModel
if rankFeature != nil {
nlpReq.RankFeature = &rankFeature
diff --git a/internal/agentic_rag/bm25_service.go b/internal/agentic_rag/bm25_service.go
index 787f97971c..30946b30fb 100644
--- a/internal/agentic_rag/bm25_service.go
+++ b/internal/agentic_rag/bm25_service.go
@@ -21,8 +21,10 @@ import (
"fmt"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/dao"
"ragflow/internal/engine"
enginetypes "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
)
// Bm25Adapter backs search_bm25_chunks with pure lexical full-text (BM25)
@@ -45,10 +47,26 @@ type Bm25Adapter struct {
}
// QueryTokenizer converts a raw query string into a tokenized match
-// expression over the tokenized fields. Implemented by the service/nlp
-// QueryBuilder (the same tokenizer family the ingestion pipeline used).
+// expression over the tokenized fields, using the dataset language ("" =
+// English). Implemented by the service/nlp QueryBuilder (the same tokenizer
+// family the ingestion pipeline used).
type QueryTokenizer interface {
- Question(txt string, tbl string, minMatch float64) (*enginetypes.MatchTextExpr, []string)
+ Question(txt string, tbl string, minMatch float64, language string) (*enginetypes.MatchTextExpr, []string)
+}
+
+// resolveDatasetsLanguage returns the language the searched datasets were
+// indexed with (see entity.KnowledgebasesLanguage), or "" when it cannot be
+// read. It is a package-level variable so unit tests can stub it without a
+// database.
+var resolveDatasetsLanguage = func(ctx context.Context, datasetIDs []string) string {
+ if dao.DB == nil || len(datasetIDs) == 0 {
+ return ""
+ }
+ kbs, err := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, datasetIDs)
+ if err != nil {
+ return ""
+ }
+ return entity.KnowledgebasesLanguage(kbs)
}
// SetQueryBuilder attaches the query tokenizer. Called at boot, after
@@ -98,6 +116,11 @@ func (b *Bm25Adapter) SearchBm25(ctx context.Context, req runtime.Bm25Request) (
ectx, ecancel := engineCallContext(ctx)
defer ecancel()
+ language := ""
+ if b.queryTokenizer != nil {
+ language = resolveDatasetsLanguage(ctx, req.DatasetIDs)
+ }
+
seen := map[string]struct{}{}
var merged []runtime.RetrievalChunk
for _, q := range queries {
@@ -105,7 +128,7 @@ func (b *Bm25Adapter) SearchBm25(ctx context.Context, req runtime.Bm25Request) (
// pipeline used (see Bm25Adapter.queryTokenizer): raw query words
// cannot match their stemmed/subword index tokens. Falls back to the
// raw text match when no query tokenizer is attached.
- matchExpr := b.buildMatchExpr(q, topN)
+ matchExpr := b.buildMatchExpr(q, topN, language)
res, err := b.docEngine.Search(ectx, &enginetypes.SearchRequest{
IndexNames: []string{fmt.Sprintf("ragflow_%s", req.TenantID)},
KbIDs: req.DatasetIDs,
@@ -164,11 +187,11 @@ func floatFromMap(m map[string]interface{}, key string) float64 {
// buildMatchExpr converts a raw query into the match expression SearchBm25
// sends to the engine: tokenized via the attached QueryTokenizer when one is
// registered, otherwise the historical raw-text match.
-func (b *Bm25Adapter) buildMatchExpr(q string, topN int) interface{} {
+func (b *Bm25Adapter) buildMatchExpr(q string, topN int, language string) interface{} {
if b == nil || b.queryTokenizer == nil {
return &enginetypes.MatchTextExpr{MatchingText: q, TopN: topN}
}
- expr, _ := b.queryTokenizer.Question(q, "", 0.0)
+ expr, _ := b.queryTokenizer.Question(q, "", 0.0, language)
if expr == nil {
return &enginetypes.MatchTextExpr{MatchingText: q, TopN: topN}
}
diff --git a/internal/agentic_rag/bm25_test.go b/internal/agentic_rag/bm25_test.go
index 5cc12f5039..2ef0bb659b 100644
--- a/internal/agentic_rag/bm25_test.go
+++ b/internal/agentic_rag/bm25_test.go
@@ -219,10 +219,14 @@ func (s *bm25StubService) SearchBm25(_ context.Context, req runtime.Bm25Request)
}
// fakeQueryTokenizer stands in for the service/nlp QueryBuilder: it "tokenizes"
-// the raw query into the word forms the ingestion pipeline would have indexed.
-type fakeQueryTokenizer struct{}
+// the raw query into the word forms the ingestion pipeline would have indexed,
+// and records the language it was asked to tokenize with.
+type fakeQueryTokenizer struct {
+ language string
+}
-func (fakeQueryTokenizer) Question(txt string, _ string, _ float64) (*enginetypes.MatchTextExpr, []string) {
+func (f *fakeQueryTokenizer) Question(txt string, _ string, _ float64, language string) (*enginetypes.MatchTextExpr, []string) {
+ f.language = language
return &enginetypes.MatchTextExpr{
Fields: []string{"content_ltks^2"},
MatchingText: "tokenized(" + txt + ")",
@@ -237,7 +241,7 @@ func (fakeQueryTokenizer) Question(txt string, _ string, _ float64) (*enginetype
func TestBm25Adapter_TokenizedMatchExpr(t *testing.T) {
fe := &grepFakeEngine{}
adapter := NewBm25Adapter(fe)
- adapter.SetQueryBuilder(fakeQueryTokenizer{})
+ adapter.SetQueryBuilder(&fakeQueryTokenizer{})
if _, err := adapter.SearchBm25(context.Background(), runtime.Bm25Request{
TenantID: "t",
@@ -257,3 +261,33 @@ func TestBm25Adapter_TokenizedMatchExpr(t *testing.T) {
t.Fatalf("TopN = %d, want caller topN 12 (tokenizer's 100 overridden)", mt.TopN)
}
}
+
+// TestBm25Adapter_TokenizesWithDatasetLanguage: the query is tokenized with
+// the language the searched datasets were indexed with, so a Slovak dataset's
+// folded index tokens are matched by a folded query.
+func TestBm25Adapter_TokenizesWithDatasetLanguage(t *testing.T) {
+ orig := resolveDatasetsLanguage
+ t.Cleanup(func() { resolveDatasetsLanguage = orig })
+ var gotIDs []string
+ resolveDatasetsLanguage = func(_ context.Context, ids []string) string {
+ gotIDs = ids
+ return "Slovak"
+ }
+
+ qt := &fakeQueryTokenizer{}
+ adapter := NewBm25Adapter(&grepFakeEngine{})
+ adapter.SetQueryBuilder(qt)
+ if _, err := adapter.SearchBm25(context.Background(), runtime.Bm25Request{
+ TenantID: "t",
+ Queries: []string{"škola"},
+ DatasetIDs: []string{"kb1"},
+ }); err != nil {
+ t.Fatalf("SearchBm25: %v", err)
+ }
+ if len(gotIDs) != 1 || gotIDs[0] != "kb1" {
+ t.Fatalf("language resolved for %v, want [kb1]", gotIDs)
+ }
+ if qt.language != "Slovak" {
+ t.Fatalf("Question language = %q, want Slovak", qt.language)
+ }
+}
diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cpp
index b606d41e2d..c3ee7330f3 100644
--- a/internal/binding/cpp/rag_analyzer.cpp
+++ b/internal/binding/cpp/rag_analyzer.cpp
@@ -699,6 +699,8 @@ RAGAnalyzer::~RAGAnalyzer() {
void RAGAnalyzer::InitStemmer(Language language) {
stemmer_->Init(language);
use_lemmatizer_ = (language == STEM_LANG_ENGLISH);
+ use_stemmer_ = true;
+ fold_diacritics_ = false;
}
void RAGAnalyzer::SetLanguage(const std::string &language) {
@@ -707,6 +709,16 @@ void RAGAnalyzer::SetLanguage(const std::string &language) {
lang_key.erase(lang_key.find_last_not_of(" \t") + 1);
lang_key.erase(0, lang_key.find_first_not_of(" \t"));
+ // Slovak and Czech have no Snowball stemmer. Instead their diacritics
+ // are folded to ASCII before tokenization, and stemming/lemmatization
+ // are disabled, matching Infinity's RAG analyzer.
+ if (lang_key == "slovak" || lang_key == "czech") {
+ fold_diacritics_ = true;
+ use_stemmer_ = false;
+ use_lemmatizer_ = false;
+ return;
+ }
+
Language stem_lang = STEM_LANG_UNKNOWN;
for (const auto &pair : SNOWBALL_LANGUAGE_MAP) {
if (pair.first == lang_key) {
@@ -716,8 +728,7 @@ void RAGAnalyzer::SetLanguage(const std::string &language) {
}
if (stem_lang != STEM_LANG_UNKNOWN) {
- stemmer_->Init(stem_lang);
- use_lemmatizer_ = (stem_lang == STEM_LANG_ENGLISH);
+ InitStemmer(stem_lang);
}
}
@@ -887,6 +898,71 @@ std::string RAGAnalyzer::StrQ2B(const std::string &input) {
return output;
}
+// ASCII base letter for each codepoint in Latin-1 Supplement (U+00C0–U+00FF)
+// and Latin Extended-A (U+0100–U+017F), indexed by codepoint - 0xC0. Only
+// letters whose NFD decomposition is exactly one ASCII letter plus combining
+// marks are folded; '\0' marks a pass-through codepoint (Æ, Ð, ×, Ø, Þ, ß,
+// Đ, Ħ, ı, IJ, ĸ, Ŀ, Ł, ʼn, Ŋ, Œ, Ŧ, ſ and their case pairs). Derived from
+// Unicode NFD, the same table Infinity's RAG analyzer folds with, so an index
+// built by either analyzer is searchable through the other.
+static constexpr char DIACRITICS_FOLD_TABLE[0x180 - 0xC0] = {
+ // clang-format off
+ // U+00C0 À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï
+ 'A', 'A', 'A', 'A', 'A', 'A', 0, 'C', 'E', 'E', 'E', 'E', 'I', 'I', 'I', 'I',
+ // U+00D0 Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß
+ 0, 'N', 'O', 'O', 'O', 'O', 'O', 0, 0, 'U', 'U', 'U', 'U', 'Y', 0, 0,
+ // U+00E0 à á â ã ä å æ ç è é ê ë ì í î ï
+ 'a', 'a', 'a', 'a', 'a', 'a', 0, 'c', 'e', 'e', 'e', 'e', 'i', 'i', 'i', 'i',
+ // U+00F0 ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ
+ 0, 'n', 'o', 'o', 'o', 'o', 'o', 0, 0, 'u', 'u', 'u', 'u', 'y', 0, 'y',
+ // U+0100 Ā ā Ă ă Ą ą Ć ć Ĉ ĉ Ċ ċ Č č Ď ď
+ 'A', 'a', 'A', 'a', 'A', 'a', 'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd',
+ // U+0110 Đ đ Ē ē Ĕ ĕ Ė ė Ę ę Ě ě Ĝ ĝ Ğ ğ
+ 0, 0, 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g',
+ // U+0120 Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĩ ĩ Ī ī Ĭ ĭ Į į
+ 'G', 'g', 'G', 'g', 'H', 'h', 0, 0, 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',
+ // U+0130 İ ı IJ ij Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ
+ 'I', 0, 0, 0, 'J', 'j', 'K', 'k', 0, 'L', 'l', 'L', 'l', 'L', 'l', 0,
+ // U+0140 ŀ Ł ł Ń ń Ņ ņ Ň ň ʼn Ŋ ŋ Ō ō Ŏ ŏ
+ 0, 0, 0, 'N', 'n', 'N', 'n', 'N', 'n', 0, 0, 0, 'O', 'o', 'O', 'o',
+ // U+0150 Ő ő Œ œ Ŕ ŕ Ŗ ŗ Ř ř Ś ś Ŝ ŝ Ş ş
+ 'O', 'o', 0, 0, 'R', 'r', 'R', 'r', 'R', 'r', 'S', 's', 'S', 's', 'S', 's',
+ // U+0160 Š š Ţ ţ Ť ť Ŧ ŧ Ũ ũ Ū ū Ŭ ŭ Ů ů
+ 'S', 's', 'T', 't', 'T', 't', 0, 0, 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u',
+ // U+0170 Ű ű Ų ų Ŵ ŵ Ŷ ŷ Ÿ Ź ź Ż ż Ž ž ſ
+ 'U', 'u', 'U', 'u', 'W', 'w', 'Y', 'y', 'Y', 'Z', 'z', 'Z', 'z', 'Z', 'z', 0,
+ // clang-format on
+};
+
+std::string RAGAnalyzer::FoldDiacritics(const std::string &input) {
+ std::string output;
+ output.reserve(input.size());
+
+ size_t i = 0;
+ while (i < input.size()) {
+ unsigned char c = input[i];
+ // Only 2-byte UTF-8 sequences can encode U+00C0–U+017F.
+ if ((c & 0xE0) == 0xC0 && i + 1 < input.size()) {
+ uint32_t codepoint = ((c & 0x1F) << 6) | (static_cast<unsigned char>(input[i + 1]) & 0x3F);
+ if (codepoint >= 0xC0 && codepoint < 0x180) {
+ if (char folded = DIACRITICS_FOLD_TABLE[codepoint - 0xC0]; folded != '\0') {
+ output += folded;
+ i += 2;
+ continue;
+ }
+ }
+ output += input[i];
+ output += input[i + 1];
+ i += 2;
+ continue;
+ }
+ output += input[i];
+ i += 1;
+ }
+
+ return output;
+}
+
int32_t RAGAnalyzer::Freq(const std::string_view key) const {
int32_t v = trie_->Get(key);
v = DecodeFreq(v);
@@ -1433,7 +1509,11 @@ void RAGAnalyzer::EnglishNormalize(const std::vector<std::string> &tokens, std::
term_to_stem = lowercase_term;
}
std::string stem_term;
- stemmer_->Stem(term_to_stem, stem_term);
+ if (use_stemmer_) {
+ stemmer_->Stem(term_to_stem, stem_term);
+ } else {
+ stem_term = term_to_stem;
+ }
res.push_back(stem_term);
} else {
res.push_back(t);
@@ -1775,8 +1855,17 @@ std::string PCRE2GlobalReplace(const std::string &text, const std::string &patte
}
std::string RAGAnalyzer::Tokenize(const std::string &line) {
+ // Fold diacritics to ASCII before any other processing when the
+ // language asks for it (Slovak, Czech).
+ std::string folded_line;
+ const std::string *input = &line;
+ if (fold_diacritics_) {
+ folded_line = FoldDiacritics(line);
+ input = &folded_line;
+ }
+
// Python-style simple tokenization: re.sub(r"\\W+", " ", line)
- std::string processed_line = PCRE2GlobalReplace(line, R"#(\W+)#", " ");
+ std::string processed_line = PCRE2GlobalReplace(*input, R"#(\W+)#", " ");
std::string str1 = StrQ2B(processed_line);
std::string strline;
opencc_->convert(str1, strline);
@@ -1806,7 +1895,11 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
term_to_stem = lowercase_term;
}
std::string stem_term;
- stemmer_->Stem(term_to_stem, stem_term);
+ if (use_stemmer_) {
+ stemmer_->Stem(term_to_stem, stem_term);
+ } else {
+ stem_term = term_to_stem;
+ }
res.push_back(stem_term);
}
continue;
@@ -1839,9 +1932,23 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
}
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) {
+ // Fold diacritics to ASCII before any other processing when the
+ // language asks for it (Slovak, Czech). Folding shrinks 2-byte UTF-8
+ // letters to 1-byte ASCII, so byte offsets shift; like the StrQ2B and
+ // OpenCC stages below, a position mapping recovers offsets on the
+ // original input.
+ std::string folded_line;
+ std::vector<unsigned> fold_pos_mapping;
+ const std::string *input = &line;
+ if (fold_diacritics_) {
+ folded_line = FoldDiacritics(line);
+ BuildPositionMapping(line, folded_line, fold_pos_mapping);
+ input = &folded_line;
+ }
+
// Python-style simple tokenization: re.sub(r"\W+", " ", line)
// Get processed line and position mapping from PCRE2GlobalReplace
- auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(line, R"#(\W+)#", " ");
+ auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(*input, R"#(\W+)#", " ");
std::string str1 = StrQ2B(processed_line);
std::string strline;
@@ -1857,7 +1964,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
std::vector<unsigned> opencc_pos_mapping;
BuildPositionMapping(str1, strline, opencc_pos_mapping);
- // Combine all position mappings: strline -> str1 -> processed_line -> line
+ // Combine all position mappings: strline -> str1 -> processed_line -> (folded_)line
std::vector<unsigned> final_pos_mapping;
final_pos_mapping.resize(strline.size() + 1);
@@ -1869,19 +1976,26 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
if (processed_pos < pcre2_pos_mapping.size()) {
final_pos_mapping[i] = pcre2_pos_mapping[processed_pos].first;
} else {
- final_pos_mapping[i] = static_cast<unsigned>(line.size());
+ final_pos_mapping[i] = static_cast<unsigned>(input->size());
}
} else {
- final_pos_mapping[i] = static_cast<unsigned>(line.size());
+ final_pos_mapping[i] = static_cast<unsigned>(input->size());
}
} else {
- final_pos_mapping[i] = static_cast<unsigned>(line.size());
+ final_pos_mapping[i] = static_cast<unsigned>(input->size());
}
}
// Fill the last position
if (strline.size() < final_pos_mapping.size()) {
- final_pos_mapping[strline.size()] = static_cast<unsigned>(line.size());
+ final_pos_mapping[strline.size()] = static_cast<unsigned>(input->size());
+ }
+
+ // Remap folded-line offsets back to the original input.
+ if (fold_diacritics_) {
+ for (auto &pos : final_pos_mapping) {
+ pos = pos < fold_pos_mapping.size() ? fold_pos_mapping[pos] : static_cast<unsigned>(line.size());
+ }
}
// Use SplitByLang to separate by language
@@ -1929,7 +2043,11 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
term_to_stem = lowercase_term;
}
std::string stem_term;
- stemmer_->Stem(term_to_stem, stem_term);
+ if (use_stemmer_) {
+ stemmer_->Stem(term_to_stem, stem_term);
+ } else {
+ stem_term = term_to_stem;
+ }
tokens.push_back(stem_term);
@@ -2258,7 +2376,11 @@ void RAGAnalyzer::EnglishNormalizeWithPosition(const std::vector<std::string> &t
term_to_stem = lowercase_term;
}
std::string stem_term;
- stemmer_->Stem(term_to_stem, stem_term);
+ if (use_stemmer_) {
+ stemmer_->Stem(term_to_stem, stem_term);
+ } else {
+ stem_term = term_to_stem;
+ }
normalize_tokens.push_back(stem_term);
normalize_positions.emplace_back(start_pos, end_pos);
diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.h
index 95da708609..c2b7c31885 100644
--- a/internal/binding/cpp/rag_analyzer.h
+++ b/internal/binding/cpp/rag_analyzer.h
@@ -102,6 +102,14 @@ private:
static std::string StrQ2B(const std::string& input);
+ // Fold Latin-1 Supplement / Latin Extended-A letters whose NFD
+ // decomposition is one ASCII letter plus combining marks to that base
+ // letter, preserving case (Š→S, ď→d). Letters without such a
+ // decomposition (æ, œ, ß, ø, ł, …) pass through unchanged. Applied to
+ // the whole input before tokenization when SetLanguage selected a
+ // diacritics-folding language (Slovak, Czech).
+ static std::string FoldDiacritics(const std::string& input);
+
static void BuildPositionMapping(const std::string& original, const std::string& converted,
std::vector<unsigned>& pos_mapping);
@@ -161,6 +169,10 @@ public:
bool use_lemmatizer_{true};
+ bool use_stemmer_{true}; // Disabled for languages without a Snowball stemmer (Slovak, Czech)
+
+ bool fold_diacritics_{false}; // Fold Latin diacritics to ASCII before tokenization (Slovak, Czech)
+
bool fine_grained_{false};
bool enable_position_{false};
diff --git a/internal/binding/rag_analyzer.go b/internal/binding/rag_analyzer.go
index c248ceff14..7a83c2b180 100644
--- a/internal/binding/rag_analyzer.go
+++ b/internal/binding/rag_analyzer.go
@@ -97,7 +97,8 @@ func (a *Analyzer) SetEnablePosition(enablePosition bool) {
}
// SetLanguage configures the Snowball stemmer for the given language (e.g. "English", "Dutch").
-// Falls back to the English Porter stemmer for unmapped languages.
+// Diacritic-folding languages ("Slovak", "Czech") disable stemming and
+// lemmatization instead; other unmapped languages keep the previous stemmer.
func (a *Analyzer) SetLanguage(language string) {
if a.handle == nil {
return
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 3693e56231..6034e9d49c 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -51,7 +51,7 @@ var jsonIterator = sonic.Config{
var memoryMessageVectorFieldRE = regexp.MustCompile(`^q_\d+_vec$`)
// CreateChunkStore creates an index
-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
if baseName == "" {
return fmt.Errorf("index name cannot be empty")
}
@@ -158,8 +158,8 @@ func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID strin
// chunks straight back (the chunk APIs, the debug endpoints) sees them right
// away. Ingestion must use InsertChunksNoRefresh instead - see that method for
// why the wait is the wrong trade there.
-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
- return e.insertChunks(ctx, chunks, baseName, datasetID, "wait_for")
+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
+ return e.insertChunks(ctx, chunks, baseName, datasetID, language, "wait_for")
}
// InsertChunksNoRefresh inserts chunks without waiting for an index refresh:
@@ -171,13 +171,16 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// This is the ingestion path's contract, and it matches Python's: the Python
// ingestion inserts chunks with refresh=False (rag/svr/task_executor_refactor/
// chunk_service.py:386 and :423), while its API default stays "wait_for".
-func (e *Engine) InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
- return e.insertChunks(ctx, chunks, baseName, datasetID, "")
+func (e *Engine) InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
+ return e.insertChunks(ctx, chunks, baseName, datasetID, language, "")
}
// insertChunks is the shared bulk-insert body. An empty refresh omits the
// parameter, which leaves the decision to the index (no forced refresh).
-func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, refresh string) ([]string, error) {
+// language is the dataset language, which only engines whose fulltext analyzer
+// is fixed at index creation need; Elasticsearch analyses client-side and
+// ignores it.
+func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string, refresh string) ([]string, error) {
common.Info("ElasticsearchConnection.InsertChunks called", zap.String("index_name", baseName), zap.Int("chunkCount", len(chunks)))
if len(chunks) == 0 {
diff --git a/internal/engine/elasticsearch/chunk_readback_integration_test.go b/internal/engine/elasticsearch/chunk_readback_integration_test.go
index dcef822925..e1669a89b4 100644
--- a/internal/engine/elasticsearch/chunk_readback_integration_test.go
+++ b/internal/engine/elasticsearch/chunk_readback_integration_test.go
@@ -59,7 +59,7 @@ func TestInsertChunks_ReadBackSuffixedFields(t *testing.T) {
"position_int": int(2),
}
- if _, err := engine.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, datasetID); err != nil {
+ if _, err := engine.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, datasetID, ""); err != nil {
t.Fatalf("InsertChunks: %v", err)
}
diff --git a/internal/engine/elasticsearch/chunk_readback_test.go b/internal/engine/elasticsearch/chunk_readback_test.go
index a7b3540ef9..8665b7507a 100644
--- a/internal/engine/elasticsearch/chunk_readback_test.go
+++ b/internal/engine/elasticsearch/chunk_readback_test.go
@@ -91,7 +91,7 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {
visible := map[string]interface{}{"id": "visible", "doc_id": "doc-1", "content_with_weight": "visible source"}
nullAvailability := map[string]interface{}{"id": "null", "doc_id": "doc-1", "available_int": nil}
enabled := map[string]interface{}{"id": "enabled", "doc_id": "doc-1", "available_int": 1}
- if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk, visible, nullAvailability, enabled}, "ragflow_chunk_readback_test", "kb-1"); err != nil {
+ if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk, visible, nullAvailability, enabled}, "ragflow_chunk_readback_test", "kb-1", ""); err != nil {
t.Fatalf("InsertChunks: %v", err)
}
diff --git a/internal/engine/elasticsearch/chunk_refresh_test.go b/internal/engine/elasticsearch/chunk_refresh_test.go
index 523d3ff34b..5441ea4747 100644
--- a/internal/engine/elasticsearch/chunk_refresh_test.go
+++ b/internal/engine/elasticsearch/chunk_refresh_test.go
@@ -55,7 +55,7 @@ func TestInsertChunksWaitsForRefresh(t *testing.T) {
var refresh []string
engine := newBulkCaptureEngine(t, &refresh)
- if _, err := engine.InsertChunks(t.Context(), oneChunk(), "ragflow_tenant", "kb-1"); err != nil {
+ if _, err := engine.InsertChunks(t.Context(), oneChunk(), "ragflow_tenant", "kb-1", ""); err != nil {
t.Fatalf("InsertChunks: %v", err)
}
if len(refresh) != 1 || refresh[0] != "wait_for" {
@@ -71,7 +71,7 @@ func TestInsertChunksNoRefreshOmitsRefreshParam(t *testing.T) {
var refresh []string
engine := newBulkCaptureEngine(t, &refresh)
- if _, err := engine.InsertChunksNoRefresh(t.Context(), oneChunk(), "ragflow_tenant", "kb-1"); err != nil {
+ if _, err := engine.InsertChunksNoRefresh(t.Context(), oneChunk(), "ragflow_tenant", "kb-1", ""); err != nil {
t.Fatalf("InsertChunksNoRefresh: %v", err)
}
if len(refresh) != 1 {
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index 74fb347eb8..cf7455801f 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -41,8 +41,14 @@ const (
// DocEngine document storage engine interface
type DocEngine interface {
// Chunk operations
- CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error
- InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)
+ // CreateChunkStore creates the chunk store for a dataset. language is the
+ // dataset language, needed by engines whose fulltext analyzer is fixed when
+ // the index is created; the others ignore it.
+ CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error
+ // InsertChunks writes chunks, creating the chunk store on first write.
+ // language is the dataset language, needed there for the same reason as in
+ // CreateChunkStore; engines that analyse client-side ignore it.
+ InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)
UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error
DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error)
Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error)
diff --git a/internal/engine/infinity/analyzer_test.go b/internal/engine/infinity/analyzer_test.go
new file mode 100644
index 0000000000..af8a451343
--- /dev/null
+++ b/internal/engine/infinity/analyzer_test.go
@@ -0,0 +1,69 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package infinity
+
+import "testing"
+
+func TestAnalyzerForLanguage(t *testing.T) {
+ tests := []struct {
+ name string
+ analyzer string
+ language string
+ want string
+ }{
+ {"slovak suffixes the coarse rag analyzer", "rag-coarse", "Slovak", "rag-coarse-slovak"},
+ {"slovak suffixes the fine rag analyzer", "rag-fine", "Slovak", "rag-fine-slovak"},
+ {"czech is handled the same way", "rag-coarse", "czech", "rag-coarse-czech"},
+ {"the bare rag analyzer takes a suffix too", "rag", "Slovak", "rag-slovak"},
+ {"case and surrounding space do not matter", "rag-fine", " CZECH ", "rag-fine-czech"},
+
+ // Languages the analyzer treats no differently must not move: their
+ // datasets are already indexed under the default analyzer, and
+ // switching a Snowball stemmer on would silently change tokenization.
+ {"english is left alone", "rag-coarse", "English", "rag-coarse"},
+ {"dutch is left alone", "rag-coarse", "Dutch", "rag-coarse"},
+ {"an unset language is left alone", "rag-coarse", "", "rag-coarse"},
+
+ // Keyword analyzers are language independent.
+ {"whitespace is left alone", "whitespace-#", "Slovak", "whitespace-#"},
+ {"rankfeatures is left alone", "rankfeatures", "Slovak", "rankfeatures"},
+ {"a rag prefix is not a rag analyzer", "ragged", "Slovak", "ragged"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := analyzerForLanguage(tt.analyzer, tt.language); got != tt.want {
+ t.Errorf("analyzerForLanguage(%q, %q) = %q, want %q", tt.analyzer, tt.language, got, tt.want)
+ }
+ })
+ }
+}
+
+// Queries and filters reference a fulltext index by name (convertMatchingField),
+// so the language must change only the analyzer, never the index name.
+func TestFulltextIndexNameIsLanguageIndependent(t *testing.T) {
+ for _, tt := range []struct{ field, analyzer, ref string }{
+ {"content", "rag-coarse", "content_ltks"},
+ {"content", "rag-fine", "content_sm_ltks"},
+ {"docnm", "rag-coarse", "docnm_kwd"},
+ } {
+ name := fulltextIndexName(tt.field, tt.analyzer)
+ if want := tt.field + "@" + name; convertMatchingField(tt.ref) != want {
+ t.Errorf("convertMatchingField(%q) = %q, want %q", tt.ref, convertMatchingField(tt.ref), want)
+ }
+ }
+}
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index e9880a5d07..bf2c29cd1c 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -55,14 +55,15 @@ var pagerankAdjustLocks [pagerankAdjustLockCount]sync.Mutex
// The full table name is built as "{baseName}_{datasetID}"
// For skill index (datasetID="skill"), tableName is just baseName and uses skill_infinity_mapping.json
// For memory message indexes (baseName prefixed "memory_"), uses message_infinity_mapping.json
-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+// language is the dataset language, which selects the fulltext analyzer; see analyzerForLanguage.
+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return fmt.Errorf("failed to get database: %w", err)
}
defer release()
- return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID)
+ return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID, language)
}
// memoryIndexPrefix marks per-memory message indexes (MemoryIndexName).
@@ -88,7 +89,7 @@ func chunkMappingFileName(baseName, datasetID, configured string) string {
}
}
-func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID string) error {
+func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID, language string) error {
vecSize := vectorSize
// Determine table name and mapping file based on index type
@@ -227,7 +228,9 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset
}
common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
- // Create full-text indexes for varchar fields with analyzers
+ // Create full-text indexes for varchar fields with analyzers. CreateTable
+ // above is ConflictTypeIgnore, so this may be an existing table: its indexes
+ // are kept as they are, and with them the analyzer it was created with.
for _, fieldName := range schema.Keys {
fieldInfo := schema.Fields[fieldName]
if fieldInfo.Type != "varchar" || fieldInfo.Analyzer == nil {
@@ -247,13 +250,10 @@ func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, dataset
}
for _, analyzer := range analyzers {
- indexNameFt := fmt.Sprintf("ft_%s_%s",
- regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(fieldName, "_"),
- regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(analyzer, "_"),
- )
+ indexNameFt := fulltextIndexName(fieldName, analyzer)
_, err = table.CreateIndex(
indexNameFt,
- infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzer}),
+ infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzerForLanguage(analyzer, language)}),
infinity.ConflictTypeIgnore,
"",
)
@@ -332,7 +332,7 @@ func (e *Engine) ensureChunkDataColumn(table *infinity.Table, exists bool) error
// Table name format: {baseName}_{datasetID}
// Auto-create the table if it doesn't exist
// Delete existing rows with matching IDs before insert
-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
tableName := buildChunkTableName(baseName, datasetID)
common.Info("InfinityConnection.InsertChunks called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))
@@ -369,8 +369,9 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
return nil, fmt.Errorf("cannot infer vector size from chunks")
}
- // Create table
- if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks)); err != nil {
+ // Create table. This is the path the ingestion pipeline actually takes,
+ // so the dataset language has to reach it here.
+ if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks), language); err != nil {
return nil, fmt.Errorf("failed to create table: %w", err)
}
diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go
index edeb53036e..466d6fa83f 100644
--- a/internal/engine/infinity/common.go
+++ b/internal/engine/infinity/common.go
@@ -23,6 +23,7 @@ import (
"fmt"
"ragflow/internal/common"
"ragflow/internal/engine/types"
+ "regexp"
"strings"
"unicode"
@@ -573,3 +574,54 @@ func buildChunkTableName(baseName, datasetID string) string {
func buildMetadataTableName(tenantID string) string {
return fmt.Sprintf("ragflow_doc_meta_%s", tenantID)
}
+
+// analyzerLanguages are the dataset languages that need their own RAG
+// analyzer. Infinity's analyzer folds their diacritics to ASCII and skips
+// stemming, so a dataset in one of them has to be indexed and queried through
+// "rag-<language>" instead of the default "rag-coarse"/"rag-fine" pair.
+//
+// They are the only two languages whose analyzer differs from the default:
+// every other name either selects a Snowball stemmer (english, dutch, german,
+// ... -- SNOWBALL_LANGUAGE_MAP in Infinity's rag_analyzer) or is a no-op
+// (Chinese, Japanese, Korean, ... use dictionary segmentation, not stemming).
+// Naming a stemmer here would switch it for every newly created dataset in
+// that language, which is a behaviour change of its own.
+var analyzerLanguages = map[string]struct{}{
+ "czech": {},
+ "slovak": {},
+}
+
+// ragAnalyzer is the analyzer family whose behaviour the dataset language
+// changes. Fields indexed with "whitespace-#" or "rankfeatures" carry
+// keywords, not prose.
+const ragAnalyzer = "rag"
+
+// analyzerForLanguage returns the analyzer to index a dataset in language
+// with. Infinity analyzes a query with the analyzer of the fulltext index it
+// matches against, so the language has to be baked into the index when it is
+// created -- there is no per-query analyzer to pass. Chunk tables are
+// per-dataset, which is what makes a per-dataset analyzer possible.
+//
+// Languages outside analyzerLanguages, and analyzers outside the rag family,
+// are returned unchanged.
+func analyzerForLanguage(analyzer, language string) string {
+ key := strings.ToLower(strings.TrimSpace(language))
+ if _, ok := analyzerLanguages[key]; !ok {
+ return analyzer
+ }
+ if base, _, _ := strings.Cut(analyzer, "-"); base != ragAnalyzer {
+ return analyzer
+ }
+ return analyzer + "-" + key
+}
+
+// indexNamePartRe matches everything that cannot appear in an index name.
+var indexNamePartRe = regexp.MustCompile(`[^a-zA-Z0-9]`)
+
+// fulltextIndexName is the name of the fulltext index on fieldName for the
+// configured analyzer. Queries and filters name the index explicitly (see
+// convertMatchingField), so the name is the same in every language; the
+// dataset language only changes the analyzer behind it.
+func fulltextIndexName(fieldName, analyzer string) string {
+ return fmt.Sprintf("ft_%s_%s", indexNamePartRe.ReplaceAllString(fieldName, "_"), indexNamePartRe.ReplaceAllString(analyzer, "_"))
+}
diff --git a/internal/engine/infinity/language_integration_test.go b/internal/engine/infinity/language_integration_test.go
new file mode 100644
index 0000000000..21db5fc9f0
--- /dev/null
+++ b/internal/engine/infinity/language_integration_test.go
@@ -0,0 +1,107 @@
+//go:build integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package infinity
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+ "ragflow/internal/server/config"
+)
+
+// TestSlovakChunkStoreMatchesWithAndWithoutDiacritics runs against a real
+// Infinity: a chunk store created with the Slovak dataset language gets the
+// folding rag-*-slovak analyzers, so an accented word in a chunk matches a query
+// typed with or without its diacritics.
+func TestSlovakChunkStoreMatchesWithAndWithoutDiacritics(t *testing.T) {
+ if err := common.InitLogger("error", common.FileOutput{}, "infinity_language_test"); err != nil {
+ t.Fatalf("init logger: %v", err)
+ }
+ uri := os.Getenv("RAGFLOW_INFINITY_URI")
+ if uri == "" {
+ uri = "localhost:23817"
+ }
+ ctx := t.Context()
+ eng, err := NewEngine(ctx, config.InfinityConfig{URI: uri, DBName: "ragflow_language_test", PostgresPort: 5432})
+ if err != nil {
+ t.Fatalf("connect to Infinity at %s: %v", uri, err)
+ }
+ t.Cleanup(func() { _ = eng.Close() })
+
+ baseName := fmt.Sprintf("ragflow_language_test_%d", time.Now().UnixNano())
+ search := func(datasetID, query string) int {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{datasetID},
+ Limit: 10,
+ SelectFields: []string{"id", "content_with_weight"},
+ MatchExprs: []interface{}{&types.MatchTextExpr{
+ Fields: []string{"content_ltks", "content_sm_ltks", "docnm_kwd"},
+ MatchingText: query,
+ TopN: 10,
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Search(%s, %q): %v", datasetID, query, err)
+ }
+ return len(res.Chunks)
+ }
+
+ content := "Deti chodia do školy každý deň."
+ for _, tt := range []struct {
+ datasetID, language string
+ unaccentedHits int
+ }{
+ {"kb_slovak", "slovak", 1},
+ // The default analyzer does not fold, which is what makes the Slovak
+ // match above the analyzer's doing.
+ {"kb_default", "", 0},
+ } {
+ if err := eng.CreateChunkStore(ctx, baseName, tt.datasetID, 2, "naive", tt.language); err != nil {
+ t.Fatalf("CreateChunkStore(%s): %v", tt.datasetID, err)
+ }
+ t.Cleanup(func() { _ = eng.DropChunkStore(context.Background(), baseName, tt.datasetID) })
+ chunk := map[string]interface{}{
+ "id": "chunk-1",
+ "doc_id": "doc-1",
+ "kb_id": tt.datasetID,
+ "docnm_kwd": "škola.txt",
+ "content_with_weight": content,
+ "content_ltks": content,
+ "content_sm_ltks": content,
+ "available_int": 1,
+ "q_2_vec": []float64{0.1, 0.2},
+ }
+ if _, err := eng.InsertChunks(ctx, []map[string]interface{}{chunk}, baseName, tt.datasetID, tt.language); err != nil {
+ t.Fatalf("InsertChunks(%s): %v", tt.datasetID, err)
+ }
+
+ if got := search(tt.datasetID, "školy"); got != 1 {
+ t.Errorf("%s: accented query returned %d chunks, want 1", tt.datasetID, got)
+ }
+ if got := search(tt.datasetID, "skoly"); got != tt.unaccentedHits {
+ t.Errorf("%s: unaccented query returned %d chunks, want %d", tt.datasetID, got, tt.unaccentedHits)
+ }
+ }
+}
diff --git a/internal/engine/oceanbase/crud.go b/internal/engine/oceanbase/crud.go
index acd1ac5760..a09b0e401e 100644
--- a/internal/engine/oceanbase/crud.go
+++ b/internal/engine/oceanbase/crud.go
@@ -26,7 +26,7 @@ import (
)
// InsertChunks writes chunks or memory messages with legacy REPLACE semantics.
-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {
+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {
if len(chunks) == 0 {
return []string{}, nil
}
@@ -48,7 +48,7 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
if vectorSize == 0 {
return nil, fmt.Errorf("cannot infer vector size from documents")
}
- if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", ""); err != nil {
return nil, err
}
} else if vectorSize > 0 {
diff --git a/internal/engine/oceanbase/crud_test.go b/internal/engine/oceanbase/crud_test.go
index c972b6baa4..8adab1565a 100644
--- a/internal/engine/oceanbase/crud_test.go
+++ b/internal/engine/oceanbase/crud_test.go
@@ -66,7 +66,7 @@ func TestInsertChunksFindsLaterVectorAndWaitsForSeekDBIndexRefresh(t *testing.T)
"id": "memory-1_2", "message_id": "2", "memory_id": "memory-1",
"content": "hello", "content_embed": []float64{0.1, 0.2},
},
- }, tableName, "memory-1")
+ }, tableName, "memory-1", "")
if err != nil {
t.Fatal(err)
}
@@ -104,7 +104,7 @@ func TestInsertChunksReturnsNormalizationErrorWithoutPanic(t *testing.T) {
_, err = engine.InsertChunks(t.Context(), []map[string]interface{}{{
"id": "memory-1_1", "message_id": "1", "q_2_vec": "invalid",
- }}, tableName, "memory-1")
+ }}, tableName, "memory-1", "")
if err == nil {
t.Fatal("InsertChunks() error = nil, want vector normalization error")
}
diff --git a/internal/engine/oceanbase/document.go b/internal/engine/oceanbase/document.go
index f46c1c7bca..a9ea37b4fc 100644
--- a/internal/engine/oceanbase/document.go
+++ b/internal/engine/oceanbase/document.go
@@ -40,7 +40,7 @@ func (e *Engine) IndexDocument(ctx context.Context, indexName, docID string, doc
}
vectorSize := vectorDimension(document)
if !ready {
- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {
return err
}
} else if vectorSize > 0 {
@@ -91,7 +91,7 @@ func (e *Engine) BulkIndex(ctx context.Context, indexName string, docs []interfa
return nil, err
}
if !ready {
- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {
return nil, err
}
} else if vectorSize > 0 {
diff --git a/internal/engine/oceanbase/integration_test.go b/internal/engine/oceanbase/integration_test.go
index d6ffb2b488..ee0cec7c6a 100644
--- a/internal/engine/oceanbase/integration_test.go
+++ b/internal/engine/oceanbase/integration_test.go
@@ -58,7 +58,7 @@ func TestLegacyStorageRoundTrip(t *testing.T) {
datasetID := "kb-" + suffix
defer cleanupChunkStore(t, engine, tableName, "")
- if err := engine.CreateChunkStore(ctx, tableName, datasetID, 2, "naive"); err != nil {
+ if err := engine.CreateChunkStore(ctx, tableName, datasetID, 2, "naive", ""); err != nil {
t.Fatal(err)
}
if _, err := engine.InsertChunks(ctx, []map[string]interface{}{{
@@ -66,7 +66,7 @@ func TestLegacyStorageRoundTrip(t *testing.T) {
"content_ltks": "hello oceanbase", "important_kwd": []string{"hello"},
"metadata": map[string]interface{}{"_group_id": "group-1", "custom": "json-value"},
"custom_field": "kept-in-extra", "q_2_vec": []float64{0.25, 0.5},
- }}, tableName, datasetID); err != nil {
+ }}, tableName, datasetID, ""); err != nil {
t.Fatal(err)
}
row, err := engine.GetChunk(ctx, tableName, "chunk-1", []string{datasetID})
@@ -127,13 +127,13 @@ func TestLegacyStorageRoundTrip(t *testing.T) {
memoryA := "memory-a-" + suffix
memoryB := "memory-b-" + suffix
defer cleanupChunkStore(t, engine, memoryTable, "")
- if err := engine.CreateChunkStore(ctx, memoryTable, memoryA, 2, ""); err != nil {
+ if err := engine.CreateChunkStore(ctx, memoryTable, memoryA, 2, "", ""); err != nil {
t.Fatal(err)
}
if _, err := engine.InsertChunks(ctx, []map[string]interface{}{
{"id": memoryA + "_1", "message_id": "1", "memory_id": memoryA, "content": "first", "content_embed": []float64{0.1, 0.2}},
{"id": memoryB + "_1", "message_id": "1", "memory_id": memoryB, "content": "second", "content_embed": []float64{0.3, 0.4}},
- }, memoryTable, memoryA); err != nil {
+ }, memoryTable, memoryA, ""); err != nil {
t.Fatal(err)
}
if err := engine.DropChunkStore(ctx, memoryTable, memoryA); err != nil {
diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.go
index 27adf734c8..8f40d4bb5e 100644
--- a/internal/engine/oceanbase/schema.go
+++ b/internal/engine/oceanbase/schema.go
@@ -159,7 +159,7 @@ var tokenizedFullTextFields = []string{"title_tks", "title_sm_tks", "important_t
// CreateChunkStore creates or upgrades the legacy shared tenant table. The
// dataset ID is a row-level discriminator for chunk and memory tables.
-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
if err := validateIdentifier(baseName); err != nil {
return err
}
diff --git a/internal/engine/serenedb/chunk.go b/internal/engine/serenedb/chunk.go
index 9f82255c28..fb836da590 100644
--- a/internal/engine/serenedb/chunk.go
+++ b/internal/engine/serenedb/chunk.go
@@ -65,7 +65,7 @@ func chunkTableDDL(tableName string, vectorSize int) []string {
// CreateChunkStore ensures the tenant chunk table and its indexes exist. All
// datasets in the tenant share this table, so it is created once and is a
// no-op for later datasets.
-func (e *serenedbEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+func (e *serenedbEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
tableName := chunkTableName(baseName)
for _, stmt := range chunkTableDDL(tableName, vectorSize) {
if err := e.exec(ctx, stmt); err != nil {
@@ -227,7 +227,7 @@ func prepareChunkRow(chunk map[string]interface{}, defaultKbID string) ([]string
// InsertChunks upserts chunks by id. Rows are grouped by identical column set
// and inserted in one multi-row statement per group. Returns an empty slice on
// success (ids are the caller's own).
-func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {
+func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {
if len(chunks) == 0 {
return []string{}, nil
}
@@ -243,7 +243,7 @@ func (e *serenedbEngine) InsertChunks(ctx context.Context, chunks []map[string]i
size, _ = strconv.Atoi(m[1])
}
}
- if err := e.CreateChunkStore(ctx, baseName, datasetID, size, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, baseName, datasetID, size, "", ""); err != nil {
return nil, err
}
}
diff --git a/internal/engine/serenedb/integration_test.go b/internal/engine/serenedb/integration_test.go
index 3bdfa35de8..3a0cd4934c 100644
--- a/internal/engine/serenedb/integration_test.go
+++ b/internal/engine/serenedb/integration_test.go
@@ -107,7 +107,7 @@ func TestIntegrationChunkLifecycle(t *testing.T) {
_ = e.DropChunkStore(ctx, base, kb)
defer func() { _ = e.DropChunkStore(ctx, base, kb) }()
- if err := e.CreateChunkStore(ctx, base, kb, 4, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, base, kb, 4, "", ""); err != nil {
t.Fatalf("CreateChunkStore: %v", err)
}
if ok, _ := e.ChunkStoreExists(ctx, base, kb); !ok {
@@ -119,7 +119,7 @@ func TestIntegrationChunkLifecycle(t *testing.T) {
{"id": "b", "doc_id": "d1", "kb_id": kb, "content_ltks": "gamma delta", "content_with_weight": "gamma delta", "q_4_vec": []float64{0, 1, 0, 0}, "important_kwd": []interface{}{"gamma"}},
{"id": "c", "doc_id": "d2", "kb_id": kb, "content_ltks": "alpha gamma", "content_with_weight": "alpha gamma", "q_4_vec": []float64{0.9, 0.1, 0, 0}, "important_kwd": []interface{}{"alpha", "gamma"}},
}
- if _, err := e.InsertChunks(ctx, chunks, base, kb); err != nil {
+ if _, err := e.InsertChunks(ctx, chunks, base, kb, ""); err != nil {
t.Fatalf("InsertChunks: %v", err)
}
diff --git a/internal/engine/vastbase/crud.go b/internal/engine/vastbase/crud.go
index 20689df643..a4c77f16b3 100644
--- a/internal/engine/vastbase/crud.go
+++ b/internal/engine/vastbase/crud.go
@@ -31,7 +31,7 @@ import (
// delete-then-insert upsert: legacy tables carry no PRIMARY KEY, so ON
// CONFLICT is not an option. The delete and the grouped inserts share one
// transaction, keeping a failed batch from leaving half-replaced rows.
-func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {
+func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName, datasetID, language string) ([]string, error) {
if len(chunks) == 0 {
return []string{}, nil
}
@@ -63,7 +63,7 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
if !tablePresent && vectorSize == 0 {
return nil, fmt.Errorf("cannot infer vector size from documents")
}
- if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", ""); err != nil {
return nil, err
}
} else if vectorSize > 0 {
diff --git a/internal/engine/vastbase/crud_test.go b/internal/engine/vastbase/crud_test.go
index 22c59d0b55..8e611ca9f6 100644
--- a/internal/engine/vastbase/crud_test.go
+++ b/internal/engine/vastbase/crud_test.go
@@ -88,7 +88,7 @@ func TestInsertChunksRunsDeleteThenInsertInOneTransaction(t *testing.T) {
inserted, err := engine.InsertChunks(context.Background(), []map[string]interface{}{{
"id": "c1", "content_ltks": "hello", "q_3_vec": []interface{}{0.1, 0.2, 0.3},
- }}, "t1", "kb-1")
+ }}, "t1", "kb-1", "")
if err != nil {
t.Fatal(err)
}
@@ -119,7 +119,7 @@ func TestInsertChunksReturnsEncodeErrorForMalformedVector(t *testing.T) {
_, err = engine.InsertChunks(context.Background(), []map[string]interface{}{{
"id": "c1", "q_3_vec": "oops",
- }}, "t1", "kb-1")
+ }}, "t1", "kb-1", "")
if err == nil || !strings.Contains(err.Error(), "encode q_3_vec") {
t.Fatalf("err = %v", err)
}
@@ -161,7 +161,7 @@ func TestInsertChunksRequiresVectorSizeForNewStore(t *testing.T) {
mock.ExpectQuery(regexp.QuoteMeta(tableCountQuery())).WithArgs("t1").
WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0))
if _, err := engine.InsertChunks(context.Background(),
- []map[string]interface{}{{"id": "c1", "content_ltks": "x"}}, "t1", "kb-1"); err == nil {
+ []map[string]interface{}{{"id": "c1", "content_ltks": "x"}}, "t1", "kb-1", ""); err == nil {
t.Fatal("creating a new store without an inferable vector dimension must fail")
}
if err := mock.ExpectationsWereMet(); err != nil {
@@ -225,7 +225,7 @@ func TestInsertChunksVectorlessBatchOnTableWithFailedFullTextIndex(t *testing.T)
if _, err := engine.InsertChunks(context.Background(), []map[string]interface{}{
{"id": "c3", "kb_id": "kb-1", "content_ltks": "third"},
- }, "t1", "kb-1"); err != nil {
+ }, "t1", "kb-1", ""); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
diff --git a/internal/engine/vastbase/document.go b/internal/engine/vastbase/document.go
index d5c659c27a..e1c104bb46 100644
--- a/internal/engine/vastbase/document.go
+++ b/internal/engine/vastbase/document.go
@@ -40,7 +40,7 @@ func (e *Engine) IndexDocument(ctx context.Context, indexName, docID string, doc
}
vectorSize := vectorDimension(document)
if !ready {
- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {
return err
}
} else if vectorSize > 0 {
@@ -94,7 +94,7 @@ func (e *Engine) BulkIndex(ctx context.Context, indexName string, docs []interfa
return nil, err
}
if !ready {
- if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, ""); err != nil {
+ if err := e.CreateChunkStore(ctx, indexName, "skill", vectorSize, "", ""); err != nil {
return nil, err
}
} else if vectorSize > 0 {
diff --git a/internal/engine/vastbase/integration_test.go b/internal/engine/vastbase/integration_test.go
index 258fe7e601..f9e7b48fa2 100644
--- a/internal/engine/vastbase/integration_test.go
+++ b/internal/engine/vastbase/integration_test.go
@@ -82,7 +82,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {
defer engine.DropChunkStore(ctx, table, "")
// --- Chunk store: DDL, dynamic columns, storage-format round trip ---
- if err := engine.CreateChunkStore(ctx, table, dataset, 3, "naive"); err != nil {
+ if err := engine.CreateChunkStore(ctx, table, dataset, 3, "naive", ""); err != nil {
t.Fatal(err)
}
exists, err := engine.ChunkStoreExists(ctx, table, dataset)
@@ -105,7 +105,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {
"content_with_weight": "second chunk", "content_ltks": "second chunk",
"available_int": 1, "q_3_vec": []interface{}{0.9, 0.8, 0.7},
},
- }, table, dataset); err != nil {
+ }, table, dataset, ""); err != nil {
t.Fatal(err)
}
row, err := engine.GetChunk(ctx, table, "c1", []string{dataset})
@@ -213,13 +213,13 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {
memoryTable := "memory_vb_it_" + suffix
memoryID := "mem-" + suffix
defer engine.DropChunkStore(ctx, memoryTable, "")
- if err := engine.CreateChunkStore(ctx, memoryTable, memoryID, 0, ""); err != nil {
+ if err := engine.CreateChunkStore(ctx, memoryTable, memoryID, 0, "", ""); err != nil {
t.Fatal(err)
}
if _, err := engine.InsertChunks(ctx, []map[string]interface{}{
{"id": memoryID + "_1", "content": "remember the vastbase meeting", "status": true,
"content_embed": []interface{}{0.25, 0.5}},
- }, memoryTable, memoryID); err != nil {
+ }, memoryTable, memoryID, ""); err != nil {
t.Fatal(err)
}
memoryText, err := engine.Search(ctx, &types.SearchRequest{
@@ -293,7 +293,7 @@ func TestVastbaseStorageRoundTrip(t *testing.T) {
// --- Drop deletes dataset rows but keeps the shared table ---
remaining, err := engine.InsertChunks(ctx, []map[string]interface{}{
{"id": "c3", "kb_id": dataset, "content_ltks": "third"},
- }, table, dataset)
+ }, table, dataset, "")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/engine/vastbase/schema.go b/internal/engine/vastbase/schema.go
index 1f43aad026..1b5cf1678a 100644
--- a/internal/engine/vastbase/schema.go
+++ b/internal/engine/vastbase/schema.go
@@ -178,7 +178,7 @@ var skillFullTextFields = []string{"name_tks", "tags_tks", "description_tks", "c
// CreateChunkStore creates or upgrades the shared baseName table. The dataset
// ID is a row-level discriminator for chunk and memory tables.
-func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
+func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
if err := validateIdentifier(baseName); err != nil {
return err
}
diff --git a/internal/engine/vastbase/schema_test.go b/internal/engine/vastbase/schema_test.go
index 64257e22f3..7e24aa7e84 100644
--- a/internal/engine/vastbase/schema_test.go
+++ b/internal/engine/vastbase/schema_test.go
@@ -95,7 +95,7 @@ func TestCreateMemoryStoreDDLGolden(t *testing.T) {
}
// vectorSize 0: no vector column or index is created.
- if err := engine.CreateChunkStore(context.Background(), "memory_t1", "mem-1", 0, ""); err != nil {
+ if err := engine.CreateChunkStore(context.Background(), "memory_t1", "mem-1", 0, "", ""); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
@@ -144,7 +144,7 @@ func TestCreateChunkStoreBModeProvisionsFullTextPerColumn(t *testing.T) {
quoteIdent(indexName), quoteIdent(field)))
}
- if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, ""); err != nil {
+ if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, "", ""); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
@@ -185,7 +185,7 @@ func TestCreateChunkStoreBModeDegradesWhenFullTextIndexFails(t *testing.T) {
// vectorSize 0: no vector column or index is created; the failed
// full-text DDL must not surface as an error.
- if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, ""); err != nil {
+ if err := engine.CreateChunkStore(context.Background(), "t1", "kb-1", 0, "", ""); err != nil {
t.Fatal(err)
}
if err := mock.ExpectationsWereMet(); err != nil {
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go
index 63b101e6f3..381cc8b854 100644
--- a/internal/entity/dataset.go
+++ b/internal/entity/dataset.go
@@ -16,7 +16,10 @@
package entity
-import "time"
+import (
+ "strings"
+ "time"
+)
// DatasetNameLimit is the maximum length for dataset name
const DatasetNameLimit = 128
@@ -147,6 +150,29 @@ func (kb *Knowledgebase) TableName() string {
return "knowledgebase"
}
+// KnowledgebasesLanguage returns the language to retrieve with over kbs, which
+// is only well defined when they all agree: one query is tokenized once, but
+// each dataset was indexed under its own language. A mixed set therefore falls
+// back to "" (English, no folding) rather than letting the first dataset in the
+// slice decide how the others are searched.
+func KnowledgebasesLanguage(kbs []*Knowledgebase) string {
+ language := ""
+ for i, kb := range kbs {
+ current := ""
+ if kb != nil && kb.Language != nil {
+ current = *kb.Language
+ }
+ if i == 0 {
+ language = current
+ continue
+ }
+ if !strings.EqualFold(strings.TrimSpace(current), strings.TrimSpace(language)) {
+ return ""
+ }
+ }
+ return language
+}
+
// ToMap converts Knowledgebase to a map for JSON response
func (kb *Knowledgebase) ToMap() map[string]interface{} {
result := map[string]interface{}{
diff --git a/internal/entity/dataset_language_test.go b/internal/entity/dataset_language_test.go
new file mode 100644
index 0000000000..8b018c14ab
--- /dev/null
+++ b/internal/entity/dataset_language_test.go
@@ -0,0 +1,54 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package entity
+
+import "testing"
+
+func kb(language *string) *Knowledgebase {
+ return &Knowledgebase{Language: language}
+}
+
+func ptr(s string) *string { return &s }
+
+func TestKnowledgebasesLanguage(t *testing.T) {
+ tests := []struct {
+ name string
+ kbs []*Knowledgebase
+ want string
+ }{
+ {"no datasets", nil, ""},
+ {"single dataset", []*Knowledgebase{kb(ptr("Slovak"))}, "Slovak"},
+ {"all agree", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr("Slovak"))}, "Slovak"},
+ {"agree ignoring case and space", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr(" slovak "))}, "Slovak"},
+ // A mixed set has no single answer: one query is tokenized once, but the
+ // datasets were indexed under different rules. Fall back to English.
+ {"mixed languages", []*Knowledgebase{kb(ptr("Slovak")), kb(ptr("English"))}, ""},
+ {"mixed with unset", []*Knowledgebase{kb(ptr("Slovak")), kb(nil)}, ""},
+ {"unset first, set second", []*Knowledgebase{kb(nil), kb(ptr("Slovak"))}, ""},
+ {"all unset", []*Knowledgebase{kb(nil), kb(nil)}, ""},
+ {"nil element", []*Knowledgebase{kb(ptr("Slovak")), nil}, ""},
+ // Order must not decide the result.
+ {"mixed, reversed", []*Knowledgebase{kb(ptr("English")), kb(ptr("Slovak"))}, ""},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := KnowledgebasesLanguage(tt.kbs); got != tt.want {
+ t.Errorf("KnowledgebasesLanguage() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/handler/dify_retrieval_handler.go b/internal/handler/dify_retrieval_handler.go
index 2da98d9249..6e826ee0e0 100644
--- a/internal/handler/dify_retrieval_handler.go
+++ b/internal/handler/dify_retrieval_handler.go
@@ -273,6 +273,7 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {
KNNTopK: topK,
SimilarityThreshold: scoreThreshold,
EmbeddingModel: embModel,
+ Language: entity.KnowledgebasesLanguage(kbs),
}
if rankFeature != nil {
sr.RankFeature = &rankFeature
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 712c682b33..2a6c74b290 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -230,7 +230,11 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
// conf/infinity_mapping.json) and merge them into the upstream input
// chunks. The component stays DB-independent and no longer routes through a
// separate writer seam: its output is plain chunks.
- compiled, err := productsToChunkDocs(out.Products)
+ // Compiled rows share the dataset's chunk table with the parsed chunks and
+ // are retrieved by the same queries, so they tokenize with the dataset
+ // language the Tokenizer component uses.
+ language := globals.GlobalOrInput(ctx, inputs, "lang", "English")
+ compiled, err := productsToChunkDocs(out.Products, language)
if err != nil {
return nil, err
}
@@ -396,7 +400,8 @@ func kindOrVariant(p common.Product) string {
// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is
// not persisted: only engine columns are written, because Infinity rejects an
// insert naming an unknown column.
-func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
+func productsToChunkDocs(products []common.Product, language string) ([]schema.ChunkDoc, error) {
+ tok := tokenizer.New(language)
docs := make([]schema.ChunkDoc, 0, len(products))
for _, p := range products {
doc := schema.ChunkDoc{
@@ -420,9 +425,9 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
indexText = d
}
}
- if ltks, err := tokenizer.Tokenize(indexText); err == nil && ltks != "" {
+ if ltks, err := tok.Tokenize(indexText); err == nil && ltks != "" {
doc.ContentLtks = ltks
- if sm, err := tokenizer.FineGrainedTokenize(ltks); err == nil && sm != "" {
+ if sm, err := tok.FineGrainedTokenize(ltks); err == nil && sm != "" {
doc.ContentSmLtks = sm
}
}
@@ -488,7 +493,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
}
}
// Per-variant fine-grained columns (conf/infinity_mapping.json §45–97).
- if err := applyVariantColumns(&doc, p); err != nil {
+ if err := applyVariantColumns(&doc, p, tok); err != nil {
return nil, err
}
docs = append(docs, doc)
@@ -499,7 +504,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
// applyVariantColumns emits the compile-specific columns defined in
// conf/infinity_mapping.json lines 45–77, driven by Product.Meta keys that
// each variant's build site populates. Unknown/absent keys are skipped.
-func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
+func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Tokenizer) error {
kind := metaString(p.Meta, "kind")
switch p.Variant {
@@ -537,7 +542,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
if err := doc.SetExtraValue("title_kwd", v); err != nil {
return err
}
- setTitleTokens(doc, v)
+ setTitleTokens(doc, v, tok)
}
if v := metaString(p.Meta, "page_type"); v != "" {
if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
@@ -706,11 +711,11 @@ func metaString(m map[string]any, key string) string {
// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no
// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into
// `docnm`, so the twin would make docnm map-order dependent.
-func setTitleTokens(doc *schema.ChunkDoc, title string) {
+func setTitleTokens(doc *schema.ChunkDoc, title string, tok tokenizer.Tokenizer) {
if title == "" {
return
}
- if tks, err := tokenizer.Tokenize(title); err == nil && tks != "" {
+ if tks, err := tok.Tokenize(title); err == nil && tks != "" {
doc.TitleTks = tks
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index c5ccf566a6..bc27a695a4 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -1463,7 +1463,7 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
Content: "Section body", ParentID: "page-id",
Meta: map[string]any{"kind": "section", "slug": "overview", "page_slug": "entity/alpha", "section_level": 1, "source_chunk_ids": []string{"c1"}},
}
- docs, err := productsToChunkDocs([]common.Product{page, section})
+ docs, err := productsToChunkDocs([]common.Product{page, section}, "")
if err != nil {
t.Fatalf("productsToChunkDocs: %v", err)
}
diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.go
index 1cb27a3fd3..a957f0dedc 100644
--- a/internal/ingestion/component/knowledge_compiler/storage_test.go
+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go
@@ -24,7 +24,7 @@ func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {
} {
t.Run(test.kind, func(t *testing.T) {
docs, err := productsToChunkDocs([]common.Product{{ID: "row", Kind: test.kind, Variant: test.variant, Content: "summary",
- Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}})
+ Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}}, "")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index ab41cfb45b..fa11a31f02 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -904,7 +904,8 @@ func declaredTitleTokens(chunks []schema.ChunkDoc) []string {
// Mirrors python tokenizer.py:130-185 and rag/nlp/__init__.py tokenize() /
// tokenize_chunks().
//
-// language sets the Snowball stemmer language, matching Python's
+// language selects the analyzer language (Snowball stemmer, or diacritics
+// folding without stemming for Slovak/Czech), matching Python's
// rag_tokenizer.tokenizer.set_language(language) call inside tokenize().
func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string) error {
tok := tokenizer.New(language)
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 3d128bf0d7..e7c2fc259e 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -85,7 +85,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
c := &Consumer{
scheduler: scheduler,
reader: engineReader{eng: engine.Get()},
- writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService()},
+ writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService(), resolveLanguage: datasetLanguageByID},
factory: defaultDeduperFactory,
contributions: newWikiContributionStore(engine.Get()),
ttl: 2 * time.Minute,
diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.go
index e0fae24d58..958bd744f8 100644
--- a/internal/ingestion/knowledge_compile/fake_engine_test.go
+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go
@@ -55,10 +55,10 @@ func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interf
// --- stubs: the reader/writer path under test never exercises these. ---
-func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _ string) ([]string, error) {
+func (f *fakeEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _, _, _ string) ([]string, error) {
f.insertedChunks = append(f.insertedChunks, chunks...)
return nil, nil
}
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index ad872543e1..6df368136f 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -107,7 +107,7 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, dataset
"content_with_weight": string(payload),
"available_int": 0,
}
- _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)
+ _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID, "")
return err
}
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index 3637b06ad2..b5a76d220d 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -18,6 +18,7 @@ package knowledge_compile
import (
"context"
"encoding/json"
+ "errors"
"strings"
"testing"
)
@@ -362,3 +363,26 @@ func TestProjectWikiGraphRowsSelfLoopSkipped(t *testing.T) {
t.Fatalf("the self-referencing entity must still exist")
}
}
+
+// TestProjectWikiGraphRowsResolvesDatasetLanguage pins that graph rows are
+// tokenized with the language of the dataset they are written to.
+func TestProjectWikiGraphRowsResolvesDatasetLanguage(t *testing.T) {
+ var resolved []string
+ w := engineWriter{resolveLanguage: func(_ context.Context, datasetID string) (string, error) {
+ resolved = append(resolved, datasetID)
+ return "slovak", nil
+ }}
+ if _, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{page("entity", "škola")}); err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ if len(resolved) != 1 || resolved[0] != kbForTest {
+ t.Fatalf("language resolved for %v, want [%s]", resolved, kbForTest)
+ }
+
+ w.resolveLanguage = func(context.Context, string) (string, error) {
+ return "", errors.New("dataset lookup failed")
+ }
+ if _, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{page("entity", "škola")}); err == nil {
+ t.Fatal("projection must fail when the dataset language cannot be resolved")
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.go
index 6c6921008e..804bd8e2e3 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go
@@ -22,6 +22,7 @@ import (
"fmt"
"strings"
+ "ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -36,6 +37,9 @@ const (
type wikiMapVersionStore struct {
engine engine.DocEngine
resolveVectorSize func(context.Context) (int, error)
+ // resolveLanguage returns the dataset language a missing Infinity chunk
+ // table is created with; nil reads it from the dataset row.
+ resolveLanguage func(ctx context.Context, datasetID string) (string, error)
}
func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error) {
@@ -84,7 +88,7 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k
"content_with_weight": string(state.Payload),
"available_int": 0,
}
- _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID)
+ _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID, "")
return err
}
@@ -124,12 +128,34 @@ func (s *wikiMapVersionStore) ensureInfinityChunkStore(ctx context.Context, tena
if vectorSize <= 0 {
return fmt.Errorf("resolve Wiki MAP vector size: got %d", vectorSize)
}
- if err := s.engine.CreateChunkStore(ctx, baseName, datasetID, vectorSize, ""); err != nil {
+ // The table is the dataset's chunk table, and Infinity fixes its fulltext
+ // analyzer here, so it has to be created with the dataset language.
+ resolveLanguage := s.resolveLanguage
+ if resolveLanguage == nil {
+ resolveLanguage = datasetLanguageByID
+ }
+ language, err := resolveLanguage(ctx, datasetID)
+ if err != nil {
+ return fmt.Errorf("resolve Wiki MAP dataset language: %w", err)
+ }
+ if err := s.engine.CreateChunkStore(ctx, baseName, datasetID, vectorSize, "", language); err != nil {
return fmt.Errorf("initialize Infinity chunk store for Wiki MAP: %w", err)
}
return nil
}
+// datasetLanguageByID reads the dataset's language, "" when it is unset.
+func datasetLanguageByID(ctx context.Context, datasetID string) (string, error) {
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)
+ if err != nil {
+ return "", err
+ }
+ if kb.Language == nil {
+ return "", nil
+ }
+ return *kb.Language, nil
+}
+
func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID, datasetID string, keys []string) (map[string][]byte, error) {
out := make(map[string][]byte, len(keys))
if len(keys) == 0 {
@@ -215,7 +241,7 @@ func (s *wikiMapVersionStore) PutWikiMapVersions(ctx context.Context, versions [
if len(rows) == 0 {
continue
}
- if _, err := s.engine.InsertChunks(ctx, rows, fmt.Sprintf("ragflow_%s", batch[0].TenantID), batch[0].DatasetID); err != nil {
+ if _, err := s.engine.InsertChunks(ctx, rows, fmt.Sprintf("ragflow_%s", batch[0].TenantID), batch[0].DatasetID, ""); err != nil {
return fmt.Errorf("save Wiki MAP versions: %w", err)
}
}
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
index 5165608a41..ceec7df3ab 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
@@ -36,6 +36,7 @@ type wikiMapStoreEngine struct {
chunkStoreChecks int
createdStores int
createdVectorSize int
+ createdLanguage string
}
func (e *wikiMapStoreEngine) GetType() string {
@@ -50,9 +51,10 @@ func (e *wikiMapStoreEngine) ChunkStoreExists(context.Context, string, string) (
return e.chunkStoreExists, nil
}
-func (e *wikiMapStoreEngine) CreateChunkStore(_ context.Context, _, _ string, vectorSize int, _ string) error {
+func (e *wikiMapStoreEngine) CreateChunkStore(_ context.Context, _, _ string, vectorSize int, _, language string) error {
e.createdStores++
e.createdVectorSize = vectorSize
+ e.createdLanguage = language
e.chunkStoreExists = true
return nil
}
@@ -126,7 +128,7 @@ func matchesStoredCompilationRow(row, filter map[string]interface{}) bool {
return true
}
-func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {
+func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName, datasetID, _ string) ([]string, error) {
e.insertBase = baseName
e.insertDataset = datasetID
for _, chunk := range chunks {
@@ -239,6 +241,14 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {
store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, func(context.Context) (int, error) {
return 768, nil
})
+ // The table is the dataset's chunk table, so it is created with the
+ // dataset language, which fixes Infinity's fulltext analyzer.
+ store.(*wikiMapVersionStore).resolveLanguage = func(_ context.Context, datasetID string) (string, error) {
+ if datasetID != "kb-1" {
+ t.Fatalf("language resolved for dataset %q", datasetID)
+ }
+ return "slovak", nil
+ }
version := kccommon.WikiMapVersion{
Key: "version-a", TenantID: "tenant-1", DatasetID: "kb-1",
DocumentID: "doc-1", ChunkID: "chunk-1", Payload: []byte(`{}`),
@@ -247,8 +257,8 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {
if err := store.PutWikiMapVersions(t.Context(), []kccommon.WikiMapVersion{version}); err != nil {
t.Fatalf("PutWikiMapVersions() error = %v", err)
}
- if engine.createdStores != 1 || engine.createdVectorSize != 768 {
- t.Fatalf("created store: count=%d vector_size=%d", engine.createdStores, engine.createdVectorSize)
+ if engine.createdStores != 1 || engine.createdVectorSize != 768 || engine.createdLanguage != "slovak" {
+ t.Fatalf("created store: count=%d vector_size=%d language=%q", engine.createdStores, engine.createdVectorSize, engine.createdLanguage)
}
}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 577c99b00b..371d95a3a1 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -126,6 +126,23 @@ type StructureBucket struct {
type engineWriter struct {
eng engine.DocEngine
commitService pageCommitter
+ // resolveLanguage returns the dataset language the dataset-level rows are
+ // tokenized with; nil tokenizes them as English.
+ resolveLanguage func(ctx context.Context, datasetID string) (string, error)
+}
+
+// datasetTokenizer returns a tokenizer in kb's language. Dataset-level rows
+// share the dataset's chunk table with the parsed chunks and are retrieved by
+// the same queries, so they tokenize like them.
+func (w engineWriter) datasetTokenizer(ctx context.Context, kb string) (tokenizer.Tokenizer, error) {
+ if w.resolveLanguage == nil {
+ return tokenizer.New(""), nil
+ }
+ language, err := w.resolveLanguage(ctx, kb)
+ if err != nil {
+ return tokenizer.Tokenizer{}, fmt.Errorf("resolve dataset language: %w", err)
+ }
+ return tokenizer.New(language), nil
}
type pageCommitter interface {
@@ -273,7 +290,7 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
zap.String("id", metaString(chunks[0], "id")),
)
}
- _, err := eng.InsertChunks(ctx, chunks, baseName, kb)
+ _, err := eng.InsertChunks(ctx, chunks, baseName, kb, "")
return err
})
}
@@ -483,6 +500,10 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
if err != nil {
return err
}
+ tok, err := w.datasetTokenizer(ctx, kb)
+ if err != nil {
+ return err
+ }
rows := make([]map[string]interface{}, 0, len(buckets))
for _, b := range buckets {
if !structureBucketWritable(b) {
@@ -529,7 +550,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
return err
}
payload := string(payloadBytes)
- contentLTKS, contentSMLTKS := tokenizeWikiGraphContent("", kccommon.PayloadDescription(payloadMap, nil))
+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(tok, "", kccommon.PayloadDescription(payloadMap, nil))
row := map[string]interface{}{
// Stable dataset-level id keyed on the template-scoped entity name or
// relation identity plus the raw compile kind.
@@ -617,7 +638,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
}
batch := rows[start:end]
jobs = append(jobs, func() error {
- _, err := eng.InsertChunks(ctx, batch, baseName, kb)
+ _, err := eng.InsertChunks(ctx, batch, baseName, kb, "")
return err
})
}
@@ -1359,17 +1380,17 @@ func wikiGraphBareKey(slug string) string {
// tokenizeWikiGraphContent tokenizes Python's entity search text,
// `slug + " " + description` (dataset_wiki_generator.py:750-773); the entity row
// gets no title token column.
-func tokenizeWikiGraphContent(slug, description string) (string, string) {
+func tokenizeWikiGraphContent(tok tokenizer.Tokenizer, slug, description string) (string, string) {
content := strings.TrimSpace(strings.Join([]string{slug, description}, " "))
if content == "" {
return "", ""
}
- contentLTKS, err := tokenizer.Tokenize(content)
+ contentLTKS, err := tok.Tokenize(content)
if err != nil || contentLTKS == "" {
contentLTKS = content
}
- contentSMLTKS, err := tokenizer.FineGrainedTokenize(contentLTKS)
+ contentSMLTKS, err := tok.FineGrainedTokenize(contentLTKS)
if err != nil || contentSMLTKS == "" {
contentSMLTKS = contentLTKS
}
@@ -1726,7 +1747,11 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string
// dangling edge, e.g. the target was pruned/deleted) is skipped, as is a
// self-loop (src == tgt) — the graph is rebuilt from a consistent read each
// time, so cross-batch dangling edges cannot accumulate.
-func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {
+func (w engineWriter) projectWikiGraphRows(ctx context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {
+ tok, err := w.datasetTokenizer(ctx, kb)
+ if err != nil {
+ return nil, err
+ }
// Sort pages deterministically by full slug before building the indexes so
// the outcome is reproducible across reprojections. The engine's row order
// is otherwise scroll-order dependent, which made bare-slug first-match
@@ -1786,7 +1811,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
if err != nil {
return nil, err
}
- contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Slug, p.Summary)
+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(tok, p.Slug, p.Summary)
rows = append(rows, map[string]interface{}{
"id": entityID,
"doc_id": kb,
@@ -1898,7 +1923,7 @@ func (w engineWriter) insertWikiGraphChunks(ctx context.Context, tenant, kb stri
}
batch := rows[start:end]
jobs = append(jobs, func() error {
- _, err := eng.InsertChunks(ctx, batch, baseName, kb)
+ _, err := eng.InsertChunks(ctx, batch, baseName, kb, "")
return err
})
}
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 4ed4e6353a..5538a5ac3d 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -305,7 +305,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
}
}
if len(rows) > 0 {
- if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID); err != nil {
+ if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID, ""); err != nil {
return err
}
}
diff --git a/internal/ingestion/task/pipeline_e2e_test.go b/internal/ingestion/task/pipeline_e2e_test.go
index 0225419ab3..90569a5492 100644
--- a/internal/ingestion/task/pipeline_e2e_test.go
+++ b/internal/ingestion/task/pipeline_e2e_test.go
@@ -114,10 +114,10 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da
_ = docEngine.DropChunkStore(ctx, baseName, datasetID)
// Create chunk store (note: vec dimension = 2 because our test chunks have q_2_vec)
- if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive"); err != nil {
+ if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive", ""); err != nil {
// If create failed, maybe it exists; try dropping and recreating
_ = docEngine.DropChunkStore(ctx, baseName, datasetID)
- if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive"); err != nil {
+ if err := docEngine.CreateChunkStore(ctx, baseName, datasetID, 2, "naive", ""); err != nil {
_ = docEngine.Close()
t.Fatalf("Could not create chunk store: %v", err)
}
@@ -256,7 +256,7 @@ func TestPipelineE2E_PipelineExecutor(t *testing.T) {
svc.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
insertChunksCalled = true
t.Logf("DocEngine InsertChunks called! baseName=%s datasetID=%s len(chunks)=%d", baseName, datasetID, len(chunks))
- ids, err := docEngine.InsertChunks(ctx, chunks, baseName, datasetID)
+ ids, err := docEngine.InsertChunks(ctx, chunks, baseName, datasetID, "")
if err != nil {
t.Logf("WARNING: InsertChunks err=%v", err)
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index f6534c323a..1b6b8caae2 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -106,7 +106,7 @@ func validateTaskContext(taskCtx *TaskContext) error {
// an engine that implements it can write chunks without waiting for an index
// refresh.
type noRefreshChunkInserter interface {
- InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)
+ InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)
}
// insertChunksForIngestion writes non-final chunks through the engine's
@@ -116,21 +116,24 @@ type noRefreshChunkInserter interface {
// Refreshing every batch adds unnecessary latency. The writer uses the regular
// inserter for its final batch, so all preceding batches become searchable
// before the document task is acknowledged and its completion event is sent.
-func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {
+//
+// The first write creates the chunk store, and on Infinity that fixes the
+// fulltext analyzer, so the dataset language has to travel with it.
+func insertChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
if bulk, ok := eng.(noRefreshChunkInserter); ok {
- return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID)
+ return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID, language)
}
- return eng.InsertChunks(ctx, chunks, baseName, datasetID)
+ return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
}
}
// insertFinalChunksForIngestion waits for the index refresh before returning.
// The final write makes every preceding no-refresh batch searchable before the
// document task is acknowledged and its completion event is published.
-func insertFinalChunksForIngestion(eng engine.DocEngine) InsertFunc {
+func insertFinalChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
- return eng.InsertChunks(ctx, chunks, baseName, datasetID)
+ return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
}
}
@@ -150,11 +153,11 @@ func NewPipelineExecutor(
canvasID: canvasID,
docBulkSize: docBulkSize,
indexWriter: newChunkIndexWriter(
- insertChunksForIngestion(engine.Get()),
+ insertChunksForIngestion(engine.Get(), datasetLanguage(taskCtx)),
fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),
taskCtx.Doc.KbID,
docBulkSize,
- ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get())),
+ ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get(), datasetLanguage(taskCtx))),
deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)
},
@@ -1267,3 +1270,13 @@ func injectDebugChunkCap(inputs map[string]any) map[string]any {
}
return inputs
}
+
+// datasetLanguage is the language of the dataset being ingested, or "" when it
+// is unset. It selects the fulltext analyzer on engines that fix it when the
+// chunk store is created.
+func datasetLanguage(taskCtx *TaskContext) string {
+ if taskCtx == nil || taskCtx.KB.Language == nil {
+ return ""
+ }
+ return *taskCtx.KB.Language
+}
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index f4dc141a45..5e18382371 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -736,6 +736,7 @@ func (s *ChatPipelineService) AsyncChat(
RerankModel: rerankModel,
EmbeddingModel: embModel,
Aggs: func() *bool { v := true; return &v }(),
+ Language: entity.KnowledgebasesLanguage(kbs),
}
result, retErr := retrievalSvc.Retrieval(ctx, req)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index a867a9af81..c0b45fb063 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -101,8 +101,8 @@ type ChunkService struct {
storeChunkImageFunc func(string, string, []byte, string) error
removeChunkImageFunc func(string, string) error
markWikiDirtyFunc func(tenantID, datasetID, documentID string, chunkIDs []string)
- tokenizeFunc func(string) (string, error)
- fineGrainedTokenizeFunc func(string) (string, error)
+ tokenizeFunc func(text, language string) (string, error)
+ fineGrainedTokenizeFunc func(text, language string) (string, error)
numTokensFunc func(string) int
}
@@ -416,6 +416,7 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrieval
RankFeature: &labels,
EmbeddingModel: embeddingModel,
Highlight: req.Highlight,
+ Language: entity.KnowledgebasesLanguage(kbRecords),
}
// Call RetrievalService to perform retrieval
@@ -851,7 +852,7 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
if queryBuilder == nil {
queryBuilder = nlp.NewQueryBuilder()
}
- if matchText, _ := queryBuilder.Question(keywords, "", 0.3); matchText != nil {
+ if matchText, _ := queryBuilder.Question(keywords, "", 0.3, datasetLanguageOf(kb)); matchText != nil {
matchExprs = append(matchExprs, matchText)
}
}
@@ -1094,6 +1095,7 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk
return err
}
targetTenantID := kb.TenantID
+ language := datasetLanguageOf(kb)
// Verify document belongs to dataset
docDAO := dao.NewDocumentDAO()
@@ -1139,8 +1141,8 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk
// Tokenize content
contentStr := d["content_with_weight"].(string)
- d["content_ltks"], _ = tokenizer.Tokenize(contentStr)
- d["content_sm_ltks"], _ = tokenizer.FineGrainedTokenize(d["content_ltks"].(string))
+ d["content_ltks"], _ = s.tokenize(contentStr, language)
+ d["content_sm_ltks"], _ = s.fineGrainedTokenize(d["content_ltks"].(string), language)
// Important keywords - convert []string to []interface{} for transformChunkFields
if req.ImportantKwd != nil {
@@ -1351,20 +1353,21 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
chunkID := common.ChunkID(req.DocumentID, req.Content)
indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)
- contentLtks, err := s.tokenize(req.Content)
+ language := datasetLanguageOf(kb)
+ contentLtks, err := s.tokenize(req.Content, language)
if err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content: %v", err)}
}
- contentSmLtks, err := s.fineGrainedTokenize(contentLtks)
+ contentSmLtks, err := s.fineGrainedTokenize(contentLtks, language)
if err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize content fine-grained: %v", err)}
}
- importantTks, err := s.tokenize(strings.Join(req.ImportantKeywords, " "))
+ importantTks, err := s.tokenize(strings.Join(req.ImportantKeywords, " "), language)
if err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize important keywords: %v", err)}
}
questionKwd := filterTrimmedStrings(req.Questions)
- questionTks, err := s.tokenize(strings.Join(req.Questions, "\n"))
+ questionTks, err := s.tokenize(strings.Join(req.Questions, "\n"), language)
if err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("tokenize questions: %v", err)}
}
@@ -1447,7 +1450,7 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
ctx, cancel := context.WithTimeout(ctx, 600*time.Second)
defer cancel()
- if _, err = s.docEngine.InsertChunks(ctx, []map[string]interface{}{chunkData}, indexName, req.DatasetID); err != nil {
+ if _, err = s.docEngine.InsertChunks(ctx, []map[string]interface{}{chunkData}, indexName, req.DatasetID, datasetLanguageOf(kb)); err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("insert chunk: %v", err)}
}
@@ -1632,18 +1635,20 @@ func filterTrimmedStrings(values []string) []string {
return filtered
}
-func (s *ChunkService) tokenize(text string) (string, error) {
+// tokenize tokenizes a manually written chunk field with the dataset language,
+// so it is indexed the way the dataset's parsed chunks are and its queries fold.
+func (s *ChunkService) tokenize(text, language string) (string, error) {
if s.tokenizeFunc != nil {
- return s.tokenizeFunc(text)
+ return s.tokenizeFunc(text, language)
}
- return tokenizer.Tokenize(text)
+ return tokenizer.New(language).Tokenize(text)
}
-func (s *ChunkService) fineGrainedTokenize(text string) (string, error) {
+func (s *ChunkService) fineGrainedTokenize(text, language string) (string, error) {
if s.fineGrainedTokenizeFunc != nil {
- return s.fineGrainedTokenizeFunc(text)
+ return s.fineGrainedTokenizeFunc(text, language)
}
- return tokenizer.FineGrainedTokenize(text)
+ return tokenizer.New(language).FineGrainedTokenize(text)
}
func (s *ChunkService) numTokens(text string) int {
@@ -1845,3 +1850,14 @@ func releaseChunkImageLock(key string) {
delete(chunkImageLocks.locks, key)
}
}
+
+// datasetLanguageOf is the dataset's language, or "" when unset. A query is
+// folded the way its dataset was indexed, and on engines whose fulltext
+// analyzer is fixed when the chunk store is created a write that may create it
+// has to carry the language too.
+func datasetLanguageOf(kb *entity.Knowledgebase) string {
+ if kb == nil || kb.Language == nil {
+ return ""
+ }
+ return *kb.Language
+}
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 7eebb0f24b..c19eed1212 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -1067,6 +1067,8 @@ func TestAddChunkSuccess(t *testing.T) {
engine := &addChunkTestEngine{}
var incrementTokenNum, incrementChunkNum int64
+ language := "Slovak"
+ var tokenizeLanguages []string
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
@@ -1078,7 +1080,7 @@ func TestAddChunkSuccess(t *testing.T) {
return permission.ErrPermissionDenied
},
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
- return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
+ return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language}, nil
},
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
@@ -1098,9 +1100,15 @@ func TestAddChunkSuccess(t *testing.T) {
incrementChunkNum = chunkNum
return nil
},
- tokenizeFunc: func(text string) (string, error) { return text, nil },
- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },
- numTokensFunc: func(text string) int { return len(text) },
+ tokenizeFunc: func(text, lang string) (string, error) {
+ tokenizeLanguages = append(tokenizeLanguages, lang)
+ return text, nil
+ },
+ fineGrainedTokenizeFunc: func(text, lang string) (string, error) {
+ tokenizeLanguages = append(tokenizeLanguages, lang)
+ return text + "_fg", nil
+ },
+ numTokensFunc: func(text string) int { return len(text) },
}
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
@@ -1142,6 +1150,19 @@ func TestAddChunkSuccess(t *testing.T) {
if inserted["img_id"] != nil {
t.Fatalf("did not expect image id in inserted chunk: %#v", inserted)
}
+ // A manual chunk is tokenized like the dataset's parsed chunks, and the
+ // write that may create the chunk store carries the dataset language.
+ if len(tokenizeLanguages) != 4 {
+ t.Fatalf("tokenizer calls = %v, want 4", tokenizeLanguages)
+ }
+ for _, lang := range tokenizeLanguages {
+ if lang != language {
+ t.Fatalf("tokenized with language %q, want %q", lang, language)
+ }
+ }
+ if engine.insertLanguage != language {
+ t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)
+ }
vec, ok := inserted["q_2_vec"].([]float64)
if !ok {
t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])
@@ -1190,8 +1211,8 @@ func TestAddChunkPersistsTagKwdOnlyWhenProvided(t *testing.T) {
return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
},
incrementChunkStatsFunc: func(_, _ string, _, _ int64, _ float64) error { return nil },
- tokenizeFunc: func(text string) (string, error) { return text, nil },
- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },
+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
}
return svc, engine
@@ -1337,8 +1358,8 @@ func TestAddChunkImageAndTagFeatureValidation(t *testing.T) {
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
},
- tokenizeFunc: func(text string) (string, error) { return text, nil },
- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },
+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
@@ -1435,8 +1456,8 @@ func TestAddChunkIncrementsStatsAfterInsert(t *testing.T) {
incrementCalls++
return nil
},
- tokenizeFunc: func(text string) (string, error) { return text, nil },
- fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },
+ tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
+ fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
}
@@ -1959,11 +1980,11 @@ type parseTestDocEngine struct {
chunkStoreExists bool // if true, ChunkStoreExists returns true
}
-func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
@@ -2064,13 +2085,15 @@ type addChunkTestEngine struct {
insertedChunks []map[string]interface{}
insertIndex string
insertDataset string
+ insertLanguage string
insertErr error
}
-func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
+func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
e.insertedChunks = chunks
e.insertIndex = baseName
e.insertDataset = datasetID
+ e.insertLanguage = language
return nil, e.insertErr
}
@@ -2377,10 +2400,10 @@ type switchChunksEngineMock struct {
updateCalls []updateChunksCall
}
-func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string) error {
+func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (m *switchChunksEngineMock) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, indexName string, datasetID string) error {
diff --git a/internal/service/dataset/fake_doc_engine_test.go b/internal/service/dataset/fake_doc_engine_test.go
index 7fa80346a8..6c98f3de85 100644
--- a/internal/service/dataset/fake_doc_engine_test.go
+++ b/internal/service/dataset/fake_doc_engine_test.go
@@ -11,10 +11,10 @@ import (
// fakeChatDocEngine is a no-op DocEngine implementation for tests.
type fakeChatDocEngine struct{}
-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 4c8351921a..14e16883c5 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -309,6 +309,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
RankFeature: &labels,
EmbeddingModel: embeddingModel,
Highlight: req.Highlight,
+ Language: entity.KnowledgebasesLanguage(kbRecords),
}
retrievalReq.Filter = knowledgeCompilationFilter(req.IncludeCompiledChunks, req.KnowledgeCompilationScope)
if len(docIDs) > 0 && (req.IncludeCompiledChunks == nil || *req.IncludeCompiledChunks) && req.KnowledgeCompilationScope != "document" {
diff --git a/internal/service/dataset_artifact_keyword_integration_test.go b/internal/service/dataset_artifact_keyword_integration_test.go
index 8ea9870d14..45bf0e066a 100644
--- a/internal/service/dataset_artifact_keyword_integration_test.go
+++ b/internal/service/dataset_artifact_keyword_integration_test.go
@@ -66,8 +66,8 @@ func TestWikiKeywordFilter_Integration(t *testing.T) {
row["kb_id"] = kbID
}
// Best effort: the store may already exist from a previous run.
- _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive")
- if _, err := de.InsertChunks(ctx, rows, idx, kbID); err != nil {
+ _ = de.CreateChunkStore(ctx, idx, kbID, 1024, "naive", "")
+ if _, err := de.InsertChunks(ctx, rows, idx, kbID, ""); err != nil {
t.Fatalf("seed wiki rows: %v", err)
}
t.Cleanup(func() {
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index cc881aeea3..d5f8f74c51 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -619,9 +619,18 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
// so a raw multi-token keyword (e.g. "桃园三结义") never equals a stored
// token and silently matches nothing. Tokenize the query like the other
// lexical callers (see navKeywordExpressions); the raw input stays on
- // original_query for engines that analyze the query themselves.
+ // original_query for engines that analyze the query themselves. The
+ // entity rows are tokenized in the dataset language, so the query is too.
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)
+ if err != nil {
+ return nil, err
+ }
+ kbLanguage := ""
+ if kb.Language != nil {
+ kbLanguage = *kb.Language
+ }
originalQuery := keywords
- if tokenized, err := tokenizer.Tokenize(keywords); err == nil && tokenized != "" {
+ if tokenized, err := tokenizer.New(kbLanguage).Tokenize(keywords); err == nil && tokenized != "" {
keywords = tokenized
}
matchExprs = []interface{}{&types.MatchTextExpr{
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 38e005727b..5cd86a8198 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -138,7 +138,7 @@ func (s *DocumentService) UpdateDocument(ctx context.Context, id string, req *Up
if err != nil {
return common.CodeServerError, err
}
- if err = s.updateDocumentNameOnly(ctx, document, kb.TenantID, *req.Name); err != nil {
+ if err = s.updateDocumentNameOnly(ctx, document, kb, *req.Name); err != nil {
return common.CodeServerError, err
}
return common.CodeSuccess, nil
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 77521913ad..e313c7f8df 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -145,7 +145,7 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat
}
if present["name"] && req.Name != nil && (doc.Name == nil || *req.Name != *doc.Name) {
- if err = s.updateDocumentNameOnly(ctx, doc, kb.TenantID, *req.Name); err != nil {
+ if err = s.updateDocumentNameOnly(ctx, doc, kb, *req.Name); err != nil {
return nil, common.CodeDataError, err
}
}
@@ -453,7 +453,9 @@ func validateMetaFields(meta map[string]any) error {
return nil
}
-func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entity.Document, tenantID, newName string) error {
+// updateDocumentNameOnly renames a document and re-tokenizes its chunks' title
+// fields with the dataset language, the way ingestion tokenized them.
+func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, newName string) error {
if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"name": newName}); err != nil {
return errors.New("database error (Document rename)")
}
@@ -470,9 +472,14 @@ func (s *DocumentService) updateDocumentNameOnly(ctx context.Context, doc *entit
return nil
}
- titleTks, _ := tokenizer.Tokenize(newName)
- titleSmTks, _ := tokenizer.FineGrainedTokenize(titleTks)
- indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ language := ""
+ if kb.Language != nil {
+ language = *kb.Language
+ }
+ tok := tokenizer.New(language)
+ titleTks, _ := tok.Tokenize(newName)
+ titleSmTks, _ := tok.FineGrainedTokenize(titleTks)
+ indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)
return s.docEngine.UpdateChunks(
ctx,
map[string]interface{}{"doc_id": doc.ID},
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index d5f22a171f..80b8a7b618 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -151,10 +151,10 @@ func (f *fakeUploadStorage) Close() error { return nil }
type fakeChatDocEngine struct{}
-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
diff --git a/internal/service/memory.go b/internal/service/memory.go
index 7406012347..b76d02cc88 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -1448,7 +1448,7 @@ func memoryMessageTextExpr(question string, similarityThreshold float64) *engine
if queryBuilder == nil {
queryBuilder = nlp.NewQueryBuilder()
}
- if built, _ := queryBuilder.Question(question, "messages", similarityThreshold); built != nil {
+ if built, _ := queryBuilder.Question(question, "messages", similarityThreshold, ""); built != nil {
matchText.MatchingText = built.MatchingText
matchText.ExtraOptions = built.ExtraOptions
if matchText.ExtraOptions == nil {
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index b79593a3ca..ca7f51b6ff 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -357,7 +357,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
return fmt.Errorf("check message index: %w", err)
}
if !exists {
- if err := s.memories.docEngine.CreateChunkStore(ctx, indexName, mem.ID, vectorDim, ""); err != nil {
+ if err := s.memories.docEngine.CreateChunkStore(ctx, indexName, mem.ID, vectorDim, "", ""); err != nil {
return fmt.Errorf("create message index: %w", err)
}
}
@@ -365,7 +365,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
for i, message := range messages {
docs[i] = mapStringAny(message)
}
- if _, err := s.memories.docEngine.InsertChunks(ctx, docs, indexName, mem.ID); err != nil {
+ if _, err := s.memories.docEngine.InsertChunks(ctx, docs, indexName, mem.ID, ""); err != nil {
return fmt.Errorf("insert message into memory: %w", err)
}
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index 04b51e18ee..a94349b4d4 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -856,7 +856,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"content_sm_ltks": contentSmLtks,
"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
}
- _, err = de.InsertChunks(ctx, []map[string]interface{}{row}, idx, in.KbID)
+ _, err = de.InsertChunks(ctx, []map[string]interface{}{row}, idx, in.KbID, "")
return err
}
@@ -910,7 +910,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"content_sm_ltks": clusterContentSmLtks,
"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
}
- _, err = de.InsertChunks(ctx, []map[string]interface{}{clusterRow}, idx, in.KbID)
+ _, err = de.InsertChunks(ctx, []map[string]interface{}{clusterRow}, idx, in.KbID, "")
if err != nil {
return err
}
@@ -935,7 +935,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"content_sm_ltks": contentSmLtks,
"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
}
- _, err = de.InsertChunks(ctx, []map[string]interface{}{docRow}, idx, in.KbID)
+ _, err = de.InsertChunks(ctx, []map[string]interface{}{docRow}, idx, in.KbID, "")
return err
}
@@ -1308,7 +1308,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
if vec := pickAnyVector(children, i2boolForTarget(spl.name, splitA)); len(vec) > 0 {
row["q_"+fmt.Sprintf("%d", len(vec))+"_vec"] = f32ToF64Slice(vec)
}
- if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID); err != nil {
+ if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID, ""); err != nil {
return err
}
}
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index d484e54ced..fc17df3cdc 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -29,7 +29,7 @@ type memNavEngine struct {
func newMemNavEngine() *memNavEngine { return &memNavEngine{} }
-func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string) ([]string, error) {
+func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string, _ string) ([]string, error) {
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
// A caller-provided row id is kept, exactly as the real engines do it
@@ -134,7 +134,7 @@ func (m *memNavEngine) Close() error { return nil }
func (m *memNavEngine) Ping(context.Context) error { return nil }
func (m *memNavEngine) GetType() string { return "mem" }
func (m *memNavEngine) SupportsPageRank() bool { return false }
-func (m *memNavEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (m *memNavEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
func (m *memNavEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {
@@ -503,7 +503,7 @@ func TestNavService_ListClusters_SearchReturnsPrunedForest(t *testing.T) {
"content_with_weight": `{"type":"nav_doc","description":"孤儿概要"}`,
},
}
- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {
t.Fatal(err)
}
@@ -807,7 +807,7 @@ func TestNavService_Search_DocScope(t *testing.T) {
row(nav.TypeNavCluster, "kb1", "covering d1+d2", []string{"d1", "d2"}, 0.4),
row(nav.TypeNavCluster, "kb1", "covering d2 only", []string{"d2"}, 1),
}
- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {
t.Fatal(err)
}
q := make([]float32, dim)
@@ -905,7 +905,7 @@ func TestNavService_Search_DocScopeBeyondAnyPoolSize(t *testing.T) {
"q_4_vec": vec(0.4),
},
)
- if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1", ""); err != nil {
t.Fatal(err)
}
q := make([]float32, dim)
@@ -1103,7 +1103,7 @@ func TestNavService_UpsertDoc_PlacementBySimilarity(t *testing.T) {
"doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,
"q_1024_vec": vec,
}
- if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1", ""); err != nil {
t.Fatal(err)
}
}
@@ -1254,7 +1254,7 @@ func TestNavService_FindBestCluster_DeepestMatchWins(t *testing.T) {
"doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,
"q_1024_vec": vec,
}
- if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1", ""); err != nil {
t.Fatal(err)
}
}
@@ -1390,7 +1390,7 @@ func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {
"doc_id": "d9", "title_kwd": "出师表全文内容与赏析",
"content_with_weight": `{"type":"nav_doc"}`,
},
- }, "", "kb1"); err != nil {
+ }, "", "kb1", ""); err != nil {
t.Fatal(err)
}
got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d9"})
@@ -1409,7 +1409,7 @@ func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {
"doc_id": rawID, "title_kwd": rawID,
"content_with_weight": `{"type":"nav_doc"}`,
},
- }, "", "kb1"); err != nil {
+ }, "", "kb1", ""); err != nil {
t.Fatal(err)
}
got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{rawID})
@@ -1505,7 +1505,7 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {
"doc_count_int": 1,
})
}
- if _, err := eng.InsertChunks(t.Context(), rows, idx, "kb1"); err != nil {
+ if _, err := eng.InsertChunks(t.Context(), rows, idx, "kb1", ""); err != nil {
t.Fatal(err)
}
if err := ns.maybeSplitCluster(t.Context(), "t1", "kb1", clusterName, ""); err != nil {
diff --git a/internal/service/nlp/query_builder.go b/internal/service/nlp/query_builder.go
index 3b9fb59812..1a471241cc 100644
--- a/internal/service/nlp/query_builder.go
+++ b/internal/service/nlp/query_builder.go
@@ -243,11 +243,15 @@ func (qb *QueryBuilder) NeedFineGrainedTokenize(tk string) bool {
}
// Question builds a full-text query expression based on input text.
-// References Python FulltextQueryer.question method.
-func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*types.MatchTextExpr, []string) {
+// References Python FulltextQueryer.question method. language is the dataset
+// language ("" = English) the query is tokenized with, so query tokens match
+// index tokens.
+func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64, language string) (*types.MatchTextExpr, []string) {
// originalQuery stores the original input text for later use in query expression.
originalQuery := txt
+ tok := tokenizer.New(language)
+
// Add space between English and Chinese
txtWithSpaces := qb.AddSpaceBetweenEngZh(txt)
@@ -278,7 +282,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
txtFinal := qb.RmWWW(txtNoStopWords)
// Tokenize using rag_tokenizer
- tokenized, err := tokenizer.Tokenize(txtFinal)
+ tokenized, err := tok.Tokenize(txtFinal)
if err != nil {
// If tokenizer fails, use simple split
tokenized = txtFinal
@@ -297,7 +301,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
// Calculate term weights using TermWeightDealer
// Reference: rag/nlp/query.py L56
// tws holds the term weight list for each token.
- tws := qb.termWeight.Weights(tks, false)
+ tws := qb.termWeight.Weights(tks, false, language)
// Clean tokens and filter
// Reference: rag/nlp/query.py L57-60
@@ -435,7 +439,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
// Get term weights
// termWeightList holds term weights for the current segment.
- termWeightList := qb.termWeight.Weights([]string{segment}, true)
+ termWeightList := qb.termWeight.Weights([]string{segment}, true, language)
// Lookup synonyms
// syns are synonyms for the current segment.
@@ -463,7 +467,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
// sm holds fine‑grained tokens for the current term.
var sm []string
if qb.NeedFineGrainedTokenize(term) {
- fineGrained, err := tokenizer.FineGrainedTokenize(term)
+ fineGrained, err := tok.FineGrainedTokenize(term)
if err == nil && fineGrained != "" {
sm = strings.Fields(fineGrained)
}
@@ -512,7 +516,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
if s == "" {
continue
}
- fg, err := tokenizer.FineGrainedTokenize(s)
+ fg, err := tok.FineGrainedTokenize(s)
if err == nil && fg != "" {
// Quote if contains space
if strings.Contains(fg, " ") {
@@ -566,7 +570,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
// Add proximity query if multiple tokens
if len(termWeightList) > 1 {
// tokenized is the tokenized version of the segment.
- tokenized, _ := tokenizer.Tokenize(segment)
+ tokenized, _ := tok.Tokenize(segment)
if tokenized != "" {
tmsStr += fmt.Sprintf(` ("%s"~2)^1.5`, tokenized)
}
@@ -579,7 +583,7 @@ func (qb *QueryBuilder) Question(txt string, tbl string, minMatch float64) (*typ
for _, s := range syns {
s = qb.SubSpecialChar(s)
if s != "" {
- tokenized, _ := tokenizer.Tokenize(s)
+ tokenized, _ := tok.Tokenize(s)
if tokenized != "" {
synParts = append(synParts, fmt.Sprintf(`"%s"`, tokenized))
}
diff --git a/internal/service/nlp/query_builder_language_test.go b/internal/service/nlp/query_builder_language_test.go
new file mode 100644
index 0000000000..a2648c4e99
--- /dev/null
+++ b/internal/service/nlp/query_builder_language_test.go
@@ -0,0 +1,79 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//go:build manual
+
+package nlp
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/tokenizer"
+)
+
+// TestQueryBuilder_Question_FoldingLanguage verifies that a Slovak query is
+// tokenized by the analyzer with diacritics folded to ASCII, so every emitted
+// keyword and the query expression match the folded index tokens.
+func TestQueryBuilder_Question_FoldingLanguage(t *testing.T) {
+ // Folding happens inside the analyzer, so the real dictionaries must be
+ // loaded (manual tier, like the analyzer-level tests in internal/tokenizer).
+ if err := tokenizer.Init(&tokenizer.PoolConfig{MinSize: 1, MaxSize: 1, IdleTimeout: 5 * time.Second, AcquireTimeout: 5 * time.Second}); err != nil {
+ t.Fatalf("Failed to initialize pool: %v", err)
+ }
+ defer tokenizer.Close()
+
+ qb := NewQueryBuilder()
+ hasDiacritics := func(s string) bool {
+ for _, r := range s {
+ if r > 127 {
+ return true
+ }
+ }
+ return false
+ }
+
+ expr, keywords := qb.Question("daňové priznanie pre živnostníkov", "test", 0.5, "Slovak")
+ if expr == nil {
+ t.Fatal("Question(Slovak) returned nil expr")
+ }
+ if hasDiacritics(expr.MatchingText) {
+ t.Errorf("Question(Slovak) query contains diacritics: %q", expr.MatchingText)
+ }
+ for _, k := range keywords {
+ if hasDiacritics(k) {
+ t.Errorf("Question(Slovak) keyword contains diacritics: %q", k)
+ }
+ }
+ if !strings.Contains(expr.MatchingText, "danove") {
+ t.Errorf("Question(Slovak) query missing folded token 'danove': %q", expr.MatchingText)
+ }
+
+ // original_query must stay untouched for highlighting.
+ if orig, _ := expr.ExtraOptions["original_query"].(string); orig != "daňové priznanie pre živnostníkov" {
+ t.Errorf("Question(Slovak) original_query = %q, want the unfolded input", orig)
+ }
+
+ // Non-folding languages keep accents untouched.
+ exprEn, _ := qb.Question("daňové priznanie pre živnostníkov", "test", 0.5, "")
+ if exprEn == nil {
+ t.Fatal("Question(English) returned nil expr")
+ }
+ if !hasDiacritics(exprEn.MatchingText) {
+ t.Errorf("Question(English) unexpectedly folded diacritics: %q", exprEn.MatchingText)
+ }
+}
diff --git a/internal/service/nlp/query_builder_test.go b/internal/service/nlp/query_builder_test.go
index 33233af3e1..233a2b7c4c 100644
--- a/internal/service/nlp/query_builder_test.go
+++ b/internal/service/nlp/query_builder_test.go
@@ -296,7 +296,7 @@ func TestQueryBuilder_Question(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- expr, keywords := qb.Question(tt.txt, tt.tbl, tt.minMatch)
+ expr, keywords := qb.Question(tt.txt, tt.tbl, tt.minMatch, "")
if tt.expectNil && expr != nil {
t.Errorf("Question(%q) expected nil expr, got %v", tt.txt, expr)
}
@@ -474,7 +474,7 @@ func TestQueryBuilder_Question_SynonymSingleQuote(t *testing.T) {
qb.synonym.dictionary = map[string][]string{
"cat": {"cat-o'-nine-tails", "'"},
}
- expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)
+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3, "")
if expr == nil {
t.Fatal("Question returned nil expr")
}
@@ -518,7 +518,7 @@ func TestQueryBuilder_Question_SynonymQuestionMark(t *testing.T) {
qb.synonym.dictionary = map[string][]string{
"cat": {"what?cat"},
}
- expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3)
+ expr, keywords := qb.Question("black cat sleeping near warm fire", "test", 0.3, "")
if expr == nil {
t.Fatal("Question returned nil expr")
}
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index bf26fa7375..abbbee19ca 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -65,10 +65,11 @@ func Rerank(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
+ language string,
) (sim []float64, tsim []float64, vsim []float64, err error) {
// If reranker model is provided and there are results, use model reranking
if rerankModel != nil && total > 0 {
- return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature)
+ return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)
}
// Otherwise, use fallback logic based on engine type
@@ -84,7 +85,7 @@ func Rerank(
}
// For Elasticsearch: need to perform reranking and apply rank features
- sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature)
+ sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature, language)
return sim, tsim, vsim, nil
}
@@ -100,6 +101,7 @@ func RerankByModel(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
+ language string,
) (sim []float64, tsim []float64, vsim []float64, err error) {
if chunks == nil || len(chunks) == 0 {
return []float64{}, []float64{}, []float64{}, nil
@@ -112,7 +114,7 @@ func RerankByModel(
// Extract keywords from query
keywords := []string{}
if qb != nil {
- _, keywords = qb.Question(query, "qa", 0.6)
+ _, keywords = qb.Question(query, "qa", 0.6, language)
}
common.Info("RerankByModel keywords extracted", zap.Any("keywords", keywords))
@@ -162,7 +164,7 @@ func RerankByModel(
}
// Calculate token similarity
- tsim = TokenSimilarity(keywords, insTw, qb)
+ tsim = TokenSimilarity(keywords, insTw, qb, language)
// Get similarity scores from reranker model
rerankResponse, err := rerankModel.Rerank(ctx, models.RerankRequest{Query: query, Documents: docs}, &models.RerankConfig{}, nil)
@@ -277,6 +279,7 @@ func RerankStandard(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
+ language string,
) (sim []float64, tsim []float64, vsim []float64) {
chunkCount := len(chunks)
if chunkCount == 0 {
@@ -287,7 +290,7 @@ func RerankStandard(
// Compute keywords fresh from query
if qb != nil && len(keywords) == 0 {
- _, keywords = qb.Question(query, "qa", 0.6)
+ _, keywords = qb.Question(query, "qa", 0.6, language)
}
common.Info("RerankStandard keywords", zap.Any("keywords", keywords))
@@ -332,7 +335,7 @@ func RerankStandard(
}
// Calculate hybrid similarity
- sim, tsim, vsim = HybridSimilarity(questionVector, insEmbd, keywords, insTw, tkWeight, vtWeight, qb)
+ sim, tsim, vsim = HybridSimilarity(questionVector, insEmbd, keywords, insTw, tkWeight, vtWeight, qb, language)
// Apply rank feature scores (tag_score * 10 + pagerank)
// Always apply pageranks, even when rankFeature is nil/empty
@@ -376,6 +379,7 @@ func HybridSimilarity(
btkss [][]string,
tkWeight, vtWeight float64,
qb *QueryBuilder,
+ language string,
) (sim []float64, tsim []float64, vsim []float64) {
// Calculate vector similarities using cosine similarity
vsim = make([]float64, len(bvecs))
@@ -383,7 +387,7 @@ func HybridSimilarity(
vsim[i] = cosineSimilarity(avec, bvec)
}
- tsim = TokenSimilarity(atks, btkss, qb)
+ tsim = TokenSimilarity(atks, btkss, qb, language)
// Check if all vector similarities are zero
allZero := true
@@ -409,11 +413,11 @@ func HybridSimilarity(
}
// TokenSimilarity calculates token-based similarity
-func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder) []float64 {
- atksDict, atksKeyOrder := tokensToDict(atks, qb)
+func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder, language string) []float64 {
+ atksDict, atksKeyOrder := tokensToDict(atks, qb, language)
btkssDicts := make([]map[string]float64, len(btkss))
for i, btks := range btkss {
- btkssDicts[i], _ = tokensToDict(btks, qb)
+ btkssDicts[i], _ = tokensToDict(btks, qb, language)
}
similarities := make([]float64, len(btkssDicts))
@@ -426,13 +430,13 @@ func TokenSimilarity(atks []string, btkss [][]string, qb *QueryBuilder) []float6
// tokensToDict converts tokens to a weighted dictionary.
// Also returns the insertion order of keys to match Python's dict insertion order.
-func tokensToDict(tks []string, qb *QueryBuilder) (map[string]float64, []string) {
+func tokensToDict(tks []string, qb *QueryBuilder, language string) (map[string]float64, []string) {
d := make(map[string]float64)
var keyOrder []string
if qb == nil || qb.termWeight == nil {
return d, keyOrder
}
- wts := qb.termWeight.Weights(tks, false)
+ wts := qb.termWeight.Weights(tks, false, language)
for i, tw := range wts {
t := tw.Term
@@ -921,6 +925,7 @@ func RerankWithKNN(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
+ language string,
) (sim []float64, tsim []float64, vsim []float64) {
if len(ids) == 0 {
return []float64{}, []float64{}, []float64{}
@@ -947,7 +952,7 @@ func RerankWithKNN(
// Extract keywords from query
keywords := []string{}
if qb != nil {
- _, keywords = qb.Question(query, "qa", 0.6)
+ _, keywords = qb.Question(query, "qa", 0.6, language)
}
common.InfoCtx(ctx, "RerankWithKNN keywords", zap.Any("keywords", keywords))
@@ -987,7 +992,7 @@ func RerankWithKNN(
}
// Calculate token similarity
- tsim = TokenSimilarity(keywords, insTw, qb)
+ tsim = TokenSimilarity(keywords, insTw, qb, language)
common.InfoCtx(ctx, "RerankWithKNN tsim", zap.Float64s("tsim", tsim))
// Build vector similarity from knnScores - matches Python's np.array([knn_scores.get(chunk_id, 0.0) for chunk_id in sres.ids])
diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.go
index 769554fcb9..0a1edfead6 100644
--- a/internal/service/nlp/reranker_question_tks_test.go
+++ b/internal/service/nlp/reranker_question_tks_test.go
@@ -57,6 +57,7 @@ func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {
"content_ltks",
nil,
nil,
+ "",
)
if len(driver.documents) != 1 {
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index e1a3f45696..9170eb9189 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -73,6 +73,9 @@ type RetrievalRequest struct {
// bridge a wording gap the query's own words cannot cross; the scoring pass
// then uses the engine's kNN score directly instead of re-scoring tokens.
VectorOnly bool
+ // Language is the dataset language ("" = English); diacritic-folding
+ // languages (Slovak, Czech) fold the query so its tokens match index tokens.
+ Language string
}
// RetrievalResult result from retrieval search
@@ -178,6 +181,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
AllowDenseFallback: req.AllowDenseFallback,
VectorOnly: req.VectorOnly,
Filter: req.Filter,
+ Language: req.Language,
}
searchResult, err := s.Search(ctx, searchReq)
if err != nil {
@@ -477,6 +481,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
"content_ltks",
qb,
*req.RankFeature,
+ req.Language,
)
}
@@ -507,6 +512,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
"content_ltks",
qb,
*req.RankFeature,
+ req.Language,
)
return sim, tsim, vsim, nil
}
@@ -563,6 +569,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
"content_ltks",
qb,
*req.RankFeature,
+ req.Language,
)
return sim, tsim, vsim, nil
}
@@ -579,6 +586,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
"content_ltks",
qb,
*req.RankFeature,
+ req.Language,
)
return sim, tsim, vsim, nil
}
@@ -605,6 +613,8 @@ type RetrievalSearchRequest struct {
EmbeddingModel *models.EmbeddingModel
VectorSimilarityWeight *float64
AllowDenseFallback *bool
+ // Language is the dataset language ("" = English).
+ Language string
}
func buildInfinityFusionExpr(topn int, vectorSimilarityWeight *float64) *types.FusionExpr {
@@ -750,7 +760,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// Non-empty question
// Compute keywords via QueryBuilder
- matchText, keywords := GetQueryBuilder().Question(req.Question, "", minMatch(req.VectorSimilarityWeight, 0.3))
+ matchText, keywords := GetQueryBuilder().Question(req.Question, "", minMatch(req.VectorSimilarityWeight, 0.3), req.Language)
for _, k := range keywords {
kwds[k] = struct{}{}
}
@@ -853,7 +863,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// and lower vector similarity threshold (0.17 vs default 0.1-0.2).
// This provides a second chance for queries that were too strict
// on the first attempt.
- matchText, _ := GetQueryBuilder().Question(req.Question, "qa", minMatch(req.VectorSimilarityWeight, 0.1))
+ matchText, _ := GetQueryBuilder().Question(req.Question, "qa", minMatch(req.VectorSimilarityWeight, 0.1), req.Language)
matchDense = cloneDenseExpr(denseTemplate)
matchDense.ExtraOptions["similarity"] = 0.17
if req.VectorOnly || matchText == nil {
@@ -885,9 +895,10 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
}
// Build kwds from keywords with fine-grained tokenization
+ tok := tokenizer.New(req.Language)
for _, k := range keywords {
kwds[k] = struct{}{}
- fgToken, _ := tokenizer.FineGrainedTokenize(k)
+ fgToken, _ := tok.FineGrainedTokenize(k)
for kk := range strings.FieldsSeq(fgToken) {
if len(kk) < 2 {
continue
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 0421ef86f6..e13e8096b2 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -268,10 +268,10 @@ func (e *retrievalCountEngine) Close() error { return nil }
func (e *retrievalCountEngine) Ping(context.Context) error { return nil }
func (e *retrievalCountEngine) GetType() string { return "elasticsearch" }
func (e *retrievalCountEngine) SupportsPageRank() bool { return false }
-func (e *retrievalCountEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (e *retrievalCountEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (e *retrievalCountEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (e *retrievalCountEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (e *retrievalCountEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go
index f99c4cdd91..156f50c044 100644
--- a/internal/service/nlp/term_weight.go
+++ b/internal/service/nlp/term_weight.go
@@ -191,12 +191,12 @@ func loadDict(fnm string) map[string]int {
return res
}
-// Pretoken preprocesses and tokenizes text
+// Pretoken preprocesses and tokenizes text with the dataset language.
// Reference: term_weight.py L92-114
-func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool) []string {
+func (d *TermWeightDealer) Pretoken(txt string, num bool, stpwd bool, lang string) []string {
warnPunctPatternOnce()
res := []string{}
- tokenized, err := tokenizer.Tokenize(txt)
+ tokenized, err := tokenizer.New(lang).Tokenize(txt)
if err != nil {
// Fallback to simple split
tokenized = txt
@@ -346,9 +346,10 @@ func (d *TermWeightDealer) Split(txt string) []string {
return tks
}
-// Weights calculates weights for tokens
+// Weights calculates weights for tokens with the dataset language ("" = English).
// Reference: term_weight.py L163-246
-func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
+func (d *TermWeightDealer) Weights(tks []string, preprocess bool, lang string) []TermWeight {
+ tok := tokenizer.New(lang)
// One dictionary lookup per distinct term. Both halves come from the same
// entry, tks repeats terms (title x2, important x5, question x6), and the
// consumers below are pure functions of the term, so caching the pair for the
@@ -426,7 +427,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
}
if s == 0 && len([]rune(t)) >= 4 {
// Try fine-grained tokenization
- fgTokens, _ := tokenizer.FineGrainedTokenize(t)
+ fgTokens, _ := tok.FineGrainedTokenize(t)
tokens := strings.Fields(fgTokens)
// Filter: only keep tokens with length > 1
@@ -473,7 +474,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
}
if len([]rune(t)) >= 4 {
// Use fine-grained tokenization
- fgTokens, _ := tokenizer.FineGrainedTokenize(t)
+ fgTokens, _ := tok.FineGrainedTokenize(t)
tokens := strings.Fields(fgTokens)
// Filter: only keep tokens with length > 1
@@ -540,7 +541,7 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
} else {
// With preprocessing
for _, tk := range tks {
- tokens := d.Pretoken(tk, true, true)
+ tokens := d.Pretoken(tk, true, true, lang)
tt := d.TokenMerge(tokens)
if len(tt) == 0 {
continue
diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.go
index 6282726545..0ab97f4a2e 100644
--- a/internal/service/nlp/term_weight_test.go
+++ b/internal/service/nlp/term_weight_test.go
@@ -76,7 +76,7 @@ func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) {
func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {
d := NewTermWeightDealer(t.TempDir())
terms := []string{"was", "largest", "supplier", "equipment"}
- weights := d.Weights(terms, false)
+ weights := d.Weights(terms, false, "")
if len(weights) != len(terms) {
t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms))
}
@@ -87,7 +87,7 @@ func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {
}
d.df["equipment"] = 1_000_000
- weights = d.Weights([]string{"supplier", "equipment"}, false)
+ weights = d.Weights([]string{"supplier", "equipment"}, false, "")
if weights[1].Weight >= weights[0].Weight {
t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights)
}
@@ -206,7 +206,7 @@ func TestPretoken(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- result := d.Pretoken(tt.txt, tt.num, tt.stpwd)
+ result := d.Pretoken(tt.txt, tt.num, tt.stpwd, "")
// Just check it doesn't panic and returns a slice
if result == nil {
t.Error("Pretoken returned nil")
@@ -392,7 +392,7 @@ func TestWeights(t *testing.T) {
t.Run("without preprocess", func(t *testing.T) {
tks := []string{"hello", "world", "123"}
- weights := d.Weights(tks, false)
+ weights := d.Weights(tks, false, "")
if len(weights) != len(tks) {
t.Errorf("Expected %d weights, got %d", len(tks), len(weights))
@@ -410,7 +410,7 @@ func TestWeights(t *testing.T) {
t.Run("with preprocess", func(t *testing.T) {
tks := []string{"hello world", "test"}
- weights := d.Weights(tks, true)
+ weights := d.Weights(tks, true, "")
// Check it doesn't panic and returns results
if weights == nil {
@@ -419,7 +419,7 @@ func TestWeights(t *testing.T) {
})
t.Run("empty input", func(t *testing.T) {
- weights := d.Weights([]string{}, false)
+ weights := d.Weights([]string{}, false, "")
if len(weights) != 0 {
t.Errorf("Expected empty weights for empty input, got %d", len(weights))
}
@@ -432,7 +432,7 @@ func TestWeights(t *testing.T) {
d2 := NewTermWeightDealer(tmpDir2)
tks := []string{"toxicterm", "normal"}
- weights := d2.Weights(tks, false)
+ weights := d2.Weights(tks, false, "")
if len(weights) != 2 {
t.Fatalf("Expected 2 weights, got %d", len(weights))
@@ -613,7 +613,7 @@ func TestWeightsNormalization(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- weights := d.Weights(tt.tks, false)
+ weights := d.Weights(tt.tks, false, "")
if len(weights) != len(tt.tks) {
t.Fatalf("Expected %d weights, got %d", len(tt.tks), len(weights))
@@ -665,7 +665,7 @@ func BenchmarkWeights(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
- d.Weights(tks, false)
+ d.Weights(tks, false, "")
}
}
@@ -721,7 +721,7 @@ func TestIntegration(t *testing.T) {
// Step 2: Pretoken
var allTokens []string
for _, s := range splitted {
- tokens := d.Pretoken(s, true, true)
+ tokens := d.Pretoken(s, true, true, "")
allTokens = append(allTokens, tokens...)
}
@@ -729,7 +729,7 @@ func TestIntegration(t *testing.T) {
merged := d.TokenMerge(allTokens)
// Step 4: Calculate weights
- weights := d.Weights(merged, false)
+ weights := d.Weights(merged, false, "")
// Verify results
if len(weights) == 0 && len(merged) > 0 {
@@ -752,7 +752,7 @@ func TestWeightsEdgeCases(t *testing.T) {
t.Run("numbers pattern", func(t *testing.T) {
tks := []string{"123,45", "abc"}
- weights := d.Weights(tks, false)
+ weights := d.Weights(tks, false, "")
if len(weights) != 2 {
t.Fatalf("Expected 2 weights, got %d", len(weights))
}
@@ -761,7 +761,7 @@ func TestWeightsEdgeCases(t *testing.T) {
t.Run("short letters pattern", func(t *testing.T) {
tks := []string{"ab", "abc"}
- weights := d.Weights(tks, false)
+ weights := d.Weights(tks, false, "")
if len(weights) != 2 {
t.Fatalf("Expected 2 weights, got %d", len(weights))
}
@@ -769,7 +769,7 @@ func TestWeightsEdgeCases(t *testing.T) {
t.Run("letter pattern with spaces", func(t *testing.T) {
tks := []string{"hello world test"}
- weights := d.Weights(tks, true)
+ weights := d.Weights(tks, true, "")
// Should not panic
if weights == nil {
t.Error("Weights returned nil for letter pattern")
@@ -782,7 +782,7 @@ func TestPretokenWithNumbers(t *testing.T) {
d := NewTermWeightDealer("")
t.Run("num=false filters single digits", func(t *testing.T) {
- result := d.Pretoken("5", false, true)
+ result := d.Pretoken("5", false, true, "")
// Single digit should be filtered when num=false
if slices.Contains(result, "5") {
t.Error("Single digit should be filtered when num=false")
@@ -790,7 +790,7 @@ func TestPretokenWithNumbers(t *testing.T) {
})
t.Run("num=true keeps single digits", func(t *testing.T) {
- result := d.Pretoken("5 123", true, true)
+ result := d.Pretoken("5 123", true, true, "")
// Check at least something is returned
if len(result) == 0 {
t.Log("Single digit may still be filtered by other rules")
@@ -803,7 +803,7 @@ func TestPretokenStopWords(t *testing.T) {
d := NewTermWeightDealer("")
t.Run("stpwd=true removes stop words", func(t *testing.T) {
- result := d.Pretoken("请问", true, true)
+ result := d.Pretoken("请问", true, true, "")
// "请问" is a stop word
for _, r := range result {
if r == "请问" {
@@ -813,7 +813,7 @@ func TestPretokenStopWords(t *testing.T) {
})
t.Run("stpwd=false keeps stop words", func(t *testing.T) {
- result := d.Pretoken("请问", true, false)
+ result := d.Pretoken("请问", true, false, "")
// With tokenizer, this might still filter it
_ = result
})
@@ -906,7 +906,7 @@ func TestWeightsDependOnlyOnTheTerm(t *testing.T) {
d := NewTermWeightDealer("")
ratio := func(tks []string, i, j int) float64 {
- weights := d.Weights(tks, false)
+ weights := d.Weights(tks, false, "")
if len(weights) != len(tks) {
t.Fatalf("got %d weights for %d terms", len(weights), len(tks))
}
diff --git a/internal/service/skill_indexer.go b/internal/service/skill_indexer.go
index b17b941ae3..3cb484f03f 100644
--- a/internal/service/skill_indexer.go
+++ b/internal/service/skill_indexer.go
@@ -931,7 +931,7 @@ func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tena
// Use the doc engine's CreateChunkStore method with skill-specific mapping
// The mapping file is loaded from conf/skill_es_mapping.json or conf/skill_infinity_mapping.json
- err := docEngine.CreateChunkStore(ctx, indexName, "skill", dimension, "")
+ err := docEngine.CreateChunkStore(ctx, indexName, "skill", dimension, "", "")
if err != nil {
common.Error("Failed to create skill index", err)
return err
diff --git a/internal/service/tenant.go b/internal/service/tenant.go
index f6ff646e86..63d91506f7 100644
--- a/internal/service/tenant.go
+++ b/internal/service/tenant.go
@@ -256,9 +256,16 @@ func (s *TenantService) CreateChunkStore(ctx context.Context, req *CreateDataset
// Build table name prefix: ragflow_<tenant_id>
tableName := fmt.Sprintf("ragflow_%s", kb.TenantID)
+ // The dataset language picks the fulltext analyzer on engines that fix it at
+ // index creation, so it has to travel with the table, not the query.
+ language := ""
+ if kb.Language != nil {
+ language = *kb.Language
+ }
+
// Call document engine to create table
// Full table name will be built as "{tableName}_{kb_id}"
- err = s.docEngine.CreateChunkStore(ctx, tableName, req.KBID, vecSize, req.ParserID)
+ err = s.docEngine.CreateChunkStore(ctx, tableName, req.KBID, vecSize, req.ParserID, language)
if err != nil {
return nil, common.CodeServerError, fmt.Errorf("failed to create dataset: %w", err)
}
diff --git a/internal/service/test_helpers_test.go b/internal/service/test_helpers_test.go
index 632dbf6397..0e860d53d8 100644
--- a/internal/service/test_helpers_test.go
+++ b/internal/service/test_helpers_test.go
@@ -94,10 +94,10 @@ func getAPIConversationForTest(t *testing.T, db *gorm.DB, id string) *entity.API
// fakeChatDocEngine is a stub engine.DocEngine used by parent-package tests.
type fakeChatDocEngine struct{}
-func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string) error {
+func (fakeChatDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
-func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {
+func (fakeChatDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (fakeChatDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go
index 669353cde4..ade35fdc12 100644
--- a/internal/tokenizer/tokenizer_test.go
+++ b/internal/tokenizer/tokenizer_test.go
@@ -362,6 +362,64 @@ func TestTokenize_DefaultLanguageResetsAnalyzerState(t *testing.T) {
}
}
+func TestTokenize_SlovakCzechFoldDiacriticsAndSkipStemming(t *testing.T) {
+ restore := saveEngineType()
+ defer restore()
+ SetEngineType("")
+
+ if err := Init(&PoolConfig{
+ DictPath: "",
+ MinSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 5 * time.Second,
+ AcquireTimeout: 5 * time.Second,
+ }); err != nil {
+ t.Fatalf("Failed to initialize pool: %v", err)
+ }
+ defer Close()
+
+ tests := []struct {
+ lang string
+ input string
+ want string
+ }{
+ // The analyzer folds diacritics before splitting, so the split regex
+ // keeps words whole, and stemming/lemmatization are off.
+ {"Slovak", "škola daňové priznanie", "skola danove priznanie"},
+ {"Czech", "příliš žluťoučký kůň", "prilis zlutoucky kun"},
+ // No stemming for folding languages: the token survives unchanged.
+ {"Slovak", "running", "running"},
+ }
+ for _, tt := range tests {
+ got, err := New(tt.lang).Tokenize(tt.input)
+ if err != nil {
+ t.Fatalf("Tokenize(%s, %q) unexpected error: %v", tt.lang, tt.input, err)
+ }
+ if got != tt.want {
+ t.Errorf("Tokenize(%s, %q) = %q, want %q", tt.lang, tt.input, got, tt.want)
+ }
+ }
+
+ // English tokenization is unchanged and the pool resets sticky Slovak
+ // state: the default-language path still stems.
+ engGot, err := Tokenize("running")
+ if err != nil {
+ t.Fatalf("Tokenize(default, \"running\") unexpected error: %v", err)
+ }
+ if engGot == "running" {
+ t.Errorf("Tokenize(default, \"running\") = %q, want a stemmed form (English stemming must stay enabled)", engGot)
+ }
+
+ // FineGrainedTokenize on folded tokens must not stem either.
+ fg, err := New("Slovak").FineGrainedTokenize("danove priznanie")
+ if err != nil {
+ t.Fatalf("FineGrainedTokenize(Slovak) unexpected error: %v", err)
+ }
+ if fg != "danove priznanie" {
+ t.Errorf("FineGrainedTokenize(Slovak, \"danove priznanie\") = %q, want unchanged tokens", fg)
+ }
+}
+
func findEnglishDutchDifferentiator(t *testing.T) languageDifferentiator {
t.Helper()
diff --git a/web/src/components/cross-language-form-field.tsx b/web/src/components/cross-language-form-field.tsx
index f2f9734bbb..2c087dcc12 100644
--- a/web/src/components/cross-language-form-field.tsx
+++ b/web/src/components/cross-language-form-field.tsx
@@ -39,6 +39,8 @@ export const Languages = [
'Arabic',
'Turkish',
'Dutch',
+ 'Slovak',
+ 'Czech',
];
export function useCrossLanguageOptions() {
diff --git a/web/src/constants/common.ts b/web/src/constants/common.ts
index abb09bd79a..2bbe9f6904 100644
--- a/web/src/constants/common.ts
+++ b/web/src/constants/common.ts
@@ -32,6 +32,8 @@ export const LanguageList = [
'Arabic',
'Turkish',
'Dutch',
+ 'Slovak',
+ 'Czech',
];
export const LanguageMap = {
English: 'English',
@@ -51,6 +53,8 @@ export const LanguageMap = {
Arabic: 'العربية',
Turkish: 'Türkçe',
Dutch: 'Nederlands',
+ Slovak: 'Slovenčina',
+ Czech: 'Čeština',
};
export enum LanguageAbbreviation {
@@ -125,6 +129,8 @@ export const LanguageTranslationMap = {
Arabic: 'ar',
Turkish: 'tr',
Dutch: 'nl',
+ Slovak: 'sk',
+ Czech: 'cs',
};
export enum FileMimeType {
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 6cfc36c667..b8a51658f4 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -46,6 +46,8 @@ export default {
arabic: 'Arabic',
turkish: 'Turkish',
dutch: 'Dutch',
+ slovak: 'Slovak',
+ czech: 'Czech',
language: 'Language',
languageMessage: 'Please input your language!',
languagePlaceholder: 'select your language',
@@ -3909,6 +3911,8 @@ Important structured information may include: names, dates, locations, events, k
arabic: 'Arabic',
turkish: 'Turkish',
dutch: 'Dutch',
+ slovak: 'Slovak',
+ czech: 'Czech',
},
pagination: {
total: 'Total {{total}}',