RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nlp): differentiate alphabetic OOV term weights (#18470)
Loong86c520a33617
diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.goindex 362f656b34..984b2df613 100644--- a/internal/service/nlp/term_weight.go+++ b/internal/service/nlp/term_weight.go@@ -23,6 +23,7 @@ import ("regexp""strconv""strings"+ "unicode""ragflow/internal/tokenizer"@@ -87,13 +88,39 @@ func initStopWords() map[string]struct{} {return stopWords}+// alphabeticOOVFrequency estimates frequency for an alphabetic out-of-vocabulary+// term. It preserves the previous frequency of 300 for short words, then halves+// it every two letters, with a floor of 10. Latin, Greek, and Cyrillic scripts+// are supported while logographic terms keep their existing tokenizer path.+func alphabeticOOVFrequency(term string) (float64, bool) {+ letterCount := 0+ for _, r := range term {+ if unicode.IsLetter(r) && unicode.In(r, unicode.Latin, unicode.Greek, unicode.Cyrillic) {+ letterCount+++ continue+ }+ if r != ' ' && r != '.' && r != '-' {+ return 0, false+ }+ }+ if letterCount == 0 {+ return 0, false+ }++ exponent := float64(max(0, letterCount-3)) / 2+ frequency := math.Round(300 / math.Pow(2, exponent))+ return math.Max(10, frequency), true+}+// loadDict loads a dictionary file// Format: term\tfreq or just termfunc loadDict(fnm string) map[string]int {res := make(map[string]int)data, err := os.ReadFile(fnm)if err != nil {- common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))+ if !os.IsNotExist(err) {+ common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))+ }return res}@@ -285,7 +312,6 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {numPattern := regexp.MustCompile("^[0-9,.]{2,}$")shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$")numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$")- letterPattern := regexp.MustCompile("^[a-z. -]+$")// ner weight functionnerWeight := func(t string) float64 {@@ -338,8 +364,10 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {}// Use tokenizer's freq functions := tokenizer.GetTermFreq(t)- if s == 0 && letterPattern.MatchString(t) {- return 300+ if s == 0 {+ if oovFrequency, ok := alphabeticOOVFrequency(t); ok {+ return oovFrequency+ }}if s == 0 && len([]rune(t)) >= 4 {// Try fine-grained tokenization@@ -385,8 +413,8 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {if v, ok := d.df[t]; ok {return float64(v) + 3}- if letterPattern.MatchString(t) {- return 300+ if oovFrequency, ok := alphabeticOOVFrequency(t); ok {+ return oovFrequency}if len([]rune(t)) >= 4 {// Use fine-grained tokenizationdiff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.goindex 078e5b0009..c3007af6b2 100644--- a/internal/service/nlp/term_weight_test.go+++ b/internal/service/nlp/term_weight_test.go@@ -15,6 +15,7 @@package nlpimport (+ "encoding/json""os""path/filepath""reflect"@@ -23,6 +24,74 @@ import ("testing")+type oovFrequencyFixture struct {+ Term string `json:"term"`+ Frequency *float64 `json:"frequency"`+}++func repositoryRoot(t *testing.T) string {+ t.Helper()+ dir, err := os.Getwd()+ if err != nil {+ t.Fatalf("get working directory: %v", err)+ }+ for {+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {+ return dir+ }+ parent := filepath.Dir(dir)+ if parent == dir {+ t.Fatal("repository root not found")+ }+ dir = parent+ }+}++func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) {+ fixturePath := filepath.Join(repositoryRoot(t), "test", "fixtures", "term_weight_oov.json")+ data, err := os.ReadFile(fixturePath)+ if err != nil {+ t.Fatalf("read OOV fixture: %v", err)+ }++ var cases []oovFrequencyFixture+ if err := json.Unmarshal(data, &cases); err != nil {+ t.Fatalf("decode OOV fixture: %v", err)+ }+ for _, testCase := range cases {+ got, ok := alphabeticOOVFrequency(testCase.Term)+ if testCase.Frequency == nil {+ if ok {+ t.Errorf("alphabeticOOVFrequency(%q) = %v, true; want no fallback", testCase.Term, got)+ }+ continue+ }+ if !ok || got != *testCase.Frequency {+ t.Errorf("alphabeticOOVFrequency(%q) = %v, %v; want %v, true", testCase.Term, got, ok, *testCase.Frequency)+ }+ }+}++func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {+ d := NewTermWeightDealer(t.TempDir())+ terms := []string{"was", "largest", "supplier", "equipment"}+ weights := d.Weights(terms, false)+ if len(weights) != len(terms) {+ t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms))+ }+ for i := 1; i < len(weights); i++ {+ if weights[i-1].Weight >= weights[i].Weight {+ t.Fatalf("weights are not increasing with OOV term length: %v", weights)+ }+ }++ d.df["equipment"] = 1_000_000+ weights = d.Weights([]string{"supplier", "equipment"}, false)+ if weights[1].Weight >= weights[0].Weight {+ t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights)+ }+}+// TestNewTermWeightDealer tests the constructorfunc TestNewTermWeightDealer(t *testing.T) {// Test with empty resPathdiff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.pyindex 43b018a429..9477945680 100644--- a/rag/nlp/term_weight.py+++ b/rag/nlp/term_weight.py@@ -14,14 +14,43 @@# limitations under the License.#+import jsonimport loggingimport math-import json-import reimport os+import re+import unicodedata+import numpy as np-from rag.nlp import rag_tokenizer+from common.file_utils import get_project_base_directory+from rag.nlp import rag_tokenizer+++def _alphabetic_oov_frequency(term):+ """Estimate frequency for a Latin, Greek, or Cyrillic OOV term."""+ letter_count = 0+ for char in term:+ if char.isascii():+ if char.isalpha():+ letter_count += 1+ elif char not in " .-":+ return None+ continue+ script_name = unicodedata.name(char, "").split(" ", 1)[0]+ if char.isalpha() and script_name in {"LATIN", "GREEK", "CYRILLIC"}:+ letter_count += 1+ elif char not in " .-":+ return None+ if not letter_count:+ return None++ # Preserve the old frequency (300) for short words, then halve it every+ # two letters. This is a bounded, language-neutral prior: longer unknown+ # words are usually more informative, without outranking known terms by an+ # unbounded amount.+ exponent = max(0, letter_count - 3) / 2+ return max(10, round(300 / (2**exponent)))class Dealer:@@ -89,10 +118,13 @@ class Dealer:self.ne = json.load(f)except Exception:logging.warning("Load ner.json FAIL!")+ freq_path = os.path.join(fnm, "term.freq")try:- self.df = load_dict(os.path.join(fnm, "term.freq"))- except Exception:- logging.warning("Load term.freq FAIL!")+ self.df = load_dict(freq_path)+ except FileNotFoundError:+ pass+ except (OSError, ValueError):+ logging.warning("Load term.freq FAIL!", exc_info=True)def pretoken(self, txt, num=False, stpwd=True):patt = [r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;‘’:“”【¥ 】…¥!、·()×`&\\/「」\\]"]@@ -161,7 +193,6 @@ class Dealer:num_pattern = re.compile(r"[0-9,.]{2,}$")short_letter_pattern = re.compile(r"[a-z]{1,2}$")num_space_pattern = re.compile(r"[0-9. -]{2,}$")- letter_pattern = re.compile(r"[a-z. -]+$")def ner(t):if num_pattern.match(t):@@ -189,8 +220,10 @@ class Dealer:if num_space_pattern.match(t):return 3s = rag_tokenizer.freq(t)- if not s and letter_pattern.match(t):- return 300+ if not s:+ oov_frequency = _alphabetic_oov_frequency(t)+ if oov_frequency is not None:+ return oov_frequencyif not s:s = 0@@ -208,9 +241,10 @@ class Dealer:return 5if t in self.df:return self.df[t] + 3- elif letter_pattern.match(t):- return 300- elif len(t) >= 4:+ oov_frequency = _alphabetic_oov_frequency(t)+ if oov_frequency is not None:+ return oov_frequency+ if len(t) >= 4:s = [tt for tt in rag_tokenizer.fine_grained_tokenize(t).split() if len(tt) > 1]if len(s) > 1:return max(3, np.min([df(tt) for tt in s]) / 6.0)diff --git a/test/fixtures/term_weight_oov.json b/test/fixtures/term_weight_oov.jsonnew file mode 100644index 0000000000..c7adbbafdc--- /dev/null+++ b/test/fixtures/term_weight_oov.json@@ -0,0 +1,17 @@+[+ {"term": "a", "frequency": 300},+ {"term": "the", "frequency": 300},+ {"term": "Café", "frequency": 212},+ {"term": "maior", "frequency": 150},+ {"term": "κόσμος", "frequency": 106},+ {"term": "привет", "frequency": 106},+ {"term": "largest", "frequency": 75},+ {"term": "supplier", "frequency": 53},+ {"term": "equipment", "frequency": 38},+ {"term": "équipement", "frequency": 27},+ {"term": "hospital-equipment", "frequency": 10},+ {"term": "北京", "frequency": null},+ {"term": "abc123", "frequency": null},+ {"term": "hello_world", "frequency": null},+ {"term": "...", "frequency": null}+]diff --git a/test/unit_test/rag/nlp/test_term_weight_oov.py b/test/unit_test/rag/nlp/test_term_weight_oov.pynew file mode 100644index 0000000000..d220bf726b--- /dev/null+++ b/test/unit_test/rag/nlp/test_term_weight_oov.py@@ -0,0 +1,74 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import json+import logging+from pathlib import Path++from rag.nlp import rag_tokenizer, term_weight+from rag.nlp.term_weight import Dealer, _alphabetic_oov_frequency+++def test_alphabetic_oov_frequency_matches_shared_fixture():+ """Keep the Python fallback aligned with the fixture consumed by Go tests."""+ fixture_path = Path(__file__).parents[3] / "fixtures" / "term_weight_oov.json"+ cases = json.loads(fixture_path.read_text(encoding="utf-8"))++ for case in cases:+ assert _alphabetic_oov_frequency(case["term"]) == case["frequency"]+++def test_oov_weights_favor_longer_content_terms(monkeypatch):+ """Avoid equal weights when no dictionary or tokenizer frequency is available."""+ monkeypatch.setattr(rag_tokenizer, "freq", lambda _term: 0)+ monkeypatch.setattr(rag_tokenizer, "tag", lambda _term: "")+ dealer = Dealer()+ dealer.df = {}++ weights = dict(dealer.weights(["was", "largest", "supplier", "equipment"], preprocess=False))++ assert weights["was"] < weights["largest"] < weights["supplier"] < weights["equipment"]++ dealer.df["equipment"] = 1_000_000+ weights = dict(dealer.weights(["supplier", "equipment"], preprocess=False))+ assert weights["equipment"] < weights["supplier"]+++def test_missing_frequency_dictionary_is_silent(tmp_path, monkeypatch, caplog):+ """Treat an absent optional frequency dictionary as an expected fallback."""+ resource_dir = tmp_path / "rag" / "res"+ resource_dir.mkdir(parents=True)+ (resource_dir / "ner.json").write_text("{}", encoding="utf-8")+ monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))++ with caplog.at_level(logging.WARNING):+ Dealer()++ assert "Load term.freq FAIL!" not in caplog.text+++def test_malformed_frequency_dictionary_logs_warning(tmp_path, monkeypatch, caplog):+ """Expose malformed dictionaries instead of silently enabling the fallback."""+ resource_dir = tmp_path / "rag" / "res"+ resource_dir.mkdir(parents=True)+ (resource_dir / "ner.json").write_text("{}", encoding="utf-8")+ (resource_dir / "term.freq").write_text("term\tnot-a-number\n", encoding="utf-8")+ monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))++ with caplog.at_level(logging.WARNING):+ Dealer()++ assert "Load term.freq FAIL!" in caplog.text