Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nlp): differentiate alphabetic OOV term weights (#18470)

Loong86c520a33617
diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go
index 362f656b34..984b2df613 100644
--- a/internal/service/nlp/term_weight.go
+++ b/internal/service/nlp/term_weight.go
@@ -23,6 +23,7 @@ import (
"regexp"
"strconv"
"strings"
+ "unicode"
"ragflow/internal/tokenizer"
@@ -87,13 +88,39 @@ func initStopWords() map[string]struct{} {
return stopWords
}
+// alphabeticOOVFrequency estimates frequency for an alphabetic out-of-vocabulary
+// term. It preserves the previous frequency of 300 for short words, then halves
+// it every two letters, with a floor of 10. Latin, Greek, and Cyrillic scripts
+// are supported while logographic terms keep their existing tokenizer path.
+func alphabeticOOVFrequency(term string) (float64, bool) {
+ letterCount := 0
+ for _, r := range term {
+ if unicode.IsLetter(r) && unicode.In(r, unicode.Latin, unicode.Greek, unicode.Cyrillic) {
+ letterCount++
+ continue
+ }
+ if r != ' ' && r != '.' && r != '-' {
+ return 0, false
+ }
+ }
+ if letterCount == 0 {
+ return 0, false
+ }
+
+ exponent := float64(max(0, letterCount-3)) / 2
+ frequency := math.Round(300 / math.Pow(2, exponent))
+ return math.Max(10, frequency), true
+}
+
// loadDict loads a dictionary file
// Format: term\tfreq or just term
func loadDict(fnm string) map[string]int {
res := make(map[string]int)
data, err := os.ReadFile(fnm)
if err != nil {
- common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))
+ if !os.IsNotExist(err) {
+ common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))
+ }
return res
}
@@ -285,7 +312,6 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
numPattern := regexp.MustCompile("^[0-9,.]{2,}$")
shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$")
numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$")
- letterPattern := regexp.MustCompile("^[a-z. -]+$")
// ner weight function
nerWeight := func(t string) float64 {
@@ -338,8 +364,10 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
}
// Use tokenizer's freq function
s := tokenizer.GetTermFreq(t)
- if s == 0 && letterPattern.MatchString(t) {
- return 300
+ if s == 0 {
+ if oovFrequency, ok := alphabeticOOVFrequency(t); ok {
+ return oovFrequency
+ }
}
if s == 0 && len([]rune(t)) >= 4 {
// Try fine-grained tokenization
@@ -385,8 +413,8 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
if v, ok := d.df[t]; ok {
return float64(v) + 3
}
- if letterPattern.MatchString(t) {
- return 300
+ if oovFrequency, ok := alphabeticOOVFrequency(t); ok {
+ return oovFrequency
}
if len([]rune(t)) >= 4 {
// Use fine-grained tokenization
diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.go
index 078e5b0009..c3007af6b2 100644
--- a/internal/service/nlp/term_weight_test.go
+++ b/internal/service/nlp/term_weight_test.go
@@ -15,6 +15,7 @@
package nlp
import (
+ "encoding/json"
"os"
"path/filepath"
"reflect"
@@ -23,6 +24,74 @@ import (
"testing"
)
+type oovFrequencyFixture struct {
+ Term string `json:"term"`
+ Frequency *float64 `json:"frequency"`
+}
+
+func repositoryRoot(t *testing.T) string {
+ t.Helper()
+ dir, err := os.Getwd()
+ if err != nil {
+ t.Fatalf("get working directory: %v", err)
+ }
+ for {
+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+ return dir
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ t.Fatal("repository root not found")
+ }
+ dir = parent
+ }
+}
+
+func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) {
+ fixturePath := filepath.Join(repositoryRoot(t), "test", "fixtures", "term_weight_oov.json")
+ data, err := os.ReadFile(fixturePath)
+ if err != nil {
+ t.Fatalf("read OOV fixture: %v", err)
+ }
+
+ var cases []oovFrequencyFixture
+ if err := json.Unmarshal(data, &cases); err != nil {
+ t.Fatalf("decode OOV fixture: %v", err)
+ }
+ for _, testCase := range cases {
+ got, ok := alphabeticOOVFrequency(testCase.Term)
+ if testCase.Frequency == nil {
+ if ok {
+ t.Errorf("alphabeticOOVFrequency(%q) = %v, true; want no fallback", testCase.Term, got)
+ }
+ continue
+ }
+ if !ok || got != *testCase.Frequency {
+ t.Errorf("alphabeticOOVFrequency(%q) = %v, %v; want %v, true", testCase.Term, got, ok, *testCase.Frequency)
+ }
+ }
+}
+
+func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {
+ d := NewTermWeightDealer(t.TempDir())
+ terms := []string{"was", "largest", "supplier", "equipment"}
+ weights := d.Weights(terms, false)
+ if len(weights) != len(terms) {
+ t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms))
+ }
+ for i := 1; i < len(weights); i++ {
+ if weights[i-1].Weight >= weights[i].Weight {
+ t.Fatalf("weights are not increasing with OOV term length: %v", weights)
+ }
+ }
+
+ d.df["equipment"] = 1_000_000
+ weights = d.Weights([]string{"supplier", "equipment"}, false)
+ if weights[1].Weight >= weights[0].Weight {
+ t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights)
+ }
+}
+
// TestNewTermWeightDealer tests the constructor
func TestNewTermWeightDealer(t *testing.T) {
// Test with empty resPath
diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.py
index 43b018a429..9477945680 100644
--- a/rag/nlp/term_weight.py
+++ b/rag/nlp/term_weight.py
@@ -14,14 +14,43 @@
# limitations under the License.
#
+import json
import logging
import math
-import json
-import re
import os
+import re
+import unicodedata
+
import numpy as np
-from rag.nlp import rag_tokenizer
+
from common.file_utils import get_project_base_directory
+from rag.nlp import rag_tokenizer
+
+
+def _alphabetic_oov_frequency(term):
+ """Estimate frequency for a Latin, Greek, or Cyrillic OOV term."""
+ letter_count = 0
+ for char in term:
+ if char.isascii():
+ if char.isalpha():
+ letter_count += 1
+ elif char not in " .-":
+ return None
+ continue
+ script_name = unicodedata.name(char, "").split(" ", 1)[0]
+ if char.isalpha() and script_name in {"LATIN", "GREEK", "CYRILLIC"}:
+ letter_count += 1
+ elif char not in " .-":
+ return None
+ if not letter_count:
+ return None
+
+ # Preserve the old frequency (300) for short words, then halve it every
+ # two letters. This is a bounded, language-neutral prior: longer unknown
+ # words are usually more informative, without outranking known terms by an
+ # unbounded amount.
+ exponent = max(0, letter_count - 3) / 2
+ return max(10, round(300 / (2**exponent)))
class Dealer:
@@ -89,10 +118,13 @@ class Dealer:
self.ne = json.load(f)
except Exception:
logging.warning("Load ner.json FAIL!")
+ freq_path = os.path.join(fnm, "term.freq")
try:
- self.df = load_dict(os.path.join(fnm, "term.freq"))
- except Exception:
- logging.warning("Load term.freq FAIL!")
+ self.df = load_dict(freq_path)
+ except FileNotFoundError:
+ pass
+ except (OSError, ValueError):
+ logging.warning("Load term.freq FAIL!", exc_info=True)
def pretoken(self, txt, num=False, stpwd=True):
patt = [r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;‘’:“”【¥ 】…¥!、·()×`&\\/「」\\]"]
@@ -161,7 +193,6 @@ class Dealer:
num_pattern = re.compile(r"[0-9,.]{2,}$")
short_letter_pattern = re.compile(r"[a-z]{1,2}$")
num_space_pattern = re.compile(r"[0-9. -]{2,}$")
- letter_pattern = re.compile(r"[a-z. -]+$")
def ner(t):
if num_pattern.match(t):
@@ -189,8 +220,10 @@ class Dealer:
if num_space_pattern.match(t):
return 3
s = rag_tokenizer.freq(t)
- if not s and letter_pattern.match(t):
- return 300
+ if not s:
+ oov_frequency = _alphabetic_oov_frequency(t)
+ if oov_frequency is not None:
+ return oov_frequency
if not s:
s = 0
@@ -208,9 +241,10 @@ class Dealer:
return 5
if t in self.df:
return self.df[t] + 3
- elif letter_pattern.match(t):
- return 300
- elif len(t) >= 4:
+ oov_frequency = _alphabetic_oov_frequency(t)
+ if oov_frequency is not None:
+ return oov_frequency
+ if len(t) >= 4:
s = [tt for tt in rag_tokenizer.fine_grained_tokenize(t).split() if len(tt) > 1]
if len(s) > 1:
return max(3, np.min([df(tt) for tt in s]) / 6.0)
diff --git a/test/fixtures/term_weight_oov.json b/test/fixtures/term_weight_oov.json
new file mode 100644
index 0000000000..c7adbbafdc
--- /dev/null
+++ b/test/fixtures/term_weight_oov.json
@@ -0,0 +1,17 @@
+[
+ {"term": "a", "frequency": 300},
+ {"term": "the", "frequency": 300},
+ {"term": "Café", "frequency": 212},
+ {"term": "maior", "frequency": 150},
+ {"term": "κόσμος", "frequency": 106},
+ {"term": "привет", "frequency": 106},
+ {"term": "largest", "frequency": 75},
+ {"term": "supplier", "frequency": 53},
+ {"term": "equipment", "frequency": 38},
+ {"term": "équipement", "frequency": 27},
+ {"term": "hospital-equipment", "frequency": 10},
+ {"term": "北京", "frequency": null},
+ {"term": "abc123", "frequency": null},
+ {"term": "hello_world", "frequency": null},
+ {"term": "...", "frequency": null}
+]
diff --git a/test/unit_test/rag/nlp/test_term_weight_oov.py b/test/unit_test/rag/nlp/test_term_weight_oov.py
new file mode 100644
index 0000000000..d220bf726b
--- /dev/null
+++ b/test/unit_test/rag/nlp/test_term_weight_oov.py
@@ -0,0 +1,74 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import json
+import logging
+from pathlib import Path
+
+from rag.nlp import rag_tokenizer, term_weight
+from rag.nlp.term_weight import Dealer, _alphabetic_oov_frequency
+
+
+def test_alphabetic_oov_frequency_matches_shared_fixture():
+ """Keep the Python fallback aligned with the fixture consumed by Go tests."""
+ fixture_path = Path(__file__).parents[3] / "fixtures" / "term_weight_oov.json"
+ cases = json.loads(fixture_path.read_text(encoding="utf-8"))
+
+ for case in cases:
+ assert _alphabetic_oov_frequency(case["term"]) == case["frequency"]
+
+
+def test_oov_weights_favor_longer_content_terms(monkeypatch):
+ """Avoid equal weights when no dictionary or tokenizer frequency is available."""
+ monkeypatch.setattr(rag_tokenizer, "freq", lambda _term: 0)
+ monkeypatch.setattr(rag_tokenizer, "tag", lambda _term: "")
+ dealer = Dealer()
+ dealer.df = {}
+
+ weights = dict(dealer.weights(["was", "largest", "supplier", "equipment"], preprocess=False))
+
+ assert weights["was"] < weights["largest"] < weights["supplier"] < weights["equipment"]
+
+ dealer.df["equipment"] = 1_000_000
+ weights = dict(dealer.weights(["supplier", "equipment"], preprocess=False))
+ assert weights["equipment"] < weights["supplier"]
+
+
+def test_missing_frequency_dictionary_is_silent(tmp_path, monkeypatch, caplog):
+ """Treat an absent optional frequency dictionary as an expected fallback."""
+ resource_dir = tmp_path / "rag" / "res"
+ resource_dir.mkdir(parents=True)
+ (resource_dir / "ner.json").write_text("{}", encoding="utf-8")
+ monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))
+
+ with caplog.at_level(logging.WARNING):
+ Dealer()
+
+ assert "Load term.freq FAIL!" not in caplog.text
+
+
+def test_malformed_frequency_dictionary_logs_warning(tmp_path, monkeypatch, caplog):
+ """Expose malformed dictionaries instead of silently enabling the fallback."""
+ resource_dir = tmp_path / "rag" / "res"
+ resource_dir.mkdir(parents=True)
+ (resource_dir / "ner.json").write_text("{}", encoding="utf-8")
+ (resource_dir / "term.freq").write_text("term\tnot-a-number\n", encoding="utf-8")
+ monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))
+
+ with caplog.at_level(logging.WARNING):
+ Dealer()
+
+ assert "Load term.freq FAIL!" in caplog.text