RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nlp): load term weight resources as UTF-8 (#18467)
Loongb299dd7f2e71
diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.pyindex 9477945680..9b73f97459 100644--- a/rag/nlp/term_weight.py+++ b/rag/nlp/term_weight.py@@ -93,7 +93,7 @@ class Dealer:def load_dict(fnm):res = {}- with open(fnm, "r") as f:+ with open(fnm, "r", encoding="utf-8") as f:while True:line = f.readline()if not line:@@ -114,7 +114,7 @@ class Dealer:fnm = os.path.join(get_project_base_directory(), "rag/res")self.ne, self.df = {}, {}try:- with open(os.path.join(fnm, "ner.json"), "r") as f:+ with open(os.path.join(fnm, "ner.json"), "r", encoding="utf-8") as f:self.ne = json.load(f)except Exception:logging.warning("Load ner.json FAIL!")diff --git a/test/unit_test/rag/nlp/test_term_weight.py b/test/unit_test/rag/nlp/test_term_weight.pynew file mode 100644index 0000000000..297cdf3516--- /dev/null+++ b/test/unit_test/rag/nlp/test_term_weight.py@@ -0,0 +1,44 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import builtins+from pathlib import Path++from rag.nlp import term_weight+++def test_dealer_loads_resource_files_as_utf8(monkeypatch, tmp_path):+ """Load non-ASCII term-weight resources with an explicit UTF-8 encoding."""+ resource_dir = tmp_path / "rag" / "res"+ resource_dir.mkdir(parents=True)+ (resource_dir / "ner.json").write_text('{"\u5317\u4eac": "loca"}', encoding="utf-8")+ (resource_dir / "term.freq").write_text("caf\u00e9\t42\n", encoding="utf-8")++ monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))+ real_open = builtins.open++ def require_explicit_utf8(file, *args, **kwargs):+ """Reject resource reads that rely on the platform default encoding."""+ if Path(file).parent == resource_dir:+ assert kwargs.get("encoding") == "utf-8"+ return real_open(file, *args, **kwargs)++ monkeypatch.setattr(builtins, "open", require_explicit_utf8)++ dealer = term_weight.Dealer()++ assert dealer.ne == {"\u5317\u4eac": "loca"}+ assert dealer.df == {"caf\u00e9": 42}