Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): normalize legacy text encodings (#2770)
Hao Zhe324f96ebb664
diff --git a/openviking/parse/parsers/base_parser.py b/openviking/parse/parsers/base_parser.pyindex be2ec403b..85911c2b1 100644--- a/openviking/parse/parsers/base_parser.py+++ b/openviking/parse/parsers/base_parser.py@@ -80,16 +80,24 @@ class BaseParser(ABC):File content as string"""path = Path(path)- encodings = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]-- for encoding in encodings:+ from openviking.parse.parsers.upload_utils import detect_and_convert_encoding++ raw = path.read_bytes()+ normalized = detect_and_convert_encoding(raw, path)+ try:+ return normalized.decode("utf-8")+ except UnicodeDecodeError:+ pass++ # Preserve the previous permissive fallback for files whose encoding is+ # unknown or rejected by text detection as likely binary/control-heavy.+ for encoding in ("utf-8-sig", "cp1252"):try:- with open(path, "r", encoding=encoding) as f:- return f.read()+ return raw.decode(encoding)except UnicodeDecodeError:continue- raise ValueError(f"Unable to decode file: {path}")+ return raw.decode("latin-1")def _get_viking_fs(self):"""diff --git a/openviking/parse/parsers/constants.py b/openviking/parse/parsers/constants.pyindex 8561e31a8..74609115e 100644--- a/openviking/parse/parsers/constants.py+++ b/openviking/parse/parsers/constants.py@@ -232,20 +232,3 @@ ADDITIONAL_TEXT_EXTENSIONS = {".lock",".in",}--# Common text encodings to try for encoding detection (in order of likelihood)-TEXT_ENCODINGS = [- "utf-8", # Most common modern encoding- "utf-8-sig", # UTF-8 with BOM- "gbk", # Chinese GBK (simplified Chinese)- "gb2312", # Chinese GB2312 (simplified Chinese)- "big5", # Traditional Chinese- "shift_jis", # Japanese- "euc-kr", # Korean- "iso-8859-1", # Latin-1 (Western European)- "cp1252", # Windows Latin-1- "latin-1", # Latin-1 alias-]--# UTF-8 variants that don't need conversion-UTF8_VARIANTS = {"utf-8", "utf-8-sig"}diff --git a/openviking/parse/parsers/text_encoding.py b/openviking/parse/parsers/text_encoding.pynew file mode 100644index 000000000..4c954d87f--- /dev/null+++ b/openviking/parse/parsers/text_encoding.py@@ -0,0 +1,289 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Text encoding normalization helpers for uploaded text files."""++import codecs+from dataclasses import dataclass+from pathlib import Path+from typing import Iterable, Optional, Union++from charset_normalizer import from_bytes++from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++_BOM_ENCODINGS = (+ (codecs.BOM_UTF32_LE, "utf-32"),+ (codecs.BOM_UTF32_BE, "utf-32"),+ (codecs.BOM_UTF16_LE, "utf-16"),+ (codecs.BOM_UTF16_BE, "utf-16"),+ (codecs.BOM_UTF8, "utf-8-sig"),+)++_DETECTION_ENCODINGS = (+ "gb18030",+ "gbk",+ "gb2312",+ "big5",+ "cp950",+ "cp932",+ "shift_jis",+ "euc_jp",+ "euc_kr",+ "cp949",+ "cp1252",+ "latin_1",+)++# These values use codecs.lookup(...).name canonical forms because candidates+# are canonicalized before set membership checks.+_CHINESE_ENCODINGS = {"gb18030", "gbk", "gb2312", "big5", "cp950"}+_CHINESE_ENCODING_PREFERENCE = ("gb18030", "gbk", "gb2312", "big5", "cp950")+_JAPANESE_ENCODINGS = {"cp932", "shift_jis", "euc_jp"}+_KOREAN_ENCODINGS = {"euc_kr", "cp949"}+_CJK_ENCODINGS = _CHINESE_ENCODINGS | _JAPANESE_ENCODINGS | _KOREAN_ENCODINGS+# Very short CJK samples can decode into plausible text under several legacy+# code pages. Require a few script-specific characters before overriding the+# detector's rank with Korean script evidence.+_MIN_STRONG_SCRIPT_CHARS = 4+++@dataclass(frozen=True)+class _ScriptProfile:+ cjk: int+ kana: int+ hangul_syllables: int+ hangul_jamo: int+++@dataclass(frozen=True)+class _EncodingCandidate:+ encoding: str+ decoded: str+ rank: int+ from_detector: bool+++def normalize_text_bytes(content: bytes, file_path: Union[str, Path] = "") -> bytes:+ """Normalize legacy text bytes to UTF-8 while leaving unknown bytes untouched."""+ try:+ return _normalize_text_bytes(content, file_path)+ except Exception as exc:+ logger.warning(f"Encoding detection failed for {file_path}: {exc}")+ return content+++def _normalize_text_bytes(content: bytes, file_path: Union[str, Path] = "") -> bytes:+ if not content:+ return content++ bom_normalized = _decode_known_bom(content)+ if bom_normalized is not None:+ return bom_normalized++ try:+ content.decode("utf-8")+ return content+ except UnicodeDecodeError:+ pass++ if _looks_binary(content):+ logger.debug(f"Detected binary-like text file {file_path}, skipping encoding detection")+ return content++ candidate = _choose_candidate(_iter_candidates(content))+ if candidate is None:+ logger.warning(f"Encoding detection failed for {file_path}: no matching encoding found")+ return content++ decoded = candidate.decoded.replace("\x00", "")+ normalized = decoded.encode("utf-8")+ logger.debug(f"Converted {file_path} from {candidate.encoding} to UTF-8")+ return normalized+++def _decode_known_bom(content: bytes) -> Optional[bytes]:+ for bom, encoding in _BOM_ENCODINGS:+ if content.startswith(bom):+ try:+ return content.decode(encoding).encode("utf-8")+ except UnicodeDecodeError:+ return None+ return None+++def _looks_binary(content: bytes) -> bool:+ sample = content[: min(8192, len(content))]+ if not sample:+ return False++ return sample.count(b"\x00") / len(sample) > 0.05+++def _iter_candidates(content: bytes) -> Iterable[_EncodingCandidate]:+ seen: set[str] = set()+ rank = 0++ for match in from_bytes(content, cp_isolation=_DETECTION_ENCODINGS):+ candidate = _decode_candidate(content, match.encoding, rank, from_detector=True)+ if candidate is None or candidate.encoding in seen:+ continue+ seen.add(candidate.encoding)+ rank += 1+ yield candidate++ for encoding in _DETECTION_ENCODINGS:+ candidate = _decode_candidate(content, encoding, rank, from_detector=False)+ if candidate is None or candidate.encoding in seen:+ continue+ seen.add(candidate.encoding)+ rank += 1+ yield candidate+++def _decode_candidate(+ content: bytes, encoding: Optional[str], rank: int, *, from_detector: bool+) -> Optional[_EncodingCandidate]:+ if not encoding:+ return None++ try:+ canonical = codecs.lookup(encoding).name+ decoded = content.decode(canonical)+ except (LookupError, UnicodeDecodeError):+ return None++ if not _is_valid_text(decoded):+ return None++ return _EncodingCandidate(+ encoding=canonical,+ decoded=decoded,+ rank=rank,+ from_detector=from_detector,+ )+++def _is_valid_text(decoded: str) -> bool:+ sample = decoded[:1000]+ if not sample:+ return True++ if "\ufffd" in sample:+ return False++ disallowed_controls = sum(1 for char in sample if _is_disallowed_control(char))+ return disallowed_controls / len(sample) <= 0.05+++def _is_disallowed_control(char: str) -> bool:+ codepoint = ord(char)+ return (codepoint < 32 and char not in "\t\n\r") or 0x7F <= codepoint <= 0x9F+++def _choose_candidate(candidates: Iterable[_EncodingCandidate]) -> Optional[_EncodingCandidate]:+ candidates = list(candidates)+ if not candidates:+ return None++ consistent = [candidate for candidate in candidates if _is_script_consistent(candidate)]+ if not consistent:+ consistent = candidates++ # Preserve detector rank for Japanese: short kanji-only Shift-JIS has no+ # kana signal, but charset-normalizer still ranks CP932 first.+ first = min(consistent, key=lambda candidate: candidate.rank)+ if first.encoding not in _CJK_ENCODINGS:+ return first++ # Short CJK-only samples can remain ambiguous across Chinese, Japanese, and+ # Korean encodings without source charset or language metadata.+ if first.encoding in _JAPANESE_ENCODINGS:+ return first++ preferred_chinese = _preferred_chinese_candidate(consistent)+ detected_chinese = _preferred_chinese_candidate(consistent, from_detector_only=True)+ for candidate in consistent:+ profile = _script_profile(candidate.decoded)+ if (+ candidate.encoding in _KOREAN_ENCODINGS+ and profile.hangul_syllables >= _MIN_STRONG_SCRIPT_CHARS+ and profile.cjk == 0+ ):+ return candidate++ first_profile = _script_profile(first.decoded)+ # Short Simplified Chinese can be misranked as mixed Hangul/CJK under CP949.+ # Only rescue to Chinese when charset-normalizer also emitted a Chinese+ # candidate and the Korean-looking decode is not Hanja-dominant.+ if (+ first.encoding in _KOREAN_ENCODINGS+ and detected_chinese is not None+ and first_profile.hangul_syllables > 0+ and first_profile.cjk > 0+ and first_profile.hangul_syllables >= first_profile.cjk+ ):+ return detected_chinese++ if first.encoding in _CHINESE_ENCODINGS and preferred_chinese is not None:+ return preferred_chinese++ return first+++def _preferred_chinese_candidate(+ candidates: Iterable[_EncodingCandidate],+ *,+ from_detector_only: bool = False,+) -> Optional[_EncodingCandidate]:+ for encoding in _CHINESE_ENCODING_PREFERENCE:+ for candidate in candidates:+ if candidate.encoding == encoding and (+ not from_detector_only or candidate.from_detector+ ):+ return candidate+ return None+++def _is_script_consistent(candidate: _EncodingCandidate) -> bool:+ profile = _script_profile(candidate.decoded)++ if candidate.encoding in _CHINESE_ENCODINGS:+ return profile.kana == 0 and profile.hangul_syllables == 0 and profile.hangul_jamo == 0++ if candidate.encoding in _JAPANESE_ENCODINGS:+ return profile.hangul_syllables == 0 and profile.hangul_jamo == 0++ if candidate.encoding in _KOREAN_ENCODINGS:+ return profile.kana == 0 and profile.hangul_jamo == 0++ return True+++def _script_profile(decoded: str) -> _ScriptProfile:+ sample = decoded[:1000]+ return _ScriptProfile(+ cjk=sum(1 for char in sample if _is_cjk(char)),+ kana=sum(1 for char in sample if _is_kana(char)),+ hangul_syllables=sum(1 for char in sample if "\uac00" <= char <= "\ud7af"),+ hangul_jamo=sum(1 for char in sample if _is_hangul_jamo(char)),+ )+++def _is_cjk(char: str) -> bool:+ return "\u3400" <= char <= "\u4dbf" or "\u4e00" <= char <= "\u9fff"+++def _is_kana(char: str) -> bool:+ return "\u3040" <= char <= "\u30ff" or "\uff66" <= char <= "\uff9f"+++def _is_hangul_jamo(char: str) -> bool:+ return (+ "\u1100" <= char <= "\u11ff"+ or "\u3130" <= char <= "\u318f"+ or "\ua960" <= char <= "\ua97f"+ or "\ud7b0" <= char <= "\ud7ff"+ )diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.pyindex 9d1f98069..54d5b09b2 100644--- a/openviking/parse/parsers/upload_utils.py+++ b/openviking/parse/parsers/upload_utils.py@@ -14,9 +14,8 @@ from openviking.parse.parsers.constants import (DOCUMENTATION_EXTENSIONS,IGNORE_DIRS,IGNORE_EXTENSIONS,- TEXT_ENCODINGS,- UTF8_VARIANTS,)+from openviking.parse.parsers.text_encoding import normalize_text_bytesfrom openviking.utils.path_safety import safe_join_viking_uri, sanitize_relative_viking_pathfrom openviking_cli.utils.logger import get_logger@@ -63,50 +62,7 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = """""Detect text encoding and normalize content to UTF-8 when needed."""if not is_text_file(file_path):return content-- try:- # Check for potential binary content (null bytes in first 8KB)- # Binary files often contain null bytes which can cause issues- sample_size = min(8192, len(content))- if b"\x00" in content[:sample_size]:- null_count = content[:sample_size].count(b"\x00")- # If more than 5% null bytes in sample, likely binary - don't process- if null_count / sample_size > 0.05:- logger.debug(- f"Detected binary content in {file_path} (null bytes: {null_count}), skipping encoding detection"- )- return content-- detected_encoding: Optional[str] = None- for encoding in TEXT_ENCODINGS:- try:- decoded = content.decode(encoding)- # Additional validation: check for control characters that suggest binary- control_chars = sum(1 for c in decoded[:1000] if ord(c) < 32 and c not in "\t\n\r")- if control_chars / min(1000, len(decoded)) > 0.05: # More than 5% control chars- continue- detected_encoding = encoding- break- except UnicodeDecodeError:- continue-- if detected_encoding is None:- logger.warning(f"Encoding detection failed for {file_path}: no matching encoding found")- return content-- if detected_encoding not in UTF8_VARIANTS:- decoded_content = content.decode(detected_encoding, errors="replace")- # Remove null bytes from decoded content as they can cause issues downstream- if "\x00" in decoded_content:- decoded_content = decoded_content.replace("\x00", "")- logger.debug(f"Removed null bytes from decoded content in {file_path}")- content = decoded_content.encode("utf-8")- logger.debug(f"Converted {file_path} from {detected_encoding} to UTF-8")-- return content- except Exception as exc:- logger.warning(f"Encoding detection failed for {file_path}: {exc}")- return content+ return normalize_text_bytes(content, file_path)def should_skip_file(diff --git a/pyproject.toml b/pyproject.tomlindex b36c306e2..61c51b5ab 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,6 +40,7 @@ dependencies = ["markdownify>=0.11.0","openai>=1.0.0","requests>=2.31.0",+ "charset-normalizer>=3.4,<4","python-docx>=1.0.0","olefile>=0.47","xlrd>=2.0.1",diff --git a/tests/parse/test_text_file_encoding.py b/tests/parse/test_text_file_encoding.pynew file mode 100644index 000000000..9cd601726--- /dev/null+++ b/tests/parse/test_text_file_encoding.py@@ -0,0 +1,125 @@+import pytest++from openviking.parse.parsers import upload_utils+from openviking.parse.parsers.markdown import MarkdownParser+from openviking.parse.parsers.upload_utils import detect_and_convert_encoding+++def test_markdown_file_read_normalizes_gb18030_text(tmp_path):+ content = "# 编码兼容测试文档\n\n这是一段中文正文,用于验证旧编码文本不会被解析成乱码。\n"+ path = tmp_path / "legacy-chinese.md"+ path.write_bytes(content.encode("gb18030"))++ assert MarkdownParser()._read_file(path) == content+++def test_markdown_file_read_normalizes_gb18030_four_byte_text(tmp_path):+ content = "# 标题 😀 emoji\n\n正文内容\n"+ path = tmp_path / "legacy-chinese-emoji.md"+ path.write_bytes(content.encode("gb18030"))++ assert MarkdownParser()._read_file(path) == content+++ ("encoding", "content"),+ [+ ("gb18030", "简体中文测试内容\n"),+ ("gb18030", "简体中文测试内容123 abc\n"),+ ("gbk", "简体中文测试内容\n"),+ ],+)+def test_markdown_file_read_normalizes_short_simplified_chinese_text(tmp_path, encoding, content):+ path = tmp_path / "short-simplified-chinese.md"+ path.write_bytes(content.encode(encoding))++ assert MarkdownParser()._read_file(path) == content+++def test_markdown_file_read_preserves_detector_first_non_cjk_text(tmp_path):+ content = "# Müller straße Zürich Köln Größe\n"+ path = tmp_path / "legacy-german.md"+ path.write_bytes(content.encode("cp1252"))++ assert MarkdownParser()._read_file(path) == content+++ "content",+ [+ "한국어 漢字 混用 문장 입니다\n",+ "大韓民國 서울特別市 政府\n",+ ],+)+def test_markdown_file_read_preserves_korean_hanja_text(tmp_path, content):+ path = tmp_path / "korean-hanja.md"+ path.write_bytes(content.encode("euc-kr"))++ assert MarkdownParser()._read_file(path) == content+++def test_markdown_file_read_strips_utf8_bom(tmp_path):+ content = "# Heading\n\nBody\n"+ path = tmp_path / "utf8-bom.md"+ path.write_bytes(b"\xef\xbb\xbf" + content.encode("utf-8"))++ assert MarkdownParser()._read_file(path) == content++[email protected]("encoding", ["utf-16", "utf-16-be", "utf-32", "utf-32-be"])+def test_markdown_file_read_normalizes_unicode_bom_text(tmp_path, encoding):+ content = "# Heading\n\nBody\n"+ path = tmp_path / f"unicode-bom-{encoding}.md"+ encoded = content.encode(encoding)+ if encoding == "utf-16-be":+ encoded = b"\xfe\xff" + encoded+ elif encoding == "utf-32-be":+ encoded = b"\x00\x00\xfe\xff" + encoded+ path.write_bytes(encoded)++ assert MarkdownParser()._read_file(path) == content+++ ("encoding", "content"),+ [+ ("shift_jis", "漢字\n"),+ ("shift_jis", "# タイトル\n本文です\n"),+ ("big5", "# 標題\n繁體中文內容\n"),+ ("euc-kr", "# 제목\n본문 내용\n"),+ ("cp1252", "Price: €10 – café\n"),+ ("cp1252", "Smart “quotes” and — dash\n"),+ ("cp1252", "œ Œ ž Ž š Š Ÿ ™\n"),+ ("latin-1", "café naïve\n"),+ ],+)+def test_markdown_file_read_normalizes_representative_legacy_encodings(tmp_path, encoding, content):+ path = tmp_path / f"legacy-{encoding}.md"+ path.write_bytes(content.encode(encoding))++ assert MarkdownParser()._read_file(path) == content+++def test_text_file_read_preserves_legacy_latin1_fallback_for_unrecognized_bytes(tmp_path):+ raw = bytes([1, 2, 3, 4, 5, 0xFF, 0xFE, 0xFD])+ path = tmp_path / "control-heavy.txt"+ path.write_bytes(raw)++ assert MarkdownParser()._read_file(path) == raw.decode("latin-1")+++def test_encoding_detection_handles_empty_text_without_warning(tmp_path, monkeypatch):+ path = tmp_path / "empty.md"+ warnings = []++ monkeypatch.setattr(upload_utils.logger, "warning", lambda message: warnings.append(message))++ assert detect_and_convert_encoding(b"", path) == b""+ assert warnings == []+++def test_encoding_detection_preserves_malformed_bom_bytes(tmp_path):+ raw = b"\xff\xfe\x00"+ path = tmp_path / "malformed-utf16.md"++ assert detect_and_convert_encoding(raw, path) == rawdiff --git a/tests/test_upload_utils.py b/tests/test_upload_utils.pyindex 045a5af7d..d665714da 100644--- a/tests/test_upload_utils.py+++ b/tests/test_upload_utils.py@@ -423,9 +423,8 @@ class TestDetectAndConvertEncodingEdgeCases:assert result.decode("utf-8") == textdef test_undecodable_content(self) -> None:- # Note: TEXT_ENCODINGS includes iso-8859-1 which can decode any byte sequence,- # so the "no matching encoding" branch is effectively unreachable.- # This test verifies that arbitrary bytes are handled gracefully regardless.+ # Arbitrary bytes should be handled gracefully even when no text+ # encoding can be selected with confidence.content = bytes(range(128, 256)) * 10result = detect_and_convert_encoding(content, "test.py")assert isinstance(result, bytes)diff --git a/uv.lock b/uv.lockindex fd1d9c03a..da10cb068 100644--- a/uv.lock+++ b/uv.lock@@ -3509,6 +3509,7 @@ source = { editable = "." }dependencies = [{ name = "apscheduler" },{ name = "argon2-cffi" },+ { name = "charset-normalizer" },{ name = "cryptography" },{ name = "ebooklib" },{ name = "fastapi" },@@ -3733,6 +3734,7 @@ requires-dist = [{ name = "beautifulsoup4", marker = "extra == 'bot'", specifier = ">=4.12.0" },{ name = "boto3", marker = "extra == 'test'", specifier = ">=1.42.44" },{ name = "build", marker = "extra == 'build'" },+ { name = "charset-normalizer", specifier = ">=3.4,<4" },{ name = "cmake", marker = "extra == 'build'", specifier = ">=3.15" },{ name = "croniter", marker = "extra == 'bot'", specifier = ">=2.0.0" },{ name = "cryptography", specifier = ">=42.0.0" },