Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): normalize legacy text encodings (#2770)

Hao Zhe324f96ebb664
diff --git a/openviking/parse/parsers/base_parser.py b/openviking/parse/parsers/base_parser.py
index be2ec403b..85911c2b1 100644
--- a/openviking/parse/parsers/base_parser.py
+++ b/openviking/parse/parsers/base_parser.py
@@ -80,16 +80,24 @@ class BaseParser(ABC):
File content as string
"""
path = Path(path)
- encodings = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
-
- for encoding in encodings:
+ from openviking.parse.parsers.upload_utils import detect_and_convert_encoding
+
+ raw = path.read_bytes()
+ normalized = detect_and_convert_encoding(raw, path)
+ try:
+ return normalized.decode("utf-8")
+ except UnicodeDecodeError:
+ pass
+
+ # Preserve the previous permissive fallback for files whose encoding is
+ # unknown or rejected by text detection as likely binary/control-heavy.
+ for encoding in ("utf-8-sig", "cp1252"):
try:
- with open(path, "r", encoding=encoding) as f:
- return f.read()
+ return raw.decode(encoding)
except UnicodeDecodeError:
continue
- raise ValueError(f"Unable to decode file: {path}")
+ return raw.decode("latin-1")
def _get_viking_fs(self):
"""
diff --git a/openviking/parse/parsers/constants.py b/openviking/parse/parsers/constants.py
index 8561e31a8..74609115e 100644
--- a/openviking/parse/parsers/constants.py
+++ b/openviking/parse/parsers/constants.py
@@ -232,20 +232,3 @@ ADDITIONAL_TEXT_EXTENSIONS = {
".lock",
".in",
}
-
-# Common text encodings to try for encoding detection (in order of likelihood)
-TEXT_ENCODINGS = [
- "utf-8", # Most common modern encoding
- "utf-8-sig", # UTF-8 with BOM
- "gbk", # Chinese GBK (simplified Chinese)
- "gb2312", # Chinese GB2312 (simplified Chinese)
- "big5", # Traditional Chinese
- "shift_jis", # Japanese
- "euc-kr", # Korean
- "iso-8859-1", # Latin-1 (Western European)
- "cp1252", # Windows Latin-1
- "latin-1", # Latin-1 alias
-]
-
-# UTF-8 variants that don't need conversion
-UTF8_VARIANTS = {"utf-8", "utf-8-sig"}
diff --git a/openviking/parse/parsers/text_encoding.py b/openviking/parse/parsers/text_encoding.py
new file mode 100644
index 000000000..4c954d87f
--- /dev/null
+++ b/openviking/parse/parsers/text_encoding.py
@@ -0,0 +1,289 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Text encoding normalization helpers for uploaded text files."""
+
+import codecs
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Optional, Union
+
+from charset_normalizer import from_bytes
+
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+_BOM_ENCODINGS = (
+ (codecs.BOM_UTF32_LE, "utf-32"),
+ (codecs.BOM_UTF32_BE, "utf-32"),
+ (codecs.BOM_UTF16_LE, "utf-16"),
+ (codecs.BOM_UTF16_BE, "utf-16"),
+ (codecs.BOM_UTF8, "utf-8-sig"),
+)
+
+_DETECTION_ENCODINGS = (
+ "gb18030",
+ "gbk",
+ "gb2312",
+ "big5",
+ "cp950",
+ "cp932",
+ "shift_jis",
+ "euc_jp",
+ "euc_kr",
+ "cp949",
+ "cp1252",
+ "latin_1",
+)
+
+# These values use codecs.lookup(...).name canonical forms because candidates
+# are canonicalized before set membership checks.
+_CHINESE_ENCODINGS = {"gb18030", "gbk", "gb2312", "big5", "cp950"}
+_CHINESE_ENCODING_PREFERENCE = ("gb18030", "gbk", "gb2312", "big5", "cp950")
+_JAPANESE_ENCODINGS = {"cp932", "shift_jis", "euc_jp"}
+_KOREAN_ENCODINGS = {"euc_kr", "cp949"}
+_CJK_ENCODINGS = _CHINESE_ENCODINGS | _JAPANESE_ENCODINGS | _KOREAN_ENCODINGS
+# Very short CJK samples can decode into plausible text under several legacy
+# code pages. Require a few script-specific characters before overriding the
+# detector's rank with Korean script evidence.
+_MIN_STRONG_SCRIPT_CHARS = 4
+
+
+@dataclass(frozen=True)
+class _ScriptProfile:
+ cjk: int
+ kana: int
+ hangul_syllables: int
+ hangul_jamo: int
+
+
+@dataclass(frozen=True)
+class _EncodingCandidate:
+ encoding: str
+ decoded: str
+ rank: int
+ from_detector: bool
+
+
+def normalize_text_bytes(content: bytes, file_path: Union[str, Path] = "") -> bytes:
+ """Normalize legacy text bytes to UTF-8 while leaving unknown bytes untouched."""
+ try:
+ return _normalize_text_bytes(content, file_path)
+ except Exception as exc:
+ logger.warning(f"Encoding detection failed for {file_path}: {exc}")
+ return content
+
+
+def _normalize_text_bytes(content: bytes, file_path: Union[str, Path] = "") -> bytes:
+ if not content:
+ return content
+
+ bom_normalized = _decode_known_bom(content)
+ if bom_normalized is not None:
+ return bom_normalized
+
+ try:
+ content.decode("utf-8")
+ return content
+ except UnicodeDecodeError:
+ pass
+
+ if _looks_binary(content):
+ logger.debug(f"Detected binary-like text file {file_path}, skipping encoding detection")
+ return content
+
+ candidate = _choose_candidate(_iter_candidates(content))
+ if candidate is None:
+ logger.warning(f"Encoding detection failed for {file_path}: no matching encoding found")
+ return content
+
+ decoded = candidate.decoded.replace("\x00", "")
+ normalized = decoded.encode("utf-8")
+ logger.debug(f"Converted {file_path} from {candidate.encoding} to UTF-8")
+ return normalized
+
+
+def _decode_known_bom(content: bytes) -> Optional[bytes]:
+ for bom, encoding in _BOM_ENCODINGS:
+ if content.startswith(bom):
+ try:
+ return content.decode(encoding).encode("utf-8")
+ except UnicodeDecodeError:
+ return None
+ return None
+
+
+def _looks_binary(content: bytes) -> bool:
+ sample = content[: min(8192, len(content))]
+ if not sample:
+ return False
+
+ return sample.count(b"\x00") / len(sample) > 0.05
+
+
+def _iter_candidates(content: bytes) -> Iterable[_EncodingCandidate]:
+ seen: set[str] = set()
+ rank = 0
+
+ for match in from_bytes(content, cp_isolation=_DETECTION_ENCODINGS):
+ candidate = _decode_candidate(content, match.encoding, rank, from_detector=True)
+ if candidate is None or candidate.encoding in seen:
+ continue
+ seen.add(candidate.encoding)
+ rank += 1
+ yield candidate
+
+ for encoding in _DETECTION_ENCODINGS:
+ candidate = _decode_candidate(content, encoding, rank, from_detector=False)
+ if candidate is None or candidate.encoding in seen:
+ continue
+ seen.add(candidate.encoding)
+ rank += 1
+ yield candidate
+
+
+def _decode_candidate(
+ content: bytes, encoding: Optional[str], rank: int, *, from_detector: bool
+) -> Optional[_EncodingCandidate]:
+ if not encoding:
+ return None
+
+ try:
+ canonical = codecs.lookup(encoding).name
+ decoded = content.decode(canonical)
+ except (LookupError, UnicodeDecodeError):
+ return None
+
+ if not _is_valid_text(decoded):
+ return None
+
+ return _EncodingCandidate(
+ encoding=canonical,
+ decoded=decoded,
+ rank=rank,
+ from_detector=from_detector,
+ )
+
+
+def _is_valid_text(decoded: str) -> bool:
+ sample = decoded[:1000]
+ if not sample:
+ return True
+
+ if "\ufffd" in sample:
+ return False
+
+ disallowed_controls = sum(1 for char in sample if _is_disallowed_control(char))
+ return disallowed_controls / len(sample) <= 0.05
+
+
+def _is_disallowed_control(char: str) -> bool:
+ codepoint = ord(char)
+ return (codepoint < 32 and char not in "\t\n\r") or 0x7F <= codepoint <= 0x9F
+
+
+def _choose_candidate(candidates: Iterable[_EncodingCandidate]) -> Optional[_EncodingCandidate]:
+ candidates = list(candidates)
+ if not candidates:
+ return None
+
+ consistent = [candidate for candidate in candidates if _is_script_consistent(candidate)]
+ if not consistent:
+ consistent = candidates
+
+ # Preserve detector rank for Japanese: short kanji-only Shift-JIS has no
+ # kana signal, but charset-normalizer still ranks CP932 first.
+ first = min(consistent, key=lambda candidate: candidate.rank)
+ if first.encoding not in _CJK_ENCODINGS:
+ return first
+
+ # Short CJK-only samples can remain ambiguous across Chinese, Japanese, and
+ # Korean encodings without source charset or language metadata.
+ if first.encoding in _JAPANESE_ENCODINGS:
+ return first
+
+ preferred_chinese = _preferred_chinese_candidate(consistent)
+ detected_chinese = _preferred_chinese_candidate(consistent, from_detector_only=True)
+ for candidate in consistent:
+ profile = _script_profile(candidate.decoded)
+ if (
+ candidate.encoding in _KOREAN_ENCODINGS
+ and profile.hangul_syllables >= _MIN_STRONG_SCRIPT_CHARS
+ and profile.cjk == 0
+ ):
+ return candidate
+
+ first_profile = _script_profile(first.decoded)
+ # Short Simplified Chinese can be misranked as mixed Hangul/CJK under CP949.
+ # Only rescue to Chinese when charset-normalizer also emitted a Chinese
+ # candidate and the Korean-looking decode is not Hanja-dominant.
+ if (
+ first.encoding in _KOREAN_ENCODINGS
+ and detected_chinese is not None
+ and first_profile.hangul_syllables > 0
+ and first_profile.cjk > 0
+ and first_profile.hangul_syllables >= first_profile.cjk
+ ):
+ return detected_chinese
+
+ if first.encoding in _CHINESE_ENCODINGS and preferred_chinese is not None:
+ return preferred_chinese
+
+ return first
+
+
+def _preferred_chinese_candidate(
+ candidates: Iterable[_EncodingCandidate],
+ *,
+ from_detector_only: bool = False,
+) -> Optional[_EncodingCandidate]:
+ for encoding in _CHINESE_ENCODING_PREFERENCE:
+ for candidate in candidates:
+ if candidate.encoding == encoding and (
+ not from_detector_only or candidate.from_detector
+ ):
+ return candidate
+ return None
+
+
+def _is_script_consistent(candidate: _EncodingCandidate) -> bool:
+ profile = _script_profile(candidate.decoded)
+
+ if candidate.encoding in _CHINESE_ENCODINGS:
+ return profile.kana == 0 and profile.hangul_syllables == 0 and profile.hangul_jamo == 0
+
+ if candidate.encoding in _JAPANESE_ENCODINGS:
+ return profile.hangul_syllables == 0 and profile.hangul_jamo == 0
+
+ if candidate.encoding in _KOREAN_ENCODINGS:
+ return profile.kana == 0 and profile.hangul_jamo == 0
+
+ return True
+
+
+def _script_profile(decoded: str) -> _ScriptProfile:
+ sample = decoded[:1000]
+ return _ScriptProfile(
+ cjk=sum(1 for char in sample if _is_cjk(char)),
+ kana=sum(1 for char in sample if _is_kana(char)),
+ hangul_syllables=sum(1 for char in sample if "\uac00" <= char <= "\ud7af"),
+ hangul_jamo=sum(1 for char in sample if _is_hangul_jamo(char)),
+ )
+
+
+def _is_cjk(char: str) -> bool:
+ return "\u3400" <= char <= "\u4dbf" or "\u4e00" <= char <= "\u9fff"
+
+
+def _is_kana(char: str) -> bool:
+ return "\u3040" <= char <= "\u30ff" or "\uff66" <= char <= "\uff9f"
+
+
+def _is_hangul_jamo(char: str) -> bool:
+ return (
+ "\u1100" <= char <= "\u11ff"
+ or "\u3130" <= char <= "\u318f"
+ or "\ua960" <= char <= "\ua97f"
+ or "\ud7b0" <= char <= "\ud7ff"
+ )
diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.py
index 9d1f98069..54d5b09b2 100644
--- a/openviking/parse/parsers/upload_utils.py
+++ b/openviking/parse/parsers/upload_utils.py
@@ -14,9 +14,8 @@ from openviking.parse.parsers.constants import (
DOCUMENTATION_EXTENSIONS,
IGNORE_DIRS,
IGNORE_EXTENSIONS,
- TEXT_ENCODINGS,
- UTF8_VARIANTS,
)
+from openviking.parse.parsers.text_encoding import normalize_text_bytes
from openviking.utils.path_safety import safe_join_viking_uri, sanitize_relative_viking_path
from openviking_cli.utils.logger import get_logger
@@ -63,50 +62,7 @@ def detect_and_convert_encoding(content: bytes, file_path: Union[str, Path] = ""
"""Detect text encoding and normalize content to UTF-8 when needed."""
if not is_text_file(file_path):
return content
-
- try:
- # Check for potential binary content (null bytes in first 8KB)
- # Binary files often contain null bytes which can cause issues
- sample_size = min(8192, len(content))
- if b"\x00" in content[:sample_size]:
- null_count = content[:sample_size].count(b"\x00")
- # If more than 5% null bytes in sample, likely binary - don't process
- if null_count / sample_size > 0.05:
- logger.debug(
- f"Detected binary content in {file_path} (null bytes: {null_count}), skipping encoding detection"
- )
- return content
-
- detected_encoding: Optional[str] = None
- for encoding in TEXT_ENCODINGS:
- try:
- decoded = content.decode(encoding)
- # Additional validation: check for control characters that suggest binary
- control_chars = sum(1 for c in decoded[:1000] if ord(c) < 32 and c not in "\t\n\r")
- if control_chars / min(1000, len(decoded)) > 0.05: # More than 5% control chars
- continue
- detected_encoding = encoding
- break
- except UnicodeDecodeError:
- continue
-
- if detected_encoding is None:
- logger.warning(f"Encoding detection failed for {file_path}: no matching encoding found")
- return content
-
- if detected_encoding not in UTF8_VARIANTS:
- decoded_content = content.decode(detected_encoding, errors="replace")
- # Remove null bytes from decoded content as they can cause issues downstream
- if "\x00" in decoded_content:
- decoded_content = decoded_content.replace("\x00", "")
- logger.debug(f"Removed null bytes from decoded content in {file_path}")
- content = decoded_content.encode("utf-8")
- logger.debug(f"Converted {file_path} from {detected_encoding} to UTF-8")
-
- return content
- except Exception as exc:
- logger.warning(f"Encoding detection failed for {file_path}: {exc}")
- return content
+ return normalize_text_bytes(content, file_path)
def should_skip_file(
diff --git a/pyproject.toml b/pyproject.toml
index b36c306e2..61c51b5ab 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,7 @@ dependencies = [
"markdownify>=0.11.0",
"openai>=1.0.0",
"requests>=2.31.0",
+ "charset-normalizer>=3.4,<4",
"python-docx>=1.0.0",
"olefile>=0.47",
"xlrd>=2.0.1",
diff --git a/tests/parse/test_text_file_encoding.py b/tests/parse/test_text_file_encoding.py
new file mode 100644
index 000000000..9cd601726
--- /dev/null
+++ b/tests/parse/test_text_file_encoding.py
@@ -0,0 +1,125 @@
+import pytest
+
+from openviking.parse.parsers import upload_utils
+from openviking.parse.parsers.markdown import MarkdownParser
+from openviking.parse.parsers.upload_utils import detect_and_convert_encoding
+
+
+def test_markdown_file_read_normalizes_gb18030_text(tmp_path):
+ content = "# 编码兼容测试文档\n\n这是一段中文正文,用于验证旧编码文本不会被解析成乱码。\n"
+ path = tmp_path / "legacy-chinese.md"
+ path.write_bytes(content.encode("gb18030"))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+def test_markdown_file_read_normalizes_gb18030_four_byte_text(tmp_path):
+ content = "# 标题 😀 emoji\n\n正文内容\n"
+ path = tmp_path / "legacy-chinese-emoji.md"
+ path.write_bytes(content.encode("gb18030"))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+ ("encoding", "content"),
+ [
+ ("gb18030", "简体中文测试内容\n"),
+ ("gb18030", "简体中文测试内容123 abc\n"),
+ ("gbk", "简体中文测试内容\n"),
+ ],
+)
+def test_markdown_file_read_normalizes_short_simplified_chinese_text(tmp_path, encoding, content):
+ path = tmp_path / "short-simplified-chinese.md"
+ path.write_bytes(content.encode(encoding))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+def test_markdown_file_read_preserves_detector_first_non_cjk_text(tmp_path):
+ content = "# Müller straße Zürich Köln Größe\n"
+ path = tmp_path / "legacy-german.md"
+ path.write_bytes(content.encode("cp1252"))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+ "content",
+ [
+ "한국어 漢字 混用 문장 입니다\n",
+ "大韓民國 서울特別市 政府\n",
+ ],
+)
+def test_markdown_file_read_preserves_korean_hanja_text(tmp_path, content):
+ path = tmp_path / "korean-hanja.md"
+ path.write_bytes(content.encode("euc-kr"))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+def test_markdown_file_read_strips_utf8_bom(tmp_path):
+ content = "# Heading\n\nBody\n"
+ path = tmp_path / "utf8-bom.md"
+ path.write_bytes(b"\xef\xbb\xbf" + content.encode("utf-8"))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
[email protected]("encoding", ["utf-16", "utf-16-be", "utf-32", "utf-32-be"])
+def test_markdown_file_read_normalizes_unicode_bom_text(tmp_path, encoding):
+ content = "# Heading\n\nBody\n"
+ path = tmp_path / f"unicode-bom-{encoding}.md"
+ encoded = content.encode(encoding)
+ if encoding == "utf-16-be":
+ encoded = b"\xfe\xff" + encoded
+ elif encoding == "utf-32-be":
+ encoded = b"\x00\x00\xfe\xff" + encoded
+ path.write_bytes(encoded)
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+ ("encoding", "content"),
+ [
+ ("shift_jis", "漢字\n"),
+ ("shift_jis", "# タイトル\n本文です\n"),
+ ("big5", "# 標題\n繁體中文內容\n"),
+ ("euc-kr", "# 제목\n본문 내용\n"),
+ ("cp1252", "Price: €10 – café\n"),
+ ("cp1252", "Smart “quotes” and — dash\n"),
+ ("cp1252", "œ Œ ž Ž š Š Ÿ ™\n"),
+ ("latin-1", "café naïve\n"),
+ ],
+)
+def test_markdown_file_read_normalizes_representative_legacy_encodings(tmp_path, encoding, content):
+ path = tmp_path / f"legacy-{encoding}.md"
+ path.write_bytes(content.encode(encoding))
+
+ assert MarkdownParser()._read_file(path) == content
+
+
+def test_text_file_read_preserves_legacy_latin1_fallback_for_unrecognized_bytes(tmp_path):
+ raw = bytes([1, 2, 3, 4, 5, 0xFF, 0xFE, 0xFD])
+ path = tmp_path / "control-heavy.txt"
+ path.write_bytes(raw)
+
+ assert MarkdownParser()._read_file(path) == raw.decode("latin-1")
+
+
+def test_encoding_detection_handles_empty_text_without_warning(tmp_path, monkeypatch):
+ path = tmp_path / "empty.md"
+ warnings = []
+
+ monkeypatch.setattr(upload_utils.logger, "warning", lambda message: warnings.append(message))
+
+ assert detect_and_convert_encoding(b"", path) == b""
+ assert warnings == []
+
+
+def test_encoding_detection_preserves_malformed_bom_bytes(tmp_path):
+ raw = b"\xff\xfe\x00"
+ path = tmp_path / "malformed-utf16.md"
+
+ assert detect_and_convert_encoding(raw, path) == raw
diff --git a/tests/test_upload_utils.py b/tests/test_upload_utils.py
index 045a5af7d..d665714da 100644
--- a/tests/test_upload_utils.py
+++ b/tests/test_upload_utils.py
@@ -423,9 +423,8 @@ class TestDetectAndConvertEncodingEdgeCases:
assert result.decode("utf-8") == text
def test_undecodable_content(self) -> None:
- # Note: TEXT_ENCODINGS includes iso-8859-1 which can decode any byte sequence,
- # so the "no matching encoding" branch is effectively unreachable.
- # This test verifies that arbitrary bytes are handled gracefully regardless.
+ # Arbitrary bytes should be handled gracefully even when no text
+ # encoding can be selected with confidence.
content = bytes(range(128, 256)) * 10
result = detect_and_convert_encoding(content, "test.py")
assert isinstance(result, bytes)
diff --git a/uv.lock b/uv.lock
index fd1d9c03a..da10cb068 100644
--- a/uv.lock
+++ b/uv.lock
@@ -3509,6 +3509,7 @@ source = { editable = "." }
dependencies = [
{ name = "apscheduler" },
{ name = "argon2-cffi" },
+ { name = "charset-normalizer" },
{ name = "cryptography" },
{ name = "ebooklib" },
{ name = "fastapi" },
@@ -3733,6 +3734,7 @@ requires-dist = [
{ name = "beautifulsoup4", marker = "extra == 'bot'", specifier = ">=4.12.0" },
{ name = "boto3", marker = "extra == 'test'", specifier = ">=1.42.44" },
{ name = "build", marker = "extra == 'build'" },
+ { name = "charset-normalizer", specifier = ">=3.4,<4" },
{ name = "cmake", marker = "extra == 'build'", specifier = ">=3.15" },
{ name = "croniter", marker = "extra == 'bot'", specifier = ">=2.0.0" },
{ name = "cryptography", specifier = ">=42.0.0" },