A Simple and Universal Swarm Intelligence Engine, Predicting Anything. 简洁通用的群体智能引擎,预测万物
Python agent-memoryfinancial-forecastingfuture-predictionknowledge-graphllmsmulti-agent-simulationpublic-opinion-analysispython3social-predictionswarm-intelligence
fix(file_parser): handle non-UTF-8 encoded text files with automatic encoding detection
666ghj390c120fef9a
diff --git a/backend/app/utils/file_parser.py b/backend/app/utils/file_parser.pyindex ac0f6365..3f1d8ed2 100644--- a/backend/app/utils/file_parser.py+++ b/backend/app/utils/file_parser.py@@ -8,6 +8,56 @@ from pathlib import Pathfrom typing import List, Optional+def _read_text_with_fallback(file_path: str) -> str:+ """+ 读取文本文件,UTF-8失败时自动探测编码。++ 采用多级回退策略:+ 1. 首先尝试 UTF-8 解码+ 2. 使用 charset_normalizer 检测编码+ 3. 回退到 chardet 检测编码+ 4. 最终使用 UTF-8 + errors='replace' 兜底++ Args:+ file_path: 文件路径++ Returns:+ 解码后的文本内容+ """+ data = Path(file_path).read_bytes()++ # 首先尝试 UTF-8+ try:+ return data.decode('utf-8')+ except UnicodeDecodeError:+ pass++ # 尝试使用 charset_normalizer 检测编码+ encoding = None+ try:+ from charset_normalizer import from_bytes+ best = from_bytes(data).best()+ if best and best.encoding:+ encoding = best.encoding+ except Exception:+ pass++ # 回退到 chardet+ if not encoding:+ try:+ import chardet+ result = chardet.detect(data)+ encoding = result.get('encoding') if result else None+ except Exception:+ pass++ # 最终兜底:使用 UTF-8 + replace+ if not encoding:+ encoding = 'utf-8'++ return data.decode(encoding, errors='replace')++class FileParser:"""文件解析器"""@@ -62,15 +112,13 @@ class FileParser:@staticmethoddef _extract_from_md(file_path: str) -> str:- """从Markdown提取文本"""- with open(file_path, 'r', encoding='utf-8') as f:- return f.read()+ """从Markdown提取文本,支持自动编码检测"""+ return _read_text_with_fallback(file_path)@staticmethoddef _extract_from_txt(file_path: str) -> str:- """从TXT提取文本"""- with open(file_path, 'r', encoding='utf-8') as f:- return f.read()+ """从TXT提取文本,支持自动编码检测"""+ return _read_text_with_fallback(file_path)@classmethoddef extract_from_multiple(cls, file_paths: List[str]) -> str:diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 8f665c05..4f5361d5 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -25,6 +25,9 @@ dependencies = [# 文件处理"PyMuPDF>=1.24.0",+ # 编码检测(支持非UTF-8编码的文本文件)+ "charset-normalizer>=3.0.0",+ "chardet>=5.0.0",# 工具库"python-dotenv>=1.0.0",diff --git a/backend/requirements.txt b/backend/requirements.txtindex cfe68e0a..4f146296 100644--- a/backend/requirements.txt+++ b/backend/requirements.txt@@ -23,6 +23,9 @@ camel-ai==0.2.78# ============= 文件处理 =============PyMuPDF>=1.24.0+# 编码检测(支持非UTF-8编码的文本文件)+charset-normalizer>=3.0.0+chardet>=5.0.0# ============= 工具库 =============# 环境变量加载diff --git a/backend/uv.lock b/backend/uv.lockindex 0b715702..f1ce4b60 100644--- a/backend/uv.lock+++ b/backend/uv.lock@@ -1244,6 +1244,8 @@ source = { editable = "." }dependencies = [{ name = "camel-ai" },{ name = "camel-oasis" },+ { name = "chardet" },+ { name = "charset-normalizer" },{ name = "flask" },{ name = "flask-cors" },{ name = "openai" },@@ -1270,6 +1272,8 @@ dev = [requires-dist = [{ name = "camel-ai", specifier = "==0.2.78" },{ name = "camel-oasis", specifier = "==0.2.5" },+ { name = "chardet", specifier = ">=5.0.0" },+ { name = "charset-normalizer", specifier = ">=3.0.0" },{ name = "flask", specifier = ">=3.0.0" },{ name = "flask-cors", specifier = ">=6.0.0" },{ name = "openai", specifier = ">=1.0.0" },