Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): 保留 Understanding 上传文件的已识别扩展名 (#4981)
bot-of-qin-ctx9b5f43d828cd
diff --git a/docs/design/resource-ingestion-routing.md b/docs/design/resource-ingestion-routing.mdindex 3b2ce604d..bf27bdb0e 100644--- a/docs/design/resource-ingestion-routing.md+++ b/docs/design/resource-ingestion-routing.md@@ -181,7 +181,9 @@ Accessor 路径中的文档图片和 Base 附件字段图片使用同一条素HTTP 资源以 HTTPAccessor 检出的 `meta.extension` 为准;本地文件或上传的临时文件可优先使用显式 `source_name` 的扩展名。这样既不会拿随机临时文件名选 Parser,也不会让用户提供的 URL 名称覆盖实际下载内容类型。-本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 作为 `source_name` 传给上传层;普通上传和分片上传都使用这个名称的 basename,本地路径只用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。+本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 和 `resolved_extension` 一起传给上传层。上传层使用来源名称的 basename;如果末尾没有已识别的扩展名(忽略大小写),则追加该扩展名,保留名称中的编号和版本信息。例如无后缀 URL `/export?id=123` 返回 `Content-Type: application/pdf` 时,上传文件名为 `export.pdf`;`2601.00014` 会补成 `2601.00014.pdf`,`report.PDF` 保持原样。内部的 MPEG-TS 路由标记转换为真实的 `.ts` 后缀。++普通上传、分片上传和 MIME 类型推断统一使用补全后的上传文件名,本地路径用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。## 无后缀 URL 怎么判断类型diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.pyindex 7d88596a5..69512653d 100644--- a/openviking/parse/parser_router.py+++ b/openviking/parse/parser_router.py@@ -137,6 +137,9 @@ class ParserRouter:if use_understanding:if isinstance(source, LocalResource):kwargs["source_name"] = source.meta["resolved_name"]+ kwargs["resolved_extension"] = (+ kwargs.get("resolved_extension") or source.meta["resolved_extension"]+ )display = source_pathif isinstance(source_path, str) and source_path.startswith(("http://", "https://")):display = "<url>"@@ -164,7 +167,13 @@ class ParserRouter:else kwargs.get("source_name"))return await self._get_understanding_api().submit_file(- source_path, source_name=source_name+ source_path,+ source_name=source_name,+ resolved_extension=(+ source.meta["resolved_extension"]+ if isinstance(source, LocalResource)+ else kwargs.get("resolved_extension", "")+ ),)if not self.should_use_understanding_api(str(source_path)):raise ValueError("source is not routed to UnderstandingAPI")@@ -174,7 +183,13 @@ class ParserRouter:"""Upload a local source file and return only the external Files API file_id."""source_path = self._extract_source_path(source)source_name = source.meta["resolved_name"] if isinstance(source, LocalResource) else None- return await self._get_understanding_api().upload_file(source_path, source_name=source_name)+ return await self._get_understanding_api().upload_file(+ source_path,+ source_name=source_name,+ resolved_extension=(+ source.meta["resolved_extension"] if isinstance(source, LocalResource) else ""+ ),+ )def _extract_source_path(self, source: Union[str, Path, LocalResource]) -> Union[str, Path]:"""Extract a filesystem path from the source."""diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.pyindex bea40b32b..fdf57616d 100644--- a/openviking/parse/understanding_api.py+++ b/openviking/parse/understanding_api.py@@ -29,7 +29,7 @@ from openviking.parse.image_rewrite import (build_artifact_image_mappings,)from openviking.parse.parsers.base_parser import BaseParser-from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS+from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS, TYPESCRIPT_MPEG_TS_EXTENSIONfrom openviking.parse.parsers.media.constants import (AUDIO_EXTENSIONS,IMAGE_EXTENSIONS,@@ -180,7 +180,11 @@ class UnderstandingAPI(BaseParser):task_meta["file_id"] = prepared_file_idresponse_obj = await self._create_response_for_file(file_id=prepared_file_id)elif url is None and local_path is not None:- file_obj = await self._create_file(local_path=local_path, source_name=source_name)+ file_obj = await self._create_file(+ local_path=local_path,+ source_name=source_name,+ resolved_extension=resolved_extension,+ )file_id_value = file_obj.get("id")if not file_id_value:raise RuntimeError(@@ -280,24 +284,36 @@ class UnderstandingAPI(BaseParser):return resultasync def upload_file(- self, source: Union[str, Path], *, source_name: Optional[str] = None+ self,+ source: Union[str, Path],+ *,+ source_name: Optional[str] = None,+ resolved_extension: str = "",) -> str:"""Upload a local file and return a durable Files API file_id."""local_path = Path(source)if not local_path.is_file():raise ValueError("UnderstandingAPI file upload requires an existing local file")- file_obj = await self._create_file(local_path=local_path, source_name=source_name)+ file_obj = await self._create_file(+ local_path=local_path, source_name=source_name, resolved_extension=resolved_extension+ )file_id = file_obj.get("id")if not file_id:raise RuntimeError(f"files api missing file_id: {self._safe_error_summary(file_obj)}")return str(file_id)async def submit_file(- self, source: Union[str, Path], *, source_name: Optional[str] = None+ self,+ source: Union[str, Path],+ *,+ source_name: Optional[str] = None,+ resolved_extension: str = "",) -> str:"""Upload a local file and submit it without retaining the local artifact."""- file_id = await self.upload_file(source, source_name=source_name)+ file_id = await self.upload_file(+ source, source_name=source_name, resolved_extension=resolved_extension+ )response_obj = await self._create_response_for_file(file_id=str(file_id))response_id = response_obj.get("id")if not response_id:@@ -443,13 +459,23 @@ class UnderstandingAPI(BaseParser):return bodyasync def _create_file(- self, *, local_path: Path, source_name: Optional[str] = None+ self,+ *,+ local_path: Path,+ source_name: Optional[str] = None,+ resolved_extension: str = "",) -> Dict[str, Any]:file_size = local_path.stat().st_sizeif file_size == 0:raise InvalidArgumentError("Understanding parser does not support empty files.")# The local path locates bytes; its temporary basename is not the source identity.file_name = Path(source_name).name if source_name else local_path.name+ extension = (resolved_extension or local_path.suffix).lower().lstrip(".")+ if extension == MPEG_TS_EXTENSION_ALIAS:+ extension = TYPESCRIPT_MPEG_TS_EXTENSION.lstrip(".")+ # Preserve dotted identifiers (e.g. 2601.00014) and existing suffix casing.+ if extension and not file_name.lower().endswith(f".{extension}"):+ file_name = f"{file_name}.{extension}"if file_size > self._upload_simple_max_bytes:if not self._enable_resumable_upload:raise ValueError(@@ -461,7 +487,7 @@ class UnderstandingAPI(BaseParser):data: Dict[str, Any] = {"purpose": "user_data"}- content_type = mimetypes.guess_type(str(local_path))[0] or "application/octet-stream"+ content_type = mimetypes.guess_type(file_name)[0] or "application/octet-stream"with open(local_path, "rb") as f:files = {"file": (file_name, f, content_type)}async with httpx.AsyncClient(timeout=1200.0, follow_redirects=True) as client:@@ -628,7 +654,7 @@ class UnderstandingAPI(BaseParser):payload = {"file_name": file_name,"file_size": file_path.stat().st_size,- "content_type": mimetypes.guess_type(str(file_path))[0] or "application/octet-stream",+ "content_type": mimetypes.guess_type(file_name)[0] or "application/octet-stream","part_size": int(self._upload_part_size_bytes),}async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.pyindex a52081662..a0a90f412 100644--- a/tests/parse/test_feishu_parser_api.py+++ b/tests/parse/test_feishu_parser_api.py@@ -358,7 +358,6 @@ async def test_legacy_accessor_output_does_not_enable_lark_protocol(tmp_path: Pafeishu_access_token="u-test",)- api._create_file.assert_awaited_once_with(local_path=markdown_path, source_name=None)api._create_response_for_file.assert_awaited_once_with(file_id="file-1")api._create_response_for_url.assert_not_awaited()diff --git a/tests/parse/test_understanding_api.py b/tests/parse/test_understanding_api.pyindex aca5bca17..00aee31c1 100644--- a/tests/parse/test_understanding_api.py+++ b/tests/parse/test_understanding_api.py@@ -37,7 +37,7 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(resolved_extension,source_format,):- source_name = f"original.{source_format}"+ source_name = "export"uploaded_names = []uploaded_content = []@@ -114,7 +114,7 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(else:assert await processor.upload_understanding_file(resource) == "file-1"- assert uploaded_names == [source_name, source_name]+ assert uploaded_names == [f"{source_name}.{source_format}"] * 2assert uploaded_content == [content, content]