Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(parse): preserve source filenames in Understanding uploads (#4947)
Qin Haojieb3104fbd0ce6
diff --git a/docs/design/resource-ingestion-routing.md b/docs/design/resource-ingestion-routing.mdindex 93237a989..3b2ce604d 100644--- a/docs/design/resource-ingestion-routing.md+++ b/docs/design/resource-ingestion-routing.md@@ -177,10 +177,12 @@ Accessor 路径中的文档图片和 Base 附件字段图片使用同一条素`UnifiedResourceProcessor.prepare` 随后冻结两个字段:- `resolved_extension`:本次 Parser 路由唯一使用的扩展名。-- `resolved_name`:用于展示和默认资源命名,不参与 HTTP 资源的类型覆盖。+- `resolved_name`:用于展示、默认资源命名和 Understanding 上传文件名,不参与 HTTP 资源的类型覆盖。HTTP 资源以 HTTPAccessor 检出的 `meta.extension` 为准;本地文件或上传的临时文件可优先使用显式 `source_name` 的扩展名。这样既不会拿随机临时文件名选 Parser,也不会让用户提供的 URL 名称覆盖实际下载内容类型。+本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 作为 `source_name` 传给上传层;普通上传和分片上传都使用这个名称的 basename,本地路径只用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。+## 无后缀 URL 怎么判断类型HTTPAccessor 按以下顺序收集和修正类型:diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.pyindex d9b889084..7d88596a5 100644--- a/openviking/parse/parser_router.py+++ b/openviking/parse/parser_router.py@@ -135,6 +135,8 @@ class ParserRouter:)if use_understanding:+ if isinstance(source, LocalResource):+ kwargs["source_name"] = source.meta["resolved_name"]display = source_pathif isinstance(source_path, str) and source_path.startswith(("http://", "https://")):display = "<url>"@@ -156,7 +158,14 @@ class ParserRouter:async def submit(self, source: Union[str, Path, LocalResource], **kwargs) -> str:source_path = self._extract_source_path(source)if Path(source_path).is_file():- return await self._get_understanding_api().submit_file(source_path)+ source_name = (+ source.meta["resolved_name"]+ if isinstance(source, LocalResource)+ else kwargs.get("source_name")+ )+ return await self._get_understanding_api().submit_file(+ source_path, source_name=source_name+ )if not self.should_use_understanding_api(str(source_path)):raise ValueError("source is not routed to UnderstandingAPI")return await self._get_understanding_api().submit_url(str(source_path), **kwargs)@@ -164,7 +173,8 @@ class ParserRouter:async def upload_file(self, source: Union[str, Path, LocalResource]) -> str:"""Upload a local source file and return only the external Files API file_id."""source_path = self._extract_source_path(source)- return await self._get_understanding_api().upload_file(source_path)+ source_name = source.meta["resolved_name"] if isinstance(source, LocalResource) else None+ return await self._get_understanding_api().upload_file(source_path, source_name=source_name)def _extract_source_path(self, source: Union[str, Path, LocalResource]) -> Union[str, Path]:"""Extract a filesystem path from the source."""diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.pyindex 06ebc9e7e..bea40b32b 100644--- a/openviking/parse/understanding_api.py+++ b/openviking/parse/understanding_api.py@@ -180,7 +180,7 @@ class UnderstandingAPI(BaseParser):task_meta["file_id"] = prepared_file_idresponse_obj = await self._create_response_for_file(file_id=prepared_file_id)elif url is None and local_path is not None:- file_obj = await self._create_file(local_path=local_path)+ file_obj = await self._create_file(local_path=local_path, source_name=source_name)file_id_value = file_obj.get("id")if not file_id_value:raise RuntimeError(@@ -279,21 +279,25 @@ class UnderstandingAPI(BaseParser):logger.info("[UnderstandingAPI] done")return result- async def upload_file(self, source: Union[str, Path]) -> str:+ async def upload_file(+ self, source: Union[str, Path], *, source_name: Optional[str] = None+ ) -> str:"""Upload a local file and return a durable Files API file_id."""local_path = Path(source)if not local_path.is_file():raise ValueError("UnderstandingAPI file upload requires an existing local file")- file_obj = await self._create_file(local_path=local_path)+ file_obj = await self._create_file(local_path=local_path, source_name=source_name)file_id = file_obj.get("id")if not file_id:raise RuntimeError(f"files api missing file_id: {self._safe_error_summary(file_obj)}")return str(file_id)- async def submit_file(self, source: Union[str, Path]) -> str:+ async def submit_file(+ self, source: Union[str, Path], *, source_name: Optional[str] = None+ ) -> str:"""Upload a local file and submit it without retaining the local artifact."""- file_id = await self.upload_file(source)+ file_id = await self.upload_file(source, source_name=source_name)response_obj = await self._create_response_for_file(file_id=str(file_id))response_id = response_obj.get("id")if not response_id:@@ -438,10 +442,14 @@ class UnderstandingAPI(BaseParser):self._raise_if_error(body, context=context)return body- async def _create_file(self, *, local_path: Path) -> Dict[str, Any]:+ async def _create_file(+ self, *, local_path: Path, source_name: Optional[str] = None+ ) -> Dict[str, Any]:file_size = local_path.stat().st_sizeif file_size == 0:raise InvalidArgumentError("Understanding parser does not support empty files.")+ # The local path locates bytes; its temporary basename is not the source identity.+ file_name = Path(source_name).name if source_name else local_path.nameif file_size > self._upload_simple_max_bytes:if not self._enable_resumable_upload:raise ValueError(@@ -449,13 +457,13 @@ class UnderstandingAPI(BaseParser):f"upload_simple_max_bytes={self._upload_simple_max_bytes}; ""enable parser_api.enable_resumable_upload to continue")- return await self._multipart_create_file(local_path)+ return await self._multipart_create_file(local_path, file_name=file_name)data: Dict[str, Any] = {"purpose": "user_data"}content_type = mimetypes.guess_type(str(local_path))[0] or "application/octet-stream"with open(local_path, "rb") as f:- files = {"file": (local_path.name, f, content_type)}+ files = {"file": (file_name, f, content_type)}async with httpx.AsyncClient(timeout=1200.0, follow_redirects=True) as client:rsp = await client.post(f"{self._api_base}/files",@@ -616,9 +624,9 @@ class UnderstandingAPI(BaseParser):return str(zip_obj["url"])return None- async def _uploads_init(self, *, file_path: Path) -> Dict[str, Any]:+ async def _uploads_init(self, *, file_path: Path, file_name: str) -> Dict[str, Any]:payload = {- "file_name": file_path.name,+ "file_name": file_name,"file_size": file_path.stat().st_size,"content_type": mimetypes.guess_type(str(file_path))[0] or "application/octet-stream","part_size": int(self._upload_part_size_bytes),@@ -663,8 +671,8 @@ class UnderstandingAPI(BaseParser):)return self._read_api_response(rsp, context="uploads complete error")- async def _multipart_create_file(self, file_path: Path) -> Dict[str, Any]:- init_obj = await self._uploads_init(file_path=file_path)+ async def _multipart_create_file(self, file_path: Path, *, file_name: str) -> Dict[str, Any]:+ init_obj = await self._uploads_init(file_path=file_path, file_name=file_name)upload_id = init_obj.get("upload_id") or init_obj.get("uploadId")object_key = init_obj.get("object_key") or init_obj.get("objectKey")part_size = int(diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.pyindex ce46242d9..a52081662 100644--- a/tests/parse/test_feishu_parser_api.py+++ b/tests/parse/test_feishu_parser_api.py@@ -358,7 +358,7 @@ async def test_legacy_accessor_output_does_not_enable_lark_protocol(tmp_path: Pafeishu_access_token="u-test",)- api._create_file.assert_awaited_once_with(local_path=markdown_path)+ api._create_file.assert_awaited_once_with(local_path=markdown_path, source_name=None)api._create_response_for_file.assert_awaited_once_with(file_id="file-1")api._create_response_for_url.assert_not_awaited()diff --git a/tests/parse/test_understanding_api.py b/tests/parse/test_understanding_api.pyindex 858ea6461..aca5bca17 100644--- a/tests/parse/test_understanding_api.py+++ b/tests/parse/test_understanding_api.py@@ -1,18 +1,24 @@-from pathlib import Path+import json+from email.parser import BytesParser+from email.policy import defaultfrom unittest.mock import AsyncMock, Mockimport httpximport pytest+from openviking.parse.accessors.base import LocalResource, SourceType+from openviking.parse.parser_router import ParserRouterfrom openviking.parse.understanding_api import (PREPARED_FILE_ID_ARG,UnderstandingAPI,UnderstandingAPIError,)+from openviking.utils.media_processor import UnifiedResourceProcessorfrom openviking_cli.exceptions import InvalidArgumentError@pytest.mark.asyncio[email protected]("entry_point", ["parse", "submit", "upload_file"])@pytest.mark.parametrize("filename,content,original_source,resolved_extension,source_format",[@@ -22,35 +28,58 @@ from openviking_cli.exceptions import InvalidArgumentError],)async def test_parse_uses_downloaded_file_and_resolved_extension(- monkeypatch, tmp_path, filename, content, original_source, resolved_extension, source_format+ monkeypatch,+ tmp_path,+ entry_point,+ filename,+ content,+ original_source,+ resolved_extension,+ source_format,):- downloaded = tmp_path / filename- downloaded.write_bytes(content)- zip_path = tmp_path / "result.zip"- zip_path.write_bytes(b"zip")- uploaded: list[Path] = []+ source_name = f"original.{source_format}"+ uploaded_names = []+ uploaded_content = []- api = UnderstandingAPI.__new__(UnderstandingAPI)- api._video_exts = {"mp4"}- api._audio_exts = {"mp3"}- api._image_exts = {"png"}-- async def create_file(*, local_path):- uploaded.append(local_path)- return {"id": "file-1"}-- async def create_response_for_file(*, file_id):- assert file_id == "file-1"- return {"id": "response-1"}-- async def poll_response(*, response_id):- assert response_id == "response-1"- return {"status": "completed"}+ def handler(request):+ if request.url.path.endswith("/files"):+ if request.url.query == b"uploads":+ uploaded_names.append(json.loads(request.content)["file_name"])+ return httpx.Response(200, json={"upload_id": "upload-1", "object_key": "obj-1"})+ if request.method == "GET":+ return httpx.Response(200, json={"parts": []})+ if request.method == "PUT":+ uploaded_content.append(request.content)+ return httpx.Response(200, json={"etag": "part-1"})+ if not request.url.query:+ message = BytesParser(policy=default).parsebytes(+ f"Content-Type: {request.headers['content-type']}\r\n\r\n".encode()+ + request.content+ )+ part = next(part for part in message.iter_parts() if part.get_filename())+ uploaded_names.append(part.get_filename())+ uploaded_content.append(part.get_payload(decode=True))+ return httpx.Response(200, json={"id": "file-1", "status": "active"})+ if request.method == "POST" and request.url.path.endswith("/responses"):+ assert json.loads(request.content)["input"][0]["content"] == [+ {"type": "file", "file": {"file_id": "file-1"}}+ ]+ return httpx.Response(200, json={"id": "response-1"})+ assert request.url.path.endswith("/responses/response-1")+ return httpx.Response(+ 200,+ json={"status": "completed", "result": {"zip_url": "https://example.com/result.zip"}},+ )- monkeypatch.setattr(api, "_create_file", create_file)- monkeypatch.setattr(api, "_create_response_for_file", create_response_for_file)- monkeypatch.setattr(api, "_poll_response", poll_response)- monkeypatch.setattr(api, "_extract_zip_url", lambda _: "https://example.com/result.zip")+ api = _api_with_transport(monkeypatch, handler)+ api._enable_resumable_upload = True+ # Exercise both upload protocols through every ingestion entry point.+ api._upload_simple_max_bytes = 1 if source_format == "pdf" else 1024+ router = ParserRouter(parser_registry=object())+ router._understanding_api = api+ processor = UnifiedResourceProcessor(vlm_processor=object())+ processor._parser_router = router+ zip_path = tmp_path / "result.zip"monkeypatch.setattr(api, "_download_zip", lambda _: _return(zip_path))monkeypatch.setattr(api,@@ -58,21 +87,39 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(lambda **_: _return("viking://temp/result"),)- result = await api.parse(- downloaded,- original_source=original_source,- resource_name="report",- resolved_extension=resolved_extension,- )-- assert uploaded == [downloaded]- assert result.source_path == original_source- assert result.source_format == source_format- assert result.root.title == "report"+ for prefix in ("tmpABC", "tmpXYZ"):+ downloaded = tmp_path / f"{prefix}-{filename}"+ downloaded.write_bytes(content)+ resource = LocalResource(+ path=downloaded,+ source_type=SourceType.HTTP,+ original_source=original_source,+ meta={"original_filename": source_name, "extension": resolved_extension},+ )+ processor._set_resolved_identity(resource, source_name=None)+ if entry_point == "parse":+ zip_path.write_bytes(b"zip")+ result = await processor.process(+ original_source,+ prepared_resource=resource,+ resource_name="report",+ source_name=None,+ parser_backend="understanding",+ )+ assert result.source_path == original_source+ assert result.source_format == source_format+ assert result.root.title == "report"+ elif entry_point == "submit":+ assert await processor.submit_understanding(resource) == "response-1"+ else:+ assert await processor.upload_understanding_file(resource) == "file-1"++ assert uploaded_names == [source_name, source_name]+ assert uploaded_content == [content, content]@pytest.mark.asyncio-async def test_upload_file_validates_input_and_returns_file_id(tmp_path):+async def test_upload_file_validates_input_and_returns_file_id(monkeypatch, tmp_path):empty_source = tmp_path / "empty.pdf"empty_source.touch()api = UnderstandingAPI.__new__(UnderstandingAPI)@@ -87,12 +134,17 @@ async def test_upload_file_validates_input_and_returns_file_id(tmp_path):source = tmp_path / "download.pdf"source.write_bytes(b"%PDF-1.7")- api._create_file = AsyncMock(return_value={"id": "file-1"})++ def handler(request):+ assert b'filename="download.pdf"' in request.content+ assert source.read_bytes() in request.content+ return httpx.Response(200, json={"id": "file-1"})++ api = _api_with_transport(monkeypatch, handler)file_id = await api.upload_file(source)assert file_id == "file-1"- api._create_file.assert_awaited_once_with(local_path=source)@pytest.mark.asyncio@@ -145,21 +197,6 @@ async def test_file_above_simple_limit_requires_resumable_upload(tmp_path):await api._create_file(local_path=source)-async def test_file_above_simple_limit_uses_multipart_when_enabled(tmp_path):- source = tmp_path / "large.pdf"- source.write_bytes(b"123456789")- api = UnderstandingAPI.__new__(UnderstandingAPI)- api._upload_simple_max_bytes = 8- api._enable_resumable_upload = True- api._multipart_create_file = AsyncMock(return_value={"id": "file-1"})-- result = await api._create_file(local_path=source)-- assert result == {"id": "file-1"}- api._multipart_create_file.assert_awaited_once_with(source)--@pytest.mark.asyncioasync def test_parse_failure_preserves_observed_remote_ids(monkeypatch, tmp_path):source = tmp_path / "report.pdf"@@ -325,7 +362,7 @@ async def test_http_errors_preserve_business_message(monkeypatch, tmp_path, meth"_create_response_for_file": {"file_id": "file-1"},"_create_response_for_url": {"url": "https://example.test/a.pdf", "doc_type": "pdf"},"_poll_response": {"response_id": "response-1"},- "_uploads_init": {"file_path": source},+ "_uploads_init": {"file_path": source, "file_name": source.name},"_uploads_status": {"upload_id": "upload-1", "object_key": "object-1"},"_uploads_put_part": {"upload_id": "upload-1",