Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): preserve source filenames in Understanding uploads (#4947)

Qin Haojieb3104fbd0ce6
diff --git a/docs/design/resource-ingestion-routing.md b/docs/design/resource-ingestion-routing.md
index 93237a989..3b2ce604d 100644
--- a/docs/design/resource-ingestion-routing.md
+++ b/docs/design/resource-ingestion-routing.md
@@ -177,10 +177,12 @@ Accessor 路径中的文档图片和 Base 附件字段图片使用同一条素
`UnifiedResourceProcessor.prepare` 随后冻结两个字段:
- `resolved_extension`:本次 Parser 路由唯一使用的扩展名。
-- `resolved_name`:用于展示和默认资源命名,不参与 HTTP 资源的类型覆盖。
+- `resolved_name`:用于展示、默认资源命名和 Understanding 上传文件名,不参与 HTTP 资源的类型覆盖。
HTTP 资源以 HTTPAccessor 检出的 `meta.extension` 为准;本地文件或上传的临时文件可优先使用显式 `source_name` 的扩展名。这样既不会拿随机临时文件名选 Parser,也不会让用户提供的 URL 名称覆盖实际下载内容类型。
+本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 作为 `source_name` 传给上传层;普通上传和分片上传都使用这个名称的 basename,本地路径只用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。
+
## 无后缀 URL 怎么判断类型
HTTPAccessor 按以下顺序收集和修正类型:
diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.py
index d9b889084..7d88596a5 100644
--- a/openviking/parse/parser_router.py
+++ b/openviking/parse/parser_router.py
@@ -135,6 +135,8 @@ class ParserRouter:
)
if use_understanding:
+ if isinstance(source, LocalResource):
+ kwargs["source_name"] = source.meta["resolved_name"]
display = source_path
if isinstance(source_path, str) and source_path.startswith(("http://", "https://")):
display = "<url>"
@@ -156,7 +158,14 @@ class ParserRouter:
async def submit(self, source: Union[str, Path, LocalResource], **kwargs) -> str:
source_path = self._extract_source_path(source)
if Path(source_path).is_file():
- return await self._get_understanding_api().submit_file(source_path)
+ source_name = (
+ source.meta["resolved_name"]
+ if isinstance(source, LocalResource)
+ else kwargs.get("source_name")
+ )
+ return await self._get_understanding_api().submit_file(
+ source_path, source_name=source_name
+ )
if not self.should_use_understanding_api(str(source_path)):
raise ValueError("source is not routed to UnderstandingAPI")
return await self._get_understanding_api().submit_url(str(source_path), **kwargs)
@@ -164,7 +173,8 @@ class ParserRouter:
async def upload_file(self, source: Union[str, Path, LocalResource]) -> str:
"""Upload a local source file and return only the external Files API file_id."""
source_path = self._extract_source_path(source)
- return await self._get_understanding_api().upload_file(source_path)
+ source_name = source.meta["resolved_name"] if isinstance(source, LocalResource) else None
+ return await self._get_understanding_api().upload_file(source_path, source_name=source_name)
def _extract_source_path(self, source: Union[str, Path, LocalResource]) -> Union[str, Path]:
"""Extract a filesystem path from the source."""
diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.py
index 06ebc9e7e..bea40b32b 100644
--- a/openviking/parse/understanding_api.py
+++ b/openviking/parse/understanding_api.py
@@ -180,7 +180,7 @@ class UnderstandingAPI(BaseParser):
task_meta["file_id"] = prepared_file_id
response_obj = await self._create_response_for_file(file_id=prepared_file_id)
elif url is None and local_path is not None:
- file_obj = await self._create_file(local_path=local_path)
+ file_obj = await self._create_file(local_path=local_path, source_name=source_name)
file_id_value = file_obj.get("id")
if not file_id_value:
raise RuntimeError(
@@ -279,21 +279,25 @@ class UnderstandingAPI(BaseParser):
logger.info("[UnderstandingAPI] done")
return result
- async def upload_file(self, source: Union[str, Path]) -> str:
+ async def upload_file(
+ self, source: Union[str, Path], *, source_name: Optional[str] = None
+ ) -> str:
"""Upload a local file and return a durable Files API file_id."""
local_path = Path(source)
if not local_path.is_file():
raise ValueError("UnderstandingAPI file upload requires an existing local file")
- file_obj = await self._create_file(local_path=local_path)
+ file_obj = await self._create_file(local_path=local_path, source_name=source_name)
file_id = file_obj.get("id")
if not file_id:
raise RuntimeError(f"files api missing file_id: {self._safe_error_summary(file_obj)}")
return str(file_id)
- async def submit_file(self, source: Union[str, Path]) -> str:
+ async def submit_file(
+ self, source: Union[str, Path], *, source_name: Optional[str] = None
+ ) -> str:
"""Upload a local file and submit it without retaining the local artifact."""
- file_id = await self.upload_file(source)
+ file_id = await self.upload_file(source, source_name=source_name)
response_obj = await self._create_response_for_file(file_id=str(file_id))
response_id = response_obj.get("id")
if not response_id:
@@ -438,10 +442,14 @@ class UnderstandingAPI(BaseParser):
self._raise_if_error(body, context=context)
return body
- async def _create_file(self, *, local_path: Path) -> Dict[str, Any]:
+ async def _create_file(
+ self, *, local_path: Path, source_name: Optional[str] = None
+ ) -> Dict[str, Any]:
file_size = local_path.stat().st_size
if file_size == 0:
raise InvalidArgumentError("Understanding parser does not support empty files.")
+ # The local path locates bytes; its temporary basename is not the source identity.
+ file_name = Path(source_name).name if source_name else local_path.name
if file_size > self._upload_simple_max_bytes:
if not self._enable_resumable_upload:
raise ValueError(
@@ -449,13 +457,13 @@ class UnderstandingAPI(BaseParser):
f"upload_simple_max_bytes={self._upload_simple_max_bytes}; "
"enable parser_api.enable_resumable_upload to continue"
)
- return await self._multipart_create_file(local_path)
+ return await self._multipart_create_file(local_path, file_name=file_name)
data: Dict[str, Any] = {"purpose": "user_data"}
content_type = mimetypes.guess_type(str(local_path))[0] or "application/octet-stream"
with open(local_path, "rb") as f:
- files = {"file": (local_path.name, f, content_type)}
+ files = {"file": (file_name, f, content_type)}
async with httpx.AsyncClient(timeout=1200.0, follow_redirects=True) as client:
rsp = await client.post(
f"{self._api_base}/files",
@@ -616,9 +624,9 @@ class UnderstandingAPI(BaseParser):
return str(zip_obj["url"])
return None
- async def _uploads_init(self, *, file_path: Path) -> Dict[str, Any]:
+ async def _uploads_init(self, *, file_path: Path, file_name: str) -> Dict[str, Any]:
payload = {
- "file_name": file_path.name,
+ "file_name": file_name,
"file_size": file_path.stat().st_size,
"content_type": mimetypes.guess_type(str(file_path))[0] or "application/octet-stream",
"part_size": int(self._upload_part_size_bytes),
@@ -663,8 +671,8 @@ class UnderstandingAPI(BaseParser):
)
return self._read_api_response(rsp, context="uploads complete error")
- async def _multipart_create_file(self, file_path: Path) -> Dict[str, Any]:
- init_obj = await self._uploads_init(file_path=file_path)
+ async def _multipart_create_file(self, file_path: Path, *, file_name: str) -> Dict[str, Any]:
+ init_obj = await self._uploads_init(file_path=file_path, file_name=file_name)
upload_id = init_obj.get("upload_id") or init_obj.get("uploadId")
object_key = init_obj.get("object_key") or init_obj.get("objectKey")
part_size = int(
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index ce46242d9..a52081662 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -358,7 +358,7 @@ async def test_legacy_accessor_output_does_not_enable_lark_protocol(tmp_path: Pa
feishu_access_token="u-test",
)
- api._create_file.assert_awaited_once_with(local_path=markdown_path)
+ api._create_file.assert_awaited_once_with(local_path=markdown_path, source_name=None)
api._create_response_for_file.assert_awaited_once_with(file_id="file-1")
api._create_response_for_url.assert_not_awaited()
diff --git a/tests/parse/test_understanding_api.py b/tests/parse/test_understanding_api.py
index 858ea6461..aca5bca17 100644
--- a/tests/parse/test_understanding_api.py
+++ b/tests/parse/test_understanding_api.py
@@ -1,18 +1,24 @@
-from pathlib import Path
+import json
+from email.parser import BytesParser
+from email.policy import default
from unittest.mock import AsyncMock, Mock
import httpx
import pytest
+from openviking.parse.accessors.base import LocalResource, SourceType
+from openviking.parse.parser_router import ParserRouter
from openviking.parse.understanding_api import (
PREPARED_FILE_ID_ARG,
UnderstandingAPI,
UnderstandingAPIError,
)
+from openviking.utils.media_processor import UnifiedResourceProcessor
from openviking_cli.exceptions import InvalidArgumentError
@pytest.mark.asyncio
[email protected]("entry_point", ["parse", "submit", "upload_file"])
@pytest.mark.parametrize(
"filename,content,original_source,resolved_extension,source_format",
[
@@ -22,35 +28,58 @@ from openviking_cli.exceptions import InvalidArgumentError
],
)
async def test_parse_uses_downloaded_file_and_resolved_extension(
- monkeypatch, tmp_path, filename, content, original_source, resolved_extension, source_format
+ monkeypatch,
+ tmp_path,
+ entry_point,
+ filename,
+ content,
+ original_source,
+ resolved_extension,
+ source_format,
):
- downloaded = tmp_path / filename
- downloaded.write_bytes(content)
- zip_path = tmp_path / "result.zip"
- zip_path.write_bytes(b"zip")
- uploaded: list[Path] = []
+ source_name = f"original.{source_format}"
+ uploaded_names = []
+ uploaded_content = []
- api = UnderstandingAPI.__new__(UnderstandingAPI)
- api._video_exts = {"mp4"}
- api._audio_exts = {"mp3"}
- api._image_exts = {"png"}
-
- async def create_file(*, local_path):
- uploaded.append(local_path)
- return {"id": "file-1"}
-
- async def create_response_for_file(*, file_id):
- assert file_id == "file-1"
- return {"id": "response-1"}
-
- async def poll_response(*, response_id):
- assert response_id == "response-1"
- return {"status": "completed"}
+ def handler(request):
+ if request.url.path.endswith("/files"):
+ if request.url.query == b"uploads":
+ uploaded_names.append(json.loads(request.content)["file_name"])
+ return httpx.Response(200, json={"upload_id": "upload-1", "object_key": "obj-1"})
+ if request.method == "GET":
+ return httpx.Response(200, json={"parts": []})
+ if request.method == "PUT":
+ uploaded_content.append(request.content)
+ return httpx.Response(200, json={"etag": "part-1"})
+ if not request.url.query:
+ message = BytesParser(policy=default).parsebytes(
+ f"Content-Type: {request.headers['content-type']}\r\n\r\n".encode()
+ + request.content
+ )
+ part = next(part for part in message.iter_parts() if part.get_filename())
+ uploaded_names.append(part.get_filename())
+ uploaded_content.append(part.get_payload(decode=True))
+ return httpx.Response(200, json={"id": "file-1", "status": "active"})
+ if request.method == "POST" and request.url.path.endswith("/responses"):
+ assert json.loads(request.content)["input"][0]["content"] == [
+ {"type": "file", "file": {"file_id": "file-1"}}
+ ]
+ return httpx.Response(200, json={"id": "response-1"})
+ assert request.url.path.endswith("/responses/response-1")
+ return httpx.Response(
+ 200,
+ json={"status": "completed", "result": {"zip_url": "https://example.com/result.zip"}},
+ )
- monkeypatch.setattr(api, "_create_file", create_file)
- monkeypatch.setattr(api, "_create_response_for_file", create_response_for_file)
- monkeypatch.setattr(api, "_poll_response", poll_response)
- monkeypatch.setattr(api, "_extract_zip_url", lambda _: "https://example.com/result.zip")
+ api = _api_with_transport(monkeypatch, handler)
+ api._enable_resumable_upload = True
+ # Exercise both upload protocols through every ingestion entry point.
+ api._upload_simple_max_bytes = 1 if source_format == "pdf" else 1024
+ router = ParserRouter(parser_registry=object())
+ router._understanding_api = api
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+ processor._parser_router = router
+ zip_path = tmp_path / "result.zip"
monkeypatch.setattr(api, "_download_zip", lambda _: _return(zip_path))
monkeypatch.setattr(
api,
@@ -58,21 +87,39 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(
lambda **_: _return("viking://temp/result"),
)
- result = await api.parse(
- downloaded,
- original_source=original_source,
- resource_name="report",
- resolved_extension=resolved_extension,
- )
-
- assert uploaded == [downloaded]
- assert result.source_path == original_source
- assert result.source_format == source_format
- assert result.root.title == "report"
+ for prefix in ("tmpABC", "tmpXYZ"):
+ downloaded = tmp_path / f"{prefix}-{filename}"
+ downloaded.write_bytes(content)
+ resource = LocalResource(
+ path=downloaded,
+ source_type=SourceType.HTTP,
+ original_source=original_source,
+ meta={"original_filename": source_name, "extension": resolved_extension},
+ )
+ processor._set_resolved_identity(resource, source_name=None)
+ if entry_point == "parse":
+ zip_path.write_bytes(b"zip")
+ result = await processor.process(
+ original_source,
+ prepared_resource=resource,
+ resource_name="report",
+ source_name=None,
+ parser_backend="understanding",
+ )
+ assert result.source_path == original_source
+ assert result.source_format == source_format
+ assert result.root.title == "report"
+ elif entry_point == "submit":
+ assert await processor.submit_understanding(resource) == "response-1"
+ else:
+ assert await processor.upload_understanding_file(resource) == "file-1"
+
+ assert uploaded_names == [source_name, source_name]
+ assert uploaded_content == [content, content]
@pytest.mark.asyncio
-async def test_upload_file_validates_input_and_returns_file_id(tmp_path):
+async def test_upload_file_validates_input_and_returns_file_id(monkeypatch, tmp_path):
empty_source = tmp_path / "empty.pdf"
empty_source.touch()
api = UnderstandingAPI.__new__(UnderstandingAPI)
@@ -87,12 +134,17 @@ async def test_upload_file_validates_input_and_returns_file_id(tmp_path):
source = tmp_path / "download.pdf"
source.write_bytes(b"%PDF-1.7")
- api._create_file = AsyncMock(return_value={"id": "file-1"})
+
+ def handler(request):
+ assert b'filename="download.pdf"' in request.content
+ assert source.read_bytes() in request.content
+ return httpx.Response(200, json={"id": "file-1"})
+
+ api = _api_with_transport(monkeypatch, handler)
file_id = await api.upload_file(source)
assert file_id == "file-1"
- api._create_file.assert_awaited_once_with(local_path=source)
@pytest.mark.asyncio
@@ -145,21 +197,6 @@ async def test_file_above_simple_limit_requires_resumable_upload(tmp_path):
await api._create_file(local_path=source)
-async def test_file_above_simple_limit_uses_multipart_when_enabled(tmp_path):
- source = tmp_path / "large.pdf"
- source.write_bytes(b"123456789")
- api = UnderstandingAPI.__new__(UnderstandingAPI)
- api._upload_simple_max_bytes = 8
- api._enable_resumable_upload = True
- api._multipart_create_file = AsyncMock(return_value={"id": "file-1"})
-
- result = await api._create_file(local_path=source)
-
- assert result == {"id": "file-1"}
- api._multipart_create_file.assert_awaited_once_with(source)
-
-
@pytest.mark.asyncio
async def test_parse_failure_preserves_observed_remote_ids(monkeypatch, tmp_path):
source = tmp_path / "report.pdf"
@@ -325,7 +362,7 @@ async def test_http_errors_preserve_business_message(monkeypatch, tmp_path, meth
"_create_response_for_file": {"file_id": "file-1"},
"_create_response_for_url": {"url": "https://example.test/a.pdf", "doc_type": "pdf"},
"_poll_response": {"response_id": "response-1"},
- "_uploads_init": {"file_path": source},
+ "_uploads_init": {"file_path": source, "file_name": source.name},
"_uploads_status": {"upload_id": "upload-1", "object_key": "object-1"},
"_uploads_put_part": {
"upload_id": "upload-1",