Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(parse): 保留 Understanding 上传文件的已识别扩展名 (#4981)

bot-of-qin-ctx9b5f43d828cd
diff --git a/docs/design/resource-ingestion-routing.md b/docs/design/resource-ingestion-routing.md
index 3b2ce604d..bf27bdb0e 100644
--- a/docs/design/resource-ingestion-routing.md
+++ b/docs/design/resource-ingestion-routing.md
@@ -181,7 +181,9 @@ Accessor 路径中的文档图片和 Base 附件字段图片使用同一条素
HTTP 资源以 HTTPAccessor 检出的 `meta.extension` 为准;本地文件或上传的临时文件可优先使用显式 `source_name` 的扩展名。这样既不会拿随机临时文件名选 Parser,也不会让用户提供的 URL 名称覆盖实际下载内容类型。
-本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 作为 `source_name` 传给上传层;普通上传和分片上传都使用这个名称的 basename,本地路径只用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。
+本地文件进入 Understanding 时,ParserRouter 将冻结的 `resolved_name` 和 `resolved_extension` 一起传给上传层。上传层使用来源名称的 basename;如果末尾没有已识别的扩展名(忽略大小写),则追加该扩展名,保留名称中的编号和版本信息。例如无后缀 URL `/export?id=123` 返回 `Content-Type: application/pdf` 时,上传文件名为 `export.pdf`;`2601.00014` 会补成 `2601.00014.pdf`,`report.PDF` 保持原样。内部的 MPEG-TS 路由标记转换为真实的 `.ts` 后缀。
+
+普通上传、分片上传和 MIME 类型推断统一使用补全后的上传文件名,本地路径用于读取文件。同步解析、提前提交解析和仅上传获取 `file_id` 使用相同规则。`resource_name` 只控制外层资源目录,例如 `README.md` 下载为 `/tmp/tmpABC.md` 后,上传文件名仍为 `README.md`。
## 无后缀 URL 怎么判断类型
diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.py
index 7d88596a5..69512653d 100644
--- a/openviking/parse/parser_router.py
+++ b/openviking/parse/parser_router.py
@@ -137,6 +137,9 @@ class ParserRouter:
if use_understanding:
if isinstance(source, LocalResource):
kwargs["source_name"] = source.meta["resolved_name"]
+ kwargs["resolved_extension"] = (
+ kwargs.get("resolved_extension") or source.meta["resolved_extension"]
+ )
display = source_path
if isinstance(source_path, str) and source_path.startswith(("http://", "https://")):
display = "<url>"
@@ -164,7 +167,13 @@ class ParserRouter:
else kwargs.get("source_name")
)
return await self._get_understanding_api().submit_file(
- source_path, source_name=source_name
+ source_path,
+ source_name=source_name,
+ resolved_extension=(
+ source.meta["resolved_extension"]
+ if isinstance(source, LocalResource)
+ else kwargs.get("resolved_extension", "")
+ ),
)
if not self.should_use_understanding_api(str(source_path)):
raise ValueError("source is not routed to UnderstandingAPI")
@@ -174,7 +183,13 @@ class ParserRouter:
"""Upload a local source file and return only the external Files API file_id."""
source_path = self._extract_source_path(source)
source_name = source.meta["resolved_name"] if isinstance(source, LocalResource) else None
- return await self._get_understanding_api().upload_file(source_path, source_name=source_name)
+ return await self._get_understanding_api().upload_file(
+ source_path,
+ source_name=source_name,
+ resolved_extension=(
+ source.meta["resolved_extension"] if isinstance(source, LocalResource) else ""
+ ),
+ )
def _extract_source_path(self, source: Union[str, Path, LocalResource]) -> Union[str, Path]:
"""Extract a filesystem path from the source."""
diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.py
index bea40b32b..fdf57616d 100644
--- a/openviking/parse/understanding_api.py
+++ b/openviking/parse/understanding_api.py
@@ -29,7 +29,7 @@ from openviking.parse.image_rewrite import (
build_artifact_image_mappings,
)
from openviking.parse.parsers.base_parser import BaseParser
-from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS
+from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS, TYPESCRIPT_MPEG_TS_EXTENSION
from openviking.parse.parsers.media.constants import (
AUDIO_EXTENSIONS,
IMAGE_EXTENSIONS,
@@ -180,7 +180,11 @@ class UnderstandingAPI(BaseParser):
task_meta["file_id"] = prepared_file_id
response_obj = await self._create_response_for_file(file_id=prepared_file_id)
elif url is None and local_path is not None:
- file_obj = await self._create_file(local_path=local_path, source_name=source_name)
+ file_obj = await self._create_file(
+ local_path=local_path,
+ source_name=source_name,
+ resolved_extension=resolved_extension,
+ )
file_id_value = file_obj.get("id")
if not file_id_value:
raise RuntimeError(
@@ -280,24 +284,36 @@ class UnderstandingAPI(BaseParser):
return result
async def upload_file(
- self, source: Union[str, Path], *, source_name: Optional[str] = None
+ self,
+ source: Union[str, Path],
+ *,
+ source_name: Optional[str] = None,
+ resolved_extension: str = "",
) -> str:
"""Upload a local file and return a durable Files API file_id."""
local_path = Path(source)
if not local_path.is_file():
raise ValueError("UnderstandingAPI file upload requires an existing local file")
- file_obj = await self._create_file(local_path=local_path, source_name=source_name)
+ file_obj = await self._create_file(
+ local_path=local_path, source_name=source_name, resolved_extension=resolved_extension
+ )
file_id = file_obj.get("id")
if not file_id:
raise RuntimeError(f"files api missing file_id: {self._safe_error_summary(file_obj)}")
return str(file_id)
async def submit_file(
- self, source: Union[str, Path], *, source_name: Optional[str] = None
+ self,
+ source: Union[str, Path],
+ *,
+ source_name: Optional[str] = None,
+ resolved_extension: str = "",
) -> str:
"""Upload a local file and submit it without retaining the local artifact."""
- file_id = await self.upload_file(source, source_name=source_name)
+ file_id = await self.upload_file(
+ source, source_name=source_name, resolved_extension=resolved_extension
+ )
response_obj = await self._create_response_for_file(file_id=str(file_id))
response_id = response_obj.get("id")
if not response_id:
@@ -443,13 +459,23 @@ class UnderstandingAPI(BaseParser):
return body
async def _create_file(
- self, *, local_path: Path, source_name: Optional[str] = None
+ self,
+ *,
+ local_path: Path,
+ source_name: Optional[str] = None,
+ resolved_extension: str = "",
) -> Dict[str, Any]:
file_size = local_path.stat().st_size
if file_size == 0:
raise InvalidArgumentError("Understanding parser does not support empty files.")
# The local path locates bytes; its temporary basename is not the source identity.
file_name = Path(source_name).name if source_name else local_path.name
+ extension = (resolved_extension or local_path.suffix).lower().lstrip(".")
+ if extension == MPEG_TS_EXTENSION_ALIAS:
+ extension = TYPESCRIPT_MPEG_TS_EXTENSION.lstrip(".")
+ # Preserve dotted identifiers (e.g. 2601.00014) and existing suffix casing.
+ if extension and not file_name.lower().endswith(f".{extension}"):
+ file_name = f"{file_name}.{extension}"
if file_size > self._upload_simple_max_bytes:
if not self._enable_resumable_upload:
raise ValueError(
@@ -461,7 +487,7 @@ class UnderstandingAPI(BaseParser):
data: Dict[str, Any] = {"purpose": "user_data"}
- content_type = mimetypes.guess_type(str(local_path))[0] or "application/octet-stream"
+ content_type = mimetypes.guess_type(file_name)[0] or "application/octet-stream"
with open(local_path, "rb") as f:
files = {"file": (file_name, f, content_type)}
async with httpx.AsyncClient(timeout=1200.0, follow_redirects=True) as client:
@@ -628,7 +654,7 @@ class UnderstandingAPI(BaseParser):
payload = {
"file_name": file_name,
"file_size": file_path.stat().st_size,
- "content_type": mimetypes.guess_type(str(file_path))[0] or "application/octet-stream",
+ "content_type": mimetypes.guess_type(file_name)[0] or "application/octet-stream",
"part_size": int(self._upload_part_size_bytes),
}
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index a52081662..a0a90f412 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -358,7 +358,6 @@ async def test_legacy_accessor_output_does_not_enable_lark_protocol(tmp_path: Pa
feishu_access_token="u-test",
)
- api._create_file.assert_awaited_once_with(local_path=markdown_path, source_name=None)
api._create_response_for_file.assert_awaited_once_with(file_id="file-1")
api._create_response_for_url.assert_not_awaited()
diff --git a/tests/parse/test_understanding_api.py b/tests/parse/test_understanding_api.py
index aca5bca17..00aee31c1 100644
--- a/tests/parse/test_understanding_api.py
+++ b/tests/parse/test_understanding_api.py
@@ -37,7 +37,7 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(
resolved_extension,
source_format,
):
- source_name = f"original.{source_format}"
+ source_name = "export"
uploaded_names = []
uploaded_content = []
@@ -114,7 +114,7 @@ async def test_parse_uses_downloaded_file_and_resolved_extension(
else:
assert await processor.upload_understanding_file(resource) == "file-1"
- assert uploaded_names == [source_name, source_name]
+ assert uploaded_names == [f"{source_name}.{source_format}"] * 2
assert uploaded_content == [content, content]