Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(model): persist file manifests and pin ModelScope snapshots

myhloli74dd405a7f75
diff --git a/docs/next/cli/mineru-kit-models.md b/docs/next/cli/mineru-kit-models.md
index f6984d9d..ee0eb368 100644
--- a/docs/next/cli/mineru-kit-models.md
+++ b/docs/next/cli/mineru-kit-models.md
@@ -89,6 +89,7 @@ mineru-kit models download --tier <basic|standard> [flags]
- `--source` 不支持 `local`
- 如果当前配置为 `model.source: local`,显式 download 会临时按 `auto` 解析远端源,不改写配置
- 下载目标固定为 `config.model.base_dir` 下的 repo local dir
+- ModelScope 使用 `modelscope>=1.40.3`;先把默认分支解析为固定 commit,再按同一 commit 查询清单和下载,避免分支更新导致清单与文件版本不一致
支持的 repo 名:
@@ -159,6 +160,8 @@ mineru-kit models verify --small-backend <auto|onnx|torch> --vlm-engine <auto|ll
- `--tier` 只接受 `basic` 和 `standard`;按 `--tier` 校验时使用独立后端选项解析出的资源组合
- 不是单纯目录存在性检查,还会检查 registry 中声明的关键路径
- 整仓和目录资源会读取 `.mineru_complete` 文件清单,检查预期文件是否存在、是否为有效文件,以及大小是否一致;正常校验无需联网,不读取大权重计算 hash
+- 单文件下载把预期大小保存到仓库根目录的 `.mineru_files.json`,后续复用同样检查缺失、截断和失效链接;这份局部记录不会把父目录或整仓标为已完成
+- 完整目录重新下载后,由目录清单接管该范围的旧文件记录;文件记录损坏时需执行 `mineru-kit models download <repo>` 重建,不能只确认一个文件就丢弃其他文件的校验约束
- 兼容旧版本的零字节完成标记:检查本地非空权重、registry 声明的必需路径以及权重索引引用的分片,通过后可直接离线复用
- 首次使用旧模型时后台尝试升级标记,包括 `model.source: local`:按缓存中的旧版本查询远端文件清单,存在性和大小全部匹配后原子发布新版清单;只查询元数据,不下载、更新或删除模型文件,也不改写来源配置
- 无网、源不可达、旧版本无法确定、核验不匹配或目录不可写时保留旧标记,不阻塞当前使用;失败后冷却 5 分钟,下次使用可重试。`models show/verify` 保持只读,不启动迁移
diff --git a/mineru/model/_download_manifest.py b/mineru/model/_download_manifest.py
index eee95cc7..b1f3af68 100644
--- a/mineru/model/_download_manifest.py
+++ b/mineru/model/_download_manifest.py
@@ -11,9 +11,10 @@ from collections.abc import Mapping
from pathlib import Path, PurePosixPath, PureWindowsPath
MODEL_COMPLETE_MARKER = ".mineru_complete"
+MODEL_FILES_MANIFEST = ".mineru_files.json"
_MANIFEST_VERSION = 1
_PROVIDER_METADATA_DIR_NAMES = {".cache", "._____temp"}
-_PROVIDER_METADATA_FILE_NAMES = {MODEL_COMPLETE_MARKER, ".msc", ".mdl", ".mv"}
+_PROVIDER_METADATA_FILE_NAMES = {MODEL_COMPLETE_MARKER, MODEL_FILES_MANIFEST, ".msc", ".mdl", ".mv"}
_WEIGHT_SUFFIXES = {".safetensors", ".bin", ".pth", ".pt", ".onnx", ".gguf", ".pdiparams"}
@@ -166,6 +167,7 @@ def write_manifest(marker: Path, files: Mapping[str, int]) -> None:
__all__ = [
"MODEL_COMPLETE_MARKER",
+ "MODEL_FILES_MANIFEST",
"is_legacy_marker",
"is_payload_path",
"legacy_payload_missing",
diff --git a/mineru/model/download.py b/mineru/model/download.py
index db4e080d..96e577aa 100644
--- a/mineru/model/download.py
+++ b/mineru/model/download.py
@@ -6,6 +6,7 @@ from collections.abc import Sequence
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Literal
+from urllib.parse import quote
import requests
from filelock import FileLock
@@ -17,6 +18,7 @@ from loguru import logger
from ..config import get_config, get_config_source, update_config_file
from ._download_manifest import (
MODEL_COMPLETE_MARKER,
+ MODEL_FILES_MANIFEST,
is_legacy_marker,
is_payload_path,
legacy_payload_missing,
@@ -133,19 +135,55 @@ ResolvedRemoteModelSource = Literal["huggingface", "modelscope"]
ResolvedModelSource = Literal["huggingface", "modelscope", "local"]
-def ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str] | None = None) -> str:
+def ms_snapshot_download(repo_id: str, *, local_dir: str, revision: str, allow_patterns: list[str] | None = None) -> str:
"""仅在实际使用 ModelScope 时导入 SDK,避免 HF/本地路径触发其 Torch 探测。"""
from modelscope import snapshot_download
- return snapshot_download(repo_id, local_dir=local_dir, allow_patterns=allow_patterns)
+ return snapshot_download(repo_id, local_dir=local_dir, revision=revision, allow_patterns=allow_patterns)
-def ms_model_files(repo_id: str) -> dict[str, int]:
- """按下载器默认版本读取 ModelScope 预期文件,仅在使用该来源时加载 SDK。"""
+def ms_model_revision(repo_id: str) -> str:
+ """把默认分支解析为固定 commit,保留 SDK 的自定义端点与已配置认证。"""
from modelscope.hub.api import HubApi
from modelscope.utils.constant import DEFAULT_MODEL_REVISION
- files = HubApi().get_model_files(repo_id, revision=DEFAULT_MODEL_REVISION, recursive=True)
+ api = HubApi()
+ url = f"{api.endpoint.rstrip('/')}/api/v1/models/{quote(repo_id, safe='/')}/repo/files"
+ with requests.get(
+ url,
+ params={"Revision": DEFAULT_MODEL_REVISION, "Recursive": "False"},
+ headers=api.headers,
+ cookies=api.get_cookies(),
+ timeout=(5, 30),
+ ) as response:
+ response.raise_for_status()
+ value = response.json()
+ data = value.get("Data") if isinstance(value, dict) and value.get("Code") == 200 else None
+ commit = data.get("LatestCommitter") if isinstance(data, dict) else None
+ revision = (commit.get("Id") or commit.get("ShortId")) if isinstance(commit, dict) else None
+ if not isinstance(revision, str) or re.fullmatch(r"[0-9a-f]{7,64}", revision) is None:
+ raise ValueError(f"ModelScope did not return a fixed commit for {repo_id}.")
+ # 部分响应只给出短 commit;同次列表若含完整值则展开,否则保留 Git 支持的固定短 commit。
+ entries = data.get("Files")
+ full_revisions: set[str] = set()
+ for item in entries if isinstance(entries, list) else ():
+ full_revision = item.get("Revision") if isinstance(item, dict) else None
+ if (
+ isinstance(full_revision, str)
+ and re.fullmatch(r"[0-9a-f]{40,64}", full_revision)
+ and full_revision.startswith(revision)
+ ):
+ full_revisions.add(full_revision)
+ if len(full_revisions) > 1:
+ raise ValueError(f"ModelScope returned an ambiguous commit for {repo_id}.")
+ return next(iter(full_revisions)) if full_revisions else revision
+
+
+def ms_model_files(repo_id: str, *, revision: str) -> dict[str, int]:
+ """读取已固定 commit 的 ModelScope 文件清单,仅在使用该来源时加载 SDK。"""
+ from modelscope.hub.api import HubApi
+
+ files = HubApi().get_model_files(repo_id, revision=revision, recursive=True)
# 新版 SDK 的兼容接口不再返回 Type;递归清单中的父路径仍能确定目录,且不会误删零字节文件。
directories = {parent.as_posix() for item in files for parent in PurePosixPath(item["Path"]).parents}
return {item["Path"]: item["Size"] for item in files if item.get("Type") != "tree" and item["Path"] not in directories}
@@ -251,8 +289,10 @@ def _snapshot_download(model_source: ResolvedRemoteModelSource, repo: ModelRepo,
if model_source == "huggingface":
info = HfApi().model_info(repo_id, files_metadata=True)
files = {item.rfilename: item.size for item in info.siblings or ()}
+ revision = info.sha
else:
- files = ms_model_files(repo_id)
+ revision = ms_model_revision(repo_id)
+ files = ms_model_files(repo_id, revision=revision)
expected_files: dict[str, int] = {}
for name in filter_repo_objects(files, allow_patterns=patterns):
if not is_payload_path(name):
@@ -270,9 +310,9 @@ def _snapshot_download(model_source: ResolvedRemoteModelSource, repo: ModelRepo,
target.unlink()
requested_files = list(expected_files)
if model_source == "huggingface":
- hf_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=info.sha)
+ hf_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=revision)
else:
- ms_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files)
+ ms_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=revision)
missing_files = missing_manifest_files(local_dir, expected_files)
if missing_files:
missing = ", ".join(missing_files)
@@ -320,6 +360,16 @@ def _model_path_missing(path: ModelPath) -> list[str]:
local_path.relative_to(repo_root)
except ValueError:
return [path.relative_path]
+ # 单文件下载记录独立于目录完成标记,局部下载不能据此宣称整个目录已完整。
+ file_marker = repo_root / MODEL_FILES_MANIFEST
+ if file_marker.exists() or file_marker.is_symlink():
+ file_manifest = read_manifest(file_marker)
+ if file_manifest is None:
+ return [MODEL_FILES_MANIFEST]
+ selected_files = manifest_files_for_path(file_manifest, path.relative_path)
+ missing_files = missing_manifest_files(repo_root, selected_files)
+ if missing_files or path.relative_path in file_manifest:
+ return missing_files
is_file = local_path.is_file()
if local_path == repo_root and is_file:
return [MODEL_COMPLETE_MARKER]
@@ -491,6 +541,26 @@ def _restore_directory_markers(
write_manifest(marker, updated)
+def _record_downloaded_files(repo: ModelRepo, paths: Sequence[ModelPath], files: dict[str, int]) -> None:
+ """持久化单文件预期大小;完整目录刷新后由目录清单接管其覆盖的旧文件记录。"""
+ marker = repo.local_dir() / MODEL_FILES_MANIFEST
+ previous = read_manifest(marker)
+ if (marker.exists() or marker.is_symlink()) and previous is None:
+ if {path.relative_path for path in paths} != {path.relative_path for path in _repo_completion_paths(repo)}:
+ # 记录损坏时无法知道其他受管文件;不能仅确认一个文件就丢弃其余文件的校验约束。
+ raise ValueError(f"Invalid {marker}; run mineru-kit models download {repo.name} to rebuild file records.")
+ recorded = previous or {}
+ for path in paths:
+ for name in manifest_files_for_path(recorded, path.relative_path):
+ del recorded[name]
+ if path.relative_path in files:
+ recorded[path.relative_path] = files[path.relative_path]
+ if recorded:
+ write_manifest(marker, recorded)
+ else:
+ marker.unlink(missing_ok=True)
+
+
def _raise_not_ready(repo: ModelRepo, result: ModelReadyResult) -> None:
"""离线校验失败时报告真实路径,并提示修复文件或完成标记。"""
missing = ", ".join(result.missing_paths)
@@ -532,6 +602,7 @@ def download_model_repo(
if not downloaded_result.ready:
_raise_not_ready(repo, downloaded_result)
_mark_directories_complete(completion_paths, files)
+ _record_downloaded_files(repo, completion_paths, files)
_restore_directory_markers(repo, removed_markers, completion_paths, files)
return downloaded_result.root
@@ -572,6 +643,7 @@ def download_model_files(
if not downloaded_result.ready:
_raise_not_ready(repo, downloaded_result)
_mark_directories_complete(model_paths, files)
+ _record_downloaded_files(repo, model_paths, files)
_restore_directory_markers(repo, removed_markers, model_paths, files)
return downloaded_result.root
diff --git a/pyproject.toml b/pyproject.toml
index ed657668..602e1953 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -53,7 +53,7 @@ dependencies = [
"pypdfium2>=5.13.0,<6",
"PyYAML>=6.0.1,<7",
"packaging",
- "modelscope>=1.26.0",
+ "modelscope>=1.40.3",
"huggingface-hub[hf_xet]>=0.32.4",
"filelock>=3.0.0",
"json-repair>=0.46.2",
diff --git a/tests/unittest/test_model_file_manifest.py b/tests/unittest/test_model_file_manifest.py
new file mode 100644
index 00000000..e260ff68
--- /dev/null
+++ b/tests/unittest/test_model_file_manifest.py
@@ -0,0 +1,182 @@
+"""单文件下载记录、损坏修复与目录完成标记隔离的回归。"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+from huggingface_hub.utils import filter_repo_objects
+
+from mineru.model import download
+
+
+def file_repo(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> download.ModelRepo:
+ """隔离一个与 ONNX 仓库相同、全部必需资源均为文件的模型仓库。"""
+ monkeypatch.setattr(download.get_config().model, "base_dir", str(tmp_path))
+ monkeypatch.setattr(download.get_config().model, "source", "modelscope")
+ return download.ModelRepo(
+ name="file-model",
+ repos={"modelscope": "Owner/file-model"},
+ paths={"config": "Layout/config.json", "weights": "Layout/model.onnx"},
+ download_mode="required_paths",
+ )
+
+
+def remote(monkeypatch: pytest.MonkeyPatch) -> Mock:
+ """模拟远端按请求范围下载完整文件,同时返回独立的预期大小。"""
+ payloads = {"Layout/config.json": b"{}", "Layout/model.onnx": b"12345678", "empty.txt": b""}
+
+ def snapshot(source: str, repo: download.ModelRepo, patterns: list[str] | None) -> dict[str, int]:
+ """按真实下载模式选择文件,便于验证局部下载和整仓下载的区别。"""
+ assert source == "modelscope"
+ sizes = {}
+ for name in filter_repo_objects(payloads, allow_patterns=patterns):
+ target = repo.local_dir() / name
+ target.parent.mkdir(parents=True, exist_ok=True)
+ if target.is_symlink():
+ target.unlink()
+ target.write_bytes(payloads[name])
+ sizes[name] = len(payloads[name])
+ return sizes
+
+ result = Mock(side_effect=snapshot)
+ monkeypatch.setattr(download, "_snapshot_download", result)
+ return result
+
+
[email protected]("entry", ["files", "repo"])
[email protected]("damage", ["truncated", "deleted", "broken_link"])
+def test_downloaded_file_damage_is_rejected_and_repaired(
+ file_repo: download.ModelRepo, remote: Mock, monkeypatch: pytest.MonkeyPatch, entry: str, damage: str
+) -> None:
+ """首次成功下载后,所有单文件入口都必须拒绝损坏缓存并允许联网修复。"""
+ if entry == "files":
+ file_repo.config.ensure()
+ file_repo.weights.ensure()
+ else:
+ file_repo.ensure()
+ assert download.verify_model_repo(file_repo).ready
+ assert not list(file_repo.local_dir().rglob(download.MODEL_COMPLETE_MARKER))
+ calls = remote.call_count
+ weight = file_repo.weights.local_path()
+ if damage == "truncated":
+ weight.write_bytes(b"1")
+ else:
+ weight.unlink()
+ if damage == "broken_link":
+ weight.symlink_to("missing.onnx")
+
+ assert not download.model_path_exists(file_repo.weights)
+ assert download.verify_model_repo(file_repo).missing_paths == ["Layout/model.onnx"]
+ monkeypatch.setattr(download.get_config().model, "source", "local")
+ with pytest.raises(FileNotFoundError, match="Layout/model.onnx"):
+ file_repo.weights.ensure()
+ assert remote.call_count == calls
+
+ file_repo.weights.ensure(source="modelscope")
+ assert remote.call_count == calls + 1
+ assert weight.read_bytes() == b"12345678"
+ assert download.verify_model_repo(file_repo).ready
+
+
+def test_file_records_do_not_complete_parent_directories(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """局部文件记录不能把父目录或原始 VLM 整仓提升为已完成状态。"""
+ file_repo.weights.ensure()
+ assert download.model_path_exists(file_repo.weights)
+ assert not download.model_path_exists(file_repo.path("Layout"))
+ assert not download.verify_model_repo(replace(file_repo, download_mode="full")).ready
+ assert not download.verify_model_repo(file_repo).ready
+ assert remote.call_count == 1
+
+
+def test_unmanaged_files_and_legacy_markers_still_work_offline(
+ file_repo: download.ModelRepo, remote: Mock, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """没有新下载记录的手工模型与旧空标记保持原有离线使用边界。"""
+ root = file_repo.local_dir()
+ weight = file_repo.weights.local_path()
+ weight.parent.mkdir(parents=True)
+ weight.write_bytes(b"manual model")
+ file_repo.config.local_path().write_bytes(b"{}")
+ monkeypatch.setattr(download.get_config().model, "source", "local")
+ assert file_repo.weights.ensure() == weight
+ (root / download.MODEL_COMPLETE_MARKER).touch()
+ assert file_repo.ensure() == root
+ assert (root / download.MODEL_COMPLETE_MARKER).read_bytes() == b""
+ remote.assert_not_called()
+
+
+def test_corrupted_file_manifest_cannot_fall_back_to_nonempty_files(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """受管记录损坏时不降级放行,重新下载确认后才能恢复就绪。"""
+ file_repo.weights.ensure()
+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST
+ marker.write_text("{", encoding="utf-8")
+ assert not download.model_path_exists(file_repo.weights)
+ with pytest.raises(ValueError, match="rebuild file records"):
+ file_repo.weights.ensure()
+ assert marker.read_text() == "{"
+ file_repo.ensure()
+ assert download.model_path_exists(file_repo.weights)
+ assert json.loads(marker.read_text())["files"] == {"Layout/config.json": 2, "Layout/model.onnx": 8}
+ assert remote.call_count == 3
+
+
+def test_directory_refresh_replaces_obsolete_file_records(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """目录经远端完整确认后移除过时局部记录,避免已删除文件阻塞新快照。"""
+ file_repo.weights.ensure()
+ file_repo.weights.local_path().unlink()
+
+ def snapshot(source: str, repo: download.ModelRepo, patterns: list[str] | None) -> dict[str, int]:
+ """模拟整仓刷新后旧权重被新文件取代。"""
+ assert patterns is None
+ target = repo.local_dir() / "new-model.onnx"
+ target.write_bytes(b"new weights")
+ return {"new-model.onnx": 11}
+
+ remote.side_effect = snapshot
+ full_repo = replace(file_repo, paths={}, download_mode="full")
+ full_repo.ensure()
+ assert download.verify_model_repo(full_repo).ready
+ assert not (full_repo.local_dir() / download.MODEL_FILES_MANIFEST).exists()
+
+
+def test_confirmed_empty_file_is_valid_and_manifest_is_not_payload(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """远端确认的合法空文件可复用,文件记录本身不能作为模型 payload。"""
+ empty = file_repo.path("empty.txt")
+ empty.ensure()
+ assert download.model_path_exists(empty)
+ assert remote.call_count == 1
+ empty.ensure()
+ assert remote.call_count == 1
+ assert not download.is_payload_path(download.MODEL_FILES_MANIFEST)
+
+
+def test_failed_repair_keeps_other_managed_file_records(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """下载失败时仍保留原文件记录,其他受管文件不会退回非空即就绪的判断。"""
+ file_repo.ensure()
+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST
+ before = marker.read_bytes()
+ file_repo.weights.local_path().write_bytes(b"short")
+ remote.side_effect = RuntimeError("offline")
+ with pytest.raises(RuntimeError, match="offline"):
+ file_repo.weights.ensure()
+ assert marker.read_bytes() == before
+ assert download.model_path_exists(file_repo.config)
+ assert not download.model_path_exists(file_repo.weights)
+
+
+def test_directory_refresh_keeps_file_records_outside_its_scope(file_repo: download.ModelRepo, remote: Mock) -> None:
+ """目录清单只接管自身范围,其他位置的文件记录继续保留。"""
+ file_repo.ensure()
+ file_repo.path("empty.txt").ensure()
+ file_repo.path("Layout").ensure()
+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST
+ assert json.loads(marker.read_text())["files"] == {"empty.txt": 0}
+ assert download.model_path_exists(file_repo.path("Layout"))
+ file_repo.weights.local_path().write_bytes(b"short")
+ assert not download.model_path_exists(file_repo.weights)
diff --git a/tests/unittest/test_model_readiness.py b/tests/unittest/test_model_readiness.py
index 08e9d17b..b54a5512 100644
--- a/tests/unittest/test_model_readiness.py
+++ b/tests/unittest/test_model_readiness.py
@@ -466,6 +466,7 @@ def test_provider_repair_replaces_wrong_size_cached_payload(
)
monkeypatch.setattr(download, "HfApi", Mock(return_value=api))
monkeypatch.setattr(download, "ms_model_files", Mock(return_value=expected))
+ monkeypatch.setattr(download, "ms_model_revision", Mock(return_value="a" * 40))
def snapshot(repo_id: str, **kwargs: object) -> str:
"""模拟增量下载只补齐缺失文件,保留已验证的配置。"""
@@ -500,6 +501,7 @@ def test_provider_manifest_checks_expected_payload_before_marking_complete(
)
monkeypatch.setattr(download, "HfApi", Mock(return_value=api))
monkeypatch.setattr(download, "ms_model_files", Mock(return_value=expected))
+ monkeypatch.setattr(download, "ms_model_revision", Mock(return_value="a" * 40))
def snapshot(repo_id: str, **kwargs: object) -> str:
"""模拟 SDK 返回成功但权重可能缺失或未完整落盘。"""
diff --git a/tests/unittest/test_models_download_utils.py b/tests/unittest/test_models_download_utils.py
index f00c150a..093c9c34 100644
--- a/tests/unittest/test_models_download_utils.py
+++ b/tests/unittest/test_models_download_utils.py
@@ -444,6 +444,8 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(
repo = ModelRepo(name="repo", repos={"modelscope": "Owner/repo"}, paths={"layout": "Layout/PP-DocLayoutV2"})
monkeypatch.setattr(models_download_utils.get_config().model, "base_dir", str(tmp_path / "models"))
layout = "Layout/PP-DocLayoutV2"
+ fixed_revision = "a" * 40
+ monkeypatch.setattr(models_download_utils, "ms_model_revision", lambda repo_id: fixed_revision)
payloads = {f"{layout}/config.json": b"{}", f"{layout}/model.safetensors": b"weights", f"{layout}/empty.txt": b""}
directories = ["Layout", layout, "Unused"]
# 目录项故意排在文件之后,确保过滤逻辑不依赖 SDK 返回顺序。
@@ -458,7 +460,7 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(
def get_model_files(self, repo_id: str, *, revision: str, recursive: bool) -> list[dict[str, Any]]:
"""返回真实接口形状的递归清单,覆盖不再包含 Type 的新版 SDK。"""
assert repo_id == "Owner/repo"
- assert revision == "master"
+ assert revision == fixed_revision
assert recursive
return entries
@@ -466,9 +468,10 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(
monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))
requested: list[str] = []
- def fake_ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str]) -> str:
+ def fake_ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str], revision: str) -> str:
"""模拟下载真实文件,目录项由 SDK 忽略,缺失和截断按测试参数注入。"""
assert repo_id == "Owner/repo"
+ assert revision == fixed_revision
requested.extend(allow_patterns)
for name, content in payloads.items():
target = Path(local_dir) / name
diff --git a/tests/unittest/test_modelscope_revision.py b/tests/unittest/test_modelscope_revision.py
new file mode 100644
index 00000000..b8444a3f
--- /dev/null
+++ b/tests/unittest/test_modelscope_revision.py
@@ -0,0 +1,105 @@
+"""ModelScope 固定 commit 的元数据解析与下载一致性回归。"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock, Mock
+
+import pytest
+
+from mineru.model import download
+
+_REVISION = "a" * 40
+
+
+ ("commit", "entries", "expected"),
+ [
+ ({"Id": _REVISION}, [], _REVISION),
+ ({"Id": "", "ShortId": _REVISION[:8]}, [{"Revision": _REVISION}], _REVISION),
+ ({"ShortId": _REVISION[:8]}, [], _REVISION[:8]),
+ ({"ShortId": _REVISION[:8]}, [{"Revision": _REVISION}, {"Revision": _REVISION[:8] + "b" * 32}], None),
+ ],
+)
+def test_modelscope_commit_resolution_preserves_endpoint_and_auth(
+ monkeypatch: pytest.MonkeyPatch, commit: dict[str, str], entries: list[dict[str, str]], expected: str | None
+) -> None:
+ """从同次仓库响应解析固定 commit,并沿用 SDK 的端点、请求头与认证。"""
+ api = SimpleNamespace(
+ endpoint="https://modelscope.example/", headers={"user-agent": "sdk"}, get_cookies=Mock(return_value={})
+ )
+ monkeypatch.setitem(sys.modules, "modelscope.hub.api", SimpleNamespace(HubApi=Mock(return_value=api)))
+ monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))
+ response = MagicMock()
+ response.__enter__.return_value = response
+ response.json.return_value = {"Code": 200, "Data": {"LatestCommitter": commit, "Files": entries}}
+ request = Mock(return_value=response)
+ monkeypatch.setattr(download.requests, "get", request)
+
+ if expected is None:
+ with pytest.raises(ValueError, match="ambiguous commit"):
+ download.ms_model_revision("Owner/repo")
+ else:
+ assert download.ms_model_revision("Owner/repo") == expected
+ request.assert_called_once_with(
+ "https://modelscope.example/api/v1/models/Owner/repo/repo/files",
+ params={"Revision": "master", "Recursive": "False"},
+ headers=api.headers,
+ cookies={},
+ timeout=(5, 30),
+ )
+ response.raise_for_status.assert_called_once()
+
+
[email protected]("commit", [{}, {"Id": "master"}, {"ShortId": "latest"}, {"Id": 123}])
+def test_modelscope_missing_commit_is_not_replaced_with_mutable_branch(
+ monkeypatch: pytest.MonkeyPatch, commit: dict[str, object]
+) -> None:
+ """无法确认固定 commit 时明确失败,不能静默回退到会变化的 master。"""
+ api = SimpleNamespace(endpoint="https://modelscope.example", headers={}, get_cookies=lambda: {})
+ monkeypatch.setitem(sys.modules, "modelscope.hub.api", SimpleNamespace(HubApi=lambda: api))
+ monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))
+ response = MagicMock()
+ response.__enter__.return_value = response
+ response.json.return_value = {"Code": 200, "Data": {"LatestCommitter": commit, "Files": []}}
+ monkeypatch.setattr(download.requests, "get", Mock(return_value=response))
+ with pytest.raises(ValueError, match="fixed commit"):
+ download.ms_model_revision("Owner/repo")
+
+
+def test_modelscope_snapshot_wrapper_forwards_fixed_commit(monkeypatch: pytest.MonkeyPatch) -> None:
+ """下载封装必须把与清单相同的 commit 传给 SDK。"""
+ snapshot = Mock(return_value="local-model")
+ monkeypatch.setitem(sys.modules, "modelscope", SimpleNamespace(snapshot_download=snapshot))
+ assert download.ms_snapshot_download("Owner/repo", local_dir="local-model", revision=_REVISION) == "local-model"
+ snapshot.assert_called_once_with("Owner/repo", local_dir="local-model", revision=_REVISION, allow_patterns=None)
+
+
+def test_modelscope_branch_update_does_not_change_selected_snapshot(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ """分支在列清单后更新时仍下载原 commit,避免完整新文件被旧清单误拒绝。"""
+ monkeypatch.setattr(download.get_config().model, "base_dir", str(tmp_path))
+ repo = download.ModelRepo(name="repo", repos={"modelscope": "Owner/repo"}, paths={"layout": "Layout/weights"})
+ branch = {"revision": _REVISION}
+ monkeypatch.setattr(download, "ms_model_revision", lambda repo_id: branch["revision"])
+
+ def files(repo_id: str, *, revision: str) -> dict[str, int]:
+ """模拟读取旧 commit 后远端分支立即推进。"""
+ assert revision == _REVISION
+ branch["revision"] = "b" * 40
+ return {"Layout/weights/model.bin": 7}
+
+ def snapshot(repo_id: str, *, local_dir: str, revision: str, allow_patterns: list[str]) -> str:
+ """按实际传入的 commit 选择内容,漏传固定版本时将得到不同大小的新权重。"""
+ assert branch["revision"] != _REVISION
+ target = Path(local_dir) / "Layout/weights/model.bin"
+ target.parent.mkdir(parents=True)
+ target.write_bytes(b"weights" if revision == _REVISION else b"new weights")
+ return local_dir
+
+ monkeypatch.setattr(download, "ms_model_files", files)
+ monkeypatch.setattr(download, "ms_snapshot_download", snapshot)
+ repo.layout.ensure(source="modelscope")
+ assert (repo.layout.local_path() / "model.bin").read_bytes() == b"weights"
+ assert download.model_path_exists(repo.layout)