Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(model): persist file manifests and pin ModelScope snapshots
myhloli74dd405a7f75
diff --git a/docs/next/cli/mineru-kit-models.md b/docs/next/cli/mineru-kit-models.mdindex f6984d9d..ee0eb368 100644--- a/docs/next/cli/mineru-kit-models.md+++ b/docs/next/cli/mineru-kit-models.md@@ -89,6 +89,7 @@ mineru-kit models download --tier <basic|standard> [flags]- `--source` 不支持 `local`- 如果当前配置为 `model.source: local`,显式 download 会临时按 `auto` 解析远端源,不改写配置- 下载目标固定为 `config.model.base_dir` 下的 repo local dir+- ModelScope 使用 `modelscope>=1.40.3`;先把默认分支解析为固定 commit,再按同一 commit 查询清单和下载,避免分支更新导致清单与文件版本不一致支持的 repo 名:@@ -159,6 +160,8 @@ mineru-kit models verify --small-backend <auto|onnx|torch> --vlm-engine <auto|ll- `--tier` 只接受 `basic` 和 `standard`;按 `--tier` 校验时使用独立后端选项解析出的资源组合- 不是单纯目录存在性检查,还会检查 registry 中声明的关键路径- 整仓和目录资源会读取 `.mineru_complete` 文件清单,检查预期文件是否存在、是否为有效文件,以及大小是否一致;正常校验无需联网,不读取大权重计算 hash+- 单文件下载把预期大小保存到仓库根目录的 `.mineru_files.json`,后续复用同样检查缺失、截断和失效链接;这份局部记录不会把父目录或整仓标为已完成+- 完整目录重新下载后,由目录清单接管该范围的旧文件记录;文件记录损坏时需执行 `mineru-kit models download <repo>` 重建,不能只确认一个文件就丢弃其他文件的校验约束- 兼容旧版本的零字节完成标记:检查本地非空权重、registry 声明的必需路径以及权重索引引用的分片,通过后可直接离线复用- 首次使用旧模型时后台尝试升级标记,包括 `model.source: local`:按缓存中的旧版本查询远端文件清单,存在性和大小全部匹配后原子发布新版清单;只查询元数据,不下载、更新或删除模型文件,也不改写来源配置- 无网、源不可达、旧版本无法确定、核验不匹配或目录不可写时保留旧标记,不阻塞当前使用;失败后冷却 5 分钟,下次使用可重试。`models show/verify` 保持只读,不启动迁移diff --git a/mineru/model/_download_manifest.py b/mineru/model/_download_manifest.pyindex eee95cc7..b1f3af68 100644--- a/mineru/model/_download_manifest.py+++ b/mineru/model/_download_manifest.py@@ -11,9 +11,10 @@ from collections.abc import Mappingfrom pathlib import Path, PurePosixPath, PureWindowsPathMODEL_COMPLETE_MARKER = ".mineru_complete"+MODEL_FILES_MANIFEST = ".mineru_files.json"_MANIFEST_VERSION = 1_PROVIDER_METADATA_DIR_NAMES = {".cache", "._____temp"}-_PROVIDER_METADATA_FILE_NAMES = {MODEL_COMPLETE_MARKER, ".msc", ".mdl", ".mv"}+_PROVIDER_METADATA_FILE_NAMES = {MODEL_COMPLETE_MARKER, MODEL_FILES_MANIFEST, ".msc", ".mdl", ".mv"}_WEIGHT_SUFFIXES = {".safetensors", ".bin", ".pth", ".pt", ".onnx", ".gguf", ".pdiparams"}@@ -166,6 +167,7 @@ def write_manifest(marker: Path, files: Mapping[str, int]) -> None:__all__ = ["MODEL_COMPLETE_MARKER",+ "MODEL_FILES_MANIFEST","is_legacy_marker","is_payload_path","legacy_payload_missing",diff --git a/mineru/model/download.py b/mineru/model/download.pyindex db4e080d..96e577aa 100644--- a/mineru/model/download.py+++ b/mineru/model/download.py@@ -6,6 +6,7 @@ from collections.abc import Sequencefrom dataclasses import dataclass, fieldfrom pathlib import Path, PurePosixPathfrom typing import TYPE_CHECKING, Literal+from urllib.parse import quoteimport requestsfrom filelock import FileLock@@ -17,6 +18,7 @@ from loguru import loggerfrom ..config import get_config, get_config_source, update_config_filefrom ._download_manifest import (MODEL_COMPLETE_MARKER,+ MODEL_FILES_MANIFEST,is_legacy_marker,is_payload_path,legacy_payload_missing,@@ -133,19 +135,55 @@ ResolvedRemoteModelSource = Literal["huggingface", "modelscope"]ResolvedModelSource = Literal["huggingface", "modelscope", "local"]-def ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str] | None = None) -> str:+def ms_snapshot_download(repo_id: str, *, local_dir: str, revision: str, allow_patterns: list[str] | None = None) -> str:"""仅在实际使用 ModelScope 时导入 SDK,避免 HF/本地路径触发其 Torch 探测。"""from modelscope import snapshot_download- return snapshot_download(repo_id, local_dir=local_dir, allow_patterns=allow_patterns)+ return snapshot_download(repo_id, local_dir=local_dir, revision=revision, allow_patterns=allow_patterns)-def ms_model_files(repo_id: str) -> dict[str, int]:- """按下载器默认版本读取 ModelScope 预期文件,仅在使用该来源时加载 SDK。"""+def ms_model_revision(repo_id: str) -> str:+ """把默认分支解析为固定 commit,保留 SDK 的自定义端点与已配置认证。"""from modelscope.hub.api import HubApifrom modelscope.utils.constant import DEFAULT_MODEL_REVISION- files = HubApi().get_model_files(repo_id, revision=DEFAULT_MODEL_REVISION, recursive=True)+ api = HubApi()+ url = f"{api.endpoint.rstrip('/')}/api/v1/models/{quote(repo_id, safe='/')}/repo/files"+ with requests.get(+ url,+ params={"Revision": DEFAULT_MODEL_REVISION, "Recursive": "False"},+ headers=api.headers,+ cookies=api.get_cookies(),+ timeout=(5, 30),+ ) as response:+ response.raise_for_status()+ value = response.json()+ data = value.get("Data") if isinstance(value, dict) and value.get("Code") == 200 else None+ commit = data.get("LatestCommitter") if isinstance(data, dict) else None+ revision = (commit.get("Id") or commit.get("ShortId")) if isinstance(commit, dict) else None+ if not isinstance(revision, str) or re.fullmatch(r"[0-9a-f]{7,64}", revision) is None:+ raise ValueError(f"ModelScope did not return a fixed commit for {repo_id}.")+ # 部分响应只给出短 commit;同次列表若含完整值则展开,否则保留 Git 支持的固定短 commit。+ entries = data.get("Files")+ full_revisions: set[str] = set()+ for item in entries if isinstance(entries, list) else ():+ full_revision = item.get("Revision") if isinstance(item, dict) else None+ if (+ isinstance(full_revision, str)+ and re.fullmatch(r"[0-9a-f]{40,64}", full_revision)+ and full_revision.startswith(revision)+ ):+ full_revisions.add(full_revision)+ if len(full_revisions) > 1:+ raise ValueError(f"ModelScope returned an ambiguous commit for {repo_id}.")+ return next(iter(full_revisions)) if full_revisions else revision+++def ms_model_files(repo_id: str, *, revision: str) -> dict[str, int]:+ """读取已固定 commit 的 ModelScope 文件清单,仅在使用该来源时加载 SDK。"""+ from modelscope.hub.api import HubApi++ files = HubApi().get_model_files(repo_id, revision=revision, recursive=True)# 新版 SDK 的兼容接口不再返回 Type;递归清单中的父路径仍能确定目录,且不会误删零字节文件。directories = {parent.as_posix() for item in files for parent in PurePosixPath(item["Path"]).parents}return {item["Path"]: item["Size"] for item in files if item.get("Type") != "tree" and item["Path"] not in directories}@@ -251,8 +289,10 @@ def _snapshot_download(model_source: ResolvedRemoteModelSource, repo: ModelRepo,if model_source == "huggingface":info = HfApi().model_info(repo_id, files_metadata=True)files = {item.rfilename: item.size for item in info.siblings or ()}+ revision = info.shaelse:- files = ms_model_files(repo_id)+ revision = ms_model_revision(repo_id)+ files = ms_model_files(repo_id, revision=revision)expected_files: dict[str, int] = {}for name in filter_repo_objects(files, allow_patterns=patterns):if not is_payload_path(name):@@ -270,9 +310,9 @@ def _snapshot_download(model_source: ResolvedRemoteModelSource, repo: ModelRepo,target.unlink()requested_files = list(expected_files)if model_source == "huggingface":- hf_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=info.sha)+ hf_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=revision)else:- ms_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files)+ ms_snapshot_download(repo_id, local_dir=str(local_dir), allow_patterns=requested_files, revision=revision)missing_files = missing_manifest_files(local_dir, expected_files)if missing_files:missing = ", ".join(missing_files)@@ -320,6 +360,16 @@ def _model_path_missing(path: ModelPath) -> list[str]:local_path.relative_to(repo_root)except ValueError:return [path.relative_path]+ # 单文件下载记录独立于目录完成标记,局部下载不能据此宣称整个目录已完整。+ file_marker = repo_root / MODEL_FILES_MANIFEST+ if file_marker.exists() or file_marker.is_symlink():+ file_manifest = read_manifest(file_marker)+ if file_manifest is None:+ return [MODEL_FILES_MANIFEST]+ selected_files = manifest_files_for_path(file_manifest, path.relative_path)+ missing_files = missing_manifest_files(repo_root, selected_files)+ if missing_files or path.relative_path in file_manifest:+ return missing_filesis_file = local_path.is_file()if local_path == repo_root and is_file:return [MODEL_COMPLETE_MARKER]@@ -491,6 +541,26 @@ def _restore_directory_markers(write_manifest(marker, updated)+def _record_downloaded_files(repo: ModelRepo, paths: Sequence[ModelPath], files: dict[str, int]) -> None:+ """持久化单文件预期大小;完整目录刷新后由目录清单接管其覆盖的旧文件记录。"""+ marker = repo.local_dir() / MODEL_FILES_MANIFEST+ previous = read_manifest(marker)+ if (marker.exists() or marker.is_symlink()) and previous is None:+ if {path.relative_path for path in paths} != {path.relative_path for path in _repo_completion_paths(repo)}:+ # 记录损坏时无法知道其他受管文件;不能仅确认一个文件就丢弃其余文件的校验约束。+ raise ValueError(f"Invalid {marker}; run mineru-kit models download {repo.name} to rebuild file records.")+ recorded = previous or {}+ for path in paths:+ for name in manifest_files_for_path(recorded, path.relative_path):+ del recorded[name]+ if path.relative_path in files:+ recorded[path.relative_path] = files[path.relative_path]+ if recorded:+ write_manifest(marker, recorded)+ else:+ marker.unlink(missing_ok=True)++def _raise_not_ready(repo: ModelRepo, result: ModelReadyResult) -> None:"""离线校验失败时报告真实路径,并提示修复文件或完成标记。"""missing = ", ".join(result.missing_paths)@@ -532,6 +602,7 @@ def download_model_repo(if not downloaded_result.ready:_raise_not_ready(repo, downloaded_result)_mark_directories_complete(completion_paths, files)+ _record_downloaded_files(repo, completion_paths, files)_restore_directory_markers(repo, removed_markers, completion_paths, files)return downloaded_result.root@@ -572,6 +643,7 @@ def download_model_files(if not downloaded_result.ready:_raise_not_ready(repo, downloaded_result)_mark_directories_complete(model_paths, files)+ _record_downloaded_files(repo, model_paths, files)_restore_directory_markers(repo, removed_markers, model_paths, files)return downloaded_result.rootdiff --git a/pyproject.toml b/pyproject.tomlindex ed657668..602e1953 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -53,7 +53,7 @@ dependencies = ["pypdfium2>=5.13.0,<6","PyYAML>=6.0.1,<7","packaging",- "modelscope>=1.26.0",+ "modelscope>=1.40.3","huggingface-hub[hf_xet]>=0.32.4","filelock>=3.0.0","json-repair>=0.46.2",diff --git a/tests/unittest/test_model_file_manifest.py b/tests/unittest/test_model_file_manifest.pynew file mode 100644index 00000000..e260ff68--- /dev/null+++ b/tests/unittest/test_model_file_manifest.py@@ -0,0 +1,182 @@+"""单文件下载记录、损坏修复与目录完成标记隔离的回归。"""++from __future__ import annotations++import json+from dataclasses import replace+from pathlib import Path+from unittest.mock import Mock++import pytest+from huggingface_hub.utils import filter_repo_objects++from mineru.model import download+++def file_repo(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> download.ModelRepo:+ """隔离一个与 ONNX 仓库相同、全部必需资源均为文件的模型仓库。"""+ monkeypatch.setattr(download.get_config().model, "base_dir", str(tmp_path))+ monkeypatch.setattr(download.get_config().model, "source", "modelscope")+ return download.ModelRepo(+ name="file-model",+ repos={"modelscope": "Owner/file-model"},+ paths={"config": "Layout/config.json", "weights": "Layout/model.onnx"},+ download_mode="required_paths",+ )+++def remote(monkeypatch: pytest.MonkeyPatch) -> Mock:+ """模拟远端按请求范围下载完整文件,同时返回独立的预期大小。"""+ payloads = {"Layout/config.json": b"{}", "Layout/model.onnx": b"12345678", "empty.txt": b""}++ def snapshot(source: str, repo: download.ModelRepo, patterns: list[str] | None) -> dict[str, int]:+ """按真实下载模式选择文件,便于验证局部下载和整仓下载的区别。"""+ assert source == "modelscope"+ sizes = {}+ for name in filter_repo_objects(payloads, allow_patterns=patterns):+ target = repo.local_dir() / name+ target.parent.mkdir(parents=True, exist_ok=True)+ if target.is_symlink():+ target.unlink()+ target.write_bytes(payloads[name])+ sizes[name] = len(payloads[name])+ return sizes++ result = Mock(side_effect=snapshot)+ monkeypatch.setattr(download, "_snapshot_download", result)+ return result++[email protected]("entry", ["files", "repo"])[email protected]("damage", ["truncated", "deleted", "broken_link"])+def test_downloaded_file_damage_is_rejected_and_repaired(+ file_repo: download.ModelRepo, remote: Mock, monkeypatch: pytest.MonkeyPatch, entry: str, damage: str+) -> None:+ """首次成功下载后,所有单文件入口都必须拒绝损坏缓存并允许联网修复。"""+ if entry == "files":+ file_repo.config.ensure()+ file_repo.weights.ensure()+ else:+ file_repo.ensure()+ assert download.verify_model_repo(file_repo).ready+ assert not list(file_repo.local_dir().rglob(download.MODEL_COMPLETE_MARKER))+ calls = remote.call_count+ weight = file_repo.weights.local_path()+ if damage == "truncated":+ weight.write_bytes(b"1")+ else:+ weight.unlink()+ if damage == "broken_link":+ weight.symlink_to("missing.onnx")++ assert not download.model_path_exists(file_repo.weights)+ assert download.verify_model_repo(file_repo).missing_paths == ["Layout/model.onnx"]+ monkeypatch.setattr(download.get_config().model, "source", "local")+ with pytest.raises(FileNotFoundError, match="Layout/model.onnx"):+ file_repo.weights.ensure()+ assert remote.call_count == calls++ file_repo.weights.ensure(source="modelscope")+ assert remote.call_count == calls + 1+ assert weight.read_bytes() == b"12345678"+ assert download.verify_model_repo(file_repo).ready+++def test_file_records_do_not_complete_parent_directories(file_repo: download.ModelRepo, remote: Mock) -> None:+ """局部文件记录不能把父目录或原始 VLM 整仓提升为已完成状态。"""+ file_repo.weights.ensure()+ assert download.model_path_exists(file_repo.weights)+ assert not download.model_path_exists(file_repo.path("Layout"))+ assert not download.verify_model_repo(replace(file_repo, download_mode="full")).ready+ assert not download.verify_model_repo(file_repo).ready+ assert remote.call_count == 1+++def test_unmanaged_files_and_legacy_markers_still_work_offline(+ file_repo: download.ModelRepo, remote: Mock, monkeypatch: pytest.MonkeyPatch+) -> None:+ """没有新下载记录的手工模型与旧空标记保持原有离线使用边界。"""+ root = file_repo.local_dir()+ weight = file_repo.weights.local_path()+ weight.parent.mkdir(parents=True)+ weight.write_bytes(b"manual model")+ file_repo.config.local_path().write_bytes(b"{}")+ monkeypatch.setattr(download.get_config().model, "source", "local")+ assert file_repo.weights.ensure() == weight+ (root / download.MODEL_COMPLETE_MARKER).touch()+ assert file_repo.ensure() == root+ assert (root / download.MODEL_COMPLETE_MARKER).read_bytes() == b""+ remote.assert_not_called()+++def test_corrupted_file_manifest_cannot_fall_back_to_nonempty_files(file_repo: download.ModelRepo, remote: Mock) -> None:+ """受管记录损坏时不降级放行,重新下载确认后才能恢复就绪。"""+ file_repo.weights.ensure()+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST+ marker.write_text("{", encoding="utf-8")+ assert not download.model_path_exists(file_repo.weights)+ with pytest.raises(ValueError, match="rebuild file records"):+ file_repo.weights.ensure()+ assert marker.read_text() == "{"+ file_repo.ensure()+ assert download.model_path_exists(file_repo.weights)+ assert json.loads(marker.read_text())["files"] == {"Layout/config.json": 2, "Layout/model.onnx": 8}+ assert remote.call_count == 3+++def test_directory_refresh_replaces_obsolete_file_records(file_repo: download.ModelRepo, remote: Mock) -> None:+ """目录经远端完整确认后移除过时局部记录,避免已删除文件阻塞新快照。"""+ file_repo.weights.ensure()+ file_repo.weights.local_path().unlink()++ def snapshot(source: str, repo: download.ModelRepo, patterns: list[str] | None) -> dict[str, int]:+ """模拟整仓刷新后旧权重被新文件取代。"""+ assert patterns is None+ target = repo.local_dir() / "new-model.onnx"+ target.write_bytes(b"new weights")+ return {"new-model.onnx": 11}++ remote.side_effect = snapshot+ full_repo = replace(file_repo, paths={}, download_mode="full")+ full_repo.ensure()+ assert download.verify_model_repo(full_repo).ready+ assert not (full_repo.local_dir() / download.MODEL_FILES_MANIFEST).exists()+++def test_confirmed_empty_file_is_valid_and_manifest_is_not_payload(file_repo: download.ModelRepo, remote: Mock) -> None:+ """远端确认的合法空文件可复用,文件记录本身不能作为模型 payload。"""+ empty = file_repo.path("empty.txt")+ empty.ensure()+ assert download.model_path_exists(empty)+ assert remote.call_count == 1+ empty.ensure()+ assert remote.call_count == 1+ assert not download.is_payload_path(download.MODEL_FILES_MANIFEST)+++def test_failed_repair_keeps_other_managed_file_records(file_repo: download.ModelRepo, remote: Mock) -> None:+ """下载失败时仍保留原文件记录,其他受管文件不会退回非空即就绪的判断。"""+ file_repo.ensure()+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST+ before = marker.read_bytes()+ file_repo.weights.local_path().write_bytes(b"short")+ remote.side_effect = RuntimeError("offline")+ with pytest.raises(RuntimeError, match="offline"):+ file_repo.weights.ensure()+ assert marker.read_bytes() == before+ assert download.model_path_exists(file_repo.config)+ assert not download.model_path_exists(file_repo.weights)+++def test_directory_refresh_keeps_file_records_outside_its_scope(file_repo: download.ModelRepo, remote: Mock) -> None:+ """目录清单只接管自身范围,其他位置的文件记录继续保留。"""+ file_repo.ensure()+ file_repo.path("empty.txt").ensure()+ file_repo.path("Layout").ensure()+ marker = file_repo.local_dir() / download.MODEL_FILES_MANIFEST+ assert json.loads(marker.read_text())["files"] == {"empty.txt": 0}+ assert download.model_path_exists(file_repo.path("Layout"))+ file_repo.weights.local_path().write_bytes(b"short")+ assert not download.model_path_exists(file_repo.weights)diff --git a/tests/unittest/test_model_readiness.py b/tests/unittest/test_model_readiness.pyindex 08e9d17b..b54a5512 100644--- a/tests/unittest/test_model_readiness.py+++ b/tests/unittest/test_model_readiness.py@@ -466,6 +466,7 @@ def test_provider_repair_replaces_wrong_size_cached_payload()monkeypatch.setattr(download, "HfApi", Mock(return_value=api))monkeypatch.setattr(download, "ms_model_files", Mock(return_value=expected))+ monkeypatch.setattr(download, "ms_model_revision", Mock(return_value="a" * 40))def snapshot(repo_id: str, **kwargs: object) -> str:"""模拟增量下载只补齐缺失文件,保留已验证的配置。"""@@ -500,6 +501,7 @@ def test_provider_manifest_checks_expected_payload_before_marking_complete()monkeypatch.setattr(download, "HfApi", Mock(return_value=api))monkeypatch.setattr(download, "ms_model_files", Mock(return_value=expected))+ monkeypatch.setattr(download, "ms_model_revision", Mock(return_value="a" * 40))def snapshot(repo_id: str, **kwargs: object) -> str:"""模拟 SDK 返回成功但权重可能缺失或未完整落盘。"""diff --git a/tests/unittest/test_models_download_utils.py b/tests/unittest/test_models_download_utils.pyindex f00c150a..093c9c34 100644--- a/tests/unittest/test_models_download_utils.py+++ b/tests/unittest/test_models_download_utils.py@@ -444,6 +444,8 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(repo = ModelRepo(name="repo", repos={"modelscope": "Owner/repo"}, paths={"layout": "Layout/PP-DocLayoutV2"})monkeypatch.setattr(models_download_utils.get_config().model, "base_dir", str(tmp_path / "models"))layout = "Layout/PP-DocLayoutV2"+ fixed_revision = "a" * 40+ monkeypatch.setattr(models_download_utils, "ms_model_revision", lambda repo_id: fixed_revision)payloads = {f"{layout}/config.json": b"{}", f"{layout}/model.safetensors": b"weights", f"{layout}/empty.txt": b""}directories = ["Layout", layout, "Unused"]# 目录项故意排在文件之后,确保过滤逻辑不依赖 SDK 返回顺序。@@ -458,7 +460,7 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(def get_model_files(self, repo_id: str, *, revision: str, recursive: bool) -> list[dict[str, Any]]:"""返回真实接口形状的递归清单,覆盖不再包含 Type 的新版 SDK。"""assert repo_id == "Owner/repo"- assert revision == "master"+ assert revision == fixed_revisionassert recursivereturn entries@@ -466,9 +468,10 @@ def test_modelscope_snapshot_excludes_directories_and_checks_real_files(monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))requested: list[str] = []- def fake_ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str]) -> str:+ def fake_ms_snapshot_download(repo_id: str, *, local_dir: str, allow_patterns: list[str], revision: str) -> str:"""模拟下载真实文件,目录项由 SDK 忽略,缺失和截断按测试参数注入。"""assert repo_id == "Owner/repo"+ assert revision == fixed_revisionrequested.extend(allow_patterns)for name, content in payloads.items():target = Path(local_dir) / namediff --git a/tests/unittest/test_modelscope_revision.py b/tests/unittest/test_modelscope_revision.pynew file mode 100644index 00000000..b8444a3f--- /dev/null+++ b/tests/unittest/test_modelscope_revision.py@@ -0,0 +1,105 @@+"""ModelScope 固定 commit 的元数据解析与下载一致性回归。"""++from __future__ import annotations++import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import MagicMock, Mock++import pytest++from mineru.model import download++_REVISION = "a" * 40+++ ("commit", "entries", "expected"),+ [+ ({"Id": _REVISION}, [], _REVISION),+ ({"Id": "", "ShortId": _REVISION[:8]}, [{"Revision": _REVISION}], _REVISION),+ ({"ShortId": _REVISION[:8]}, [], _REVISION[:8]),+ ({"ShortId": _REVISION[:8]}, [{"Revision": _REVISION}, {"Revision": _REVISION[:8] + "b" * 32}], None),+ ],+)+def test_modelscope_commit_resolution_preserves_endpoint_and_auth(+ monkeypatch: pytest.MonkeyPatch, commit: dict[str, str], entries: list[dict[str, str]], expected: str | None+) -> None:+ """从同次仓库响应解析固定 commit,并沿用 SDK 的端点、请求头与认证。"""+ api = SimpleNamespace(+ endpoint="https://modelscope.example/", headers={"user-agent": "sdk"}, get_cookies=Mock(return_value={})+ )+ monkeypatch.setitem(sys.modules, "modelscope.hub.api", SimpleNamespace(HubApi=Mock(return_value=api)))+ monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))+ response = MagicMock()+ response.__enter__.return_value = response+ response.json.return_value = {"Code": 200, "Data": {"LatestCommitter": commit, "Files": entries}}+ request = Mock(return_value=response)+ monkeypatch.setattr(download.requests, "get", request)++ if expected is None:+ with pytest.raises(ValueError, match="ambiguous commit"):+ download.ms_model_revision("Owner/repo")+ else:+ assert download.ms_model_revision("Owner/repo") == expected+ request.assert_called_once_with(+ "https://modelscope.example/api/v1/models/Owner/repo/repo/files",+ params={"Revision": "master", "Recursive": "False"},+ headers=api.headers,+ cookies={},+ timeout=(5, 30),+ )+ response.raise_for_status.assert_called_once()++[email protected]("commit", [{}, {"Id": "master"}, {"ShortId": "latest"}, {"Id": 123}])+def test_modelscope_missing_commit_is_not_replaced_with_mutable_branch(+ monkeypatch: pytest.MonkeyPatch, commit: dict[str, object]+) -> None:+ """无法确认固定 commit 时明确失败,不能静默回退到会变化的 master。"""+ api = SimpleNamespace(endpoint="https://modelscope.example", headers={}, get_cookies=lambda: {})+ monkeypatch.setitem(sys.modules, "modelscope.hub.api", SimpleNamespace(HubApi=lambda: api))+ monkeypatch.setitem(sys.modules, "modelscope.utils.constant", SimpleNamespace(DEFAULT_MODEL_REVISION="master"))+ response = MagicMock()+ response.__enter__.return_value = response+ response.json.return_value = {"Code": 200, "Data": {"LatestCommitter": commit, "Files": []}}+ monkeypatch.setattr(download.requests, "get", Mock(return_value=response))+ with pytest.raises(ValueError, match="fixed commit"):+ download.ms_model_revision("Owner/repo")+++def test_modelscope_snapshot_wrapper_forwards_fixed_commit(monkeypatch: pytest.MonkeyPatch) -> None:+ """下载封装必须把与清单相同的 commit 传给 SDK。"""+ snapshot = Mock(return_value="local-model")+ monkeypatch.setitem(sys.modules, "modelscope", SimpleNamespace(snapshot_download=snapshot))+ assert download.ms_snapshot_download("Owner/repo", local_dir="local-model", revision=_REVISION) == "local-model"+ snapshot.assert_called_once_with("Owner/repo", local_dir="local-model", revision=_REVISION, allow_patterns=None)+++def test_modelscope_branch_update_does_not_change_selected_snapshot(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:+ """分支在列清单后更新时仍下载原 commit,避免完整新文件被旧清单误拒绝。"""+ monkeypatch.setattr(download.get_config().model, "base_dir", str(tmp_path))+ repo = download.ModelRepo(name="repo", repos={"modelscope": "Owner/repo"}, paths={"layout": "Layout/weights"})+ branch = {"revision": _REVISION}+ monkeypatch.setattr(download, "ms_model_revision", lambda repo_id: branch["revision"])++ def files(repo_id: str, *, revision: str) -> dict[str, int]:+ """模拟读取旧 commit 后远端分支立即推进。"""+ assert revision == _REVISION+ branch["revision"] = "b" * 40+ return {"Layout/weights/model.bin": 7}++ def snapshot(repo_id: str, *, local_dir: str, revision: str, allow_patterns: list[str]) -> str:+ """按实际传入的 commit 选择内容,漏传固定版本时将得到不同大小的新权重。"""+ assert branch["revision"] != _REVISION+ target = Path(local_dir) / "Layout/weights/model.bin"+ target.parent.mkdir(parents=True)+ target.write_bytes(b"weights" if revision == _REVISION else b"new weights")+ return local_dir++ monkeypatch.setattr(download, "ms_model_files", files)+ monkeypatch.setattr(download, "ms_snapshot_download", snapshot)+ repo.layout.ensure(source="modelscope")+ assert (repo.layout.local_path() / "model.bin").read_bytes() == b"weights"+ assert download.model_path_exists(repo.layout)