Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
Merge Transformers 5 migration and integrate Python 3.14 document and backend support
myhlolie8a3fe1d3759
diff --cc .github/workflows/transformers5.ymlindex 00000000,84945261..491c15e1mode 000000,100644..100644--- a/.github/workflows/transformers5.yml+++ b/.github/workflows/transformers5.yml@@@ -1,0 -1,105 +1,118 @@@+ name: Transformers 5 compatibility++ on:+ pull_request:+ branches: [next]+ push:+ branches: [next]+ workflow_dispatch:+ inputs:++ docvortex-wheel-url:++ description: Optional DocVortex 0.3.x wheel URL for joint pre-release validation++ required: false++ type: string+ utils-wheel-url:+ description: Optional mineru-vl-utils 2.x wheel URL for joint pre-release validation+ required: false+ type: string++ permissions:+ contents: read++ jobs:+ compatibility:+ name: ${{ matrix.os }} / Python ${{ matrix.python }} / Transformers ${{ matrix.transformers }}+ runs-on: ${{ matrix.os }}+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest]- python: ['3.10', '3.11', '3.12', '3.13']- transformers: ['5.10.1', '5.16.1']++ python: ['3.10', '3.11', '3.12', '3.13', '3.14']++ transformers: ['5.10.1', '5.14.0', '5.16.1']+ include:+ - {os: ubuntu-latest, python: '3.10', transformers: '5.10.1', torch: '2.6.0', torchvision: '0.21.0'}+ - {os: windows-latest, python: '3.12', transformers: '5.10.1'}+ - {os: windows-latest, python: '3.12', transformers: '5.16.1'}+ - {os: macos-latest, python: '3.12', transformers: '5.10.1'}+ - {os: macos-latest, python: '3.12', transformers: '5.16.1'}++ - {os: macos-latest, python: '3.14', transformers: '5.16.1'}++ - {os: windows-latest, python: '3.14', transformers: '5.16.1'}+ defaults:+ run:+ shell: bash+ steps:+ - uses: actions/checkout@v6+ - uses: actions/setup-python@v6+ with:+ python-version: ${{ matrix.python }}+ - uses: astral-sh/setup-uv@v7+ - name: Install dependencies and optional joint-validation wheel+ env:+ UTILS_WHEEL_URL: ${{ inputs.utils-wheel-url }}++ DOCVORTEX_WHEEL_URL: ${{ inputs.docvortex-wheel-url }}+ TRANSFORMERS_VERSION: ${{ matrix.transformers }}+ TORCH_VERSION: ${{ matrix.torch }}+ TORCHVISION_VERSION: ${{ matrix.torchvision }}+ run: |+ uv venv++ if [[ -n "$DOCVORTEX_WHEEL_URL" ]]; then++ uv pip install "$DOCVORTEX_WHEEL_URL"++ fi+ if [[ -n "$UTILS_WHEEL_URL" ]]; then+ uv pip install "$UTILS_WHEEL_URL"+ fi+ if [[ -n "$TORCH_VERSION" ]]; then+ uv pip install "torch==$TORCH_VERSION" "torchvision==$TORCHVISION_VERSION"+ fi+ uv pip install '.[torch,test]' "transformers==$TRANSFORMERS_VERSION"+ uv pip check+ - name: Check real model graphs, cache and public import boundaries+ run: |+ if [[ -f .venv/Scripts/activate ]]; then source .venv/Scripts/activate; else source .venv/bin/activate; fi+ python -m pytest -q -o addopts='' \+ tests/unittest/test_transformers5_models.py \+ tests/unittest/test_transformers5_runtime.py \+ tests/unittest/test_pp_doclayoutv2_postprocess.py \+ tests/unittest/test_mfr_latex_utils.py \+ tests/unittest/test_vlm_config.py \+ tests/unittest/test_vlm_remote.py \+ tests/unittest/test_parser_model_preload.py \+ tests/unittest/test_models_download_utils.py \+ tests/unittest/test_pdf_memory_lifecycle.py++ dependency-resolution:+ name: Installation matrix / ${{ matrix.platform }} / Transformers ${{ matrix.transformers }}+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ platform: [linux, windows, macos]- transformers: ['5.10.1', '5.16.1']++ transformers: ['5.10.1', '5.14.0', '5.16.1']+ steps:+ - uses: actions/checkout@v6+ - uses: actions/setup-python@v6+ with:+ python-version: '3.12'+ - uses: astral-sh/setup-uv@v7+ - name: Build wheels for metadata validation+ env:+ UTILS_WHEEL_URL: ${{ inputs.utils-wheel-url }}++ DOCVORTEX_WHEEL_URL: ${{ inputs.docvortex-wheel-url }}+ run: |+ python -m pip install build+ python -m build --wheel+ mkdir -p dependency-wheels+ python -m pip download --no-deps --dest dependency-wheels "${UTILS_WHEEL_URL:-mineru-vl-utils>=2,<3}"- - name: Resolve base, torch, full and all on Python 3.10 through 3.13++ python -m pip download --no-deps --dest dependency-wheels "${DOCVORTEX_WHEEL_URL:-docvortex>=0.3.0,<1}"++ - name: Resolve base, torch, full and all on Python 3.10 through 3.14+ run: |+ python scripts/check_transformers5_dependencies.py \- --mineru-wheel dist/*.whl --utils-wheel dependency-wheels/*.whl \++ --mineru-wheel dist/*.whl --utils-wheel dependency-wheels/mineru_vl_utils-*.whl \++ --docvortex-wheel dependency-wheels/docvortex-*.whl --utils-matrix --check-wheels \+ --platform '${{ matrix.platform }}' --transformers '${{ matrix.transformers }}' \+ --output dependency-matrix.json+ - uses: actions/upload-artifact@v6+ if: always()+ with:+ name: dependencies-${{ matrix.platform }}-${{ matrix.transformers }}+ path: dependency-matrix.jsondiff --cc docs/next/transformers5-migration.mdindex 00000000,8e505008..b556bab3mode 000000,100644..100644--- a/docs/next/transformers5-migration.md+++ b/docs/next/transformers5-migration.md@@@ -1,0 -1,79 +1,92 @@@+ # Transformers 5 推理栈+-本分支只支持 `transformers>=5.10.1,<6`,回归锚点为 5.10.1 与 5.16.1。++本分支只支持 `transformers>=5.10.1,<6`,回归锚点为 5.10.1、5.14.0 与 5.16.1;macOS full/all 的 MLX-VLM 0.7 只允许 >=5.14。+ `mineru-vl-utils` 配套版本为 2.0.0;该版本提高框架最低版本,并将 LMDeploy 参数契约更新为公开的 `Pipeline`。+ 基础安装仍使用轻量模型栈,不安装或提前导入 Torch/Transformers。++ ## 依赖和实现边界++ | 组件 | 范围 |+ | --- | --- |+ | Transformers | `>=5.10.1,<6` |+ | Tokenizers | `>=0.22.0,<0.24`;5.10.1 与 5.16.1 由解析器选择不同兼容版本 |+ | PyTorch | `>=2.6,<3`,TorchVision 与其配套 |-| vLLM | `>=0.21.0,<0.22` |++| vLLM | `>=0.19.1,<0.29.0` |+ | LMDeploy | `>=0.17.0,<0.18` |-| MLX-VLM | `>=0.3.12,<0.4`,MLX 保留 `<=0.31.1` |++| MLX-VLM | `>=0.7.0,<0.8.0`,MLX 版本由其依赖决定 |+ | mineru-vl-utils | `>=2.0.0,<3` |++ - UniMERNet 保留现有权重、词表、Q/K 压缩、token 上限及分批规则。组合配置显式构造 Swin/MBart,不修改全局 Auto 注册表。+ - 自注意力和交叉注意力使用 `Cache`/`EncoderDecoderCache`;权重初始化使用 Transformers 5 的保护机制,防止已加载参数被重新随机初始化。+ - PP-DocLayoutV2 复用官方模型,保留 MinerU 的预处理、阈值、类别、阅读顺序和后处理。MPS 的正弦位置编码在 CPU 调用官方实现,然后送回模型设备,避开 float64 限制。+ - Qwen2-VL 从文本子配置读取上下文长度,并合并生成配置、文本配置及 tokenizer 的特殊 token。+ - LMDeploy 同步/异步客户端使用同一个 `Pipeline.infer`。异步调用在线程中执行;取消时等待在途同步调用结束后再释放名额,防止共享引擎被提前卸载。+ - vLLM 客户端不再安装依赖已删除 `AnyTokenizer` 的导入时全局 logprobs 补丁。++ ## 本地联合验证+-utils 2.0.0 正式发布前,先构建两个工作树的 wheel,并在独立环境中一起安装,避免解析到旧版 utils:++utils 2.0.0 正式发布前,先构建 DocVortex 0.3.0、utils 2.0.0 和 MinerU 三个工作树的 wheel,并在独立环境中一起安装,避免解析到旧版 DocVortex 或 utils:++ ```bash+ # 在 mineru-vl-utils 工作树中+ uv build --wheel++ # 在 MinerU 工作树中+ uv build --wheel+ uv venv .venv-validation+ uv pip install --python .venv-validation/bin/python \++ /path/to/docvortex/dist/docvortex-0.3.0-py3-none-any.whl \+ /path/to/mineru-vl-utils/dist/mineru_vl_utils-2.0.0-py3-none-any.whl \+ './dist/mineru-4.0.0b1-py3-none-any.whl[torch]' \+ 'transformers==5.16.1' pytest+ uv pip check --python .venv-validation/bin/python+ ```+-安装矩阵直接读取已构建 wheel 的元数据,覆盖三个平台、Python 3.10–3.13 与基础/torch/full/all 模式:++安装矩阵直接读取已构建 wheel 的元数据,覆盖三个平台、Python 3.10–3.14 与基础/torch/full/all 模式:++ ```bash+ python scripts/check_transformers5_dependencies.py \+ --mineru-wheel dist/mineru-4.0.0b1-py3-none-any.whl \+ --utils-wheel /path/to/mineru_vl_utils-2.0.0-py3-none-any.whl \++ --docvortex-wheel /path/to/docvortex-0.3.0-py3-none-any.whl --utils-matrix --check-wheels \+ --platform linux --transformers 5.10.1 --output dependency-matrix.json+ ```++ Linux 解析目标使用 glibc 2.34,Apple Silicon 使用 macOS 14;这是所选二进制依赖的安装基线。+ 元数据可解不代表 GPU 推理已通过。++ 真实模型验证工具为 `scripts/validate_transformers_migration.py`。输入清单包含+ `pages`(每项有 `name`、`pdf`、一基 `page`、渲染后 `image` 路径)及 `formulas`(固定公式裁剪路径列表)。+ 同一清单分别交给保存的 4.57.6 基线源码和迁移源码,输出模型结果、Markdown/MiddleJson、耗时及内存采样:++ ```bash+ python scripts/validate_transformers_migration.py \+ --repo . --manifest /path/to/inputs.json --kind parse --tier basic \+ --device mps --rounds 3 --output validation/parse-basic.json+ ```++ `layout`/`mfr` 验证独立模型;`vlm-transformers`/`vlm-mlx` 验证真实 VLM 的短生成链路。+ VLM 组件 smoke 将图像最长边限制为 448 像素,不代替 `parse` 场景的原分辨率整页质量验收。+ 内存报告中的 `rss_bytes` 为主进程 RSS,`torch_device_bytes` 为每 20ms 采样的 Torch 活跃设备内存。+ Transformers 5.16 使用 safetensors 的 MPS `pread` 路径,框架 allocator 计数不包含所有权重存储;RSS 与 GPU allocator 数值不能直接跨加载方式比较。+ macOS 同时记录系统 `proc_pid_rusage` 的 `physical_footprint_bytes`,用于统一内存的真实占用比较。保留 RSS 和框架计数供诊断,不把它们误报为 MLX 或外部 MPS 存储的 GPU 峰值。++ ## CI 与发布门槛++ `transformers5.yml` 对 next/PR 运行模型图、增量缓存、重载、惰性导入、HTTP VLM 与生命周期回归,并解析安装矩阵。-手动触发时可传入 `utils-wheel-url` 验证未发布的联合构建。utils 仓库也提供客户端双版本回归。++手动触发时可传入 `utils-wheel-url` 与 `docvortex-wheel-url` 验证未发布的联合构建。utils 仓库也提供客户端双版本回归。++ 发布前需要:固定权重和输入的内容回归通过;同一设备预热后三轮耗时、内存/显存无超过 10% 的回退;Linux CUDA、Windows GPU、macOS 实机推理有对应证据。+ 单元测试替身和安装解析不替代实机结果。旧国产硬件镜像继续使用旧发布系列,单独迁移。+ 当前工作按用户要求暂缓发布,不据此文档声明所有平台已经通过。++++++## Python 3.14 联合升级++++- 三仓 Requires-Python 均为 `>=3.10,<3.15`;不包含 free-threaded ABI。++- MinerU 依赖 `docvortex>=0.3.0,<1`,共享纯字符边界拼接,完整移除 fast-langdetect/fasttext-predict 下载链。++- 同步 vLLM.generate 自行渲染 raw prompt;异步入口保留 EngineInput。corex CompilationConfig 使用 mode,不使用旧 level。++- MLX 同模型生成串行化,取消任务时等待在途线程结束;异步模型锁等待不在后台遗留 acquire 线程。++- 检查脚本显式验证 wheel 的 Requires-Python 和每个目标 ABI 的发行文件;jieba 为唯一允许的纯 Python sdist。++- macOS full/all + Transformers 5.10.1 应拒绝;Python 3.14 + vLLM 0.19.1 应拒绝,其余范围由解析器选择兼容版本。++- 发布顺序为 DocVortex、utils、MinerU。本次只做本地合入、构建与验证,未发布依赖必须显式提供三个 wheel。diff --cc mineru/backend/analysis/pdf/text/content.pyindex ac458a94,ac458a94..0db0e5f0--- a/mineru/backend/analysis/pdf/text/content.py+++ b/mineru/backend/analysis/pdf/text/content.py@@@ -9,7 -9,7 +9,6 @@@ import numpy as nfrom PIL import Imagefrom .....model.runtime.hybrid import HybridLocalModelContext, run_ocr_inferencefrom .....types import BBox, BlockType, ContentType--from docvortex.foundation.language import detect_langfrom .....model.ocr.image import rotate_vertical_crop_if_neededfrom .....model.ocr.results import OcrConfidencefrom docvortex.document.pdf.document import PDFPage@@@ -309,14 -309,14 +308,7 @@@ def _lines_to_block_content(lines: listif block_type == BlockType.INDEX or block_type in CODE_CONTENT_BLOCK_TYPES:return "\n".join(rendered_lines).strip()-- text_for_language = "".join(-- content for parts in content_lines for span_type, content in parts if span_type == ContentType.TEXT-- )-- block_language = detect_lang(text_for_language)-- return merge_text_line_contents(-- rendered_lines,-- block_language=block_language,-- )++ return merge_text_line_contents(rendered_lines)def _lines_have_native_script_markup(lines: list[_AnalyzeLine]) -> bool:diff --cc mineru/model/vlm/engine_utils.pyindex 0b4c9590,0b4c9590..fdec0e7e--- a/mineru/model/vlm/engine_utils.py+++ b/mineru/model/vlm/engine_utils.py@@@ -141,7 -141,7 +141,7 @@@ def _get_device_config(device_type: str"corex": {"compilation_config_dict": {"cudagraph_mode": "FULL_DECODE_ONLY",-- "level": 0,++ "mode": 0,},},"kxpu": {diff --cc mineru/model/vlm/runtime.pyindex d23a9bfc,064208c8..2f072252--- a/mineru/model/vlm/runtime.py+++ b/mineru/model/vlm/runtime.py@@@ -329,7 -314,7 +314,10 @@@ async def _get_model_asyncserver_url: str | None,**kwargs: Any,) -> MinerUClient:-- return await asyncio.to_thread(++ """模型初始化被取消时等待后台加载结束,防止清理后又写入迟到的模型句柄。"""++ from mineru_vl_utils.vlm_client.utils import run_in_thread_until_complete++++ return await run_in_thread_until_complete(ModelSingleton().get_model,backend,model_path,@@@ -455,11 -440,11 +443,13 @@@ def predictor_execution_guard(predictor@asynccontextmanagerasync def aio_predictor_execution_guard(predictor: MinerUClient) -> AsyncIterator[None]:++ """可取消地等待串行模型锁,避免被取消的后台 acquire 在线程中遗留持锁者。"""lock = getattr(predictor, "_mineru_execution_lock", None)if lock is None:yieldreturn-- await asyncio.to_thread(lock.acquire)++ while not lock.acquire(blocking=False):++ await asyncio.sleep(0.01)try:yieldfinally:diff --cc pyproject.tomlindex cdeb55f1,bfd670de..a1c66f0b--- a/pyproject.toml+++ b/pyproject.toml@@@ -9,7 -9,7 +9,7 @@@ license = "LicenseRef-MinerU-Open-Sourclicense-files = ["LICENSE.md"]description = "A practical document parsing tool for converting PDF, OFD, EPUB, HTML, images, CSV, RTF, OOXML, and OpenDocument files into Markdown and JSON"readme = "README.md"--requires-python = ">=3.10,<3.14"++requires-python = ">=3.10,<3.15"keywords = ["mineru","magic-pdf",@@@ -36,11 -36,12 +36,12 @@@ classifiers ="Programming Language :: Python :: 3.11","Programming Language :: Python :: 3.12","Programming Language :: Python :: 3.13",++ "Programming Language :: Python :: 3.14",]dependencies = [- "docvortex>=0.2.6,<1.0.0",- "docvortex>=0.2.2,<1.0.0",++ "docvortex>=0.3.0,<1","mineru-vl-utils>=2.0.0,<3",- "metafile-render>=0.3.0,<1.0.0",-- "mineru-llama-cpp>=0.1.0,<1",++ "mineru-llama-cpp>=0.1.1,<1","click>=8.1.7","loguru>=0.7.2","numpy>=1.21.6",@@@ -48,9 -49,12 +49,9 @@@"requests","httpx","pillow>=11.0.0",-- "pypdfium2>=5.10.1",- "pypdf>=5.6.0",++ "pypdfium2>=5.10.1,<6","PyYAML>=6.0.1,<7","packaging",- "reportlab",- "ziamath>=0.13,<0.14","modelscope>=1.26.0","huggingface-hub[hf_xet]>=0.32.4","filelock>=3.0.0",diff --cc scripts/check_transformers5_dependencies.pyindex 00000000,a48c5964..2dc47864mode 000000,100644..100644--- a/scripts/check_transformers5_dependencies.py+++ b/scripts/check_transformers5_dependencies.py@@@ -1,0 -1,78 +1,181 @@@+ #!/usr/bin/env python3-"""从已构建 wheel 验证跨平台依赖可解性,不安装 GPU 引擎或修改目标环境。"""++"""从三个真实 wheel 检查 Python 声明、平台依赖和二进制覆盖;不安装 GPU 引擎。"""++ from __future__ import annotations++ import argparse++import email+ import json+ import os+ import subprocess++import urllib.request++import zipfile+ from pathlib import Path+++from packaging.requirements import Requirement++from packaging.specifiers import SpecifierSet++from packaging.tags import compatible_tags, cpython_tags, mac_platforms++from packaging.utils import canonicalize_name, parse_wheel_filename++++PLATFORMS = {"linux": "x86_64-manylinux_2_34", "windows": "x86_64-pc-windows-msvc", "macos": "aarch64-apple-darwin"}++FORBIDDEN = {"fast-langdetect", "fasttext-predict", "robust-downloader"}++++++def read_wheel(path: Path) -> tuple[str, str, str]:++ """读取真实发行元数据,显式校验直接 URL 依赖也必须遵守的 Python 范围。"""++ with zipfile.ZipFile(path) as archive:++ name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA"))++ metadata = email.message_from_bytes(archive.read(name))++ return canonicalize_name(metadata["Name"]), metadata["Requires-Python"], path.resolve().as_uri()++++++def target_tags(platform: str, python: str) -> set:++ """生成普通 CPython 的目标 ABI 标签,不把 free-threaded wheel 当作可用包。"""++ version = tuple(map(int, python.split(".")))++ if platform == "linux":++ platforms = [f"manylinux_2_{minor}_x86_64" for minor in range(34, 4, -1)]++ platforms += ["manylinux2014_x86_64", "manylinux2010_x86_64", "manylinux1_x86_64", "linux_x86_64"]++ elif platform == "windows":++ platforms = ["win_amd64"]++ else:++ platforms = list(mac_platforms((14, 0), "arm64"))++ interpreter = "cp" + python.replace(".", "")++ return set(cpython_tags(version, abis=[interpreter], platforms=platforms)) | set(++ compatible_tags(version, interpreter=interpreter, platforms=platforms)++ )++++++def verify_wheels(resolution: str, platform: str, python: str, cache: Path) -> list[str]:++ """检查每个已解析发布包的 Python 范围及 wheel;只允许已知纯 Python jieba 从源码安装。"""++ tags = target_tags(platform, python)++ failures = []++ cache.mkdir(parents=True, exist_ok=True)++ for line in resolution.splitlines():++ if "==" not in line or line.startswith((" ", "#")):++ continue++ requirement = Requirement(line)++ name = canonicalize_name(requirement.name)++ version = next(iter(requirement.specifier)).version++ cached = cache / f"{name}-{version}.json"++ if not cached.exists():++ with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=30) as response:++ temporary = cached.with_suffix(f".{os.getpid()}.tmp")++ temporary.write_bytes(response.read())++ temporary.replace(cached)++ data = json.loads(cached.read_text())++ requires_python = data["info"].get("requires_python")++ if requires_python and python not in SpecifierSet(requires_python):++ failures.append(f"{name}=={version}: Python {python} excluded by {requires_python}")++ available = any(++ item["filename"].endswith(".whl") and bool(parse_wheel_filename(item["filename"])[3] & tags)++ for item in data["urls"]++ )++ if not available and name != "jieba":++ failures.append(f"{name}=={version}: no wheel for {platform} Python {python}")++ return failures++++ def main() -> None:- """在四个 Python 版本和基础/可选安装模式下解析真实发布元数据。"""++ """运行有效组合并明确记录预期冲突,结果包含完整求解输出和发行包证据。"""+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--mineru-wheel", type=Path, required=True)+ parser.add_argument("--utils-wheel", type=Path, required=True)- parser.add_argument("--platform", choices=("linux", "windows", "macos"), required=True)- parser.add_argument("--transformers", choices=("5.10.1", "5.16.1"), required=True)- parser.add_argument("--python", nargs="+", default=["3.10", "3.11", "3.12", "3.13"])++ parser.add_argument("--docvortex-wheel", type=Path, required=True)++ parser.add_argument("--platform", choices=tuple(PLATFORMS), required=True)++ parser.add_argument("--python", nargs="+", default=["3.10", "3.11", "3.12", "3.13", "3.14"])++ parser.add_argument("--transformers", choices=["5.10.1", "5.14.0", "5.16.1"])++ parser.add_argument("--extras", nargs="+", default=["base", "torch", "full", "all"])++ parser.add_argument("--utils-matrix", action="store_true")++ parser.add_argument("--check-wheels", action="store_true")++ parser.add_argument("--requirement", action="append", default=[])++ parser.add_argument("--expect-failure", action="store_true")+ parser.add_argument("--output", type=Path, required=True)+ args = parser.parse_args()- platforms = {"linux": "x86_64-manylinux_2_34", "windows": "x86_64-pc-windows-msvc", "macos": "aarch64-apple-darwin"}- mineru = args.mineru_wheel.resolve().as_uri()- utils = args.utils_wheel.resolve().as_uri()++ roots = [read_wheel(path) for path in (args.mineru_wheel, args.utils_wheel, args.docvortex_wheel)]++ by_name = {name: (requires, uri) for name, requires, uri in roots}++ cases = [("mineru", extra) for extra in args.extras]++ if args.utils_matrix:++ backend = {"linux": "vllm", "windows": "lmdeploy", "macos": "mlx"}[args.platform]++ cases += [("mineru-vl-utils", extra) for extra in ("base", "transformers", "llama-cpp", backend)]+ records = []- failed = False++ args.output.parent.mkdir(parents=True, exist_ok=True)+ for python in args.python:- for extra in ("", "torch", "full", "all"):- requirement = f"mineru[{extra}]" if extra else "mineru"- source = f"{requirement} @ {mineru}\nmineru-vl-utils @ {utils}\n"- if extra:- source += f"transformers=={args.transformers}\n"++ for package, extra in cases:++ root_names = [name for name, _, _ in roots] if package == "mineru" else ["mineru-vl-utils"]++ requirements = [++ f"{name}{('[' + extra + ']') if name == package and extra != 'base' else ''} @ {by_name[name][1]}"++ for name in root_names++ ]++ if args.transformers and extra != "base":++ requirements.append(f"transformers=={args.transformers}")++ requirements.extend(args.requirement)++ invalid_roots = [name for name in root_names if python not in SpecifierSet(by_name[name][0])]+ process = subprocess.run(+ [+ "uv",+ "pip",+ "compile",+ "-",+ "--python-version",+ python,+ "--python-platform",- platforms[args.platform],++ PLATFORMS[args.platform],++ "--index-url",++ "https://pypi.org/simple",+ "--no-header",+ "--no-annotate",+ "--no-progress",+ ],- input=source,++ input="\n".join(requirements) + "\n",+ text=True,+ capture_output=True,+ check=False,+ env={**os.environ, "MACOSX_DEPLOYMENT_TARGET": "14.0"},++ timeout=180,+ )- # 基础安装的解析结果也必须保持无 Torch/Transformers 依赖。- heavy_in_base = not extra and any(- line.startswith(("torch==", "transformers==")) for line in process.stdout.splitlines()++ mlx_conflict = (++ args.transformers == "5.10.1"++ and args.platform == "macos"++ and ((package == "mineru" and extra in {"full", "all"}) or extra == "mlx")+ )- success = process.returncode == 0 and not heavy_in_base- failed |= not success++ expected_failure = args.expect_failure or mlx_conflict++ resolved = process.returncode == 0 and not invalid_roots++ checks = []++ if resolved:++ selected = {++ canonicalize_name(Requirement(line).name)++ for line in process.stdout.splitlines()++ if line and not line.startswith((" ", "#"))++ }++ checks += [f"forbidden dependency: {name}" for name in selected & FORBIDDEN]++ if extra == "base" and not args.requirement:++ checks += [f"heavy base dependency: {name}" for name in selected & {"torch", "transformers"}]++ if args.check_wheels:++ checks += verify_wheels(process.stdout, args.platform, python, args.output.parent / "pypi-metadata")++ rejection_confirmed = bool(invalid_roots) or "No solution found" in process.stderr++ success = (not resolved and rejection_confirmed if expected_failure else resolved) and not checks+ records.append(+ {++ "package": package,+ "python": python,- "extra": extra or "base",++ "extra": extra,+ "success": success,++ "expected_failure": expected_failure,++ "resolved": resolved,++ "invalid_roots": invalid_roots,++ "wheel_or_dependency_errors": checks,+ "resolution": process.stdout,+ "diagnostics": process.stderr,+ }+ )- print(f"{args.platform} Python {python} {extra or 'base'}: {'PASS' if success else 'FAIL'}", flush=True)- args.output.parent.mkdir(parents=True, exist_ok=True)- args.output.write_text(- json.dumps({"platform": args.platform, "transformers": args.transformers, "results": records}, indent=2) + "\n"- )- raise SystemExit(1 if failed else 0)++ print(++ f"{args.platform} Python {python} {package}[{extra}]: {'PASS' if success else 'FAIL'}"++ f"{' (expected rejection)' if expected_failure else ''}",++ flush=True,++ )++ args.output.write_text(json.dumps({"platform": args.platform, "roots": roots, "results": records}, indent=2) + "\n")++ raise SystemExit(not all(record["success"] for record in records))+++ if __name__ == "__main__":+ main()diff --cc scripts/validate_transformers_migration.pyindex 00000000,c3ed7b2f..0b881a62mode 000000,100644..100644--- a/scripts/validate_transformers_migration.py+++ b/scripts/validate_transformers_migration.py@@@ -1,0 -1,231 +1,235 @@@+ #!/usr/bin/env python3+ """在指定 checkout 和真实模型上保存 Transformers 迁移的输出、耗时及内存证据。"""++ from __future__ import annotations++ import argparse+ import importlib.metadata+ import json+ import os+ import statistics+ import sys+ import threading+ import time+ from collections.abc import Callable+ from pathlib import Path+ from typing import Any+++ def _arguments() -> argparse.Namespace:+ """解析只读输入、模型目录和独立结果目录,不改变用户配置。"""+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--repo", type=Path, required=True)+ parser.add_argument("--manifest", type=Path, required=True)+ parser.add_argument("--output", type=Path, required=True)+ parser.add_argument("--model-root", type=Path, default=Path.home() / ".mineru/models")+ parser.add_argument("--kind", choices=("layout", "mfr", "parse", "vlm-transformers", "vlm-mlx"), required=True)+ parser.add_argument("--device", default="cpu")+ parser.add_argument("--tier", choices=("basic", "standard"), default="basic")+ parser.add_argument("--rounds", type=int, default=3)+ parser.add_argument("--warmup", type=int, default=1)+ args = parser.parse_args()+ if args.rounds < 1 or args.warmup < 0:+ parser.error("rounds must be positive and warmup must be nonnegative")+ return args+++ def _synchronize(device: str) -> None:+ """等待设备完成计算,避免把异步提交时间误认为推理耗时。"""+ import torch++ if device.startswith("cuda"):+ torch.cuda.synchronize(device)+ elif device.startswith("mps"):+ torch.mps.synchronize()+++ def _physical_footprint_reader() -> Callable[[], int] | None:+ """读取 macOS 统一内存的物理占用,避免 MPS 外部存储改变 RSS/框架计数口径。"""+ if sys.platform != "darwin":+ return None+ import ctypes++ class UsageInfo(ctypes.Structure):+ """对应系统 sys/resource.h 中稳定的 rusage_info_v0 结构。"""++ _fields_ = [("uuid", ctypes.c_uint8 * 16)] + [+ (name, ctypes.c_uint64)+ for name in (+ "user_time",+ "system_time",+ "pkg_idle_wkups",+ "interrupt_wkups",+ "pageins",+ "wired_size",+ "resident_size",+ "phys_footprint",+ "proc_start_abstime",+ "proc_exit_abstime",+ )+ ]++ function = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True).proc_pid_rusage+ function.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p]+ function.restype = ctypes.c_int+ pid = os.getpid()++ def read() -> int:+ """只读取当前进程统计,系统调用失败时使本次测量明确失败。"""+ info = UsageInfo()+ if function(pid, 0, ctypes.byref(info)) != 0:+ raise OSError(ctypes.get_errno(), "proc_pid_rusage failed")+ return info.phys_footprint++ return read+++ def _measure(operation: Callable[[], Any], device: str, rounds: int, warmup: int) -> tuple[Any, dict[str, Any]]:+ """预热后测量三轮,并每 20ms 采样进程 RSS 与 Torch 设备占用。"""+ import psutil+ import torch++ for _ in range(warmup):+ operation()+ _synchronize(device)+ process = psutil.Process()+ peaks = {"rss_bytes": 0, "torch_device_bytes": 0}+ footprint_reader = _physical_footprint_reader()+ if footprint_reader is not None:+ peaks["physical_footprint_bytes"] = 0+ sampling_errors = []+ stop = threading.Event()++ def sample() -> None:+ """持续采样同一进程,所有后端使用相同的采样间隔。"""+ while not stop.is_set():+ peaks["rss_bytes"] = max(peaks["rss_bytes"], process.memory_info().rss)+ if footprint_reader is not None:+ try:+ peaks["physical_footprint_bytes"] = max(peaks["physical_footprint_bytes"], footprint_reader())+ except OSError as exc:+ sampling_errors.append(exc)+ return+ allocated = 0+ if device.startswith("mps"):+ allocated = torch.mps.current_allocated_memory()+ elif device.startswith("cuda"):+ allocated = torch.cuda.memory_allocated(device)+ peaks["torch_device_bytes"] = max(peaks["torch_device_bytes"], allocated)+ stop.wait(0.02)++ sampler = threading.Thread(target=sample, daemon=True)+ sampler.start()+ durations = []+ result = None+ try:+ for _ in range(rounds):+ started = time.perf_counter()+ result = operation()+ _synchronize(device)+ durations.append(time.perf_counter() - started)+ finally:+ stop.set()+ sampler.join()+ if sampling_errors:+ raise sampling_errors[0]+ return result, {"seconds": durations, "median_seconds": statistics.median(durations), **peaks}+++ def _operation(args: argparse.Namespace, manifest: dict[str, Any]) -> Callable[[], Any]:+ """通过实际产品模型和解析器构造测量操作,保留默认生成与分批策略。"""+ from PIL import Image+ import torch++ model_root = args.model_root.expanduser()+ if args.kind in {"vlm-transformers", "vlm-mlx"}:+ from mineru.model.vlm.runtime import ModelSingleton+ from mineru_vl_utils.vlm_client.base_client import SamplingParams++ backend = "transformers" if args.kind == "vlm-transformers" else "mlx-engine"+ predictor = ModelSingleton().get_model(backend, str(model_root / "MinerU2.5-Pro-2605-1.2B"), None, batch_size=2)+ predictor.client.use_tqdm = False+ images = [Image.open(item["image"]).convert("RGB") for item in manifest["pages"][:2]]+ # 组件 smoke 只验证真实模型的协议链路;整页质量另由 parse 场景在原始分辨率验收。+ for image in images:+ image.thumbnail((448, 448))+ return lambda: predictor.client.batch_predict(+ images, "Read the text in this image.", SamplingParams(max_new_tokens=16, temperature=0)+ )+ if args.kind == "layout":+ from mineru.model.layout.pp_doclayoutv2 import PPDocLayoutV2LayoutModel++ model = PPDocLayoutV2LayoutModel(str(model_root / "PDF-Extract-Kit-1.0/models/Layout/PP-DocLayoutV2"), args.device)+ images = [Image.open(item["image"]).convert("RGB") for item in manifest["pages"]]+ return lambda: model.batch_predict(images, batch_size=2)+ if args.kind == "mfr":+ from mineru.model.mfr.unimernet.Unimernet import UnimernetModel++ model = UnimernetModel(str(model_root / "PDF-Extract-Kit-1.0/models/MFR/unimernet_hf_small_2503"), args.device)+ images = [Image.open(path).convert("RGB") for path in manifest["formulas"]]+ if not images:+ raise ValueError("The manifest must contain at least one formula crop")+ pixels = torch.stack([model.model.transform(image) for image in images]).to(device=args.device, dtype=model.model.dtype)++ def recognize() -> Any:+ """使用模型原有 generate 包装,输出 token、原文和修复后的公式文本。"""+ with torch.inference_mode():+ return model.model.generate({"image": pixels}, batch_size=len(images))++ return recognize++ from mineru.config import VlmConfig, config+ from mineru.parser.mineru_parser import MinerUParser++ config.model.stack = "full"+ config.llm_aided.features.title_leveling = False+ config.llm_aided.features.cross_page_table_cell_merge = False+ parser = MinerUParser(tier=args.tier, parse_mode="ocr", vlm_config=VlmConfig())++ def parse() -> list[dict[str, Any]]:+ """按清单页码解析真实 PDF,保存完整语义树与最终 Markdown。"""+ results = []+ for item in manifest["pages"]:+ result = parser.parse(item["pdf"], page_range=str(item["page"]))+ results.append({"name": item["name"], "markdown": result.markdown(), "middle": result.to_dict(skip_defaults=False)})+ return results++ return parse+++ def _json_value(value: Any) -> Any:+ """将模型返回的 NumPy/Torch 数据转换成可比较的 JSON。"""+ if hasattr(value, "tolist"):+ return value.tolist()+ raise TypeError(f"Unsupported JSON value: {type(value).__name__}")+++ def main() -> None:+ """保存版本、测量口径和推理结果,便于在两个版本之间复核。"""+ args = _arguments()+ sys.path.insert(0, str(args.repo.resolve()))+ os.environ["MINERU_DEVICE_MODE"] = args.device+ manifest = json.loads(args.manifest.read_text())+ operation = _operation(args, manifest)+ result, metrics = _measure(operation, args.device, args.rounds, args.warmup)+ versions = {name: importlib.metadata.version(name) for name in ("transformers", "torch", "tokenizers", "docvortex")}+ report = {+ "versions": versions,++ "source_paths": {++ name: sys.modules[name].__file__ for name in ("mineru", "docvortex", "mineru_vl_utils") if name in sys.modules++ },++ "python": sys.version,+ "kind": args.kind,+ "device": args.device,+ "sample_interval_ms": 20,+ "warmup_rounds": args.warmup,+ "metrics": metrics,+ "result": result,+ }+ args.output.parent.mkdir(parents=True, exist_ok=True)+ args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2, default=_json_value) + "\n")+ print(json.dumps({"output": str(args.output), **metrics}, ensure_ascii=False), flush=True)+++ if __name__ == "__main__":+ main()diff --cc tests/unittest/test_transformers5_runtime.pyindex 00000000,54a04f73..92174ecbmode 000000,100644..100644--- a/tests/unittest/test_transformers5_runtime.py+++ b/tests/unittest/test_transformers5_runtime.py@@@ -1,0 -1,76 +1,144 @@@+ """验证新推理栈的惰性加载边界及 LMDeploy Pipeline 生命周期。"""++ from __future__ import annotations++ import logging+ import subprocess+ import sys+ from pathlib import Path+ from types import ModuleType, SimpleNamespace+ from typing import Any++ import pytest+++ def test_public_imports_keep_optional_model_dependencies_lazy() -> None:+ """新进程中的公共入口导入不得尝试加载任何可选推理引擎。"""+ code = """+ import importlib.abc+ import sys++ class RejectModelImports(importlib.abc.MetaPathFinder):+ def find_spec(self, fullname, path=None, target=None):+ # 显式阻止重依赖,检查公共导入是否越过模型加载边界。+ if fullname.split('.')[0] in {'torch', 'transformers', 'vllm', 'lmdeploy', 'mlx', 'mlx_vlm'}:+ raise AssertionError('Eager optional dependency: ' + fullname)++ sys.meta_path.insert(0, RejectModelImports())+ import mineru+ import mineru.parser+ import mineru.cli.main+ """+ process = subprocess.run(+ [sys.executable, "-c", code], capture_output=True, text=True, cwd=Path(__file__).resolve().parents[2], check=False+ )+ assert process.returncode == 0, process.stderr+++ def test_runtime_constructs_caches_and_closes_pipeline(monkeypatch: pytest.MonkeyPatch) -> None:+ """模型门面须缓存公开 Pipeline 句柄,并通过其 close 方法释放资源。"""+ from mineru.model.vlm import runtime++ calls: list[dict[str, Any]] = []++ class Pipeline:+ """记录工厂参数和关闭次数的引擎替身。"""++ def __init__(self, model_path: str, **kwargs: Any) -> None:+ """捕获设备配置,不触发实际 GPU 分配。"""+ calls.append({"model_path": model_path, **kwargs})+ self.closed = 0++ def close(self) -> None:+ """模拟公开的资源释放接口。"""+ self.closed += 1++ module = ModuleType("lmdeploy")+ module.PytorchEngineConfig = SimpleNamespace+ module.TurbomindEngineConfig = SimpleNamespace+ module.pipeline = Pipeline+ utilities = ModuleType("lmdeploy.utils")+ utilities.get_logger = logging.getLogger+ monkeypatch.setitem(sys.modules, "lmdeploy", module)+ monkeypatch.setitem(sys.modules, "lmdeploy.utils", utilities)+ monkeypatch.setattr(runtime, "MinerUClient", lambda **kwargs: SimpleNamespace(**kwargs))+ monkeypatch.setattr(runtime.ModelSingleton, "_models", {})+ singleton = runtime.ModelSingleton()+ kwargs = {"lmdeploy_backend": "pytorch", "lmdeploy_device": "cuda", "cache_max_entry_count": 0.2}+ client = singleton.get_model("lmdeploy-engine", "local-model", None, **kwargs)+ assert singleton.get_model("lmdeploy-engine", "local-model", None, **kwargs) is client+ assert len(calls) == 1+ assert calls[0]["backend_config"].device_type == "cuda"+ assert calls[0]["backend_config"].cache_max_entry_count == 0.2+ assert calls[0]["log_level"] == "ERROR"+ handle = client._mineru_runtime_handles["lmdeploy_engine"]+ runtime._shutdown_runtime_handle(handle)+ assert handle.closed == 1++++++def test_cancelled_mlx_guard_does_not_leave_a_background_lock_owner() -> None:++ """等待模型锁时取消请求,不得有后台线程随后取得锁并永远阻塞后续请求。"""++ import asyncio++ import threading++++ from mineru.model.vlm.runtime import aio_predictor_execution_guard++++ lock = threading.Lock()++ predictor = SimpleNamespace(_mineru_execution_lock=lock)++++ async def guarded() -> None:++ """模拟进入共享 MLX 模型的推理范围。"""++ async with aio_predictor_execution_guard(predictor):++ await asyncio.sleep(0)++++ async def run() -> None:++ """先占用锁,取消等待者,再验证下一请求仍可正常进入。"""++ lock.acquire()++ task = asyncio.create_task(guarded())++ await asyncio.sleep(0.03)++ task.cancel()++ try:++ await task++ except asyncio.CancelledError:++ pass++ finally:++ lock.release()++ await asyncio.wait_for(guarded(), timeout=1)++ assert not lock.locked()++++ asyncio.run(run())++++++def test_cancelled_model_initialization_waits_before_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:++ """取消加载不能让后台线程在调用方清理之后重新写回模型缓存。"""++ import asyncio++ import threading++++ from mineru.model.vlm import runtime++++ entered, release, finished = threading.Event(), threading.Event(), threading.Event()++++ def load(*args: object, **kwargs: object) -> object:++ """模拟无法被 Future.cancel 中断的模型加载。"""++ entered.set()++ assert release.wait(5)++ finished.set()++ return SimpleNamespace()++++ monkeypatch.setattr(runtime.ModelSingleton, "get_model", load)++++ async def run() -> None:++ """等待加载开始后取消,确认协程只有在模型加载结束后才退出。"""++ task = asyncio.create_task(runtime._get_model_async("mlx-engine", "local", None))++ try:++ assert await asyncio.to_thread(entered.wait, 5)++ task.cancel()++ await asyncio.sleep(0)++ assert not task.done()++ finally:++ release.set()++ with pytest.raises(asyncio.CancelledError):++ await task++ assert finished.is_set()++++ asyncio.run(run())