Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
test(pdf): validate exact native engine wheels across host platforms
myhlolie149025536d2
diff --git a/.github/workflows/docvortex-boundary.yml b/.github/workflows/docvortex-boundary.ymlindex f8659e34..c281ee5f 100644--- a/.github/workflows/docvortex-boundary.yml+++ b/.github/workflows/docvortex-boundary.yml@@ -4,8 +4,12 @@ on:workflow_dispatch:inputs:docvortex-wheel-url:- description: DocVortex 0.4.x wheel URL for joint validation before publication- required: true+ description: Native ABI3 DocVortex wheel URL; mutually exclusive with docvortex-ref+ required: false+ type: string+ docvortex-ref:+ description: SHA or ref of myhloli/DocVortex to build; mutually exclusive with wheel URL+ required: falsetype: stringutils-wheel-url:description: Optional mineru-vl-utils 2.0.1+ wheel URL for unpublished dependency validation@@ -15,6 +19,11 @@ on:permissions:contents: read+env:+ PYTHONUTF8: '1'+ DOCVORTEX_COMPUTE_BACKEND: rust+ DOCVORTEX_PDF_RENDER_BACKEND: legacy+jobs:integration:strategy:@@ -32,21 +41,147 @@ jobs:with:python-version: ${{ matrix.python }}- uses: astral-sh/setup-uv@v7- - name: Install matching engine wheel and host+ - name: Validate mutually exclusive engine inputsenv:DOCVORTEX_WHEEL_URL: ${{ inputs.docvortex-wheel-url }}+ DOCVORTEX_REF: ${{ inputs.docvortex-ref }}+ run: |+ python - <<'PY'+ import os+ # 必须指定一种引擎来源,不能在输入缺失时回退到索引版本。+ wheel = os.environ["DOCVORTEX_WHEEL_URL"].strip()+ ref = os.environ["DOCVORTEX_REF"].strip()+ if bool(wheel) == bool(ref):+ raise SystemExit("Provide exactly one of docvortex-wheel-url or docvortex-ref")+ if ref and (ref != os.environ["DOCVORTEX_REF"] or any(char.isspace() for char in ref)):+ raise SystemExit("docvortex-ref must be a SHA or Git ref without whitespace")+ PY+ - name: Checkout selected engine revision+ if: ${{ inputs.docvortex-ref != '' }}+ uses: actions/checkout@v6+ with:+ repository: myhloli/DocVortex+ ref: ${{ inputs.docvortex-ref }}+ path: .boundary/docvortex-src+ persist-credentials: false+ - name: Read the selected revision's Rust toolchain+ if: ${{ inputs.docvortex-ref != '' }}+ id: rust-version+ run: |+ python - <<'PY'+ import os+ from pathlib import Path+ import re+ # 使用引擎所声明的工具链;Python 3.10 无需为读取单一字段额外安装 TOML 包。+ text = Path(".boundary/docvortex-src/rust-toolchain.toml").read_text(encoding="utf-8")+ match = re.search(r'^\s*channel\s*=\s*["\']([A-Za-z0-9][A-Za-z0-9.+-]*)["\']', text, re.MULTILINE)+ if match is None:+ raise SystemExit("Selected DocVortex revision must declare a Rust toolchain channel")+ with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:+ output.write(f"toolchain={match.group(1)}\n")+ PY+ - name: Install matching Rust toolchain+ if: ${{ inputs.docvortex-ref != '' }}+ uses: dtolnay/rust-toolchain@stable+ with:+ toolchain: ${{ steps.rust-version.outputs.toolchain }}+ - name: Build the selected native wheel+ if: ${{ inputs.docvortex-ref != '' }}+ env:+ DOCVORTEX_BUILD_NATIVE: '1'+ run: |+ uv build --wheel --python "$(python -c 'import sys; print(sys.executable)')" \+ --out-dir "$GITHUB_WORKSPACE/.boundary/wheelhouse" .boundary/docvortex-src+ - name: Download the supplied engine wheel+ if: ${{ inputs.docvortex-wheel-url != '' }}+ env:+ DOCVORTEX_WHEEL_URL: ${{ inputs.docvortex-wheel-url }}+ run: |+ python -m pip download --no-deps --only-binary=:all: \+ --dest .boundary/wheelhouse "$DOCVORTEX_WHEEL_URL"+ - name: Freeze the exact native artifact+ env:+ DOCVORTEX_REF: ${{ inputs.docvortex-ref }}+ run: |+ python - <<'PY'+ import hashlib+ import json+ import os+ from pathlib import Path+ import subprocess+ from zipfile import ZipFile+ # URL 与源码构建统一为一个本地 wheel,安装前拒绝纯 Python 或非 ABI3 产物。+ wheels = list(Path(".boundary/wheelhouse").glob("*.whl"))+ assert len(wheels) == 1, wheels+ wheel = wheels[0].resolve()+ with ZipFile(wheel) as archive:+ metadata = [name for name in archive.namelist() if name.endswith(".dist-info/WHEEL")]+ assert len(metadata) == 1, metadata+ tags = [line for line in archive.read(metadata[0]).decode().splitlines() if line.startswith("Tag:")]+ assert tags and all("-abi3-" in tag for tag in tags), tags+ assert any(name.startswith("docvortex/_native") and name.endswith((".so", ".pyd")) for name in archive.namelist())+ ref = os.environ["DOCVORTEX_REF"]+ record = {+ "source": "ref" if ref else "wheel-url",+ "requested_ref": ref or None,+ "resolved_commit": subprocess.check_output(["git", "-C", ".boundary/docvortex-src", "rev-parse", "HEAD"], text=True).strip() if ref else None,+ "host_commit": os.environ["GITHUB_SHA"],+ "wheel": str(wheel),+ "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(),+ "wheel_tags": tags,+ }+ Path(".boundary/engine-artifact.json").write_text(json.dumps(record, indent=2), encoding="utf-8")+ with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as output:+ output.write(f"DOCVORTEX_WHEEL={wheel}\n")+ PY+ - name: Install matching engine wheel and host+ env:UTILS_WHEEL_URL: ${{ inputs.utils-wheel-url }}run: |- uv venv- uv pip install "$DOCVORTEX_WHEEL_URL"- if [[ -n "$UTILS_WHEEL_URL" ]]; then uv pip install "$UTILS_WHEEL_URL"; fi- uv pip install '.[test]'+ uv venv --python "$(python -c 'import sys; print(sys.executable)')"+ packages=("$DOCVORTEX_WHEEL" '.[test]' psutil)+ if [[ -n "$UTILS_WHEEL_URL" ]]; then packages+=("$UTILS_WHEEL_URL"); fi+ # 单次求解固定引擎产物,避免安装宿主时换成索引中的另一个 DocVortex。+ uv pip install "${packages[@]}"uv pip check+ - name: Verify installed native ABI3 provenance and protocol+ run: |+ if [[ -f .venv/Scripts/activate ]]; then source .venv/Scripts/activate; else source .venv/bin/activate; fi+ python -I - <<'PY'+ import hashlib+ import importlib.metadata+ import json+ from pathlib import Path+ import sysconfig+ import docvortex+ import docvortex._native as native+ import mineru+ from docvortex._compute_backend import backend_info+ # 隔离导入必须来自已安装 wheel 的 site-packages,而非两个源码 checkout。+ site_roots = {Path(sysconfig.get_paths()[name]).resolve() for name in ("purelib", "platlib")}+ origins = {"docvortex": Path(docvortex.__file__).resolve(), "native": Path(native.__file__).resolve(), "mineru": Path(mineru.__file__).resolve()}+ for name, path in origins.items():+ assert any(path.is_relative_to(root) for root in site_roots), (name, path, site_roots)+ info = backend_info()+ assert info["backend"] == "rust", info+ assert native.PROTOCOL_VERSION >= 11 and info["protocol"] == native.PROTOCOL_VERSION, info+ assert info["extension_sha256"] == hashlib.sha256(origins["native"].read_bytes()).hexdigest(), info+ artifact = json.loads(Path(".boundary/engine-artifact.json").read_text(encoding="utf-8"))+ distribution = importlib.metadata.distribution("docvortex")+ provenance = json.loads(distribution.read_text("direct_url.json") or "{}")+ archive = provenance.get("archive_info", {})+ installed_hash = archive.get("hashes", {}).get("sha256") or archive.get("hash", "").removeprefix("sha256=")+ assert installed_hash == artifact["wheel_sha256"], (provenance, artifact)+ wheel_metadata = distribution.read_text("WHEEL") or ""+ assert any(line.startswith("Tag:") and "-abi3-" in line for line in wheel_metadata.splitlines())+ record = {"versions": {name: importlib.metadata.version(name) for name in ("docvortex", "mineru")}, "origins": {name: str(path) for name, path in origins.items()}, "compute": info, "artifact": artifact}+ Path(".boundary/installed-contract.json").write_text(json.dumps(record, indent=2), encoding="utf-8")+ print(json.dumps(record, indent=2))+ PY- name: Verify public contracts and deterministic inference replayrun: |if [[ -f .venv/Scripts/activate ]]; then source .venv/Scripts/activate; else source .venv/bin/activate; fi- python -I -c "import docvortex; print(docvortex.__file__); from docvortex.version import __version__; assert __version__.startswith('0.4.')"- python -m pytest -q -o addopts='' \+ python -I -m pytest -q -o addopts='' --import-mode=importlib \tests/unittest/test_docvortex_import_boundary.py \tests/unittest/test_backend_architecture.py \tests/unittest/test_docvortex_protocol.py \@@ -55,4 +190,19 @@ jobs:tests/unittest/test_pdf_analyze_block_content.py \tests/unittest/test_native_pdf_table_pipeline.py \tests/unittest/test_pdf_analyze_visual_blocks.py \- tests/unittest/test_pdf_model_inputs.py+ tests/unittest/test_pdf_model_inputs.py \+ tests/unittest/test_pdf_cpu_reuse.py \+ tests/unittest/test_pdf_memory_lifecycle.py \+ tests/unittest/test_pdf_render_session_integration.py \+ tests/unittest/test_flash_pdf_render_scheduling.py \+ tests/unittest/test_process_heap_memory.py+ - name: Retain exact wheel and installation evidence+ if: ${{ always() }}+ uses: actions/upload-artifact@v6+ with:+ name: docvortex-boundary-${{ matrix.os }}-py${{ matrix.python }}+ path: |+ .boundary/wheelhouse/*.whl+ .boundary/engine-artifact.json+ .boundary/installed-contract.json+ if-no-files-found: warndiff --git a/tests/unittest/test_flash_pdf_render_scheduling.py b/tests/unittest/test_flash_pdf_render_scheduling.pyindex 4c1dceee..e9329cce 100644--- a/tests/unittest/test_flash_pdf_render_scheduling.py+++ b/tests/unittest/test_flash_pdf_render_scheduling.py@@ -76,12 +76,28 @@ def test_flash_sparse_render_keeps_page_mapping_and_title_gate(result.append({"img_pil": image})return result+ def raster_crops(+ data: bytes,+ prepared_pages: list[list[tuple[int, dict[str, Any]]]],+ **options: Any,+ ) -> list[list[tuple[int, str | None]]]:+ """在当前裁图服务边界替换跨进程传输,保留真实裁剪编码与图片释放。"""+ from docvortex.document.pdf.visuals import _attach_prepared_visual_block_images++ page_images = raster(data, **options)+ try:+ assert len(prepared_pages) == len(page_images)+ _attach_prepared_visual_block_images(prepared_pages, page_images, options["start_page_id"])+ return [[(index, block.get("image_base64")) for index, block in page] for page in prepared_pages]+ finally:+ images._close_image_dicts(page_images)+monkeypatch.setenv("MINERU_PROCESSING_WINDOW_SIZE", "2")monkeypatch.setenv("MINERU_PDF_RENDER_TIMEOUT", "19")monkeypatch.setenv("MINERU_PDF_RENDER_THREADS", "2")monkeypatch.setenv("DOCVORTEX_PDF_RENDER_TIMEOUT", "31")monkeypatch.setattr(PdfModel, "predict", predict)- monkeypatch.setattr(images, "load_images_from_pdf_bytes_range", raster)+ monkeypatch.setattr(images, "_load_visual_crops_from_pdf_bytes_range", raster_crops)forbidden = MagicMock(side_effect=AssertionError("Flash TXT must skip inference windows"))monkeypatch.setattr(window, "_get_window_pdf_pages", forbidden)title = AsyncMock()@@ -126,6 +142,7 @@ def test_flash_text_only_does_not_render(monkeypatch: pytest.MonkeyPatch, tmp_pamonkeypatch.setattr(PDFDocument, "classify", classify)forbidden = MagicMock(side_effect=AssertionError("Text-only PDF must not rasterize"))monkeypatch.setattr(images, "load_images_from_pdf_bytes_range", forbidden)+ monkeypatch.setattr(images, "_load_visual_crops_from_pdf_bytes_range", forbidden)result = parse(_source(tmp_path / "text.pdf"), tier="flash", ocr_mode=parse_mode, image_analysis=False)assert len(result.pages) == 5assert classify.call_count == int(parse_mode == "auto")