Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
feat: remove deprecated alpha release workflow and related scripts
myhloli32ff57435f58
diff --git a/.github/scripts/alpha_release.py b/.github/scripts/alpha_release.pydeleted file mode 100644index 335eadf5..00000000--- a/.github/scripts/alpha_release.py+++ /dev/null@@ -1,121 +0,0 @@-from __future__ import annotations--import argparse-import ast-import re-import zipfile-from dataclasses import dataclass-from email.parser import Parser-from pathlib import Path---ALPHA_TAG_PATTERN = re.compile(r"^v(?P<version>(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)a[1-9]\d*)$")---@dataclass(frozen=True)-class WheelMetadata:- name: str- version: str---def parse_alpha_tag(tag: str) -> str:- match = ALPHA_TAG_PATTERN.fullmatch(tag)- if match is None:- raise ValueError(f"Invalid Alpha release tag {tag!r}; expected canonical format vX.Y.ZaN, for example v4.0.0a1.")- return match.group("version")---def read_declared_version(version_file: str | Path) -> str:- path = Path(version_file)- module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))- declared_versions: list[str] = []-- for statement in module.body:- if not isinstance(statement, ast.Assign) or len(statement.targets) != 1:- continue- target = statement.targets[0]- if not isinstance(target, ast.Name) or target.id != "__version__":- continue- if not isinstance(statement.value, ast.Constant) or not isinstance(statement.value.value, str):- raise ValueError(f"{path} must assign __version__ to a string literal.")- declared_versions.append(statement.value.value)-- if len(declared_versions) != 1:- raise ValueError(f"{path} must contain exactly one top-level __version__ assignment.")- return declared_versions[0]---def validate_source(tag: str, version_file: str | Path) -> str:- tag_version = parse_alpha_tag(tag)- declared_version = read_declared_version(version_file)- if declared_version != tag_version:- raise ValueError(- f"Release tag {tag!r} declares version {tag_version!r}, but {version_file} declares {declared_version!r}."- )- return tag_version---def read_wheel_metadata(wheel: str | Path) -> WheelMetadata:- path = Path(wheel)- with zipfile.ZipFile(path) as archive:- metadata_files = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")]- if len(metadata_files) != 1:- raise ValueError(f"{path} must contain exactly one .dist-info/METADATA file.")- metadata_text = archive.read(metadata_files[0]).decode("utf-8")-- metadata = Parser().parsestr(metadata_text)- name = metadata.get("Name")- version = metadata.get("Version")- if not name or not version:- raise ValueError(f"{path} metadata must contain Name and Version fields.")- return WheelMetadata(name=name, version=version)---def validate_wheel(wheel: str | Path, *, expected_name: str, expected_version: str) -> WheelMetadata:- metadata = read_wheel_metadata(wheel)- if metadata.name != expected_name:- raise ValueError(f"Wheel project name is {metadata.name!r}; expected {expected_name!r}.")- if metadata.version != expected_version:- raise ValueError(f"Wheel version is {metadata.version!r}; expected {expected_version!r}.")- return metadata---def _build_parser() -> argparse.ArgumentParser:- parser = argparse.ArgumentParser(description="Validate MinerU Alpha release source and wheel metadata.")- subparsers = parser.add_subparsers(dest="command", required=True)-- source_parser = subparsers.add_parser("validate-source", help="Validate the release tag against mineru/version.py.")- source_parser.add_argument("--tag", required=True)- source_parser.add_argument("--version-file", required=True)-- wheel_parser = subparsers.add_parser("validate-wheel", help="Validate the built wheel name and version.")- wheel_parser.add_argument("--wheel", required=True)- wheel_parser.add_argument("--expected-name", default="mineru")- wheel_parser.add_argument("--expected-version", required=True)-- return parser---def main() -> int:- parser = _build_parser()- args = parser.parse_args()-- if args.command == "validate-source":- print(validate_source(args.tag, args.version_file))- return 0-- if args.command == "validate-wheel":- metadata = validate_wheel(- args.wheel,- expected_name=args.expected_name,- expected_version=args.expected_version,- )- print(f"Validated {metadata.name} {metadata.version}.")- return 0-- parser.error(f"Unsupported command: {args.command}")- return 2---if __name__ == "__main__":- raise SystemExit(main())diff --git a/.github/workflows/alpha-release.yml b/.github/workflows/alpha-release.ymldeleted file mode 100644index 8aad033a..00000000--- a/.github/workflows/alpha-release.yml+++ /dev/null@@ -1,148 +0,0 @@-name: MinerU Alpha release--on:- push:- tags:- - "v*.*.*a*"--permissions:- contents: read--concurrency:- group: mineru-alpha-release-${{ github.ref }}- cancel-in-progress: false--jobs:- build:- name: Validate and build Alpha wheel- runs-on: ubuntu-latest- outputs:- version: ${{ steps.source.outputs.version }}- steps:- - name: Checkout tagged source- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6- with:- ref: ${{ github.ref }}- fetch-depth: 0- persist-credentials: false-- - name: Set up Python- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6- with:- python-version: "3.12"-- - name: Verify annotated tag and next branch ancestry- run: |- if [[ "$(git cat-file -t "$GITHUB_REF")" != "tag" ]]; then- echo "Alpha releases require an annotated Git tag." >&2- exit 1- fi-- git fetch --no-tags origin next:refs/remotes/origin/next- if ! git merge-base --is-ancestor HEAD refs/remotes/origin/next; then- echo "Alpha release tag must point to a commit contained in origin/next." >&2- exit 1- fi-- - name: Validate Alpha tag and source version- id: source- run: |- version="$(python .github/scripts/alpha_release.py validate-source \- --tag "$GITHUB_REF_NAME" \- --version-file mineru/version.py)"- echo "version=$version" >> "$GITHUB_OUTPUT"-- - name: Install build tooling- run: python -m pip install --upgrade build twine-- - name: Build and check wheel- run: |- python -m build --wheel- python -m twine check --strict dist/*-- wheels=(dist/*.whl)- if [[ "${#wheels[@]}" -ne 1 ]]; then- echo "Expected exactly one wheel, found ${#wheels[@]}." >&2- exit 1- fi- python .github/scripts/alpha_release.py validate-wheel \- --wheel "${wheels[0]}" \- --expected-name mineru \- --expected-version "${{ steps.source.outputs.version }}"-- - name: Smoke-test wheel installation- env:- EXPECTED_VERSION: ${{ steps.source.outputs.version }}- run: |- python -m venv "$RUNNER_TEMP/mineru-alpha-smoke"- "$RUNNER_TEMP/mineru-alpha-smoke/bin/python" -m pip install --no-deps dist/*.whl- "$RUNNER_TEMP/mineru-alpha-smoke/bin/python" - <<'PY'- import os- from importlib.metadata import version-- installed_version = version("mineru")- expected_version = os.environ["EXPECTED_VERSION"]- if installed_version != expected_version:- raise SystemExit(f"Installed mineru version {installed_version!r} != {expected_version!r}")- PY-- - name: Upload validated wheel- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6- with:- name: mineru-${{ steps.source.outputs.version }}- path: dist/*.whl- if-no-files-found: error- retention-days: 30-- publish-pypi:- name: Publish Alpha wheel to PyPI- needs: build- runs-on: ubuntu-latest- environment:- name: pypi- url: https://pypi.org/project/mineru/${{ needs.build.outputs.version }}/- steps:- - name: Set up Python- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6- with:- python-version: "3.12"-- - name: Download validated wheel- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7- with:- name: mineru-${{ needs.build.outputs.version }}- path: dist-- - name: Publish wheel to PyPI- env:- TWINE_USERNAME: __token__- TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}- run: |- python -m pip install --upgrade twine "packaging>=24.2"- python -m twine check --strict dist/*- python -m twine upload dist/*-- github-release:- name: Create GitHub pre-release- needs: [build, publish-pypi]- runs-on: ubuntu-latest- permissions:- contents: write- steps:- - name: Download published wheel- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7- with:- name: mineru-${{ needs.build.outputs.version }}- path: dist-- - name: Create GitHub pre-release- env:- GH_TOKEN: ${{ github.token }}- VERSION: ${{ needs.build.outputs.version }}- run: |- gh release create "$GITHUB_REF_NAME" dist/*.whl \- --repo "$GITHUB_REPOSITORY" \- --verify-tag \- --prerelease \- --title "MinerU $VERSION" \- --generate-notesdiff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.ymlindex 654af47b..eaff3e01 100644--- a/.github/workflows/mkdocs.yml+++ b/.github/workflows/mkdocs.yml@@ -18,7 +18,7 @@ jobs:- name: Install script test toolsrun: sudo apt-get update && sudo apt-get install -y --no-install-recommends jq- name: Set up Python- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: "3.12"- name: Install project and docs dependencies@@ -48,7 +48,7 @@ jobs:- name: Checkoutuses: actions/checkout@v6- name: Set up Python- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: "3.12"- name: Install docs dependenciesdiff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.ymlindex 576af9e2..62e9bfad 100644--- a/.github/workflows/python-package.yml+++ b/.github/workflows/python-package.yml@@ -9,6 +9,9 @@ on:- '*released'workflow_dispatch:+permissions:+ contents: read+jobs:@@ -28,6 +31,7 @@ jobs:- name: Update version.pyrun: |+ python -m pip install packagingpython update_version.py- name: Verify version.py@@ -36,19 +40,21 @@ jobs:cat mineru/version.py- name: Commit changes+ id: commit_changesrun: |- git config --local user.email "[email protected]"- git config --local user.name "myhloli"+ git config --local user.name "github-actions[bot]"+ git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"git add mineru/version.pyif git diff-index --quiet HEAD; thenecho "No changes to commit"+ echo "committed=false" >> "$GITHUB_OUTPUT"elsegit commit -m "Update version.py with new version"+ echo "committed=true" >> "$GITHUB_OUTPUT"fi- id: commit_changes- name: Push changes- if: steps.commit_changes.outcome == 'success'+ if: steps.commit_changes.outputs.committed == 'true'env:GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}run: |@@ -100,15 +106,19 @@ jobs:ref: masterfetch-depth: 0- - name: Install wheel+ - name: Install build toolsrun: |python -m pip install wheel- pip install build+ pip install build twine- name: Build wheelrun: |python -m build --wheel+ - name: Check wheel metadata+ run: |+ python -m twine check --strict dist/*.whl+- name: Upload artifactuses: actions/upload-artifact@v6with:@@ -116,13 +126,16 @@ jobs:path: dist/*.whlretention-days: 30- release:+ publish:needs: [ build ]+ if: startsWith(github.ref, 'refs/tags/')runs-on: ubuntu-latest+ environment:+ name: pypi+ url: https://pypi.org/project/mineru/+ permissions:+ id-token: writesteps:- - name: Checkout code- uses: actions/checkout@v6-- name: Download artifactuses: actions/download-artifact@v7with:@@ -138,7 +151,4 @@ jobs:GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}- name: Publish distribution to PyPI- run: |- pip install -U twine id keyring packaging readme-renderer requests requests-toolbelt rfc3986 rich urllib3- twine check dist/*- twine upload dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }}+ uses: pypa/gh-action-pypi-publish@release/v1diff --git a/tests/unittest/test_alpha_release.py b/tests/unittest/test_alpha_release.pydeleted file mode 100644index 80b24c75..00000000--- a/tests/unittest/test_alpha_release.py+++ /dev/null@@ -1,114 +0,0 @@-from __future__ import annotations--import importlib.util-import sys-import zipfile-from pathlib import Path-from types import ModuleType--import pytest---def _load_release_module() -> ModuleType:- module_path = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "alpha_release.py"- spec = importlib.util.spec_from_file_location("alpha_release", module_path)- assert spec is not None- assert spec.loader is not None- module = importlib.util.module_from_spec(spec)- sys.modules[spec.name] = module- spec.loader.exec_module(module)- return module---def _write_wheel(path: Path, *, name: str = "mineru", version: str = "4.0.0a1") -> None:- metadata = f"Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n"- with zipfile.ZipFile(path, mode="w") as archive:- archive.writestr(f"{name}-{version}.dist-info/METADATA", metadata)--[email protected]("tag", ["v4.0.0a1", "v4.0.0a12", "v10.20.30a4"])-def test_parse_alpha_tag_accepts_canonical_tags(tag: str) -> None:- module = _load_release_module()-- assert module.parse_alpha_tag(tag) == tag.removeprefix("v")--- "tag",- [- "4.0.0a1",- "v4.0.0",- "v4.0.0a0",- "v4.0.0b1",- "v4.0a1",- "v04.0.0a1",- "v4.0.0-alpha.1",- "v4.0.0a1-extra",- ],-)-def test_parse_alpha_tag_rejects_noncanonical_tags(tag: str) -> None:- module = _load_release_module()-- with pytest.raises(ValueError, match="expected canonical format"):- module.parse_alpha_tag(tag)---def test_read_declared_version_uses_static_top_level_assignment(tmp_path: Path) -> None:- module = _load_release_module()- version_file = tmp_path / "version.py"- version_file.write_text('__version__ = "4.0.0a1"\n', encoding="utf-8")-- assert module.read_declared_version(version_file) == "4.0.0a1"--- "contents",- [- "OTHER = '4.0.0a1'\n",- "__version__ = get_version()\n",- "__version__ = '4.0.0a1'\n__version__ = '4.0.0a2'\n",- ],-)-def test_read_declared_version_rejects_ambiguous_files(tmp_path: Path, contents: str) -> None:- module = _load_release_module()- version_file = tmp_path / "version.py"- version_file.write_text(contents, encoding="utf-8")-- with pytest.raises(ValueError, match="__version__"):- module.read_declared_version(version_file)---def test_validate_source_requires_tag_and_file_versions_to_match(tmp_path: Path) -> None:- module = _load_release_module()- version_file = tmp_path / "version.py"- version_file.write_text('__version__ = "4.0.0a2"\n', encoding="utf-8")-- with pytest.raises(ValueError, match="4.0.0a1.*4.0.0a2"):- module.validate_source("v4.0.0a1", version_file)---def test_validate_wheel_accepts_expected_metadata(tmp_path: Path) -> None:- module = _load_release_module()- wheel = tmp_path / "mineru-4.0.0a1-py3-none-any.whl"- _write_wheel(wheel)-- metadata = module.validate_wheel(wheel, expected_name="mineru", expected_version="4.0.0a1")-- assert metadata.name == "mineru"- assert metadata.version == "4.0.0a1"--- ("name", "version", "message"),- [- ("mineru-alpha", "4.0.0a1", "project name"),- ("mineru", "4.0.0a2", "Wheel version"),- ],-)-def test_validate_wheel_rejects_unexpected_metadata(tmp_path: Path, name: str, version: str, message: str) -> None:- module = _load_release_module()- wheel = tmp_path / "release.whl"- _write_wheel(wheel, name=name, version=version)-- with pytest.raises(ValueError, match=message):- module.validate_wheel(wheel, expected_name="mineru", expected_version="4.0.0a1")diff --git a/update_version.py b/update_version.pyindex eb8be946..b8e678a6 100644--- a/update_version.py+++ b/update_version.py@@ -1,28 +1,50 @@# Copyright (c) Opendatalab. All rights reserved.import os-import subprocess+import re+import sys+from pathlib import Path+from packaging.version import Version-def get_version():- command = ["git", "describe", "--tags"]+VERSION_FILE = Path(__file__).parent / "mineru" / "version.py"+TAG_PATTERN = re.compile(r"^mineru-(.+)-released$")+VERSION_PATTERN = re.compile(r'^__version__ = "(.+)"$', re.MULTILINE)+++def get_tag_version() -> str | None:+ """Return the version carried by the tag that triggered the workflow, or None when not triggered by a release tag."""+ ref_name = os.environ.get("GITHUB_REF_NAME", "").strip()+ match = TAG_PATTERN.match(ref_name)+ if match:+ return match.group(1)+ if os.environ.get("GITHUB_REF_TYPE") == "tag":+ raise ValueError(f"Invalid release tag {ref_name!r}. Expected format is mineru-<version>-released.")+ return None+++def get_file_version() -> str:+ match = VERSION_PATTERN.search(VERSION_FILE.read_text())+ if match is None:+ raise ValueError(f"No __version__ assignment found in {VERSION_FILE}.")+ return match.group(1)+++def main() -> None:+ tag_version = get_tag_version()+ if tag_version is None:+ print("Not triggered by a release tag; leaving mineru/version.py unchanged.")+ return+ file_version = get_file_version()+ if Version(file_version) >= Version(tag_version):+ print(f"mineru/version.py ({file_version}) is not lower than the tag ({tag_version}); leaving it unchanged.")+ return+ VERSION_FILE.write_text(f'__version__ = "{tag_version}"\n')+ print(f"Updated mineru/version.py: {file_version} -> {tag_version}")+++if __name__ == "__main__":try:- version = subprocess.check_output(command).decode().strip()- version_parts = version.split("-")- if len(version_parts) > 1 and version_parts[0].startswith("mineru"):- return version_parts[1]- else:- raise ValueError(f"Invalid version tag {version}. Expected format is mineru-<version>-released.")- except Exception as e:- print(e)- return "0.0.0"---def write_version_to_commons(version):- commons_path = os.path.join(os.path.dirname(__file__), 'mineru', 'version.py')- with open(commons_path, 'w') as f:- f.write(f'__version__ = "{version}"\n')---if __name__ == '__main__':- version_name = get_version()- write_version_to_commons(version_name)+ main()+ except ValueError as e:+ print(f"error: {e}", file=sys.stderr)+ raise SystemExit(1)