A fast, helpful, and open-source document parser
cleaner fix
Logan Markewich6dcdfde7fd18
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.ymlindex e9e3c99..55b26bf 100644--- a/.github/workflows/release-python.yml+++ b/.github/workflows/release-python.yml@@ -106,6 +106,16 @@ jobs:targetkey: ${{ runner.os }}-${{ matrix.target }}-cargo-python-${{ hashFiles('**/Cargo.lock') }}+ - name: Stage pdfium for wheel inclusion+ if: ${{ !matrix.musl }}+ shell: bash+ # Pre-download pdfium and place it at packages/python/liteparse/<dylib>+ # so the `[tool.maturin] include` rule packs it into the wheel during+ # the build below. This is what gets the dylib listed in the wheel's+ # RECORD manifest — appending it after the build leaves RECORD out of+ # sync, which Poetry warns about.+ run: bash packages/python/scripts/download-pdfium.sh ${{ matrix.target }}+- name: Build wheels (glibc)if: ${{ !matrix.musl }}uses: PyO3/maturin-action@v1@@ -143,54 +153,6 @@ jobs:# read/upload the produced wheel.sudo chmod -R a+rw packages/python/dist/ || true- - name: Copy pdfium dylib into wheel (Unix)- if: runner.os != 'Windows'- working-directory: packages/python- run: |- # Fix permissions from Docker/maturin build- sudo chmod -R a+rw dist/ || true- # Find pdfium in target/ (mounted from Docker) or cache- PDFIUM_LIB=$(find ../../target -name "libpdfium.so" -type f 2>/dev/null | head -1 || true)- if [ -z "$PDFIUM_LIB" ]; then- PDFIUM_LIB=$(find ~/.cache/pdfium-rs -name "libpdfium.so" -type f 2>/dev/null | head -1 || true)- fi- if [ -n "$PDFIUM_LIB" ]; then- export PDFIUM_LIB_PATH="$(dirname "$PDFIUM_LIB")"- fi- bash scripts/copy-pdfium.sh- if [ "$(uname -s)" = "Darwin" ]; then- DYLIB="libpdfium.dylib"- else- DYLIB="libpdfium.so"- fi- for whl in dist/*.whl; do- python scripts/inject_dylib.py "$whl" "liteparse/$DYLIB" "liteparse/$DYLIB"- done-- - name: Copy pdfium dylib into wheel (Windows)- if: runner.os == 'Windows'- working-directory: packages/python- shell: pwsh- run: |- $cacheBase = "$env:LOCALAPPDATA\pdfium-rs"- if (-not (Test-Path $cacheBase)) {- $cacheBase = "$env:USERPROFILE\.cache\pdfium-rs"- }- $dll = Get-ChildItem -Path $cacheBase -Recurse -Filter "pdfium.dll" | Select-Object -First 1- if (-not $dll) {- $dll = Get-ChildItem -Path "..\..\target" -Recurse -Filter "pdfium.dll" | Select-Object -First 1- }- if ($dll) {- Copy-Item $dll.FullName -Destination "liteparse\pdfium.dll"- Write-Host "Copied $($dll.FullName) -> liteparse\pdfium.dll"- foreach ($whl in Get-ChildItem dist\*.whl) {- python scripts/inject_dylib.py $whl.FullName "liteparse/pdfium.dll" "liteparse/pdfium.dll"- }- } else {- Write-Error "Could not find pdfium.dll"- exit 1- }-- name: Upload wheelsuses: actions/upload-artifact@v4with:diff --git a/packages/python/scripts/download-pdfium.sh b/packages/python/scripts/download-pdfium.shnew file mode 100755index 0000000..b134da2--- /dev/null+++ b/packages/python/scripts/download-pdfium.sh@@ -0,0 +1,65 @@+#!/usr/bin/env bash+#+# Download the prebuilt pdfium binary for a given target triple and stage it+# at `packages/python/liteparse/<dylib>` so maturin's `[tool.maturin] include`+# rule packs it into the wheel (and records it in RECORD) at build time.+#+# Usage: ./scripts/download-pdfium.sh <target-triple>+#+# The release tag is parsed from `crates/pdfium-sys/build.rs` so we only have+# one source of truth for the pdfium version.++set -euo pipefail++TARGET="${1:?usage: download-pdfium.sh <target-triple>}"+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"+PKG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"+REPO_ROOT="$(cd "$PKG_DIR/../.." && pwd)"++BUILD_RS="$REPO_ROOT/crates/pdfium-sys/build.rs"+TAG=$(grep -E 'const PDFIUM_RELEASE_TAG' "$BUILD_RS" | sed -E 's/.*"([^"]+)".*/\1/')+[ -n "$TAG" ] || { echo "Could not parse PDFIUM_RELEASE_TAG from $BUILD_RS" >&2; exit 1; }++case "$TARGET" in+ aarch64-apple-darwin) ASSET="pdfium-mac-arm64"; DYLIB="libpdfium.dylib" ;;+ x86_64-apple-darwin) ASSET="pdfium-mac-x64"; DYLIB="libpdfium.dylib" ;;+ x86_64-unknown-linux-gnu) ASSET="pdfium-linux-x64"; DYLIB="libpdfium.so" ;;+ aarch64-unknown-linux-gnu) ASSET="pdfium-linux-arm64"; DYLIB="libpdfium.so" ;;+ x86_64-unknown-linux-musl) ASSET="pdfium-linux-musl-x64"; DYLIB="libpdfium.so" ;;+ aarch64-unknown-linux-musl) ASSET="pdfium-linux-arm64"; DYLIB="libpdfium.so" ;;+ x86_64-pc-windows-msvc) ASSET="pdfium-win-x64"; DYLIB="pdfium.dll" ;;+ aarch64-pc-windows-msvc) ASSET="pdfium-win-arm64"; DYLIB="pdfium.dll" ;;+ *) echo "Unsupported target: $TARGET" >&2; exit 1 ;;+esac++URL_TAG="${TAG//\//%2F}"+URL="https://github.com/run-llama/pdfium-binaries/releases/download/${URL_TAG}/${ASSET}.tgz"++TMP=$(mktemp -d)+trap 'rm -rf "$TMP"' EXIT++echo "Downloading $URL"+curl -fsSL "$URL" -o "$TMP/${ASSET}.tgz"+tar -xzf "$TMP/${ASSET}.tgz" -C "$TMP"++# pdfium-binaries layout: lib/<DYLIB> on unix, bin/pdfium.dll on windows+SRC=""+for candidate in "$TMP/lib/$DYLIB" "$TMP/bin/$DYLIB"; do+ if [ -f "$candidate" ]; then+ SRC="$candidate"+ break+ fi+done+[ -n "$SRC" ] || { echo "Could not find $DYLIB in archive" >&2; ls -R "$TMP" >&2; exit 1; }++# Mirror the install-name fix in crates/pdfium-sys/build.rs so the dylib resolves+# via @rpath at runtime. pdfium-binaries ships macOS dylibs with install name+# `./libpdfium.dylib` which won't be found via our rpath.+if [ "$DYLIB" = "libpdfium.dylib" ] && command -v install_name_tool >/dev/null 2>&1; then+ install_name_tool -id "@rpath/libpdfium.dylib" "$SRC"+fi++DEST_DIR="$PKG_DIR/liteparse"+mkdir -p "$DEST_DIR"+cp "$SRC" "$DEST_DIR/$DYLIB"+echo "Staged $DEST_DIR/$DYLIB"diff --git a/packages/python/scripts/inject_dylib.py b/packages/python/scripts/inject_dylib.pydeleted file mode 100644index c8dd9e9..0000000--- a/packages/python/scripts/inject_dylib.py+++ /dev/null@@ -1,81 +0,0 @@-"""Inject the pdfium shared library into a built wheel and update its RECORD.--Maturin's `[tool.maturin] include` directive only bundles files that exist at-build time. Because libpdfium is produced as a side effect of compiling-pdfium-sys and copied in afterwards by CI, we have to splice it into the-finished wheel ourselves. Appending with `zipfile.ZipFile(..., 'a')` works for-the bytes, but leaves the `*.dist-info/RECORD` manifest out of sync — Poetry-and other strict installers warn about that. This script rewrites the wheel so-the dylib is present in both the archive and RECORD.-"""--from __future__ import annotations--import base64-import hashlib-import sys-import zipfile-from pathlib import Path---def _record_hash(data: bytes) -> str:- digest = hashlib.sha256(data).digest()- return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")---def inject(wheel_path: Path, dylib_path: Path, arcname: str) -> None:- if not dylib_path.exists():- print(f"Warning: {dylib_path} not found, skipping {wheel_path}")- return-- dylib_bytes = dylib_path.read_bytes()- dylib_hash = _record_hash(dylib_bytes)- dylib_size = str(len(dylib_bytes))-- tmp_path = wheel_path.with_suffix(wheel_path.suffix + ".tmp")-- with zipfile.ZipFile(wheel_path, "r") as zin:- record_name = next(- (n for n in zin.namelist() if n.endswith(".dist-info/RECORD")), None- )- if record_name is None:- raise RuntimeError(f"No RECORD file found in {wheel_path}")-- record_text = zin.read(record_name).decode("utf-8")- record_lines = [ln for ln in record_text.splitlines() if ln.strip()]- # Drop any pre-existing entries for this dylib or the RECORD itself- # (RECORD's own line always carries empty hash/size).- record_lines = [- ln- for ln in record_lines- if not ln.startswith(arcname + ",")- and not ln.startswith(record_name + ",")- ]- record_lines.append(f"{arcname},{dylib_hash},{dylib_size}")- record_lines.append(f"{record_name},,")- new_record = ("\n".join(record_lines) + "\n").encode("utf-8")-- with zipfile.ZipFile(- tmp_path, "w", compression=zipfile.ZIP_DEFLATED- ) as zout:- for item in zin.infolist():- if item.filename == record_name or item.filename == arcname:- continue- zout.writestr(item, zin.read(item.filename))- zout.writestr(arcname, dylib_bytes)- zout.writestr(record_name, new_record)-- tmp_path.replace(wheel_path)- print(f"Injected {arcname} ({dylib_size} bytes) into {wheel_path}")---def main() -> int:- if len(sys.argv) != 4:- print("usage: inject_dylib.py <wheel> <dylib> <arcname>", file=sys.stderr)- return 2- inject(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3])- return 0---if __name__ == "__main__":- raise SystemExit(main())