Skip to content

A fast, helpful, and open-source document parser

cleaner fix

Logan Markewich6dcdfde7fd18
diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
index e9e3c99..55b26bf 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -106,6 +106,16 @@ jobs:
target
key: ${{ runner.os }}-${{ matrix.target }}-cargo-python-${{ hashFiles('**/Cargo.lock') }}
+ - name: Stage pdfium for wheel inclusion
+ if: ${{ !matrix.musl }}
+ shell: bash
+ # Pre-download pdfium and place it at packages/python/liteparse/<dylib>
+ # so the `[tool.maturin] include` rule packs it into the wheel during
+ # the build below. This is what gets the dylib listed in the wheel's
+ # RECORD manifest — appending it after the build leaves RECORD out of
+ # sync, which Poetry warns about.
+ run: bash packages/python/scripts/download-pdfium.sh ${{ matrix.target }}
+
- name: Build wheels (glibc)
if: ${{ !matrix.musl }}
uses: PyO3/maturin-action@v1
@@ -143,54 +153,6 @@ jobs:
# read/upload the produced wheel.
sudo chmod -R a+rw packages/python/dist/ || true
- - name: Copy pdfium dylib into wheel (Unix)
- if: runner.os != 'Windows'
- working-directory: packages/python
- run: |
- # Fix permissions from Docker/maturin build
- sudo chmod -R a+rw dist/ || true
- # Find pdfium in target/ (mounted from Docker) or cache
- PDFIUM_LIB=$(find ../../target -name "libpdfium.so" -type f 2>/dev/null | head -1 || true)
- if [ -z "$PDFIUM_LIB" ]; then
- PDFIUM_LIB=$(find ~/.cache/pdfium-rs -name "libpdfium.so" -type f 2>/dev/null | head -1 || true)
- fi
- if [ -n "$PDFIUM_LIB" ]; then
- export PDFIUM_LIB_PATH="$(dirname "$PDFIUM_LIB")"
- fi
- bash scripts/copy-pdfium.sh
- if [ "$(uname -s)" = "Darwin" ]; then
- DYLIB="libpdfium.dylib"
- else
- DYLIB="libpdfium.so"
- fi
- for whl in dist/*.whl; do
- python scripts/inject_dylib.py "$whl" "liteparse/$DYLIB" "liteparse/$DYLIB"
- done
-
- - name: Copy pdfium dylib into wheel (Windows)
- if: runner.os == 'Windows'
- working-directory: packages/python
- shell: pwsh
- run: |
- $cacheBase = "$env:LOCALAPPDATA\pdfium-rs"
- if (-not (Test-Path $cacheBase)) {
- $cacheBase = "$env:USERPROFILE\.cache\pdfium-rs"
- }
- $dll = Get-ChildItem -Path $cacheBase -Recurse -Filter "pdfium.dll" | Select-Object -First 1
- if (-not $dll) {
- $dll = Get-ChildItem -Path "..\..\target" -Recurse -Filter "pdfium.dll" | Select-Object -First 1
- }
- if ($dll) {
- Copy-Item $dll.FullName -Destination "liteparse\pdfium.dll"
- Write-Host "Copied $($dll.FullName) -> liteparse\pdfium.dll"
- foreach ($whl in Get-ChildItem dist\*.whl) {
- python scripts/inject_dylib.py $whl.FullName "liteparse/pdfium.dll" "liteparse/pdfium.dll"
- }
- } else {
- Write-Error "Could not find pdfium.dll"
- exit 1
- }
-
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
diff --git a/packages/python/scripts/download-pdfium.sh b/packages/python/scripts/download-pdfium.sh
new file mode 100755
index 0000000..b134da2
--- /dev/null
+++ b/packages/python/scripts/download-pdfium.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# Download the prebuilt pdfium binary for a given target triple and stage it
+# at `packages/python/liteparse/<dylib>` so maturin's `[tool.maturin] include`
+# rule packs it into the wheel (and records it in RECORD) at build time.
+#
+# Usage: ./scripts/download-pdfium.sh <target-triple>
+#
+# The release tag is parsed from `crates/pdfium-sys/build.rs` so we only have
+# one source of truth for the pdfium version.
+
+set -euo pipefail
+
+TARGET="${1:?usage: download-pdfium.sh <target-triple>}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PKG_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO_ROOT="$(cd "$PKG_DIR/../.." && pwd)"
+
+BUILD_RS="$REPO_ROOT/crates/pdfium-sys/build.rs"
+TAG=$(grep -E 'const PDFIUM_RELEASE_TAG' "$BUILD_RS" | sed -E 's/.*"([^"]+)".*/\1/')
+[ -n "$TAG" ] || { echo "Could not parse PDFIUM_RELEASE_TAG from $BUILD_RS" >&2; exit 1; }
+
+case "$TARGET" in
+ aarch64-apple-darwin) ASSET="pdfium-mac-arm64"; DYLIB="libpdfium.dylib" ;;
+ x86_64-apple-darwin) ASSET="pdfium-mac-x64"; DYLIB="libpdfium.dylib" ;;
+ x86_64-unknown-linux-gnu) ASSET="pdfium-linux-x64"; DYLIB="libpdfium.so" ;;
+ aarch64-unknown-linux-gnu) ASSET="pdfium-linux-arm64"; DYLIB="libpdfium.so" ;;
+ x86_64-unknown-linux-musl) ASSET="pdfium-linux-musl-x64"; DYLIB="libpdfium.so" ;;
+ aarch64-unknown-linux-musl) ASSET="pdfium-linux-arm64"; DYLIB="libpdfium.so" ;;
+ x86_64-pc-windows-msvc) ASSET="pdfium-win-x64"; DYLIB="pdfium.dll" ;;
+ aarch64-pc-windows-msvc) ASSET="pdfium-win-arm64"; DYLIB="pdfium.dll" ;;
+ *) echo "Unsupported target: $TARGET" >&2; exit 1 ;;
+esac
+
+URL_TAG="${TAG//\//%2F}"
+URL="https://github.com/run-llama/pdfium-binaries/releases/download/${URL_TAG}/${ASSET}.tgz"
+
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+echo "Downloading $URL"
+curl -fsSL "$URL" -o "$TMP/${ASSET}.tgz"
+tar -xzf "$TMP/${ASSET}.tgz" -C "$TMP"
+
+# pdfium-binaries layout: lib/<DYLIB> on unix, bin/pdfium.dll on windows
+SRC=""
+for candidate in "$TMP/lib/$DYLIB" "$TMP/bin/$DYLIB"; do
+ if [ -f "$candidate" ]; then
+ SRC="$candidate"
+ break
+ fi
+done
+[ -n "$SRC" ] || { echo "Could not find $DYLIB in archive" >&2; ls -R "$TMP" >&2; exit 1; }
+
+# Mirror the install-name fix in crates/pdfium-sys/build.rs so the dylib resolves
+# via @rpath at runtime. pdfium-binaries ships macOS dylibs with install name
+# `./libpdfium.dylib` which won't be found via our rpath.
+if [ "$DYLIB" = "libpdfium.dylib" ] && command -v install_name_tool >/dev/null 2>&1; then
+ install_name_tool -id "@rpath/libpdfium.dylib" "$SRC"
+fi
+
+DEST_DIR="$PKG_DIR/liteparse"
+mkdir -p "$DEST_DIR"
+cp "$SRC" "$DEST_DIR/$DYLIB"
+echo "Staged $DEST_DIR/$DYLIB"
diff --git a/packages/python/scripts/inject_dylib.py b/packages/python/scripts/inject_dylib.py
deleted file mode 100644
index c8dd9e9..0000000
--- a/packages/python/scripts/inject_dylib.py
+++ /dev/null
@@ -1,81 +0,0 @@
-"""Inject the pdfium shared library into a built wheel and update its RECORD.
-
-Maturin's `[tool.maturin] include` directive only bundles files that exist at
-build time. Because libpdfium is produced as a side effect of compiling
-pdfium-sys and copied in afterwards by CI, we have to splice it into the
-finished wheel ourselves. Appending with `zipfile.ZipFile(..., 'a')` works for
-the bytes, but leaves the `*.dist-info/RECORD` manifest out of sync — Poetry
-and other strict installers warn about that. This script rewrites the wheel so
-the dylib is present in both the archive and RECORD.
-"""
-
-from __future__ import annotations
-
-import base64
-import hashlib
-import sys
-import zipfile
-from pathlib import Path
-
-
-def _record_hash(data: bytes) -> str:
- digest = hashlib.sha256(data).digest()
- return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
-
-
-def inject(wheel_path: Path, dylib_path: Path, arcname: str) -> None:
- if not dylib_path.exists():
- print(f"Warning: {dylib_path} not found, skipping {wheel_path}")
- return
-
- dylib_bytes = dylib_path.read_bytes()
- dylib_hash = _record_hash(dylib_bytes)
- dylib_size = str(len(dylib_bytes))
-
- tmp_path = wheel_path.with_suffix(wheel_path.suffix + ".tmp")
-
- with zipfile.ZipFile(wheel_path, "r") as zin:
- record_name = next(
- (n for n in zin.namelist() if n.endswith(".dist-info/RECORD")), None
- )
- if record_name is None:
- raise RuntimeError(f"No RECORD file found in {wheel_path}")
-
- record_text = zin.read(record_name).decode("utf-8")
- record_lines = [ln for ln in record_text.splitlines() if ln.strip()]
- # Drop any pre-existing entries for this dylib or the RECORD itself
- # (RECORD's own line always carries empty hash/size).
- record_lines = [
- ln
- for ln in record_lines
- if not ln.startswith(arcname + ",")
- and not ln.startswith(record_name + ",")
- ]
- record_lines.append(f"{arcname},{dylib_hash},{dylib_size}")
- record_lines.append(f"{record_name},,")
- new_record = ("\n".join(record_lines) + "\n").encode("utf-8")
-
- with zipfile.ZipFile(
- tmp_path, "w", compression=zipfile.ZIP_DEFLATED
- ) as zout:
- for item in zin.infolist():
- if item.filename == record_name or item.filename == arcname:
- continue
- zout.writestr(item, zin.read(item.filename))
- zout.writestr(arcname, dylib_bytes)
- zout.writestr(record_name, new_record)
-
- tmp_path.replace(wheel_path)
- print(f"Injected {arcname} ({dylib_size} bytes) into {wheel_path}")
-
-
-def main() -> int:
- if len(sys.argv) != 4:
- print("usage: inject_dylib.py <wheel> <dylib> <arcname>", file=sys.stderr)
- return 2
- inject(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3])
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())