A fast, helpful, and open-source document parser
cicd
Logan Markewich9467cddcfd4f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 40c3187..33ea9ef 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -6,63 +6,70 @@ on:pull_request:branches: [main]+env:+ CARGO_TERM_COLOR: always+jobs:- lint-and-build:+ fmt:runs-on: ubuntu-latest-- strategy:- matrix:- node-version: [18, 20, 22]-steps:- - name: Checkout repository- uses: actions/checkout@v4-- - name: Setup Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v4- with:- node-version: ${{ matrix.node-version }}- cache: "npm"-- - name: Install dependencies- run: npm ci-- - name: Check formatting- run: npm run format:check-- - name: Run linter- run: npm run lint-- - name: Build- run: npm run build+ - uses: actions/checkout@v4+ - run: rustup toolchain install stable --profile minimal --component rustfmt+ - run: cargo fmt --all -- --check- test:+ clippy:runs-on: ubuntu-latest-+ steps:+ - uses: actions/checkout@v4+ - name: Install system dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ target+ key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}+ - run: rustup toolchain install stable --profile minimal --component clippy+ - run: cargo clippy --all-targets -- -D warnings++ build-and-test:strategy:matrix:- node-version: [18, 20, 22]-+ os: [ubuntu-latest, macos-latest]+ runs-on: ${{ matrix.os }}steps:- - name: Checkout repository- uses: actions/checkout@v4-- - name: Setup Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v4- with:- node-version: ${{ matrix.node-version }}- cache: "npm"+ - uses: actions/checkout@v4- - name: Install dependencies- run: npm ci+ - name: Install system dependencies (Linux)+ if: runner.os == 'Linux'+ run: |+ sudo apt-get update+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev- - name: Run tests- run: npm run test+ - name: Install system dependencies (macOS)+ if: runner.os == 'macOS'+ run: brew install tesseract leptonica- # test that docker builds+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ ~/Library/Caches/pdfium-rs+ target+ key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}++ - run: cargo build --all-targets+ - run: cargo test --all++ # Test that docker buildsbuild-docker-image:runs-on: ubuntu-latest-steps:- uses: actions/checkout@v4- name: Build the base Docker imagediff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.ymlindex 34059cc..6367b8b 100644--- a/.github/workflows/docker-image.yml+++ b/.github/workflows/docker-image.yml@@ -31,6 +31,9 @@ jobs:- name: Set up Docker Buildxuses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0+ - name: Set up QEMU+ uses: docker/setup-qemu-action@v4+- name: Log into registry ${{ env.REGISTRY }}if: github.event_name != 'pull_request'uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0@@ -60,7 +63,6 @@ jobs:- name: Sign the published Docker imageif: ${{ github.event_name != 'pull_request' }}env:- # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variableTAGS: ${{ steps.meta.outputs.tags }}DIGEST: ${{ steps.build-and-push.outputs.digest }}run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}@@ -118,7 +120,6 @@ jobs:- name: Sign the published Docker imageif: ${{ github.event_name != 'pull_request' }}env:- # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variableTAGS: ${{ steps.meta.outputs.tags }}DIGEST: ${{ steps.build-and-push.outputs.digest }}run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.ymlindex 324b8ef..3cade7b 100644--- a/.github/workflows/e2e-output.yml+++ b/.github/workflows/e2e-output.yml@@ -24,20 +24,33 @@ jobs:- name: Install system dependenciesrun: |sudo apt-get update- sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev \+ tesseract-ocr libreoffice imagemagick ghostscript+ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ target+ key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.lock') }}++ - name: Build+ run: cargo build --release++ # The comparison scripts (compare-dataset.ts, upload-dataset.ts) still use+ # Node/tsx and need to be ported to Rust or rewritten as shell scripts.+ # For now, we install Node to run them.- name: Setup Node.jsuses: actions/setup-node@v4with:node-version: 20cache: "npm"- - name: Install dependencies+ - name: Install Node dependenciesrun: npm ci- - name: Build- run: npm run build-- name: Download from HuggingFaceenv:HF_TOKEN: ${{ secrets.HF_TOKEN }}@@ -107,7 +120,20 @@ jobs:- name: Install system dependenciesrun: |sudo apt-get update- sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev \+ tesseract-ocr libreoffice imagemagick ghostscript++ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ target+ key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.lock') }}++ - name: Build+ run: cargo build --release- name: Setup Node.jsuses: actions/setup-node@v4@@ -125,9 +151,6 @@ jobs:npm cipip install huggingface_hub- - name: Build- run: npm run build-- name: Download existing dataset from HuggingFaceenv:HF_TOKEN: ${{ secrets.HF_TOKEN }}diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 5bf71d3..97e1179 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -2,51 +2,142 @@ name: Releaseon:push:- branches:- - main+ tags: ["v*.*.*"]-concurrency: ${{ github.workflow }}-${{ github.ref }}+permissions:+ contents: write++env:+ CARGO_TERM_COLOR: alwaysjobs:- release:- name: Release+ publish-crates:runs-on: ubuntu-latest- permissions:- contents: write- pull-requests: write- id-token: write- actions: writesteps:- - name: Checkout repository- uses: actions/checkout@v4+ - uses: actions/checkout@v4++ - name: Install system dependencies+ run: |+ sudo apt-get update+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev++ - uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ target+ key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}++ - name: Publish to crates.io+ run: |+ cargo publish -p pdfium-sys --allow-dirty+ cargo publish -p pdfium --allow-dirty+ cargo publish -p liteparse-rs --allow-dirty+ env:+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}++ build-binaries:+ strategy:+ matrix:+ include:+ - target: x86_64-unknown-linux-gnu+ os: ubuntu-latest+ artifact: lit-linux-x64+ - target: aarch64-unknown-linux-gnu+ os: ubuntu-latest+ artifact: lit-linux-arm64+ - target: x86_64-apple-darwin+ os: macos-latest+ artifact: lit-darwin-x64+ - target: aarch64-apple-darwin+ os: macos-latest+ artifact: lit-darwin-arm64++ runs-on: ${{ matrix.os }}++ steps:+ - uses: actions/checkout@v4++ - name: Install system dependencies (Linux native)+ if: runner.os == 'Linux' && matrix.target == 'x86_64-unknown-linux-gnu'+ run: |+ sudo apt-get update+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev++ - name: Install system dependencies (Linux cross arm64)+ if: runner.os == 'Linux' && matrix.target == 'aarch64-unknown-linux-gnu'+ run: |+ sudo apt-get update+ sudo apt-get install -y libclang-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \+ libtesseract-dev:arm64 libleptonica-dev:arm64 || true+ # Cross compilation may need additional setup+ sudo dpkg --add-architecture arm64+ sudo apt-get update+ sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libclang-dev+ rustup target add aarch64-unknown-linux-gnu++ - name: Install system dependencies (macOS)+ if: runner.os == 'macOS'+ run: brew install tesseract leptonica- - name: Setup Node.js- uses: actions/setup-node@v4+ - uses: actions/cache@v4with:- node-version: 20- cache: "npm"- registry-url: "https://registry.npmjs.org"+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ ~/.cache/pdfium-rs+ ~/Library/Caches/pdfium-rs+ target+ key: ${{ runner.os }}-${{ matrix.target }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}- - name: Install dependencies- run: npm ci+ - name: Add target+ run: rustup target add ${{ matrix.target }}- name: Build- run: npm run build+ run: cargo build --release --target ${{ matrix.target }}- - name: Create Release Pull Request or Publish- id: changesets- uses: changesets/action@v1+ - name: Package binary+ run: |+ mkdir -p dist+ cp target/${{ matrix.target }}/release/lit dist/${{ matrix.artifact }}+ chmod +x dist/${{ matrix.artifact }}+ tar -czf dist/${{ matrix.artifact }}.tar.gz -C dist ${{ matrix.artifact }}++ - name: Upload artifact+ uses: actions/upload-artifact@v4with:- publish: npm run release- title: "chore: version packages"- commit: "chore: version packages"- env:- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}+ name: ${{ matrix.artifact }}+ path: dist/${{ matrix.artifact }}.tar.gz++ github-release:+ needs: [build-binaries, publish-crates]+ runs-on: ubuntu-latest+ permissions:+ contents: write++ steps:+ - uses: actions/checkout@v4- - name: Dispatch post-publish workflow- if: steps.changesets.outputs.published == 'true'- uses: actions/github-script@v7+ - name: Download all artifacts+ uses: actions/download-artifact@v4+ with:+ path: artifacts++ - name: Create GitHub Release+ uses: softprops/action-gh-release@v2+ with:+ generate_release_notes: true+ files: artifacts/**/*.tar.gz++ dispatch-homebrew:+ needs: github-release+ runs-on: ubuntu-latest+ permissions:+ actions: write+ steps:+ - uses: actions/github-script@v7with:script: |await github.rest.actions.createWorkflowDispatch({diff --git a/Dockerfile b/Dockerfileindex db53583..3dd7652 100644--- a/Dockerfile+++ b/Dockerfile@@ -1,34 +1,35 @@-# Stage 1: Build from source (needs build tools for native addons like sharp, pdfium)-FROM node:24-trixie AS builder+# Stage 1: Build from source+FROM rust:1-bookworm AS builder++RUN apt-get update && apt-get install -y \+ libclang-dev \+ libtesseract-dev \+ libleptonica-dev \+ && rm -rf /var/lib/apt/lists/*WORKDIR /build-COPY package.json package-lock.json ./-RUN npm ci --ignore-scripts=false+COPY Cargo.toml Cargo.lock ./+COPY crates/ ./crates/-COPY src/ ./src/-COPY cli/ ./cli/-COPY tsconfig.json ./-RUN npm run build+RUN cargo build --release# Stage 2: Minimal runtime image-FROM node:24-trixie-slim+FROM debian:bookworm-slim-# sharp and @hyzyla/pdfium ship prebuilt binaries but may need these shared libs at runtimeRUN apt-get update && apt-get install -y --no-install-recommends \- libvips42 \+ libtesseract5 \+ libleptonica6 \+ tesseract-ocr-eng \ca-certificates \&& rm -rf /var/lib/apt/lists/*-WORKDIR /usr/local/lib/liteparse-COPY --from=builder /build/package.json ./-COPY --from=builder /build/dist/ ./dist/-COPY --from=builder /build/src/vendor/pdfjs ./src/vendor/pdfjs/-COPY --from=builder /build/node_modules/ ./node_modules/+COPY --from=builder /build/target/release/lit /usr/local/bin/lit+# pdfium shared library+COPY --from=builder /root/.cache/pdfium-rs/ /usr/local/lib/pdfium-rs/-# Make the CLI executable + globally available via symlinks-RUN chmod +x /usr/local/lib/liteparse/dist/src/index.js-RUN ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/lit \- && ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/liteparse+# Ensure pdfium is discoverable at runtime+ENV LD_LIBRARY_PATH="/usr/local/lib/pdfium-rs"+RUN ln -s /usr/local/bin/lit /usr/local/bin/liteparseCMD ["/bin/sh"]diff --git a/full.Dockerfile b/full.Dockerfileindex 2e0a5a7..60d14e0 100644--- a/full.Dockerfile+++ b/full.Dockerfile@@ -1,36 +1,37 @@-# Stage 1: Build from source (needs build tools for native addons like sharp, pdfium)-FROM node:24-trixie AS builder+# Stage 1: Build from source+FROM rust:1-bookworm AS builder++RUN apt-get update && apt-get install -y \+ libclang-dev \+ libtesseract-dev \+ libleptonica-dev \+ && rm -rf /var/lib/apt/lists/*WORKDIR /build-COPY package.json package-lock.json ./-RUN npm ci --ignore-scripts=false+COPY Cargo.toml Cargo.lock ./+COPY crates/ ./crates/-COPY src/ ./src/-COPY cli/ ./cli/-COPY tsconfig.json ./-RUN npm run build+RUN cargo build --release-# Stage 2: Minimal runtime image-FROM node:24-trixie-slim+# Stage 2: Runtime image with conversion tools+FROM debian:bookworm-slim-# sharp and @hyzyla/pdfium ship prebuilt binaries but may need these shared libs at runtimeRUN apt-get update && apt-get install -y --no-install-recommends \- libvips42 \+ libtesseract5 \+ libleptonica6 \+ tesseract-ocr-eng \ca-certificates \libreoffice \imagemagick \&& rm -rf /var/lib/apt/lists/*-WORKDIR /usr/local/lib/liteparse-COPY --from=builder /build/package.json ./-COPY --from=builder /build/dist/ ./dist/-COPY --from=builder /build/src/vendor/pdfjs ./src/vendor/pdfjs/-COPY --from=builder /build/node_modules/ ./node_modules/+COPY --from=builder /build/target/release/lit /usr/local/bin/lit+# pdfium shared library+COPY --from=builder /root/.cache/pdfium-rs/ /usr/local/lib/pdfium-rs/-# Make the CLI executable + globally available via symlinks-RUN chmod +x /usr/local/lib/liteparse/dist/src/index.js-RUN ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/lit \- && ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/liteparse+# Ensure pdfium is discoverable at runtime+ENV LD_LIBRARY_PATH="/usr/local/lib/pdfium-rs"+RUN ln -s /usr/local/bin/lit /usr/local/bin/liteparseCMD ["/bin/sh"]