Skip to content

A fast, helpful, and open-source document parser

cicd

Logan Markewich9467cddcfd4f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 40c3187..33ea9ef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,63 +6,70 @@ on:
pull_request:
branches: [main]
+env:
+ CARGO_TERM_COLOR: always
+
jobs:
- lint-and-build:
+ fmt:
runs-on: ubuntu-latest
-
- strategy:
- matrix:
- node-version: [18, 20, 22]
-
steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Setup Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v4
- with:
- node-version: ${{ matrix.node-version }}
- cache: "npm"
-
- - name: Install dependencies
- run: npm ci
-
- - name: Check formatting
- run: npm run format:check
-
- - name: Run linter
- run: npm run lint
-
- - name: Build
- run: npm run build
+ - uses: actions/checkout@v4
+ - run: rustup toolchain install stable --profile minimal --component rustfmt
+ - run: cargo fmt --all -- --check
- test:
+ clippy:
runs-on: ubuntu-latest
-
+ steps:
+ - uses: actions/checkout@v4
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev
+ - uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ target
+ key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
+ - run: rustup toolchain install stable --profile minimal --component clippy
+ - run: cargo clippy --all-targets -- -D warnings
+
+ build-and-test:
strategy:
matrix:
- node-version: [18, 20, 22]
-
+ os: [ubuntu-latest, macos-latest]
+ runs-on: ${{ matrix.os }}
steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Setup Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v4
- with:
- node-version: ${{ matrix.node-version }}
- cache: "npm"
+ - uses: actions/checkout@v4
- - name: Install dependencies
- run: npm ci
+ - name: Install system dependencies (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev
- - name: Run tests
- run: npm run test
+ - name: Install system dependencies (macOS)
+ if: runner.os == 'macOS'
+ run: brew install tesseract leptonica
- # test that docker builds
+ - uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ ~/Library/Caches/pdfium-rs
+ target
+ key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
+
+ - run: cargo build --all-targets
+ - run: cargo test --all
+
+ # Test that docker builds
build-docker-image:
runs-on: ubuntu-latest
-
steps:
- uses: actions/checkout@v4
- name: Build the base Docker image
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 34059cc..6367b8b 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -31,6 +31,9 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v4
+
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
@@ -60,7 +63,6 @@ jobs:
- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
env:
- # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build-and-push.outputs.digest }}
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
@@ -118,7 +120,6 @@ jobs:
- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
env:
- # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build-and-push.outputs.digest }}
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
diff --git a/.github/workflows/e2e-output.yml b/.github/workflows/e2e-output.yml
index 324b8ef..3cade7b 100644
--- a/.github/workflows/e2e-output.yml
+++ b/.github/workflows/e2e-output.yml
@@ -24,20 +24,33 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
- sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev \
+ tesseract-ocr libreoffice imagemagick ghostscript
+ - uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ target
+ key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.lock') }}
+
+ - name: Build
+ run: cargo build --release
+
+ # The comparison scripts (compare-dataset.ts, upload-dataset.ts) still use
+ # Node/tsx and need to be ported to Rust or rewritten as shell scripts.
+ # For now, we install Node to run them.
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 20
cache: "npm"
- - name: Install dependencies
+ - name: Install Node dependencies
run: npm ci
- - name: Build
- run: npm run build
-
- name: Download from HuggingFace
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -107,7 +120,20 @@ jobs:
- name: Install system dependencies
run: |
sudo apt-get update
- sudo apt-get install -y libreoffice imagemagick ghostscript tesseract-ocr
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev \
+ tesseract-ocr libreoffice imagemagick ghostscript
+
+ - uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ target
+ key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.lock') }}
+
+ - name: Build
+ run: cargo build --release
- name: Setup Node.js
uses: actions/setup-node@v4
@@ -125,9 +151,6 @@ jobs:
npm ci
pip install huggingface_hub
- - name: Build
- run: npm run build
-
- name: Download existing dataset from HuggingFace
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5bf71d3..97e1179 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -2,51 +2,142 @@ name: Release
on:
push:
- branches:
- - main
+ tags: ["v*.*.*"]
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+permissions:
+ contents: write
+
+env:
+ CARGO_TERM_COLOR: always
jobs:
- release:
- name: Release
+ publish-crates:
runs-on: ubuntu-latest
- permissions:
- contents: write
- pull-requests: write
- id-token: write
- actions: write
steps:
- - name: Checkout repository
- uses: actions/checkout@v4
+ - uses: actions/checkout@v4
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev
+
+ - uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ target
+ key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
+
+ - name: Publish to crates.io
+ run: |
+ cargo publish -p pdfium-sys --allow-dirty
+ cargo publish -p pdfium --allow-dirty
+ cargo publish -p liteparse-rs --allow-dirty
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+ build-binaries:
+ strategy:
+ matrix:
+ include:
+ - target: x86_64-unknown-linux-gnu
+ os: ubuntu-latest
+ artifact: lit-linux-x64
+ - target: aarch64-unknown-linux-gnu
+ os: ubuntu-latest
+ artifact: lit-linux-arm64
+ - target: x86_64-apple-darwin
+ os: macos-latest
+ artifact: lit-darwin-x64
+ - target: aarch64-apple-darwin
+ os: macos-latest
+ artifact: lit-darwin-arm64
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install system dependencies (Linux native)
+ if: runner.os == 'Linux' && matrix.target == 'x86_64-unknown-linux-gnu'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libclang-dev libtesseract-dev libleptonica-dev
+
+ - name: Install system dependencies (Linux cross arm64)
+ if: runner.os == 'Linux' && matrix.target == 'aarch64-unknown-linux-gnu'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libclang-dev gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
+ libtesseract-dev:arm64 libleptonica-dev:arm64 || true
+ # Cross compilation may need additional setup
+ sudo dpkg --add-architecture arm64
+ sudo apt-get update
+ sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libclang-dev
+ rustup target add aarch64-unknown-linux-gnu
+
+ - name: Install system dependencies (macOS)
+ if: runner.os == 'macOS'
+ run: brew install tesseract leptonica
- - name: Setup Node.js
- uses: actions/setup-node@v4
+ - uses: actions/cache@v4
with:
- node-version: 20
- cache: "npm"
- registry-url: "https://registry.npmjs.org"
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ ~/.cache/pdfium-rs
+ ~/Library/Caches/pdfium-rs
+ target
+ key: ${{ runner.os }}-${{ matrix.target }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
- - name: Install dependencies
- run: npm ci
+ - name: Add target
+ run: rustup target add ${{ matrix.target }}
- name: Build
- run: npm run build
+ run: cargo build --release --target ${{ matrix.target }}
- - name: Create Release Pull Request or Publish
- id: changesets
- uses: changesets/action@v1
+ - name: Package binary
+ run: |
+ mkdir -p dist
+ cp target/${{ matrix.target }}/release/lit dist/${{ matrix.artifact }}
+ chmod +x dist/${{ matrix.artifact }}
+ tar -czf dist/${{ matrix.artifact }}.tar.gz -C dist ${{ matrix.artifact }}
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@v4
with:
- publish: npm run release
- title: "chore: version packages"
- commit: "chore: version packages"
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+ name: ${{ matrix.artifact }}
+ path: dist/${{ matrix.artifact }}.tar.gz
+
+ github-release:
+ needs: [build-binaries, publish-crates]
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+
+ steps:
+ - uses: actions/checkout@v4
- - name: Dispatch post-publish workflow
- if: steps.changesets.outputs.published == 'true'
- uses: actions/github-script@v7
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: artifacts
+
+ - name: Create GitHub Release
+ uses: softprops/action-gh-release@v2
+ with:
+ generate_release_notes: true
+ files: artifacts/**/*.tar.gz
+
+ dispatch-homebrew:
+ needs: github-release
+ runs-on: ubuntu-latest
+ permissions:
+ actions: write
+ steps:
+ - uses: actions/github-script@v7
with:
script: |
await github.rest.actions.createWorkflowDispatch({
diff --git a/Dockerfile b/Dockerfile
index db53583..3dd7652 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,34 +1,35 @@
-# Stage 1: Build from source (needs build tools for native addons like sharp, pdfium)
-FROM node:24-trixie AS builder
+# Stage 1: Build from source
+FROM rust:1-bookworm AS builder
+
+RUN apt-get update && apt-get install -y \
+ libclang-dev \
+ libtesseract-dev \
+ libleptonica-dev \
+ && rm -rf /var/lib/apt/lists/*
WORKDIR /build
-COPY package.json package-lock.json ./
-RUN npm ci --ignore-scripts=false
+COPY Cargo.toml Cargo.lock ./
+COPY crates/ ./crates/
-COPY src/ ./src/
-COPY cli/ ./cli/
-COPY tsconfig.json ./
-RUN npm run build
+RUN cargo build --release
# Stage 2: Minimal runtime image
-FROM node:24-trixie-slim
+FROM debian:bookworm-slim
-# sharp and @hyzyla/pdfium ship prebuilt binaries but may need these shared libs at runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
- libvips42 \
+ libtesseract5 \
+ libleptonica6 \
+ tesseract-ocr-eng \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
-WORKDIR /usr/local/lib/liteparse
-COPY --from=builder /build/package.json ./
-COPY --from=builder /build/dist/ ./dist/
-COPY --from=builder /build/src/vendor/pdfjs ./src/vendor/pdfjs/
-COPY --from=builder /build/node_modules/ ./node_modules/
+COPY --from=builder /build/target/release/lit /usr/local/bin/lit
+# pdfium shared library
+COPY --from=builder /root/.cache/pdfium-rs/ /usr/local/lib/pdfium-rs/
-# Make the CLI executable + globally available via symlinks
-RUN chmod +x /usr/local/lib/liteparse/dist/src/index.js
-RUN ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/lit \
- && ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/liteparse
+# Ensure pdfium is discoverable at runtime
+ENV LD_LIBRARY_PATH="/usr/local/lib/pdfium-rs"
+RUN ln -s /usr/local/bin/lit /usr/local/bin/liteparse
CMD ["/bin/sh"]
diff --git a/full.Dockerfile b/full.Dockerfile
index 2e0a5a7..60d14e0 100644
--- a/full.Dockerfile
+++ b/full.Dockerfile
@@ -1,36 +1,37 @@
-# Stage 1: Build from source (needs build tools for native addons like sharp, pdfium)
-FROM node:24-trixie AS builder
+# Stage 1: Build from source
+FROM rust:1-bookworm AS builder
+
+RUN apt-get update && apt-get install -y \
+ libclang-dev \
+ libtesseract-dev \
+ libleptonica-dev \
+ && rm -rf /var/lib/apt/lists/*
WORKDIR /build
-COPY package.json package-lock.json ./
-RUN npm ci --ignore-scripts=false
+COPY Cargo.toml Cargo.lock ./
+COPY crates/ ./crates/
-COPY src/ ./src/
-COPY cli/ ./cli/
-COPY tsconfig.json ./
-RUN npm run build
+RUN cargo build --release
-# Stage 2: Minimal runtime image
-FROM node:24-trixie-slim
+# Stage 2: Runtime image with conversion tools
+FROM debian:bookworm-slim
-# sharp and @hyzyla/pdfium ship prebuilt binaries but may need these shared libs at runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
- libvips42 \
+ libtesseract5 \
+ libleptonica6 \
+ tesseract-ocr-eng \
ca-certificates \
libreoffice \
imagemagick \
&& rm -rf /var/lib/apt/lists/*
-WORKDIR /usr/local/lib/liteparse
-COPY --from=builder /build/package.json ./
-COPY --from=builder /build/dist/ ./dist/
-COPY --from=builder /build/src/vendor/pdfjs ./src/vendor/pdfjs/
-COPY --from=builder /build/node_modules/ ./node_modules/
+COPY --from=builder /build/target/release/lit /usr/local/bin/lit
+# pdfium shared library
+COPY --from=builder /root/.cache/pdfium-rs/ /usr/local/lib/pdfium-rs/
-# Make the CLI executable + globally available via symlinks
-RUN chmod +x /usr/local/lib/liteparse/dist/src/index.js
-RUN ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/lit \
- && ln -s /usr/local/lib/liteparse/dist/src/index.js /usr/local/bin/liteparse
+# Ensure pdfium is discoverable at runtime
+ENV LD_LIBRARY_PATH="/usr/local/lib/pdfium-rs"
+RUN ln -s /usr/local/bin/lit /usr/local/bin/liteparse
CMD ["/bin/sh"]