Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

revert: simplify build system by removing manylinux/cibuildwheel

Andy Lee4aef1d814e04
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml
index 8ab5bb2..c2e7958 100644
--- a/.github/workflows/build-reusable.yml
+++ b/.github/workflows/build-reusable.yml
@@ -13,107 +13,42 @@ jobs:
build:
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
strategy:
- fail-fast: false
matrix:
include:
- os: ubuntu-latest
python: '3.9'
- container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.10'
- container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.11'
- container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.12'
- container: 'quay.io/pypa/manylinux2014_x86_64'
- - os: ubuntu-latest
- python: '3.13'
- container: 'quay.io/pypa/manylinux2014_x86_64'
- os: macos-latest
python: '3.9'
- container: ''
- os: macos-latest
python: '3.10'
- container: ''
- os: macos-latest
python: '3.11'
- container: ''
- os: macos-latest
python: '3.12'
- container: ''
- - os: macos-latest
- python: '3.13'
- container: ''
runs-on: ${{ matrix.os }}
- container: ${{ matrix.container }}
steps:
- # For manylinux2014 compatibility, we'll handle checkout differently
- uses: actions/checkout@v4
- if: matrix.container == ''
with:
ref: ${{ inputs.ref }}
submodules: recursive
- # Manual checkout for containers to avoid Node.js compatibility issues
- - name: Manual checkout in container
- if: matrix.container != ''
- run: |
- # Install git if not available
- yum install -y git || true
-
- # Configure git to handle the directory ownership issue
- git config --global --add safe.directory ${GITHUB_WORKSPACE}
- git config --global --add safe.directory /__w/LEANN/LEANN
- git config --global --add safe.directory /github/workspace
- git config --global --add safe.directory $(pwd)
-
- # Clone the repository manually in the container
- git init
- git remote add origin https://github.com/${GITHUB_REPOSITORY}.git
-
- # Fetch the appropriate ref
- if [ -n "${{ inputs.ref }}" ]; then
- git fetch --depth=1 origin ${{ inputs.ref }}
- else
- git fetch --depth=1 origin ${GITHUB_SHA}
- fi
- git checkout FETCH_HEAD
-
- # Initialize submodules
- git submodule update --init --recursive
-
- - name: Setup Python (macOS and regular Ubuntu)
- if: matrix.container == ''
+ - name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- - name: Setup Python (manylinux container)
- if: matrix.container != ''
- run: |
- # Use the pre-installed Python version in manylinux container
- # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
- PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
- /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
- # Create symlinks for convenience
- ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
- ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
-
- - name: Install uv (macOS and regular Ubuntu)
- if: matrix.container == ''
+ - name: Install uv
uses: astral-sh/setup-uv@v4
- - name: Install uv (manylinux container)
- if: matrix.container != ''
- run: |
- curl -LsSf https://astral.sh/uv/install.sh | sh
- echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-
- - name: Install system dependencies (Ubuntu - regular)
- if: runner.os == 'Linux' && matrix.container == ''
+ - name: Install system dependencies (Ubuntu)
+ if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
@@ -126,64 +61,6 @@ jobs:
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
- - name: Install system dependencies (manylinux container)
- if: runner.os == 'Linux' && matrix.container != ''
- run: |
- # manylinux2014 uses yum instead of apt
- # Update yum cache first
- yum clean all
- yum makecache
-
- # Install EPEL repository
- yum install -y epel-release || true
-
- # Update cache again after EPEL
- yum makecache || true
-
- # Install development packages
- # Note: Some packages might have different names in CentOS 7
- yum install -y \
- gcc-c++ \
- boost-devel \
- protobuf-compiler \
- protobuf-devel \
- zeromq-devel \
- pkgconfig \
- openblas-devel \
- cmake || {
- echo "Some packages failed to install, trying alternatives..."
- # Try alternative package names
- yum install -y libzmq3-devel || true
- yum install -y libzmq-devel || true
- }
-
- # Install optional packages that might not be available
- yum install -y libaio-devel || echo "libaio-devel not available, continuing..."
-
- # Verify zmq installation and create pkg-config file if needed
- if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then
- echo "Creating libzmq.pc file..."
- mkdir -p /usr/lib64/pkgconfig
- cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'
- prefix=/usr
- exec_prefix=${prefix}
- libdir=${exec_prefix}/lib64
- includedir=${prefix}/include
-
- Name: libzmq
- Description: ZeroMQ library
- Version: 4.1.4
- Libs: -L${libdir} -lzmq
- Cflags: -I${includedir}
- EOF
- fi
-
- # Update PKG_CONFIG_PATH
- echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV
-
- # Build tools are pre-installed in manylinux
- # MKL is more complex in container, skip for now and use OpenBLAS
-
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
@@ -191,65 +68,44 @@ jobs:
- name: Install build dependencies
run: |
- if [[ -n "${{ matrix.container }}" ]]; then
- # In manylinux container, use regular pip
- pip install scikit-build-core numpy swig Cython pybind11 auditwheel
+ uv pip install --system scikit-build-core numpy swig Cython pybind11
+ if [[ "$RUNNER_OS" == "Linux" ]]; then
+ uv pip install --system auditwheel
else
- # Regular environment, use uv
- uv pip install --system scikit-build-core numpy swig Cython pybind11
- if [[ "$RUNNER_OS" == "Linux" ]]; then
- uv pip install --system auditwheel
- else
- uv pip install --system delocate
- fi
+ uv pip install --system delocate
fi
- name: Build packages
run: |
- # Choose build command based on environment
- if [[ -n "${{ matrix.container }}" ]]; then
- BUILD_CMD="pip wheel . --no-deps -w dist"
- else
- BUILD_CMD="uv build --wheel --python python"
- fi
-
# Build core (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann-core
- if [[ -n "${{ matrix.container }}" ]]; then
- pip wheel . --no-deps -w dist
- else
- uv build
- fi
+ uv build
cd ../..
fi
# Build HNSW backend
cd packages/leann-backend-hnsw
if [ "${{ matrix.os }}" == "macos-latest" ]; then
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
else
- eval $BUILD_CMD
+ uv build --wheel --python python
fi
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
if [ "${{ matrix.os }}" == "macos-latest" ]; then
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
else
- eval $BUILD_CMD
+ uv build --wheel --python python
fi
cd ../..
# Build meta package (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann
- if [[ -n "${{ matrix.container }}" ]]; then
- pip wheel . --no-deps -w dist
- else
- uv build
- fi
+ uv build
cd ../..
fi
@@ -259,9 +115,6 @@ jobs:
# Repair HNSW wheel
cd packages/leann-backend-hnsw
if [ -d dist ]; then
- # Show what platform auditwheel will use
- auditwheel show dist/*.whl || true
- # Let auditwheel auto-detect the appropriate manylinux tag
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
@@ -271,9 +124,6 @@ jobs:
# Repair DiskANN wheel
cd packages/leann-backend-diskann
if [ -d dist ]; then
- # Show what platform auditwheel will use
- auditwheel show dist/*.whl || true
- # Let auditwheel auto-detect the appropriate manylinux tag
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
@@ -309,5 +159,5 @@ jobs:
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
- name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }}
+ name: packages-${{ matrix.os }}-py${{ matrix.python }}
path: packages/*/dist/
\ No newline at end of file
diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml
index 9a61b86..027489d 100644
--- a/.github/workflows/release-manual.yml
+++ b/.github/workflows/release-manual.yml
@@ -7,11 +7,6 @@ on:
description: 'Version to release (e.g., 0.1.2)'
required: true
type: string
- use_cibuildwheel:
- description: 'Use cibuildwheel for better compatibility (recommended for Colab)'
- required: false
- type: boolean
- default: false
jobs:
update-version:
@@ -47,26 +42,17 @@ jobs:
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
echo "✅ Pushed version update: $COMMIT_SHA"
- build-packages-reusable:
- name: Build packages (Standard)
+ build-packages:
+ name: Build packages
needs: update-version
- if: ${{ !inputs.use_cibuildwheel }}
uses: ./.github/workflows/build-reusable.yml
with:
ref: ${{ needs.update-version.outputs.commit-sha }}
-
- build-packages-cibuildwheel:
- name: Build packages (cibuildwheel)
- needs: update-version
- if: ${{ inputs.use_cibuildwheel }}
- uses: ./.github/workflows/build-cibuildwheel.yml
- with:
- ref: ${{ needs.update-version.outputs.commit-sha }}
publish:
name: Publish and Release
- needs: [update-version, build-packages-reusable, build-packages-cibuildwheel]
- if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success')
+ needs: [update-version, build-packages]
+ if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'
runs-on: ubuntu-latest
permissions:
contents: write