Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: use manylinux2014 containers for better Linux compatibility

Andy Lee6df7893173ee
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml
index dfdbea2..6c804b7 100644
--- a/.github/workflows/build-reusable.yml
+++ b/.github/workflows/build-reusable.yml
@@ -17,21 +17,30 @@ jobs:
include:
- os: ubuntu-latest
python: '3.9'
+ container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.10'
+ container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.11'
+ container: 'quay.io/pypa/manylinux2014_x86_64'
- os: ubuntu-latest
python: '3.12'
+ container: 'quay.io/pypa/manylinux2014_x86_64'
- os: macos-latest
python: '3.9'
+ container: ''
- os: macos-latest
python: '3.10'
+ container: ''
- os: macos-latest
python: '3.11'
+ container: ''
- os: macos-latest
python: '3.12'
+ container: ''
runs-on: ${{ matrix.os }}
+ container: ${{ matrix.container }}
steps:
- uses: actions/checkout@v4
@@ -39,16 +48,35 @@ jobs:
ref: ${{ inputs.ref }}
submodules: recursive
- - name: Setup Python
+ - name: Setup Python (macOS and regular Ubuntu)
+ if: matrix.container == ''
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- - name: Install uv
+ - name: Setup Python (manylinux container)
+ if: matrix.container != ''
+ run: |
+ # Use the pre-installed Python version in manylinux container
+ # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)
+ PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')
+ /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip
+ # Create symlinks for convenience
+ ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python
+ ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip
+
+ - name: Install uv (macOS and regular Ubuntu)
+ if: matrix.container == ''
uses: astral-sh/setup-uv@v4
- - name: Install system dependencies (Ubuntu)
- if: runner.os == 'Linux'
+ - name: Install uv (manylinux container)
+ if: matrix.container != ''
+ run: |
+ curl -LsSf https://astral.sh/uv/install.sh | sh
+ echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+ - name: Install system dependencies (Ubuntu - regular)
+ if: runner.os == 'Linux' && matrix.container == ''
run: |
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
@@ -61,6 +89,17 @@ jobs:
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+ - name: Install system dependencies (manylinux container)
+ if: runner.os == 'Linux' && matrix.container != ''
+ run: |
+ # manylinux2014 uses yum instead of apt
+ yum install -y epel-release
+ yum install -y boost-devel protobuf-compiler zeromq-devel \
+ pkg-config openblas-devel libaio-devel protobuf-devel
+
+ # Build tools are pre-installed in manylinux
+ # MKL is more complex in container, skip for now and use OpenBLAS
+
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
@@ -68,44 +107,65 @@ jobs:
- name: Install build dependencies
run: |
- uv pip install --system scikit-build-core numpy swig Cython pybind11
- if [[ "$RUNNER_OS" == "Linux" ]]; then
- uv pip install --system auditwheel
+ if [[ -n "${{ matrix.container }}" ]]; then
+ # In manylinux container, use regular pip
+ pip install scikit-build-core numpy swig Cython pybind11 auditwheel
else
- uv pip install --system delocate
+ # Regular environment, use uv
+ uv pip install --system scikit-build-core numpy swig Cython pybind11
+ if [[ "$RUNNER_OS" == "Linux" ]]; then
+ uv pip install --system auditwheel
+ else
+ uv pip install --system delocate
+ fi
fi
- name: Build packages
run: |
+ # Choose build command based on environment
+ if [[ -n "${{ matrix.container }}" ]]; then
+ BUILD_CMD="pip wheel . --no-deps -w dist"
+ else
+ BUILD_CMD="uv build --wheel --python python"
+ fi
+
# Build core (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann-core
- uv build
+ if [[ -n "${{ matrix.container }}" ]]; then
+ pip wheel . --no-deps -w dist
+ else
+ uv build
+ fi
cd ../..
fi
# Build HNSW backend
cd packages/leann-backend-hnsw
if [ "${{ matrix.os }}" == "macos-latest" ]; then
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
else
- uv build --wheel --python python
+ eval $BUILD_CMD
fi
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
if [ "${{ matrix.os }}" == "macos-latest" ]; then
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD
else
- uv build --wheel --python python
+ eval $BUILD_CMD
fi
cd ../..
# Build meta package (platform independent)
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
cd packages/leann
- uv build
+ if [[ -n "${{ matrix.container }}" ]]; then
+ pip wheel . --no-deps -w dist
+ else
+ uv build
+ fi
cd ../..
fi