[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
revert: simplify build system by removing manylinux/cibuildwheel
Andy Lee4aef1d814e04
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.ymlindex 8ab5bb2..c2e7958 100644--- a/.github/workflows/build-reusable.yml+++ b/.github/workflows/build-reusable.yml@@ -13,107 +13,42 @@ jobs:build:name: Build ${{ matrix.os }} Python ${{ matrix.python }}strategy:- fail-fast: falsematrix:include:- os: ubuntu-latestpython: '3.9'- container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.10'- container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.11'- container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.12'- container: 'quay.io/pypa/manylinux2014_x86_64'- - os: ubuntu-latest- python: '3.13'- container: 'quay.io/pypa/manylinux2014_x86_64'- os: macos-latestpython: '3.9'- container: ''- os: macos-latestpython: '3.10'- container: ''- os: macos-latestpython: '3.11'- container: ''- os: macos-latestpython: '3.12'- container: ''- - os: macos-latest- python: '3.13'- container: ''runs-on: ${{ matrix.os }}- container: ${{ matrix.container }}steps:- # For manylinux2014 compatibility, we'll handle checkout differently- uses: actions/checkout@v4- if: matrix.container == ''with:ref: ${{ inputs.ref }}submodules: recursive- # Manual checkout for containers to avoid Node.js compatibility issues- - name: Manual checkout in container- if: matrix.container != ''- run: |- # Install git if not available- yum install -y git || true-- # Configure git to handle the directory ownership issue- git config --global --add safe.directory ${GITHUB_WORKSPACE}- git config --global --add safe.directory /__w/LEANN/LEANN- git config --global --add safe.directory /github/workspace- git config --global --add safe.directory $(pwd)-- # Clone the repository manually in the container- git init- git remote add origin https://github.com/${GITHUB_REPOSITORY}.git-- # Fetch the appropriate ref- if [ -n "${{ inputs.ref }}" ]; then- git fetch --depth=1 origin ${{ inputs.ref }}- else- git fetch --depth=1 origin ${GITHUB_SHA}- fi- git checkout FETCH_HEAD-- # Initialize submodules- git submodule update --init --recursive-- - name: Setup Python (macOS and regular Ubuntu)- if: matrix.container == ''+ - name: Setup Pythonuses: actions/setup-python@v5with:python-version: ${{ matrix.python }}- - name: Setup Python (manylinux container)- if: matrix.container != ''- run: |- # Use the pre-installed Python version in manylinux container- # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)- PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')- /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip- # Create symlinks for convenience- ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python- ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip-- - name: Install uv (macOS and regular Ubuntu)- if: matrix.container == ''+ - name: Install uvuses: astral-sh/setup-uv@v4- - name: Install uv (manylinux container)- if: matrix.container != ''- run: |- curl -LsSf https://astral.sh/uv/install.sh | sh- echo "$HOME/.cargo/bin" >> $GITHUB_PATH-- - name: Install system dependencies (Ubuntu - regular)- if: runner.os == 'Linux' && matrix.container == ''+ - name: Install system dependencies (Ubuntu)+ if: runner.os == 'Linux'run: |sudo apt-get updatesudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \@@ -126,64 +61,6 @@ jobs:echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENVecho "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV- - name: Install system dependencies (manylinux container)- if: runner.os == 'Linux' && matrix.container != ''- run: |- # manylinux2014 uses yum instead of apt- # Update yum cache first- yum clean all- yum makecache-- # Install EPEL repository- yum install -y epel-release || true-- # Update cache again after EPEL- yum makecache || true-- # Install development packages- # Note: Some packages might have different names in CentOS 7- yum install -y \- gcc-c++ \- boost-devel \- protobuf-compiler \- protobuf-devel \- zeromq-devel \- pkgconfig \- openblas-devel \- cmake || {- echo "Some packages failed to install, trying alternatives..."- # Try alternative package names- yum install -y libzmq3-devel || true- yum install -y libzmq-devel || true- }-- # Install optional packages that might not be available- yum install -y libaio-devel || echo "libaio-devel not available, continuing..."-- # Verify zmq installation and create pkg-config file if needed- if [ ! -f /usr/lib64/pkgconfig/libzmq.pc ] && [ ! -f /usr/lib/pkgconfig/libzmq.pc ]; then- echo "Creating libzmq.pc file..."- mkdir -p /usr/lib64/pkgconfig- cat > /usr/lib64/pkgconfig/libzmq.pc << 'EOF'- prefix=/usr- exec_prefix=${prefix}- libdir=${exec_prefix}/lib64- includedir=${prefix}/include-- Name: libzmq- Description: ZeroMQ library- Version: 4.1.4- Libs: -L${libdir} -lzmq- Cflags: -I${includedir}- EOF- fi-- # Update PKG_CONFIG_PATH- echo "PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH" >> $GITHUB_ENV-- # Build tools are pre-installed in manylinux- # MKL is more complex in container, skip for now and use OpenBLAS-- name: Install system dependencies (macOS)if: runner.os == 'macOS'run: |@@ -191,65 +68,44 @@ jobs:- name: Install build dependenciesrun: |- if [[ -n "${{ matrix.container }}" ]]; then- # In manylinux container, use regular pip- pip install scikit-build-core numpy swig Cython pybind11 auditwheel+ uv pip install --system scikit-build-core numpy swig Cython pybind11+ if [[ "$RUNNER_OS" == "Linux" ]]; then+ uv pip install --system auditwheelelse- # Regular environment, use uv- uv pip install --system scikit-build-core numpy swig Cython pybind11- if [[ "$RUNNER_OS" == "Linux" ]]; then- uv pip install --system auditwheel- else- uv pip install --system delocate- fi+ uv pip install --system delocatefi- name: Build packagesrun: |- # Choose build command based on environment- if [[ -n "${{ matrix.container }}" ]]; then- BUILD_CMD="pip wheel . --no-deps -w dist"- else- BUILD_CMD="uv build --wheel --python python"- fi-# Build core (platform independent)if [ "${{ matrix.os }}" == "ubuntu-latest" ]; thencd packages/leann-core- if [[ -n "${{ matrix.container }}" ]]; then- pip wheel . --no-deps -w dist- else- uv build- fi+ uv buildcd ../..fi# Build HNSW backendcd packages/leann-backend-hnswif [ "${{ matrix.os }}" == "macos-latest" ]; then- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python pythonelse- eval $BUILD_CMD+ uv build --wheel --python pythonficd ../..# Build DiskANN backendcd packages/leann-backend-diskannif [ "${{ matrix.os }}" == "macos-latest" ]; then- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMD+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python pythonelse- eval $BUILD_CMD+ uv build --wheel --python pythonficd ../..# Build meta package (platform independent)if [ "${{ matrix.os }}" == "ubuntu-latest" ]; thencd packages/leann- if [[ -n "${{ matrix.container }}" ]]; then- pip wheel . --no-deps -w dist- else- uv build- fi+ uv buildcd ../..fi@@ -259,9 +115,6 @@ jobs:# Repair HNSW wheelcd packages/leann-backend-hnswif [ -d dist ]; then- # Show what platform auditwheel will use- auditwheel show dist/*.whl || true- # Let auditwheel auto-detect the appropriate manylinux tagauditwheel repair dist/*.whl -w dist_repairedrm -rf distmv dist_repaired dist@@ -271,9 +124,6 @@ jobs:# Repair DiskANN wheelcd packages/leann-backend-diskannif [ -d dist ]; then- # Show what platform auditwheel will use- auditwheel show dist/*.whl || true- # Let auditwheel auto-detect the appropriate manylinux tagauditwheel repair dist/*.whl -w dist_repairedrm -rf distmv dist_repaired dist@@ -309,5 +159,5 @@ jobs:- name: Upload artifactsuses: actions/upload-artifact@v4with:- name: packages-${{ matrix.os }}-py${{ matrix.python }}${{ matrix.container && '-manylinux' || '' }}+ name: packages-${{ matrix.os }}-py${{ matrix.python }}path: packages/*/dist/\ No newline at end of filediff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.ymlindex 9a61b86..027489d 100644--- a/.github/workflows/release-manual.yml+++ b/.github/workflows/release-manual.yml@@ -7,11 +7,6 @@ on:description: 'Version to release (e.g., 0.1.2)'required: truetype: string- use_cibuildwheel:- description: 'Use cibuildwheel for better compatibility (recommended for Colab)'- required: false- type: boolean- default: falsejobs:update-version:@@ -47,26 +42,17 @@ jobs:echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUTecho "✅ Pushed version update: $COMMIT_SHA"- build-packages-reusable:- name: Build packages (Standard)+ build-packages:+ name: Build packagesneeds: update-version- if: ${{ !inputs.use_cibuildwheel }}uses: ./.github/workflows/build-reusable.ymlwith:ref: ${{ needs.update-version.outputs.commit-sha }}-- build-packages-cibuildwheel:- name: Build packages (cibuildwheel)- needs: update-version- if: ${{ inputs.use_cibuildwheel }}- uses: ./.github/workflows/build-cibuildwheel.yml- with:- ref: ${{ needs.update-version.outputs.commit-sha }}publish:name: Publish and Release- needs: [update-version, build-packages-reusable, build-packages-cibuildwheel]- if: always() && needs.update-version.result == 'success' && (needs.build-packages-reusable.result == 'success' || needs.build-packages-cibuildwheel.result == 'success')+ needs: [update-version, build-packages]+ if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'runs-on: ubuntu-latestpermissions:contents: write