[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
refactor: use reusable workflow to avoid code duplication
Andy Leee8d2ecab031e
diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.ymlindex c19ed2f..d8689ff 100644--- a/.github/workflows/build-and-publish.yml+++ b/.github/workflows/build-and-publish.yml@@ -1,262 +1,11 @@-name: CI - Build Multi-Platform Packages+name: CIon:push:branches: [ main ]pull_request:branches: [ main ]- workflow_dispatch:- inputs:- publish:- description: 'Publish to PyPI (only use for emergency fixes)'- required: true- default: 'false'- type: choice- options:- - 'false'- - 'test'- - 'prod'jobs:- # Build pure Python package: leann-core- build-core:- name: Build leann-core- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v4-- - name: Set up Python- uses: actions/setup-python@v5- with:- python-version: '3.11'-- - name: Install uv- uses: astral-sh/setup-uv@v4-- - name: Install build dependencies- run: |- uv pip install --system build twine-- - name: Build package- run: |- cd packages/leann-core- uv build-- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: leann-core-dist- path: packages/leann-core/dist/-- # Build binary package: leann-backend-hnsw (default backend)- build-hnsw:- name: Build leann-backend-hnsw- strategy:- matrix:- os: [ubuntu-latest, macos-latest]- python-version: ['3.9', '3.10', '3.11', '3.12']- runs-on: ${{ matrix.os }}- steps:- - uses: actions/checkout@v4- with:- submodules: recursive-- - name: Set up Python- uses: actions/setup-python@v5- with:- python-version: ${{ matrix.python-version }}-- - name: Install uv- uses: astral-sh/setup-uv@v4-- - name: Install system dependencies (Ubuntu)- if: runner.os == 'Linux'- run: |- sudo apt-get update- sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \- pkg-config libopenblas-dev patchelf-- - name: Install system dependencies (macOS)- if: runner.os == 'macOS'- run: |- brew install libomp boost zeromq-- - name: Install build dependencies- run: |- uv pip install --system scikit-build-core numpy swig- uv pip install --system auditwheel delocate-- - name: Build wheel- run: |- cd packages/leann-backend-hnsw- uv build --wheel --python python-- - name: Repair wheel (Linux)- if: runner.os == 'Linux'- run: |- cd packages/leann-backend-hnsw- auditwheel repair dist/*.whl -w dist_repaired- rm -rf dist- mv dist_repaired dist-- - name: Repair wheel (macOS)- if: runner.os == 'macOS'- run: |- cd packages/leann-backend-hnsw- delocate-wheel -w dist_repaired -v dist/*.whl- rm -rf dist- mv dist_repaired dist-- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}- path: packages/leann-backend-hnsw/dist/-- # Build binary package: leann-backend-diskann (multi-platform)- build-diskann:- name: Build leann-backend-diskann- strategy:- matrix:- os: [ubuntu-latest, macos-latest]- python-version: ['3.9', '3.10', '3.11', '3.12']- runs-on: ${{ matrix.os }}- steps:- - uses: actions/checkout@v4- with:- submodules: recursive-- - name: Set up Python- uses: actions/setup-python@v5- with:- python-version: ${{ matrix.python-version }}-- - name: Install uv- uses: astral-sh/setup-uv@v4-- - name: Install system dependencies (Ubuntu)- if: runner.os == 'Linux'- run: |- sudo apt-get update- sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \- protobuf-compiler libprotobuf-dev libabsl-dev patchelf-- # Install Intel MKL using Intel's installer- wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh- sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s- source /opt/intel/oneapi/setvars.sh- echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV- echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV-- - name: Install system dependencies (macOS)- if: runner.os == 'macOS'- run: |- brew install libomp boost zeromq protobuf- # MKL is not available on Homebrew, but DiskANN can work without it-- - name: Install build dependencies- run: |- uv pip install --system scikit-build-core numpy Cython pybind11- if [[ "$RUNNER_OS" == "Linux" ]]; then- uv pip install --system auditwheel- else- uv pip install --system delocate- fi-- - name: Build wheel- run: |- cd packages/leann-backend-diskann- uv build --wheel --python python-- - name: Repair wheel (Linux)- if: runner.os == 'Linux'- run: |- cd packages/leann-backend-diskann- auditwheel repair dist/*.whl -w dist_repaired- rm -rf dist- mv dist_repaired dist-- - name: Repair wheel (macOS)- if: runner.os == 'macOS'- run: |- cd packages/leann-backend-diskann- delocate-wheel -w dist_repaired -v dist/*.whl- rm -rf dist- mv dist_repaired dist-- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}- path: packages/leann-backend-diskann/dist/-- # Build meta-package: leann (build last)- build-meta:- name: Build leann meta-package- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v4-- - name: Set up Python- uses: actions/setup-python@v5- with:- python-version: '3.11'-- - name: Install uv- uses: astral-sh/setup-uv@v4-- - name: Install build dependencies- run: |- uv pip install --system build-- - name: Build package- run: |- cd packages/leann- uv build-- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: leann-meta-dist- path: packages/leann/dist/-- # Publish to PyPI (only for emergency fixes or manual triggers)- publish:- name: Publish to PyPI (Emergency)- needs: [build-core, build-hnsw, build-diskann, build-meta]- runs-on: ubuntu-latest- if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false'-- steps:- - name: Download all artifacts- uses: actions/download-artifact@v4- with:- path: dist-- - name: Flatten directory structure- run: |- mkdir -p all_wheels- find dist -name "*.whl" -exec cp {} all_wheels/ \;- find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;-- - name: Show what will be published- run: |- echo "📦 Packages to be published:"- ls -la all_wheels/-- - name: Publish to Test PyPI- if: github.event.inputs.publish == 'test'- uses: pypa/gh-action-pypi-publish@release/v1- with:- password: ${{ secrets.TEST_PYPI_API_TOKEN }}- repository-url: https://test.pypi.org/legacy/- packages-dir: all_wheels/- skip-existing: true-- - name: Publish to PyPI- if: github.event.inputs.publish == 'prod'- uses: pypa/gh-action-pypi-publish@release/v1- with:- password: ${{ secrets.PYPI_API_TOKEN }}- packages-dir: all_wheels/- skip-existing: true\ No newline at end of file+ build:+ uses: ./.github/workflows/build-reusable.yml\ No newline at end of filediff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.ymlnew file mode 100644index 0000000..76fbf2d--- /dev/null+++ b/.github/workflows/build-reusable.yml@@ -0,0 +1,92 @@+name: Reusable Build++on:+ workflow_call:+ inputs:+ ref:+ description: 'Git ref to build'+ required: false+ type: string+ default: ''++jobs:+ build:+ name: Build ${{ matrix.os }}+ strategy:+ matrix:+ include:+ - os: ubuntu-latest+ python: '3.11'+ - os: macos-latest+ python: '3.11'+ runs-on: ${{ matrix.os }}++ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ inputs.ref }}+ submodules: recursive++ - name: Setup Python+ uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python }}++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Install system dependencies (Ubuntu)+ if: runner.os == 'Linux'+ run: |+ sudo apt-get update+ sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev++ - name: Install system dependencies (macOS)+ if: runner.os == 'macOS'+ run: |+ brew install llvm libomp boost protobuf zeromq++ - name: Build packages+ run: |+ # Build core (platform independent)+ if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then+ cd packages/leann-core+ uv build+ cd ../..+ fi++ # Build HNSW backend+ cd packages/leann-backend-hnsw+ uv pip install --system -r pyproject.toml --extra build+ if [ "${{ matrix.os }}" == "macos-latest" ]; then+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build+ else+ uv build+ fi+ cd ../..++ # Build DiskANN backend+ cd packages/leann-backend-diskann+ uv pip install --system -r pyproject.toml --extra build+ if [ "${{ matrix.os }}" == "macos-latest" ]; then+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build+ else+ uv build+ fi+ cd ../..++ # Build meta package (platform independent)+ if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then+ cd packages/leann+ uv build+ cd ../..+ fi++ echo "📦 Built packages:"+ find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort++ - name: Upload artifacts+ uses: actions/upload-artifact@v4+ with:+ name: packages-${{ matrix.os }}-${{ matrix.python }}+ path: packages/*/dist/\ No newline at end of filediff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.ymlindex e0f9d47..27d0f25 100644--- a/.github/workflows/release-manual.yml+++ b/.github/workflows/release-manual.yml@@ -45,79 +45,9 @@ jobs:build-packages:name: Build packagesneeds: update-version- strategy:- matrix:- include:- - os: ubuntu-latest- python: '3.11'- - os: macos-latest- python: '3.11'- runs-on: ${{ matrix.os }}-- steps:- - uses: actions/checkout@v4- with:- ref: ${{ needs.update-version.outputs.commit-sha }}- submodules: recursive-- - name: Setup Python- uses: actions/setup-python@v5- with:- python-version: ${{ matrix.python }}-- - name: Install uv- uses: astral-sh/setup-uv@v4-- - name: Install system dependencies (Ubuntu)- if: runner.os == 'Linux'- run: |- sudo apt-get update- sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev-- - name: Install system dependencies (macOS)- if: runner.os == 'macOS'- run: |- brew install llvm libomp boost protobuf zeromq-- - name: Build packages- run: |- # Build core (platform independent)- if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then- cd packages/leann-core- uv build- cd ../..- fi-- # Build HNSW backend- cd packages/leann-backend-hnsw- uv pip install --system -r pyproject.toml --extra build- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build- cd ../..-- # Build DiskANN backend- cd packages/leann-backend-diskann- uv pip install --system -r pyproject.toml --extra build- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build- cd ../..-- # Build meta package (platform independent)- if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then- cd packages/leann- uv build- cd ../..- fi-- echo "📦 Built packages:"- find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort- env:- CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }}- CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }}-- - name: Upload artifacts- uses: actions/upload-artifact@v4- with:- name: packages-${{ matrix.os }}-${{ matrix.python }}- path: packages/*/dist/+ uses: ./.github/workflows/build-reusable.yml+ with:+ ref: ${{ needs.update-version.outputs.commit-sha }}publish:name: Publish and Release