Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

refactor: use reusable workflow to avoid code duplication

Andy Leee8d2ecab031e
diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml
index c19ed2f..d8689ff 100644
--- a/.github/workflows/build-and-publish.yml
+++ b/.github/workflows/build-and-publish.yml
@@ -1,262 +1,11 @@
-name: CI - Build Multi-Platform Packages
+name: CI
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
- workflow_dispatch:
- inputs:
- publish:
- description: 'Publish to PyPI (only use for emergency fixes)'
- required: true
- default: 'false'
- type: choice
- options:
- - 'false'
- - 'test'
- - 'prod'
jobs:
- # Build pure Python package: leann-core
- build-core:
- name: Build leann-core
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.11'
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Install build dependencies
- run: |
- uv pip install --system build twine
-
- - name: Build package
- run: |
- cd packages/leann-core
- uv build
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: leann-core-dist
- path: packages/leann-core/dist/
-
- # Build binary package: leann-backend-hnsw (default backend)
- build-hnsw:
- name: Build leann-backend-hnsw
- strategy:
- matrix:
- os: [ubuntu-latest, macos-latest]
- python-version: ['3.9', '3.10', '3.11', '3.12']
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- with:
- submodules: recursive
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Install system dependencies (Ubuntu)
- if: runner.os == 'Linux'
- run: |
- sudo apt-get update
- sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \
- pkg-config libopenblas-dev patchelf
-
- - name: Install system dependencies (macOS)
- if: runner.os == 'macOS'
- run: |
- brew install libomp boost zeromq
-
- - name: Install build dependencies
- run: |
- uv pip install --system scikit-build-core numpy swig
- uv pip install --system auditwheel delocate
-
- - name: Build wheel
- run: |
- cd packages/leann-backend-hnsw
- uv build --wheel --python python
-
- - name: Repair wheel (Linux)
- if: runner.os == 'Linux'
- run: |
- cd packages/leann-backend-hnsw
- auditwheel repair dist/*.whl -w dist_repaired
- rm -rf dist
- mv dist_repaired dist
-
- - name: Repair wheel (macOS)
- if: runner.os == 'macOS'
- run: |
- cd packages/leann-backend-hnsw
- delocate-wheel -w dist_repaired -v dist/*.whl
- rm -rf dist
- mv dist_repaired dist
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}
- path: packages/leann-backend-hnsw/dist/
-
- # Build binary package: leann-backend-diskann (multi-platform)
- build-diskann:
- name: Build leann-backend-diskann
- strategy:
- matrix:
- os: [ubuntu-latest, macos-latest]
- python-version: ['3.9', '3.10', '3.11', '3.12']
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- with:
- submodules: recursive
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Install system dependencies (Ubuntu)
- if: runner.os == 'Linux'
- run: |
- sudo apt-get update
- sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \
- protobuf-compiler libprotobuf-dev libabsl-dev patchelf
-
- # Install Intel MKL using Intel's installer
- wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
- sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
- source /opt/intel/oneapi/setvars.sh
- echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
- echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
-
- - name: Install system dependencies (macOS)
- if: runner.os == 'macOS'
- run: |
- brew install libomp boost zeromq protobuf
- # MKL is not available on Homebrew, but DiskANN can work without it
-
- - name: Install build dependencies
- run: |
- uv pip install --system scikit-build-core numpy Cython pybind11
- if [[ "$RUNNER_OS" == "Linux" ]]; then
- uv pip install --system auditwheel
- else
- uv pip install --system delocate
- fi
-
- - name: Build wheel
- run: |
- cd packages/leann-backend-diskann
- uv build --wheel --python python
-
- - name: Repair wheel (Linux)
- if: runner.os == 'Linux'
- run: |
- cd packages/leann-backend-diskann
- auditwheel repair dist/*.whl -w dist_repaired
- rm -rf dist
- mv dist_repaired dist
-
- - name: Repair wheel (macOS)
- if: runner.os == 'macOS'
- run: |
- cd packages/leann-backend-diskann
- delocate-wheel -w dist_repaired -v dist/*.whl
- rm -rf dist
- mv dist_repaired dist
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}
- path: packages/leann-backend-diskann/dist/
-
- # Build meta-package: leann (build last)
- build-meta:
- name: Build leann meta-package
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.11'
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Install build dependencies
- run: |
- uv pip install --system build
-
- - name: Build package
- run: |
- cd packages/leann
- uv build
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: leann-meta-dist
- path: packages/leann/dist/
-
- # Publish to PyPI (only for emergency fixes or manual triggers)
- publish:
- name: Publish to PyPI (Emergency)
- needs: [build-core, build-hnsw, build-diskann, build-meta]
- runs-on: ubuntu-latest
- if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'false'
-
- steps:
- - name: Download all artifacts
- uses: actions/download-artifact@v4
- with:
- path: dist
-
- - name: Flatten directory structure
- run: |
- mkdir -p all_wheels
- find dist -name "*.whl" -exec cp {} all_wheels/ \;
- find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;
-
- - name: Show what will be published
- run: |
- echo "📦 Packages to be published:"
- ls -la all_wheels/
-
- - name: Publish to Test PyPI
- if: github.event.inputs.publish == 'test'
- uses: pypa/gh-action-pypi-publish@release/v1
- with:
- password: ${{ secrets.TEST_PYPI_API_TOKEN }}
- repository-url: https://test.pypi.org/legacy/
- packages-dir: all_wheels/
- skip-existing: true
-
- - name: Publish to PyPI
- if: github.event.inputs.publish == 'prod'
- uses: pypa/gh-action-pypi-publish@release/v1
- with:
- password: ${{ secrets.PYPI_API_TOKEN }}
- packages-dir: all_wheels/
- skip-existing: true
\ No newline at end of file
+ build:
+ uses: ./.github/workflows/build-reusable.yml
\ No newline at end of file
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml
new file mode 100644
index 0000000..76fbf2d
--- /dev/null
+++ b/.github/workflows/build-reusable.yml
@@ -0,0 +1,92 @@
+name: Reusable Build
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ description: 'Git ref to build'
+ required: false
+ type: string
+ default: ''
+
+jobs:
+ build:
+ name: Build ${{ matrix.os }}
+ strategy:
+ matrix:
+ include:
+ - os: ubuntu-latest
+ python: '3.11'
+ - os: macos-latest
+ python: '3.11'
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.ref }}
+ submodules: recursive
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+
+ - name: Install system dependencies (Ubuntu)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev
+
+ - name: Install system dependencies (macOS)
+ if: runner.os == 'macOS'
+ run: |
+ brew install llvm libomp boost protobuf zeromq
+
+ - name: Build packages
+ run: |
+ # Build core (platform independent)
+ if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+ cd packages/leann-core
+ uv build
+ cd ../..
+ fi
+
+ # Build HNSW backend
+ cd packages/leann-backend-hnsw
+ uv pip install --system -r pyproject.toml --extra build
+ if [ "${{ matrix.os }}" == "macos-latest" ]; then
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
+ else
+ uv build
+ fi
+ cd ../..
+
+ # Build DiskANN backend
+ cd packages/leann-backend-diskann
+ uv pip install --system -r pyproject.toml --extra build
+ if [ "${{ matrix.os }}" == "macos-latest" ]; then
+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
+ else
+ uv build
+ fi
+ cd ../..
+
+ # Build meta package (platform independent)
+ if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+ cd packages/leann
+ uv build
+ cd ../..
+ fi
+
+ echo "📦 Built packages:"
+ find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: packages-${{ matrix.os }}-${{ matrix.python }}
+ path: packages/*/dist/
\ No newline at end of file
diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml
index e0f9d47..27d0f25 100644
--- a/.github/workflows/release-manual.yml
+++ b/.github/workflows/release-manual.yml
@@ -45,79 +45,9 @@ jobs:
build-packages:
name: Build packages
needs: update-version
- strategy:
- matrix:
- include:
- - os: ubuntu-latest
- python: '3.11'
- - os: macos-latest
- python: '3.11'
- runs-on: ${{ matrix.os }}
-
- steps:
- - uses: actions/checkout@v4
- with:
- ref: ${{ needs.update-version.outputs.commit-sha }}
- submodules: recursive
-
- - name: Setup Python
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python }}
-
- - name: Install uv
- uses: astral-sh/setup-uv@v4
-
- - name: Install system dependencies (Ubuntu)
- if: runner.os == 'Linux'
- run: |
- sudo apt-get update
- sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev
-
- - name: Install system dependencies (macOS)
- if: runner.os == 'macOS'
- run: |
- brew install llvm libomp boost protobuf zeromq
-
- - name: Build packages
- run: |
- # Build core (platform independent)
- if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
- cd packages/leann-core
- uv build
- cd ../..
- fi
-
- # Build HNSW backend
- cd packages/leann-backend-hnsw
- uv pip install --system -r pyproject.toml --extra build
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
- cd ../..
-
- # Build DiskANN backend
- cd packages/leann-backend-diskann
- uv pip install --system -r pyproject.toml --extra build
- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build
- cd ../..
-
- # Build meta package (platform independent)
- if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
- cd packages/leann
- uv build
- cd ../..
- fi
-
- echo "📦 Built packages:"
- find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
- env:
- CC: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang' || '' }}
- CXX: ${{ runner.os == 'macOS' && '$(brew --prefix llvm)/bin/clang++' || '' }}
-
- - name: Upload artifacts
- uses: actions/upload-artifact@v4
- with:
- name: packages-${{ matrix.os }}-${{ matrix.python }}
- path: packages/*/dist/
+ uses: ./.github/workflows/build-reusable.yml
+ with:
+ ref: ${{ needs.update-version.outputs.commit-sha }}
publish:
name: Publish and Release