[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat: use manylinux2014 containers for better Linux compatibility
Andy Lee6df7893173ee
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.ymlindex dfdbea2..6c804b7 100644--- a/.github/workflows/build-reusable.yml+++ b/.github/workflows/build-reusable.yml@@ -17,21 +17,30 @@ jobs:include:- os: ubuntu-latestpython: '3.9'+ container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.10'+ container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.11'+ container: 'quay.io/pypa/manylinux2014_x86_64'- os: ubuntu-latestpython: '3.12'+ container: 'quay.io/pypa/manylinux2014_x86_64'- os: macos-latestpython: '3.9'+ container: ''- os: macos-latestpython: '3.10'+ container: ''- os: macos-latestpython: '3.11'+ container: ''- os: macos-latestpython: '3.12'+ container: ''runs-on: ${{ matrix.os }}+ container: ${{ matrix.container }}steps:- uses: actions/checkout@v4@@ -39,16 +48,35 @@ jobs:ref: ${{ inputs.ref }}submodules: recursive- - name: Setup Python+ - name: Setup Python (macOS and regular Ubuntu)+ if: matrix.container == ''uses: actions/setup-python@v5with:python-version: ${{ matrix.python }}- - name: Install uv+ - name: Setup Python (manylinux container)+ if: matrix.container != ''+ run: |+ # Use the pre-installed Python version in manylinux container+ # Convert Python version format (3.9 -> 39, 3.10 -> 310, etc.)+ PY_VER=$(echo "${{ matrix.python }}" | sed 's/\.//g')+ /opt/python/cp${PY_VER}-*/bin/python -m pip install --upgrade pip+ # Create symlinks for convenience+ ln -sf /opt/python/cp${PY_VER}-*/bin/python /usr/local/bin/python+ ln -sf /opt/python/cp${PY_VER}-*/bin/pip /usr/local/bin/pip++ - name: Install uv (macOS and regular Ubuntu)+ if: matrix.container == ''uses: astral-sh/setup-uv@v4- - name: Install system dependencies (Ubuntu)- if: runner.os == 'Linux'+ - name: Install uv (manylinux container)+ if: matrix.container != ''+ run: |+ curl -LsSf https://astral.sh/uv/install.sh | sh+ echo "$HOME/.cargo/bin" >> $GITHUB_PATH++ - name: Install system dependencies (Ubuntu - regular)+ if: runner.os == 'Linux' && matrix.container == ''run: |sudo apt-get updatesudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \@@ -61,6 +89,17 @@ jobs:echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENVecho "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV+ - name: Install system dependencies (manylinux container)+ if: runner.os == 'Linux' && matrix.container != ''+ run: |+ # manylinux2014 uses yum instead of apt+ yum install -y epel-release+ yum install -y boost-devel protobuf-compiler zeromq-devel \+ pkg-config openblas-devel libaio-devel protobuf-devel++ # Build tools are pre-installed in manylinux+ # MKL is more complex in container, skip for now and use OpenBLAS+- name: Install system dependencies (macOS)if: runner.os == 'macOS'run: |@@ -68,44 +107,65 @@ jobs:- name: Install build dependenciesrun: |- uv pip install --system scikit-build-core numpy swig Cython pybind11- if [[ "$RUNNER_OS" == "Linux" ]]; then- uv pip install --system auditwheel+ if [[ -n "${{ matrix.container }}" ]]; then+ # In manylinux container, use regular pip+ pip install scikit-build-core numpy swig Cython pybind11 auditwheelelse- uv pip install --system delocate+ # Regular environment, use uv+ uv pip install --system scikit-build-core numpy swig Cython pybind11+ if [[ "$RUNNER_OS" == "Linux" ]]; then+ uv pip install --system auditwheel+ else+ uv pip install --system delocate+ fifi- name: Build packagesrun: |+ # Choose build command based on environment+ if [[ -n "${{ matrix.container }}" ]]; then+ BUILD_CMD="pip wheel . --no-deps -w dist"+ else+ BUILD_CMD="uv build --wheel --python python"+ fi+# Build core (platform independent)if [ "${{ matrix.os }}" == "ubuntu-latest" ]; thencd packages/leann-core- uv build+ if [[ -n "${{ matrix.container }}" ]]; then+ pip wheel . --no-deps -w dist+ else+ uv build+ ficd ../..fi# Build HNSW backendcd packages/leann-backend-hnswif [ "${{ matrix.os }}" == "macos-latest" ]; then- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMDelse- uv build --wheel --python python+ eval $BUILD_CMDficd ../..# Build DiskANN backendcd packages/leann-backend-diskannif [ "${{ matrix.os }}" == "macos-latest" ]; then- CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv build --wheel --python python+ CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ $BUILD_CMDelse- uv build --wheel --python python+ eval $BUILD_CMDficd ../..# Build meta package (platform independent)if [ "${{ matrix.os }}" == "ubuntu-latest" ]; thencd packages/leann- uv build+ if [[ -n "${{ matrix.container }}" ]]; then+ pip wheel . --no-deps -w dist+ else+ uv build+ ficd ../..fi