[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
chore: build and package
Andy Lee7add391b2cc7
diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.ymlnew file mode 100644index 0000000..43a6b15--- /dev/null+++ b/.github/workflows/build-and-publish.yml@@ -0,0 +1,256 @@+name: Build and Publish to PyPI++on:+ release:+ types: [published]+ push:+ tags:+ - 'v*'+ workflow_dispatch:+ inputs:+ publish:+ description: 'Publish to PyPI'+ required: true+ default: 'false'+ type: choice+ options:+ - 'false'+ - 'test'+ - 'prod'++jobs:+ # Build pure Python package: leann-core+ build-core:+ name: Build leann-core+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: '3.11'++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Install build dependencies+ run: |+ uv pip install --system build twine++ - name: Build package+ run: |+ cd packages/leann-core+ uv build++ - name: Upload artifacts+ uses: actions/upload-artifact@v4+ with:+ name: leann-core-dist+ path: packages/leann-core/dist/++ # Build binary package: leann-backend-hnsw (default backend)+ build-hnsw:+ name: Build leann-backend-hnsw+ strategy:+ matrix:+ os: [ubuntu-latest, macos-latest]+ python-version: ['3.9', '3.10', '3.11', '3.12']+ runs-on: ${{ matrix.os }}+ steps:+ - uses: actions/checkout@v4+ with:+ submodules: recursive++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python-version }}++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Install system dependencies (Ubuntu)+ if: runner.os == 'Linux'+ run: |+ sudo apt-get update+ sudo apt-get install -y libomp-dev libboost-all-dev libzmq3-dev \+ pkg-config patchelf++ - name: Install system dependencies (macOS)+ if: runner.os == 'macOS'+ run: |+ brew install libomp boost zeromq++ - name: Install build dependencies+ run: |+ uv pip install --system scikit-build-core numpy swig+ uv pip install --system auditwheel delocate++ - name: Build wheel+ run: |+ cd packages/leann-backend-hnsw+ uv pip wheel . --no-deps -w dist++ - name: Repair wheel (Linux)+ if: runner.os == 'Linux'+ run: |+ cd packages/leann-backend-hnsw+ auditwheel repair dist/*.whl -w dist_repaired+ rm -rf dist+ mv dist_repaired dist++ - name: Repair wheel (macOS)+ if: runner.os == 'macOS'+ run: |+ cd packages/leann-backend-hnsw+ delocate-wheel -w dist_repaired -v dist/*.whl+ rm -rf dist+ mv dist_repaired dist++ - name: Upload artifacts+ uses: actions/upload-artifact@v4+ with:+ name: hnsw-${{ matrix.os }}-py${{ matrix.python-version }}+ path: packages/leann-backend-hnsw/dist/++ # Build binary package: leann-backend-diskann (multi-platform)+ build-diskann:+ name: Build leann-backend-diskann+ strategy:+ matrix:+ os: [ubuntu-latest, macos-latest]+ python-version: ['3.9', '3.10', '3.11', '3.12']+ runs-on: ${{ matrix.os }}+ steps:+ - uses: actions/checkout@v4+ with:+ submodules: recursive++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python-version }}++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Install system dependencies (Ubuntu)+ if: runner.os == 'Linux'+ run: |+ sudo apt-get update+ sudo apt-get install -y libomp-dev libboost-all-dev libaio-dev libzmq3-dev \+ pkg-config protobuf-compiler libprotobuf-dev libabsl-dev patchelf++ # Install Intel MKL using Intel's installer+ wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh+ sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s+ source /opt/intel/oneapi/setvars.sh+ echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV+ echo "LD_LIBRARY_PATH=/opt/intel/oneapi/mkl/latest/lib/intel64:$LD_LIBRARY_PATH" >> $GITHUB_ENV++ - name: Install system dependencies (macOS)+ if: runner.os == 'macOS'+ run: |+ brew install libomp boost zeromq+ # MKL is not available on Homebrew, but DiskANN can work without it++ - name: Install build dependencies+ run: |+ uv pip install --system scikit-build-core numpy Cython pybind11+ if [[ "$RUNNER_OS" == "Linux" ]]; then+ uv pip install --system auditwheel+ else+ uv pip install --system delocate+ fi++ - name: Build wheel+ run: |+ cd packages/leann-backend-diskann+ uv pip wheel . --no-deps -w dist++ - name: Repair wheel (Linux)+ if: runner.os == 'Linux'+ run: |+ cd packages/leann-backend-diskann+ auditwheel repair dist/*.whl -w dist_repaired+ rm -rf dist+ mv dist_repaired dist++ - name: Repair wheel (macOS)+ if: runner.os == 'macOS'+ run: |+ cd packages/leann-backend-diskann+ delocate-wheel -w dist_repaired -v dist/*.whl+ rm -rf dist+ mv dist_repaired dist++ - name: Upload artifacts+ uses: actions/upload-artifact@v4+ with:+ name: diskann-${{ matrix.os }}-py${{ matrix.python-version }}+ path: packages/leann-backend-diskann/dist/++ # Build meta-package: leann (build last)+ build-meta:+ name: Build leann meta-package+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: '3.11'++ - name: Install uv+ uses: astral-sh/setup-uv@v4++ - name: Install build dependencies+ run: |+ uv pip install --system build++ - name: Build package+ run: |+ cd packages/leann+ uv build++ - name: Upload artifacts+ uses: actions/upload-artifact@v4+ with:+ name: leann-meta-dist+ path: packages/leann/dist/++ # Publish to PyPI+ publish:+ name: Publish to PyPI+ needs: [build-core, build-hnsw, build-diskann, build-meta]+ runs-on: ubuntu-latest+ if: github.event_name == 'release' || github.event.inputs.publish != 'false'++ steps:+ - name: Download all artifacts+ uses: actions/download-artifact@v4+ with:+ path: dist++ - name: Flatten directory structure+ run: |+ mkdir -p all_wheels+ find dist -name "*.whl" -exec cp {} all_wheels/ \;+ find dist -name "*.tar.gz" -exec cp {} all_wheels/ \;++ - name: Publish to Test PyPI+ if: github.event.inputs.publish == 'test' || github.event_name == 'workflow_dispatch'+ uses: pypa/gh-action-pypi-publish@release/v1+ with:+ password: ${{ secrets.TEST_PYPI_API_TOKEN }}+ repository-url: https://test.pypi.org/legacy/+ packages-dir: all_wheels/++ - name: Publish to PyPI+ if: github.event_name == 'release' || github.event.inputs.publish == 'prod'+ uses: pypa/gh-action-pypi-publish@release/v1+ with:+ password: ${{ secrets.PYPI_API_TOKEN }}+ packages-dir: all_wheels/\ No newline at end of filediff --git a/docs/RELEASE.md b/docs/RELEASE.mdnew file mode 100644index 0000000..72bb251--- /dev/null+++ b/docs/RELEASE.md@@ -0,0 +1,54 @@+# Release Guide++## One-line Release 🚀++```bash+./scripts/release.sh 0.1.1+```++That's it! This script will:+1. Update all package versions+2. Commit and push changes+3. Create GitHub release+4. CI automatically builds and publishes to PyPI++## Manual Testing Before Release++For testing specific packages locally (especially DiskANN on macOS):++```bash+# Build specific package locally+./scripts/build_and_test.sh diskann # or hnsw, core, meta, all++# Test installation in a clean environment+python -m venv test_env+source test_env/bin/activate+pip install packages/*/dist/*.whl++# Upload to Test PyPI (optional)+./scripts/upload_to_pypi.sh test++# Upload to Production PyPI (use with caution)+./scripts/upload_to_pypi.sh prod+```++### Why Manual Build for DiskANN?++DiskANN's complex dependencies (protobuf, abseil, etc.) sometimes require local testing before release. The build script will:+- Compile the C++ extension+- Use `delocate` (macOS) or `auditwheel` (Linux) to bundle system libraries+- Create a self-contained wheel with no external dependencies++## First-time setup++1. Install GitHub CLI:+ ```bash+ brew install gh+ gh auth login+ ```++2. Set PyPI token in GitHub:+ ```bash+ gh secret set PYPI_API_TOKEN+ # Paste your PyPI token when prompted+ ```\ No newline at end of filediff --git a/packages/leann-backend-diskann/third_party/DiskANN b/packages/leann-backend-diskann/third_party/DiskANNindex af2a264..25339b0 160000--- a/packages/leann-backend-diskann/third_party/DiskANN+++ b/packages/leann-backend-diskann/third_party/DiskANN@@ -1 +1 @@-Subproject commit af2a26481e65232b57b82d96e68833cdee9f7635+Subproject commit 25339b03413b5067c25b6092ea3e0f77ef8515c8diff --git a/packages/leann-backend-hnsw/pyproject.toml b/packages/leann-backend-hnsw/pyproject.tomlindex 274f2b4..3aabd89 100644--- a/packages/leann-backend-hnsw/pyproject.toml+++ b/packages/leann-backend-hnsw/pyproject.toml@@ -8,7 +8,12 @@ build-backend = "scikit_build_core.build"name = "leann-backend-hnsw"version = "0.1.0"description = "Custom-built HNSW (Faiss) backend for the Leann toolkit."-dependencies = ["leann-core==0.1.0", "numpy"]+dependencies = [+ "leann-core==0.1.0",+ "numpy",+ "pyzmq>=23.0.0",+ "msgpack>=1.0.0",+][tool.scikit-build]wheel.packages = ["leann_backend_hnsw"]diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.tomlindex 08d2b4e..6a3606a 100644--- a/packages/leann-core/pyproject.toml+++ b/packages/leann-core/pyproject.toml@@ -5,14 +5,22 @@ build-backend = "setuptools.build_meta"[project]name = "leann-core"version = "0.1.0"-description = "Core API and plugin system for Leann."+description = "Core API and plugin system for LEANN"readme = "README.md"requires-python = ">=3.9"license = { text = "MIT" }+# All required dependencies includeddependencies = ["numpy>=1.20.0",- "tqdm>=4.60.0"+ "tqdm>=4.60.0",+ "psutil>=5.8.0",+ "pyzmq>=23.0.0",+ "msgpack>=1.0.0",+ "torch>=2.0.0",+ "sentence-transformers>=2.2.0",+ "llama-index-core>=0.12.0",+ "python-dotenv>=1.0.0",][project.scripts]diff --git a/packages/leann/README.md b/packages/leann/README.mdnew file mode 100644index 0000000..a1e831f--- /dev/null+++ b/packages/leann/README.md@@ -0,0 +1,40 @@+# LEANN - The smallest vector index in the world++LEANN is a revolutionary vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.++## Installation++```bash+# Default installation (HNSW backend, recommended)+uv pip install leann++# With DiskANN backend (for large-scale deployments)+uv pip install leann[diskann]+```++## Quick Start++```python+from leann import LeannBuilder, LeannSearcher, LeannChat++# Build an index+builder = LeannBuilder(backend_name="hnsw")+builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")+builder.build_index("my_index.leann")++# Search+searcher = LeannSearcher("my_index.leann")+results = searcher.search("storage savings", top_k=3)++# Chat with your data+chat = LeannChat("my_index.leann", llm_config={"type": "ollama", "model": "llama3.2:1b"})+response = chat.ask("How much storage does LEANN save?")+```++## Documentation++For full documentation, visit [https://leann.readthedocs.io](https://leann.readthedocs.io)++## License++MIT License\ No newline at end of filediff --git a/packages/leann/__init__.py b/packages/leann/__init__.pynew file mode 100644index 0000000..a629a58--- /dev/null+++ b/packages/leann/__init__.py@@ -0,0 +1,12 @@+"""+LEANN - Low-storage Embedding Approximation for Neural Networks++A revolutionary vector database that democratizes personal AI.+"""++__version__ = "0.1.0"++# Re-export main API from leann-core+from leann_core import LeannBuilder, LeannSearcher, LeannChat++__all__ = ["LeannBuilder", "LeannSearcher", "LeannChat"]diff --git a/packages/leann/pyproject.toml b/packages/leann/pyproject.tomlnew file mode 100644index 0000000..54a470c--- /dev/null+++ b/packages/leann/pyproject.toml@@ -0,0 +1,42 @@+[build-system]+requires = ["setuptools>=61.0"]+build-backend = "setuptools.build_meta"++[project]+name = "leann"+version = "0.1.0"+description = "LEANN - The smallest vector index in the world. RAG Everything with LEANN!"+readme = "README.md"+requires-python = ">=3.9"+license = { text = "MIT" }+authors = [+ { name = "LEANN Team" }+]+keywords = ["vector-database", "rag", "embeddings", "search", "ai"]+classifiers = [+ "Development Status :: 4 - Beta",+ "Intended Audience :: Developers",+ "License :: OSI Approved :: MIT License",+ "Programming Language :: Python :: 3",+ "Programming Language :: Python :: 3.9",+ "Programming Language :: Python :: 3.10",+ "Programming Language :: Python :: 3.11",+ "Programming Language :: Python :: 3.12",+]++# Default installation: core + hnsw+dependencies = [+ "leann-core>=0.1.0",+ "leann-backend-hnsw>=0.1.0",+]++[project.optional-dependencies]+diskann = [+ "leann-backend-diskann>=0.1.0",+]++[project.urls]+Homepage = "https://github.com/yourusername/leann"+Documentation = "https://leann.readthedocs.io"+Repository = "https://github.com/yourusername/leann"+Issues = "https://github.com/yourusername/leann/issues"\ No newline at end of file