[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat(backend): add GPU FlashLib IVF backend (flashlib_ivf) + IVF-vs-IVF benchmark (#348)
Shuo Yangad551cf01f6b
diff --git a/benchmarks/flashlib_ivf_vs_faiss_ivf.py b/benchmarks/flashlib_ivf_vs_faiss_ivf.pynew file mode 100644index 0000000..1a515ad--- /dev/null+++ b/benchmarks/flashlib_ivf_vs_faiss_ivf.py@@ -0,0 +1,411 @@+#!/usr/bin/env python3+"""+FlashLib IVF (GPU) vs FAISS IVF (CPU) head-to-head for LEANN.++This is the apples-to-apples *approximate* comparison: both backends are IVF-Flat+(inverted file) indexes that coarse-quantize the corpus into ``nlist`` cells and, at+search time, scan only the ``nprobe`` nearest cells. At a fixed ``(nlist, nprobe)``+the two probe (almost) the same candidate set, so recall is comparable - the only+difference is GPU vs CPU kernels. (Contrast with ``flashlib_vs_hnsw_speed_comparison.py``,+which compares *exact* GPU k-NN against exact CPU flat search.)++Backends compared, per corpus size, at a shared ``nlist`` and across an ``nprobe`` sweep:++- ``flashlib_ivf (GPU)`` -> the LEANN ``flashlib_ivf`` backend+ (``packages/leann-backend-flashlib-ivf``): FlashLib ``flash_ivf_flat`` on CUDA tensors.+- ``ivf (CPU)`` -> the LEANN ``ivf`` backend+ (``packages/leann-backend-ivf``): FAISS ``IndexIVFFlat`` on CPU.++Both are driven through the LEANN backend registry (the real builders/searchers), so+this measures what each backend actually does. Distance metric is cosine (vectors are+L2-normalized; FlashLib IVF ranks by squared-L2, FAISS by inner product - equivalent+on normalized vectors).++Metrics per (size, nprobe): single-query latency (median ms), batched throughput+(queries/s), recall@k vs exact ground truth (for BOTH backends), and the GPU/CPU+speedup. Build time and index size are reported once per (size, nlist).++Data is a mixture-of-Gaussians (clustered + L2-normalized) to mimic the local+structure of real embeddings, so IVF coarse quantization behaves realistically.++Requirements: a CUDA GPU, ``flashlib``, ``torch``+CUDA, ``faiss-cpu``, ``leann-core``,+``leann-backend-ivf`` and ``leann-backend-flashlib-ivf``.++Examples:++ # laptop-like CPU budget (8 threads) for the FAISS baseline+ python benchmarks/flashlib_ivf_vs_faiss_ivf.py --sizes 100000 1000000 --cpu-threads 8++ # single 1M run, custom nlist and nprobe sweep+ python benchmarks/flashlib_ivf_vs_faiss_ivf.py --sizes 1000000 \+ --nlist 4096 --nprobe-sweep 1 8 32 128++Note: importing ``leann`` pulls in ``leann_backend_hnsw`` (LEANN's API imports it at+module load). From a source checkout whose compiled HNSW backend is not installed+(e.g. glibc < 2.35), put the pure-Python package on the path first:++ PYTHONPATH=packages/leann-backend-hnsw python benchmarks/flashlib_ivf_vs_faiss_ivf.py+"""++# ruff: noqa: E402 (BLAS env vars must be set before importing numpy / faiss)+import os+import sys+++def _argv_value(flag: str, default: str) -> str:+ """Read ``--flag value`` from argv before argparse, so we can pin BLAS thread+ counts BEFORE numpy/faiss import (their thread pools are fixed at import time)."""+ if flag in sys.argv:+ i = sys.argv.index(flag)+ if i + 1 < len(sys.argv):+ return sys.argv[i + 1]+ return default+++# FAISS CPU search latency is governed by the BLAS thread pool, which is read at+# import time - so pin it here, before importing numpy/faiss, to the requested CPU+# budget. ``--cpu-threads 0`` means "all cores" (capped at 32: 192-thread OpenBLAS+# both crashes with "too many memory regions" and yields no benefit here).+_cpu_threads = int(_argv_value("--cpu-threads", "0"))+_blas = str(_cpu_threads) if _cpu_threads > 0 else str(min(os.cpu_count() or 1, 32))+for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", "OMP_NUM_THREADS"):+ os.environ[_v] = _blas++import argparse+import gc+import json+import math+import tempfile+import time+from pathlib import Path+from typing import Any++import numpy as np+++def _fail(msg: str) -> None:+ print(f"\n[ERROR] {msg}")+ sys.exit(1)+++def _normalize(x: np.ndarray) -> np.ndarray:+ norms = np.linalg.norm(x, axis=1, keepdims=True)+ norms[norms == 0] = 1.0+ return np.ascontiguousarray(x / norms)+++def make_clustered_data(n_db: int, n_query: int, dim: int, seed: int, cluster_std: float):+ """Mixture-of-Gaussians, L2-normalized: a stand-in for real embeddings that have+ local cluster structure (so IVF coarse quantization is representative)."""+ rng = np.random.default_rng(seed)+ n_clusters = max(16, min(n_db // 100, 8192))+ centers = rng.standard_normal((n_clusters, dim), dtype=np.float32)+ centers /= np.linalg.norm(centers, axis=1, keepdims=True)++ assign = rng.integers(0, n_clusters, size=n_db)+ db = centers[assign] + cluster_std * rng.standard_normal((n_db, dim)).astype(np.float32)++ q_assign = rng.integers(0, n_clusters, size=n_query)+ queries = centers[q_assign] + cluster_std * rng.standard_normal((n_query, dim)).astype(+ np.float32+ )+ return _normalize(db.astype(np.float32)), _normalize(queries.astype(np.float32))+++def exact_ground_truth(db: np.ndarray, queries: np.ndarray, top_k: int):+ """Exact top-k by cosine (== inner product on normalized vectors), on GPU,+ chunked over queries to bound memory."""+ import torch++ db_t = torch.from_numpy(db).cuda()+ q_t = torch.from_numpy(queries).cuda()+ out = np.empty((queries.shape[0], top_k), dtype=np.int64)+ step = 256+ for i in range(0, q_t.shape[0], step):+ scores = q_t[i : i + step] @ db_t.T+ out[i : i + step] = scores.topk(top_k, dim=1, largest=True).indices.cpu().numpy()+ del db_t, q_t+ torch.cuda.empty_cache()+ return out+++def recall_at_k(found: np.ndarray, truth: np.ndarray) -> float:+ k = truth.shape[1]+ return float(np.mean([len(set(found[i]) & set(truth[i])) / k for i in range(truth.shape[0])]))+++def best_time(fn, n_repeat: int) -> float:+ best = float("inf")+ for _ in range(n_repeat):+ t = time.perf_counter()+ fn()+ best = min(best, time.perf_counter() - t)+ return best+++def measure(search_fn, queries: np.ndarray, n_single: int, n_repeat: int):+ """Single-query latency (median ms over n_single individual queries) and batched+ throughput (q/s, best of n_repeat for all queries at once)."""+ for _ in range(3): # warmup (FlashLib JIT-compiles per batch shape)+ search_fn(queries[:1])+ search_fn(queries)++ per_query = []+ for i in range(min(n_single, queries.shape[0])):+ t = time.perf_counter()+ search_fn(queries[i : i + 1])+ per_query.append((time.perf_counter() - t) * 1000.0)+ single_ms = float(np.median(per_query))++ batch_time = best_time(lambda: search_fn(queries), n_repeat)+ return single_ms, queries.shape[0] / batch_time+++def auto_nlist(n_db: int) -> int:+ """A sane default nlist ~ 4*sqrt(N), clamped, rounded to a power of two."""+ target = 4 * math.sqrt(max(n_db, 1))+ p = 2 ** round(math.log2(max(target, 256)))+ return int(max(256, min(p, 16384)))+++def _write_meta(index_path: str, backend_name: str, dim: int) -> None:+ Path(f"{index_path}.meta.json").write_text(+ json.dumps(+ {+ "version": "1.0",+ "backend_name": backend_name,+ "embedding_model": "synthetic",+ "dimensions": dim,+ "backend_kwargs": {"distance_metric": "cosine"},+ "embedding_mode": "sentence-transformers",+ "passage_sources": [],+ }+ )+ )+++def _index_size_mb(index_path: str) -> float:+ stem = Path(index_path).stem+ parent = Path(index_path).parent+ total = 0+ for p in parent.glob(f"{stem}.*"):+ if p.name.endswith(".meta.json"):+ continue+ total += p.stat().st_size+ return total / (1024 * 1024)+++def build_backend(name: str, db, ids, index_path: str, nlist: int, nprobe: int) -> dict[str, Any]:+ from leann.registry import BACKEND_REGISTRY++ kwargs = {+ "dimensions": db.shape[1],+ "distance_metric": "cosine",+ "nlist": nlist,+ "nprobe": nprobe,+ }+ t = time.perf_counter()+ BACKEND_REGISTRY[name].builder(**kwargs).build(db, ids, index_path, **kwargs)+ build_time = time.perf_counter() - t+ _write_meta(index_path, name, db.shape[1])+ return {"build_time": build_time, "index_size_mb": _index_size_mb(index_path)}+++def make_searcher(name: str, index_path: str):+ from leann.registry import BACKEND_REGISTRY++ return BACKEND_REGISTRY[name].searcher(index_path, enable_warmup=False, use_daemon=False)+++def run_search(searcher, queries, top_k, truth, nprobe, n_single, n_repeat) -> dict[str, Any]:+ def search(x):+ return searcher.search(x, top_k=top_k, nprobe=nprobe, recompute_embeddings=False)++ single_ms, qps = measure(search, queries, n_single, n_repeat)+ out = search(queries)+ found = np.array(+ [[int(x) if x.lstrip("-").isdigit() else -1 for x in row] for row in out["labels"]],+ dtype=np.int64,+ )+ recall = recall_at_k(found, truth)+ return {"single_ms": single_ms, "throughput_qps": qps, "recall": recall}+++def run_size(n_db: int, args) -> dict[str, Any]:+ import faiss+ import torch++ nlist = args.nlist if args.nlist > 0 else auto_nlist(n_db)+ print(f"\n{'=' * 80}")+ print(+ f"Corpus: {n_db:,} vectors x {args.dim} dims | cosine | nlist={nlist} | "+ f"{args.queries} queries | top_k={args.top_k}"+ )+ print(f"{'=' * 80}")++ db, queries = make_clustered_data(n_db, args.queries, args.dim, args.seed, args.cluster_std)+ ids = [str(i) for i in range(n_db)]+ print("Computing exact ground truth (GPU)...")+ truth = exact_ground_truth(db, queries, args.top_k)++ nprobe_sweep = [p for p in args.nprobe_sweep if p <= nlist]+ result: dict[str, Any] = {+ "n_db": n_db,+ "nlist": nlist,+ "nprobe_sweep": nprobe_sweep,+ "rows": [],+ }++ with tempfile.TemporaryDirectory() as tmp:+ # ---- Build both backends once (build cost is a one-time offline step). ----+ faiss.omp_set_num_threads(min(os.cpu_count() or 1, 64)) # build uses many cores+ gpu_path = str(Path(tmp) / "flashlib_ivf.leann")+ cpu_path = str(Path(tmp) / "ivf.leann")++ print("Building flashlib_ivf (GPU)...")+ gb = build_backend("flashlib_ivf", db, ids, gpu_path, nlist, max(nprobe_sweep))+ print(f" build {gb['build_time']:.2f}s | index {gb['index_size_mb']:.1f} MB")++ print("Building ivf (FAISS, CPU)...")+ cb = build_backend("ivf", db, ids, cpu_path, nlist, max(nprobe_sweep))+ print(f" build {cb['build_time']:.2f}s | index {cb['index_size_mb']:.1f} MB")+ result["flashlib_ivf_build"] = gb+ result["ivf_build"] = cb++ # ---- Sweep nprobe; reuse a single searcher per backend across the sweep. ----+ gpu_searcher = make_searcher("flashlib_ivf", gpu_path)+ cpu_searcher = make_searcher("ivf", cpu_path)+ faiss.omp_set_num_threads(args.n_threads) # constrain SEARCH to CPU budget++ for nprobe in nprobe_sweep:+ g = run_search(+ gpu_searcher, queries, args.top_k, truth, nprobe, args.single_queries, args.repeat+ )+ c = run_search(+ cpu_searcher, queries, args.top_k, truth, nprobe, args.single_queries, args.repeat+ )+ row = {"nprobe": nprobe, "gpu": g, "cpu": c}+ result["rows"].append(row)+ lat = c["single_ms"] / g["single_ms"] if g["single_ms"] else float("nan")+ tpt = g["throughput_qps"] / c["throughput_qps"] if c["throughput_qps"] else float("nan")+ print(+ f" nprobe={nprobe:<4} | "+ f"GPU {g['single_ms']:7.3f}ms {g['throughput_qps']:>10,.0f}q/s r{g['recall']:.3f} | "+ f"CPU {c['single_ms']:7.3f}ms {c['throughput_qps']:>10,.0f}q/s r{c['recall']:.3f} | "+ f"speedup {lat:5.1f}x lat {tpt:6.1f}x tpt"+ )++ del gpu_searcher, cpu_searcher+ gc.collect()+ torch.cuda.empty_cache()++ return result+++def print_summary(rows: list[dict[str, Any]], args) -> None:+ print(f"\n\n{'#' * 84}")+ print("# SUMMARY: flashlib_ivf (GPU) vs ivf (FAISS, CPU) - matched nlist, nprobe sweep")+ print(f"# CPU baseline used {args.n_threads} thread(s); metric=cosine; top_k={args.top_k}")+ print(f"{'#' * 84}")++ rcol = f"R@{args.top_k}"+ hdr = (+ f"{'nprobe':>6} | {'GPU ms':>8} {'GPU q/s':>11} {rcol:>6} | "+ f"{'CPU ms':>8} {'CPU q/s':>11} {rcol:>6} | {'lat x':>6} {'tpt x':>6}"+ )+ for r in rows:+ gb, cb = r["flashlib_ivf_build"], r["ivf_build"]+ print(f"\n{'-' * len(hdr)}")+ print(+ f"Corpus {r['n_db']:,} x {args.dim}d | nlist={r['nlist']} | "+ f"build: GPU {gb['build_time']:.1f}s ({gb['index_size_mb']:.0f}MB) vs "+ f"CPU {cb['build_time']:.1f}s ({cb['index_size_mb']:.0f}MB)"+ )+ print("-" * len(hdr))+ print(hdr)+ print("-" * len(hdr))+ for row in r["rows"]:+ g, c = row["gpu"], row["cpu"]+ lat = c["single_ms"] / g["single_ms"] if g["single_ms"] else float("nan")+ tpt = g["throughput_qps"] / c["throughput_qps"] if c["throughput_qps"] else float("nan")+ print(+ f"{row['nprobe']:>6} | {g['single_ms']:>8.3f} {g['throughput_qps']:>11,.0f} "+ f"{g['recall']:>6.3f} | {c['single_ms']:>8.3f} {c['throughput_qps']:>11,.0f} "+ f"{c['recall']:>6.3f} | {lat:>6.1f} {tpt:>6.1f}"+ )+++def main() -> None:+ p = argparse.ArgumentParser(+ description="FlashLib IVF (GPU) vs FAISS IVF (CPU) comparison for LEANN.",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )+ p.add_argument("--sizes", type=int, nargs="+", default=[100_000, 1_000_000])+ p.add_argument("--dim", type=int, default=768)+ p.add_argument("--queries", type=int, default=1000)+ p.add_argument("--top-k", type=int, default=10)+ p.add_argument("--nlist", type=int, default=0, help="IVF partitions (0 = auto ~4*sqrt(N))")+ p.add_argument("--nprobe-sweep", type=int, nargs="+", default=[1, 4, 8, 16, 32, 64])+ p.add_argument(+ "--cpu-threads",+ type=int,+ default=0,+ help="FAISS CPU search threads (0 = all cores, capped at 32).",+ )+ p.add_argument("--cluster-std", type=float, default=0.1, help="Cluster spread (lower=tighter)")+ p.add_argument("--single-queries", type=int, default=200)+ p.add_argument("--repeat", type=int, default=5)+ p.add_argument("--seed", type=int, default=42)+ args = p.parse_args()++ try:+ import torch+ except ImportError:+ _fail("PyTorch is required (with CUDA).")+ if not torch.cuda.is_available():+ _fail("FlashLib IVF is GPU-only, but no CUDA GPU is available.")+ try:+ import faiss+ import flashlib+ except ImportError as e:+ _fail(f"Missing dependency: {e}. Need 'flashlib' and 'faiss-cpu'.")++ from leann.registry import BACKEND_REGISTRY, autodiscover_backends++ autodiscover_backends()+ for need in ("ivf", "flashlib_ivf"):+ if need not in BACKEND_REGISTRY:+ _fail(+ f"Backend '{need}' not registered. Install it: "+ f"pip install -e packages/leann-backend-{need.replace('_', '-')}"+ )++ all_cores = os.cpu_count() or 1+ args.n_threads = args.cpu_threads if args.cpu_threads > 0 else min(all_cores, 32)++ print("FlashLib IVF (GPU) vs FAISS IVF (CPU) comparison for LEANN")+ print(f"GPU: {torch.cuda.get_device_name(0)} | CPU cores available: {all_cores}")+ print(+ f"flashlib {flashlib.__version__} | faiss {faiss.__version__} | torch {torch.__version__}"+ )+ print(+ f"Config: dim={args.dim}, queries={args.queries}, top_k={args.top_k}, "+ f"FAISS search threads={args.n_threads} (build uses up to 64), "+ f"nprobe_sweep={args.nprobe_sweep}"+ )++ rows = [run_size(n, args) for n in args.sizes]+ print_summary(rows, args)+ print("\nDone.")+++if __name__ == "__main__":+ try:+ main()+ except KeyboardInterrupt:+ print("\nInterrupted.")+ sys.exit(130)+ finally:+ sys.stdout.flush()+ sys.stderr.flush()+ os._exit(0)diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdnew file mode 100644index 0000000..2030b8e--- /dev/null+++ b/docs/CHANGELOG.md@@ -0,0 +1,20 @@+# Changelog++Append-only log of major changes to LEANN (new features, breaking changes, important+fixes). Newest entries at the bottom.++## 2026-06-02: GPU FlashLib IVF backend (`flashlib_ivf`)++- Add `leann-backend-flashlib-ivf`, a GPU IVF-Flat (inverted file) approximate-NN+ backend built on FlashLib (`flash_ivf_flat`, Triton/CuteDSL) — the GPU counterpart+ of the FAISS `ivf` backend. Registered as backend name `flashlib_ivf`; install via+ `uv sync --extra flashlib-ivf` or `pip install leann-backend-flashlib-ivf`. Shares+ the `nlist`/`nprobe` recall knobs with the `ivf` backend, so the two are drop-in+ comparable. Requires a CUDA GPU at build (k-means) and search.+- Add `benchmarks/flashlib_ivf_vs_faiss_ivf.py`: head-to-head `flashlib_ivf` (GPU) vs+ `ivf` (FAISS, CPU) at matched `nlist` across an `nprobe` sweep (build time,+ single-query latency, batched throughput, recall@k vs exact ground truth). On an+ NVIDIA H200 at 1M x 768 vectors (nlist=4096, 8 CPU threads): ~13x faster build and,+ at nprobe=32, ~6.5x lower single-query latency / ~75x higher batched throughput at+ comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).+- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.diff --git a/docs/flashlib_backend_guide.md b/docs/flashlib_backend_guide.mdindex f903af9..ed5a155 100644--- a/docs/flashlib_backend_guide.md+++ b/docs/flashlib_backend_guide.md@@ -31,6 +31,7 @@ distances, indices = index.kneighbors(torch.randn(10_000, 128, device="cuda"), n| `diskann` | Larger-than-memory datasets | On-disk graph | CPU || `ivf` | Incremental add/remove without rebuild | Full vectors (FAISS) | CPU || **`flashlib`** | **High-throughput search on a CUDA GPU** | Full vectors (`.npy`) | **CUDA GPU** |+| **`flashlib_ivf`** | **GPU IVF-Flat (approximate) — the GPU counterpart of `ivf`** | Full vectors (`.pt`) | **CUDA GPU** |Use FlashLib when you already have a GPU and want fast IVF-Flat search. It storesthe full float32 vectors rather than a pruned graph, so it trades LEANN's storage@@ -108,6 +109,66 @@ constraint), and `nprobe` is derived from the search `complexity` knob.| `distance_metric` (build) | `"mips"` | `mips`, `cosine`, or `l2`. || `nprobe` (search) | derived from `complexity` | Partitions probed per query — the recall knob (higher = better recall, slower). |+## FlashLib IVF backend (`flashlib_ivf`)++The **`flashlib_ivf`** backend (`leann-backend-flashlib-ivf`) is the GPU counterpart+of LEANN's FAISS `ivf` backend: an IVF-Flat (inverted file) index that+coarse-quantizes the corpus into `nlist` cells with k-means and, at search time,+scans only the `nprobe` nearest cells — entirely on CUDA tensors via FlashLib's+`flash_ivf_flat`. At a fixed `(nlist, nprobe)` the GPU and CPU IVF probe nearly the+same candidate set, so recall is comparable; the only difference is GPU vs CPU+kernels. `nprobe` is the recall knob (defaults to `min(complexity, nlist)`).++```bash+uv sync --extra flashlib-ivf # or: pip install leann-backend-flashlib-ivf+```++```python+from leann import LeannBuilder, LeannSearcher++builder = LeannBuilder(backend_name="flashlib_ivf", nlist=4096, distance_metric="cosine")+builder.add_text("LEANN recomputes embeddings on the fly to cut storage by ~97%.")+builder.build_index("demo.leann")++searcher = LeannSearcher("demo.leann")+results = searcher.search("How does LEANN save storage?", top_k=10, complexity=32) # nprobe=32+```++**How it works:** the builder trains the coarse quantizer on the GPU and persists the+built index tensors with `torch.save` (`<index>.flashlib_ivf.pt`) plus an id map+(`<index>.flashlib_ivf_id_map.json`); the searcher reloads them onto the GPU once (no+k-means re-train). A CUDA GPU is required at **both** build (k-means) and search time.+FlashLib IVF ranks by squared L2, so `mips`/`cosine` L2-normalize the vectors (squared-L2+ranking then matches inner-product/cosine).++### Speed — IVF (GPU) vs IVF (CPU)++```bash+python benchmarks/flashlib_ivf_vs_faiss_ivf.py \+ --sizes 100000 1000000 --nprobe-sweep 1 8 32 128 --cpu-threads 8+```++`flashlib_ivf` (GPU) vs the FAISS `ivf` backend (CPU) at the **same `nlist`**, sweeping+`nprobe` (NVIDIA H200, `faiss-cpu` at 8 threads, 768-dim, top-k=10). GPU latency stays+~flat while CPU latency grows linearly with `nprobe`, so the GPU lead widens exactly as+you raise recall:++| Corpus | nprobe | GPU lat | CPU lat | GPU q/s | CPU q/s | Recall (GPU/CPU) | Speedup (lat / tpt) |+|--------|--------|---------|---------|---------|---------|------------------|---------------------|+| 1M | 8 | 0.45 ms | 1.14 ms | 107k | 5.9k | 0.340 / 0.321 | 2.6× / 18× |+| 1M | 32 | 0.46 ms | 3.00 ms | 141k | 1.9k | 0.400 / 0.350 | 6.5× / 75× |+| 1M | 128 | 0.55 ms | 9.91 ms | 95k | 0.6k | 0.539 / 0.423 | 18× / 159× |++Build (1M, `nlist=4096`): GPU **10.6 s** vs FAISS CPU **140.7 s** — a **13× faster+build** (GPU k-means vs CPU k-means training).++Honest caveats: at very low `nprobe` (e.g. 1) single-query GPU latency (~0.44 ms) is+*higher* than CPU, because the per-query work is tiny and GPU kernel-launch overhead+dominates; the GPU advantage grows with `nprobe` (higher recall), batch size, and corpus+size. The absolute recall above is low because the synthetic mixture-of-Gaussians corpus+has more clusters than `nlist`; on real embeddings recall is far higher — this benchmark+isolates the GPU-vs-CPU *relative* comparison at matched `(nlist, nprobe)`.+## Notes & limitations- GPU-only at search time; building an index works on a CPU-only machine.diff --git a/packages/leann-backend-flashlib-ivf/README.md b/packages/leann-backend-flashlib-ivf/README.mdnew file mode 100644index 0000000..79fb38a--- /dev/null+++ b/packages/leann-backend-flashlib-ivf/README.md@@ -0,0 +1,71 @@+# leann-backend-flashlib-ivf++GPU-accelerated [FlashLib](https://github.com/FlashML-org/flashlib) IVF-Flat+(inverted file) backend for LEANN - the GPU counterpart of the FAISS+[`leann-backend-ivf`](../leann-backend-ivf) backend.++FlashLib is a GPU library of classical ML operators built on Triton / CuteDSL.+Its IVF-Flat index coarse-quantizes the corpus into `nlist` cells and, at search+time, scans only the `nprobe` nearest cells - entirely on CUDA tensors. At a fixed+`(nlist, nprobe)` it probes the same candidate set as a reference IVF-Flat+(FAISS / cuVS), so recall is comparable; the difference is GPU vs CPU kernels.++This is registered as the `flashlib_ivf` backend, distinct from the exact GPU+k-NN `flashlib` backend (which does brute-force `NearestNeighbors`, not IVF).++## Requirements++- A CUDA GPU (required at **build** time for k-means training and at **search** time).+- `pip install flashlib` and `torch`.++## Install++```bash+# from a LEANN checkout+uv sync --extra flashlib-ivf+# or+pip install leann-backend-flashlib-ivf+```++## Usage++```python+from leann import LeannBuilder, LeannSearcher++builder = LeannBuilder(backend_name="flashlib_ivf", nlist=1024, distance_metric="cosine")+builder.add_text("LEANN recomputes embeddings to save storage.")+builder.build_index("demo.leann")++searcher = LeannSearcher("demo.leann")+print(searcher.search("How does LEANN save storage?", top_k=3, complexity=32)) # nprobe=min(32, nlist)+```++Or from the example apps:++```bash+python -m apps.document_rag --query "What are the main techniques LEANN explores?" \+ --backend-name flashlib_ivf+```++## How it works++The built IVF-Flat index is a small set of torch tensors (centroids,+cell-contiguous data, row ids, CSR offsets), so this backend persists it with+`torch.save` (`<index>.flashlib_ivf.pt`) plus an id map+(`<index>.flashlib_ivf_id_map.json`) and reloads it onto the GPU at searcher+start-up - no k-means re-train.++FlashLib's only distance metric is squared L2. For `mips` / `cosine` the vectors+are L2-normalized at build and query time, on which squared-L2 ranking is+equivalent to inner-product / cosine ranking.++## Parameters++| kwarg | default | meaning |+|-------|---------|---------|+| `nlist` | `1024` | number of IVF partitions / coarse centroids (clamped to corpus size) |+| `nprobe` (build) | `16` | default partitions probed per query (recall knob) |+| `niter` | `20` | Lloyd k-means iterations for the coarse quantizer |+| `seed` | `0` | RNG seed (deterministic build) |+| `distance_metric` | `"mips"` | `mips`, `cosine`, or `l2` |+| `complexity` (search) | `64` | sets `nprobe = min(complexity, nlist)` when `nprobe` is not given |diff --git a/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.py b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.pynew file mode 100644index 0000000..25fdeaa--- /dev/null+++ b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.py@@ -0,0 +1,13 @@+"""LEANN FlashLib IVF backend: GPU-accelerated IVF-Flat (Triton/CuteDSL) approximate-NN search."""++from .flashlib_ivf_backend import (+ FlashlibIVFBackend,+ FlashlibIVFBuilder,+ FlashlibIVFSearcher,+)++__all__ = [+ "FlashlibIVFBackend",+ "FlashlibIVFBuilder",+ "FlashlibIVFSearcher",+]diff --git a/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.py b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.pynew file mode 100644index 0000000..71eff52--- /dev/null+++ b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.py@@ -0,0 +1,266 @@+"""+FlashLib IVF backend: GPU-accelerated IVF-Flat (inverted file) ANN search.++FlashLib (https://github.com/FlashML-org/flashlib) is a GPU library of classical+ML primitives. This backend uses its IVF-Flat index, which runs an *approximate*+nearest-neighbor search entirely on CUDA tensors:++ from flashlib import flash_ivf_flat_build, flash_ivf_flat_search+ index = flash_ivf_flat_build(db_cuda, nlist, nprobe=..., niter=...) # (M, D) CUDA+ vals, ids = flash_ivf_flat_search(index, queries_cuda, k, nprobe=...)++This is the GPU counterpart of the FAISS ``ivf`` backend (FAISS ``IndexIVFFlat``):+both coarse-quantize the corpus into ``nlist`` cells and, at search time, scan only+the ``nprobe`` nearest cells. At a fixed ``(nlist, nprobe)`` FlashLib probes the same+candidate set as a reference IVF-Flat, so recall is comparable; the difference is GPU+vs CPU kernels.++The built index is a small set of torch tensors (centroids, cell-contiguous data,+row ids, CSR offsets), so we persist it with ``torch.save`` (``<index>.flashlib_ivf.pt``)+plus an id map (``<index>.flashlib_ivf_id_map.json``) and reload it onto the GPU at+searcher start-up (no k-means re-train).++FlashLib ranks by squared L2. For ``mips`` / ``cosine`` we L2-normalize both the+database and query vectors, on which squared-L2 ranking is equivalent to+inner-product / cosine ranking.++Requires a CUDA GPU at both build (k-means training) and search time.+"""++import json+import logging+from pathlib import Path+from typing import Any, Optional++import numpy as np+from leann.interface import (+ LeannBackendBuilderInterface,+ LeannBackendFactoryInterface,+ LeannBackendSearcherInterface,+)+from leann.registry import register_backend+from leann.searcher_base import BaseSearcher++logger = logging.getLogger(__name__)++INDEX_SUFFIX = "flashlib_ivf.pt"+ID_MAP_SUFFIX = "flashlib_ivf_id_map.json"++# IvfFlatIndex dataclass fields, split by how they serialize.+_TENSOR_FIELDS = ("centroids", "data", "ids", "list_offsets")+_SCALAR_FIELDS = ("metric", "D", "Dp", "nlist", "nprobe", "max_list_len")+++def _import_flashlib():+ try:+ import torch # noqa: F401+ from flashlib import (+ IvfFlatIndex,+ flash_ivf_flat_build,+ flash_ivf_flat_search,+ )+ except ImportError as e:+ raise ImportError(+ "The FlashLib IVF backend requires 'flashlib' and 'torch' with CUDA. "+ "Install with: pip install flashlib (a CUDA GPU is required at build and search time)."+ ) from e+ return IvfFlatIndex, flash_ivf_flat_build, flash_ivf_flat_search+++def _normalize_l2(data: np.ndarray) -> np.ndarray:+ norms = np.linalg.norm(data, axis=1, keepdims=True)+ norms[norms == 0] = 1+ return data / norms+++def _needs_normalize(distance_metric: str) -> bool:+ return distance_metric.lower() in ("mips", "cosine")+++def _index_path(index_dir: Path, index_prefix: str) -> Path:+ return index_dir / f"{index_prefix}.{INDEX_SUFFIX}"+++def _id_map_path(index_dir: Path, index_prefix: str) -> Path:+ return index_dir / f"{index_prefix}.{ID_MAP_SUFFIX}"+++def _save_id_map(index_dir: Path, index_prefix: str, ids: list[str]) -> None:+ with open(_id_map_path(index_dir, index_prefix), "w", encoding="utf-8") as f:+ json.dump({"ids": ids}, f)+++def _load_id_map(index_dir: Path, index_prefix: str) -> list[str]:+ path = _id_map_path(index_dir, index_prefix)+ if not path.exists():+ raise FileNotFoundError(f"FlashLib IVF id map not found at {path}")+ with open(path, encoding="utf-8") as f:+ return json.load(f)["ids"]+++def _save_index(index, path: Path) -> None:+ import torch++ state: dict[str, Any] = {f: getattr(index, f).detach().cpu() for f in _TENSOR_FIELDS}+ for f in _SCALAR_FIELDS:+ state[f] = getattr(index, f)+ torch.save(state, str(path))+++def _load_index(path: Path, device: str = "cuda"):+ import torch++ IvfFlatIndex, _, _ = _import_flashlib()+ state = torch.load(str(path), map_location=device, weights_only=False)+ kwargs: dict[str, Any] = {f: state[f].to(device) for f in _TENSOR_FIELDS}+ for f in _SCALAR_FIELDS:+ kwargs[f] = state[f]+ return IvfFlatIndex(**kwargs)+++@register_backend("flashlib_ivf")+class FlashlibIVFBackend(LeannBackendFactoryInterface):+ @staticmethod+ def builder(**kwargs) -> LeannBackendBuilderInterface:+ return FlashlibIVFBuilder(**kwargs)++ @staticmethod+ def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:+ return FlashlibIVFSearcher(index_path, **kwargs)+++class FlashlibIVFBuilder(LeannBackendBuilderInterface):+ def __init__(self, **kwargs):+ self.build_params = kwargs.copy()+ self.distance_metric = self.build_params.setdefault("distance_metric", "mips")+ self.nlist = self.build_params.setdefault("nlist", 1024)+ self.nprobe = self.build_params.setdefault("nprobe", 16)+ self.niter = self.build_params.setdefault("niter", 20)+ self.seed = self.build_params.setdefault("seed", 0)+ self.dimensions = self.build_params.get("dimensions")++ def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:+ import torch++ _, flash_ivf_flat_build, _ = _import_flashlib()+ if not torch.cuda.is_available():+ raise RuntimeError(+ "FlashLib IVF backend requires a CUDA GPU at build time (k-means training), "+ "but none is available."+ )++ path = Path(index_path)+ index_dir = path.parent+ index_prefix = path.stem+ index_dir.mkdir(parents=True, exist_ok=True)++ if data.dtype != np.float32:+ data = data.astype(np.float32)+ data = np.ascontiguousarray(data)+ if _needs_normalize(self.distance_metric):+ data = _normalize_l2(data)++ n = data.shape[0]+ nlist = int(min(self.nlist, n)) if n > 0 else int(self.nlist)++ db = torch.from_numpy(data).cuda()+ index = flash_ivf_flat_build(+ db,+ nlist,+ metric="l2",+ nprobe=int(self.nprobe),+ niter=int(self.niter),+ seed=int(self.seed),+ )+ _save_index(index, _index_path(index_dir, index_prefix))+ _save_id_map(index_dir, index_prefix, list(ids))+ logger.info(+ "FlashLib IVF build: %d vectors (dim=%d, metric=%s, nlist=%d, nprobe=%d) at %s",+ n,+ data.shape[1],+ self.distance_metric,+ nlist,+ self.nprobe,+ _index_path(index_dir, index_prefix),+ )+++class FlashlibIVFSearcher(BaseSearcher):+ def __init__(self, index_path: str, **kwargs):+ # Reuse the HNSW embedding server (if present) to embed queries, exactly like+ # the other non-recompute backends; falls back to direct model loading.+ super().__init__(+ index_path,+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server",+ **kwargs,+ )+ backend_kwargs = self.meta.get("backend_kwargs", {})+ self.distance_metric = backend_kwargs.get("distance_metric", "mips").lower()++ index_prefix = self.index_path.stem+ index_file = _index_path(self.index_dir, index_prefix)+ if not index_file.exists():+ raise FileNotFoundError(f"FlashLib IVF index file not found at {index_file}")++ import torch++ if not torch.cuda.is_available():+ raise RuntimeError(+ "FlashLib IVF backend requires a CUDA GPU at search time, but none is available."+ )++ self._ids = _load_id_map(self.index_dir, index_prefix)+ self._index = _load_index(index_file, device="cuda")+ self._nlist = int(self._index.nlist)+ self._ntotal = int(self._index.data.shape[0])+ logger.info(+ "FlashLib IVF searcher ready: %d vectors, nlist=%d, metric=%s",+ self._ntotal,+ self._nlist,+ self.distance_metric,+ )++ def search(+ self,+ query: np.ndarray,+ top_k: int,+ complexity: int = 64,+ nprobe: Optional[int] = None,+ **kwargs,+ ) -> dict[str, Any]:+ import torch++ _, _, flash_ivf_flat_search = _import_flashlib()+ if query.dtype != np.float32:+ query = query.astype(np.float32)+ if _needs_normalize(self.distance_metric):+ query = _normalize_l2(query)++ # complexity is the recall knob shared with the FAISS ivf backend.+ nprobe = nprobe or min(complexity, self._nlist)+ k = min(top_k, self._ntotal)+ q = torch.from_numpy(np.ascontiguousarray(query)).cuda()+ distances, indices = flash_ivf_flat_search(self._index, q, k, nprobe=int(nprobe))+ distances_np = distances.detach().cpu().numpy().astype(np.float32)+ indices_np = indices.detach().cpu().numpy()++ def map_label(i: int) -> str:+ # flash_ivf_flat_search pads short candidate lists with -1.+ return self._ids[i] if i >= 0 else "-1"++ string_labels = [[map_label(int(i)) for i in row] for row in indices_np]+ return {"labels": string_labels, "distances": distances_np}++ def compute_query_embedding(+ self,+ query: str,+ use_server_if_available: bool = True,+ zmq_port: Optional[int] = None,+ query_template: Optional[str] = None,+ ) -> np.ndarray:+ return super().compute_query_embedding(+ query,+ use_server_if_available=use_server_if_available,+ zmq_port=zmq_port,+ query_template=query_template,+ )diff --git a/packages/leann-backend-flashlib-ivf/pyproject.toml b/packages/leann-backend-flashlib-ivf/pyproject.tomlnew file mode 100644index 0000000..0437b70--- /dev/null+++ b/packages/leann-backend-flashlib-ivf/pyproject.toml@@ -0,0 +1,20 @@+[project]+name = "leann-backend-flashlib-ivf"+version = "0.3.6"+description = "GPU-accelerated FlashLib IVF-Flat (Triton/CuteDSL) approximate-NN backend for LEANN."+readme = "README.md"+requires-python = ">=3.10"+dependencies = [+ "leann-core>=0.3.6",+ "numpy>=1.20.0",+ "flashlib",+ "torch",+]++[project.optional-dependencies]+# Optional: use the HNSW embedding server for query embedding (same as other backends).+query-server = ["leann-backend-hnsw>=0.3.6", "pyzmq>=23.0.0", "msgpack>=1.0.0"]++[tool.setuptools.packages.find]+where = ["."]+include = ["leann_backend_flashlib_ivf*"]diff --git a/pyproject.toml b/pyproject.tomlindex 482e1f9..f96b96d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -66,11 +66,16 @@ diskann = ["leann-backend-diskann",]-# GPU-accelerated FlashLib (IVFFlat on Triton/CuteDSL) backend. Requires a CUDA GPU.+# GPU-accelerated FlashLib (exact k-NN on Triton/CuteDSL) backend. Requires a CUDA GPU.flashlib = ["leann-backend-flashlib",]+# GPU-accelerated FlashLib IVF-Flat (approximate-NN) backend. Requires a CUDA GPU.+flashlib-ivf = [+ "leann-backend-flashlib-ivf",+]+# Add a new optional dependency group for document processingdocuments = ["beautifulsoup4>=4.13.0", # For HTML parsing@@ -94,6 +99,7 @@ leann-core = { path = "packages/leann-core", editable = true }leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }leann-backend-flashlib = { path = "packages/leann-backend-flashlib", editable = true }+leann-backend-flashlib-ivf = { path = "packages/leann-backend-flashlib-ivf", editable = true }astchunk = { path = "packages/astchunk-leann", editable = true }[dependency-groups]diff --git a/tests/test_flashlib_ivf_backend.py b/tests/test_flashlib_ivf_backend.pynew file mode 100644index 0000000..d5e04a6--- /dev/null+++ b/tests/test_flashlib_ivf_backend.py@@ -0,0 +1,125 @@+"""+Correctness tests for the FlashLib IVF backend (``flashlib_ivf``).++Registry-level (no embedding model needed): builds a small clustered corpus through+the real LEANN backend builders/searchers and checks that ``flashlib_ivf`` (GPU)+registers, persists/reloads its index, and returns recall comparable to the FAISS+``ivf`` backend and to exact ground truth at a matched ``(nlist, nprobe)``.++Requires a CUDA GPU + ``flashlib`` (skipped otherwise, e.g. in CI).+"""++import json+import tempfile+from pathlib import Path++import numpy as np+import pytest+++def _cuda_or_skip():+ torch = pytest.importorskip("torch")+ pytest.importorskip("flashlib")+ if not torch.cuda.is_available():+ pytest.skip("FlashLib IVF backend requires a CUDA GPU")+++def _registry():+ from leann.registry import BACKEND_REGISTRY, autodiscover_backends++ autodiscover_backends()+ return BACKEND_REGISTRY+++def _clustered(n: int, dim: int, seed: int) -> np.ndarray:+ rng = np.random.default_rng(seed)+ n_clusters = 64+ centers = rng.standard_normal((n_clusters, dim), dtype=np.float32)+ centers /= np.linalg.norm(centers, axis=1, keepdims=True)+ assign = rng.integers(0, n_clusters, size=n)+ db = centers[assign] + 0.05 * rng.standard_normal((n, dim)).astype(np.float32)+ db /= np.linalg.norm(db, axis=1, keepdims=True)+ return np.ascontiguousarray(db.astype(np.float32))+++def _exact_gt(db: np.ndarray, q: np.ndarray, k: int) -> np.ndarray:+ scores = q @ db.T+ return np.argsort(-scores, axis=1)[:, :k]+++def _recall(found: np.ndarray, truth: np.ndarray) -> float:+ k = truth.shape[1]+ return float(np.mean([len(set(found[i]) & set(truth[i])) / k for i in range(len(truth))]))+++def _write_meta(index_path: str, backend: str, dim: int) -> None:+ Path(f"{index_path}.meta.json").write_text(+ json.dumps(+ {+ "version": "1.0",+ "backend_name": backend,+ "embedding_model": "synthetic",+ "dimensions": dim,+ "backend_kwargs": {"distance_metric": "cosine"},+ "embedding_mode": "sentence-transformers",+ "passage_sources": [],+ }+ )+ )+++def _labels_to_int(out: dict) -> np.ndarray:+ return np.array(+ [[int(x) if x.lstrip("-").isdigit() else -1 for x in row] for row in out["labels"]],+ dtype=np.int64,+ )+++def test_flashlib_ivf_recall_parity_with_faiss_ivf():+ _cuda_or_skip()+ reg = _registry()+ assert "flashlib_ivf" in reg, "flashlib_ivf backend not registered"+ if "ivf" not in reg:+ pytest.skip("faiss ivf backend not installed")++ dim, n, k, nlist, nprobe = 64, 20_000, 10, 128, 16+ db = _clustered(n, dim, seed=0)+ queries = db[:100].copy()+ ids = [str(i) for i in range(n)]+ truth = _exact_gt(db, queries, k)++ kw = {"dimensions": dim, "distance_metric": "cosine", "nlist": nlist, "nprobe": nprobe}+ with tempfile.TemporaryDirectory() as tmp:+ gpu_path = str(Path(tmp) / "g.leann")+ cpu_path = str(Path(tmp) / "c.leann")++ reg["flashlib_ivf"].builder(**kw).build(db, ids, gpu_path, **kw)+ reg["ivf"].builder(**kw).build(db, ids, cpu_path, **kw)++ # Persistence: the GPU index + id map are written to disk.+ assert (Path(tmp) / "g.flashlib_ivf.pt").exists()+ assert (Path(tmp) / "g.flashlib_ivf_id_map.json").exists()++ _write_meta(gpu_path, "flashlib_ivf", dim)+ _write_meta(cpu_path, "ivf", dim)++ gpu = reg["flashlib_ivf"].searcher(gpu_path, enable_warmup=False, use_daemon=False)+ cpu = reg["ivf"].searcher(cpu_path, enable_warmup=False, use_daemon=False)++ gpu_found = _labels_to_int(+ gpu.search(queries, top_k=k, nprobe=nprobe, recompute_embeddings=False)+ )+ cpu_found = _labels_to_int(+ cpu.search(queries, top_k=k, nprobe=nprobe, recompute_embeddings=False)+ )++ gpu_recall = _recall(gpu_found, truth)+ cpu_recall = _recall(cpu_found, truth)++ # At matched (nlist, nprobe) both IVF backends probe ~the same candidate set, so+ # recall should be high and comparable (GPU is often a touch higher: independent+ # coarse-quantizer training, not "more exact").+ assert gpu_recall > 0.7, f"flashlib_ivf recall too low: {gpu_recall:.3f}"+ assert abs(gpu_recall - cpu_recall) < 0.15, (+ f"recall parity off: gpu {gpu_recall:.3f} vs cpu {cpu_recall:.3f}"+ )