Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat(backend): add GPU FlashLib IVF backend (flashlib_ivf) + IVF-vs-IVF benchmark (#348)

Shuo Yangad551cf01f6b
diff --git a/benchmarks/flashlib_ivf_vs_faiss_ivf.py b/benchmarks/flashlib_ivf_vs_faiss_ivf.py
new file mode 100644
index 0000000..1a515ad
--- /dev/null
+++ b/benchmarks/flashlib_ivf_vs_faiss_ivf.py
@@ -0,0 +1,411 @@
+#!/usr/bin/env python3
+"""
+FlashLib IVF (GPU) vs FAISS IVF (CPU) head-to-head for LEANN.
+
+This is the apples-to-apples *approximate* comparison: both backends are IVF-Flat
+(inverted file) indexes that coarse-quantize the corpus into ``nlist`` cells and, at
+search time, scan only the ``nprobe`` nearest cells. At a fixed ``(nlist, nprobe)``
+the two probe (almost) the same candidate set, so recall is comparable - the only
+difference is GPU vs CPU kernels. (Contrast with ``flashlib_vs_hnsw_speed_comparison.py``,
+which compares *exact* GPU k-NN against exact CPU flat search.)
+
+Backends compared, per corpus size, at a shared ``nlist`` and across an ``nprobe`` sweep:
+
+- ``flashlib_ivf (GPU)`` -> the LEANN ``flashlib_ivf`` backend
+ (``packages/leann-backend-flashlib-ivf``): FlashLib ``flash_ivf_flat`` on CUDA tensors.
+- ``ivf (CPU)`` -> the LEANN ``ivf`` backend
+ (``packages/leann-backend-ivf``): FAISS ``IndexIVFFlat`` on CPU.
+
+Both are driven through the LEANN backend registry (the real builders/searchers), so
+this measures what each backend actually does. Distance metric is cosine (vectors are
+L2-normalized; FlashLib IVF ranks by squared-L2, FAISS by inner product - equivalent
+on normalized vectors).
+
+Metrics per (size, nprobe): single-query latency (median ms), batched throughput
+(queries/s), recall@k vs exact ground truth (for BOTH backends), and the GPU/CPU
+speedup. Build time and index size are reported once per (size, nlist).
+
+Data is a mixture-of-Gaussians (clustered + L2-normalized) to mimic the local
+structure of real embeddings, so IVF coarse quantization behaves realistically.
+
+Requirements: a CUDA GPU, ``flashlib``, ``torch``+CUDA, ``faiss-cpu``, ``leann-core``,
+``leann-backend-ivf`` and ``leann-backend-flashlib-ivf``.
+
+Examples:
+
+ # laptop-like CPU budget (8 threads) for the FAISS baseline
+ python benchmarks/flashlib_ivf_vs_faiss_ivf.py --sizes 100000 1000000 --cpu-threads 8
+
+ # single 1M run, custom nlist and nprobe sweep
+ python benchmarks/flashlib_ivf_vs_faiss_ivf.py --sizes 1000000 \
+ --nlist 4096 --nprobe-sweep 1 8 32 128
+
+Note: importing ``leann`` pulls in ``leann_backend_hnsw`` (LEANN's API imports it at
+module load). From a source checkout whose compiled HNSW backend is not installed
+(e.g. glibc < 2.35), put the pure-Python package on the path first:
+
+ PYTHONPATH=packages/leann-backend-hnsw python benchmarks/flashlib_ivf_vs_faiss_ivf.py
+"""
+
+# ruff: noqa: E402 (BLAS env vars must be set before importing numpy / faiss)
+import os
+import sys
+
+
+def _argv_value(flag: str, default: str) -> str:
+ """Read ``--flag value`` from argv before argparse, so we can pin BLAS thread
+ counts BEFORE numpy/faiss import (their thread pools are fixed at import time)."""
+ if flag in sys.argv:
+ i = sys.argv.index(flag)
+ if i + 1 < len(sys.argv):
+ return sys.argv[i + 1]
+ return default
+
+
+# FAISS CPU search latency is governed by the BLAS thread pool, which is read at
+# import time - so pin it here, before importing numpy/faiss, to the requested CPU
+# budget. ``--cpu-threads 0`` means "all cores" (capped at 32: 192-thread OpenBLAS
+# both crashes with "too many memory regions" and yields no benefit here).
+_cpu_threads = int(_argv_value("--cpu-threads", "0"))
+_blas = str(_cpu_threads) if _cpu_threads > 0 else str(min(os.cpu_count() or 1, 32))
+for _v in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", "OMP_NUM_THREADS"):
+ os.environ[_v] = _blas
+
+import argparse
+import gc
+import json
+import math
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+def _fail(msg: str) -> None:
+ print(f"\n[ERROR] {msg}")
+ sys.exit(1)
+
+
+def _normalize(x: np.ndarray) -> np.ndarray:
+ norms = np.linalg.norm(x, axis=1, keepdims=True)
+ norms[norms == 0] = 1.0
+ return np.ascontiguousarray(x / norms)
+
+
+def make_clustered_data(n_db: int, n_query: int, dim: int, seed: int, cluster_std: float):
+ """Mixture-of-Gaussians, L2-normalized: a stand-in for real embeddings that have
+ local cluster structure (so IVF coarse quantization is representative)."""
+ rng = np.random.default_rng(seed)
+ n_clusters = max(16, min(n_db // 100, 8192))
+ centers = rng.standard_normal((n_clusters, dim), dtype=np.float32)
+ centers /= np.linalg.norm(centers, axis=1, keepdims=True)
+
+ assign = rng.integers(0, n_clusters, size=n_db)
+ db = centers[assign] + cluster_std * rng.standard_normal((n_db, dim)).astype(np.float32)
+
+ q_assign = rng.integers(0, n_clusters, size=n_query)
+ queries = centers[q_assign] + cluster_std * rng.standard_normal((n_query, dim)).astype(
+ np.float32
+ )
+ return _normalize(db.astype(np.float32)), _normalize(queries.astype(np.float32))
+
+
+def exact_ground_truth(db: np.ndarray, queries: np.ndarray, top_k: int):
+ """Exact top-k by cosine (== inner product on normalized vectors), on GPU,
+ chunked over queries to bound memory."""
+ import torch
+
+ db_t = torch.from_numpy(db).cuda()
+ q_t = torch.from_numpy(queries).cuda()
+ out = np.empty((queries.shape[0], top_k), dtype=np.int64)
+ step = 256
+ for i in range(0, q_t.shape[0], step):
+ scores = q_t[i : i + step] @ db_t.T
+ out[i : i + step] = scores.topk(top_k, dim=1, largest=True).indices.cpu().numpy()
+ del db_t, q_t
+ torch.cuda.empty_cache()
+ return out
+
+
+def recall_at_k(found: np.ndarray, truth: np.ndarray) -> float:
+ k = truth.shape[1]
+ return float(np.mean([len(set(found[i]) & set(truth[i])) / k for i in range(truth.shape[0])]))
+
+
+def best_time(fn, n_repeat: int) -> float:
+ best = float("inf")
+ for _ in range(n_repeat):
+ t = time.perf_counter()
+ fn()
+ best = min(best, time.perf_counter() - t)
+ return best
+
+
+def measure(search_fn, queries: np.ndarray, n_single: int, n_repeat: int):
+ """Single-query latency (median ms over n_single individual queries) and batched
+ throughput (q/s, best of n_repeat for all queries at once)."""
+ for _ in range(3): # warmup (FlashLib JIT-compiles per batch shape)
+ search_fn(queries[:1])
+ search_fn(queries)
+
+ per_query = []
+ for i in range(min(n_single, queries.shape[0])):
+ t = time.perf_counter()
+ search_fn(queries[i : i + 1])
+ per_query.append((time.perf_counter() - t) * 1000.0)
+ single_ms = float(np.median(per_query))
+
+ batch_time = best_time(lambda: search_fn(queries), n_repeat)
+ return single_ms, queries.shape[0] / batch_time
+
+
+def auto_nlist(n_db: int) -> int:
+ """A sane default nlist ~ 4*sqrt(N), clamped, rounded to a power of two."""
+ target = 4 * math.sqrt(max(n_db, 1))
+ p = 2 ** round(math.log2(max(target, 256)))
+ return int(max(256, min(p, 16384)))
+
+
+def _write_meta(index_path: str, backend_name: str, dim: int) -> None:
+ Path(f"{index_path}.meta.json").write_text(
+ json.dumps(
+ {
+ "version": "1.0",
+ "backend_name": backend_name,
+ "embedding_model": "synthetic",
+ "dimensions": dim,
+ "backend_kwargs": {"distance_metric": "cosine"},
+ "embedding_mode": "sentence-transformers",
+ "passage_sources": [],
+ }
+ )
+ )
+
+
+def _index_size_mb(index_path: str) -> float:
+ stem = Path(index_path).stem
+ parent = Path(index_path).parent
+ total = 0
+ for p in parent.glob(f"{stem}.*"):
+ if p.name.endswith(".meta.json"):
+ continue
+ total += p.stat().st_size
+ return total / (1024 * 1024)
+
+
+def build_backend(name: str, db, ids, index_path: str, nlist: int, nprobe: int) -> dict[str, Any]:
+ from leann.registry import BACKEND_REGISTRY
+
+ kwargs = {
+ "dimensions": db.shape[1],
+ "distance_metric": "cosine",
+ "nlist": nlist,
+ "nprobe": nprobe,
+ }
+ t = time.perf_counter()
+ BACKEND_REGISTRY[name].builder(**kwargs).build(db, ids, index_path, **kwargs)
+ build_time = time.perf_counter() - t
+ _write_meta(index_path, name, db.shape[1])
+ return {"build_time": build_time, "index_size_mb": _index_size_mb(index_path)}
+
+
+def make_searcher(name: str, index_path: str):
+ from leann.registry import BACKEND_REGISTRY
+
+ return BACKEND_REGISTRY[name].searcher(index_path, enable_warmup=False, use_daemon=False)
+
+
+def run_search(searcher, queries, top_k, truth, nprobe, n_single, n_repeat) -> dict[str, Any]:
+ def search(x):
+ return searcher.search(x, top_k=top_k, nprobe=nprobe, recompute_embeddings=False)
+
+ single_ms, qps = measure(search, queries, n_single, n_repeat)
+ out = search(queries)
+ found = np.array(
+ [[int(x) if x.lstrip("-").isdigit() else -1 for x in row] for row in out["labels"]],
+ dtype=np.int64,
+ )
+ recall = recall_at_k(found, truth)
+ return {"single_ms": single_ms, "throughput_qps": qps, "recall": recall}
+
+
+def run_size(n_db: int, args) -> dict[str, Any]:
+ import faiss
+ import torch
+
+ nlist = args.nlist if args.nlist > 0 else auto_nlist(n_db)
+ print(f"\n{'=' * 80}")
+ print(
+ f"Corpus: {n_db:,} vectors x {args.dim} dims | cosine | nlist={nlist} | "
+ f"{args.queries} queries | top_k={args.top_k}"
+ )
+ print(f"{'=' * 80}")
+
+ db, queries = make_clustered_data(n_db, args.queries, args.dim, args.seed, args.cluster_std)
+ ids = [str(i) for i in range(n_db)]
+ print("Computing exact ground truth (GPU)...")
+ truth = exact_ground_truth(db, queries, args.top_k)
+
+ nprobe_sweep = [p for p in args.nprobe_sweep if p <= nlist]
+ result: dict[str, Any] = {
+ "n_db": n_db,
+ "nlist": nlist,
+ "nprobe_sweep": nprobe_sweep,
+ "rows": [],
+ }
+
+ with tempfile.TemporaryDirectory() as tmp:
+ # ---- Build both backends once (build cost is a one-time offline step). ----
+ faiss.omp_set_num_threads(min(os.cpu_count() or 1, 64)) # build uses many cores
+ gpu_path = str(Path(tmp) / "flashlib_ivf.leann")
+ cpu_path = str(Path(tmp) / "ivf.leann")
+
+ print("Building flashlib_ivf (GPU)...")
+ gb = build_backend("flashlib_ivf", db, ids, gpu_path, nlist, max(nprobe_sweep))
+ print(f" build {gb['build_time']:.2f}s | index {gb['index_size_mb']:.1f} MB")
+
+ print("Building ivf (FAISS, CPU)...")
+ cb = build_backend("ivf", db, ids, cpu_path, nlist, max(nprobe_sweep))
+ print(f" build {cb['build_time']:.2f}s | index {cb['index_size_mb']:.1f} MB")
+ result["flashlib_ivf_build"] = gb
+ result["ivf_build"] = cb
+
+ # ---- Sweep nprobe; reuse a single searcher per backend across the sweep. ----
+ gpu_searcher = make_searcher("flashlib_ivf", gpu_path)
+ cpu_searcher = make_searcher("ivf", cpu_path)
+ faiss.omp_set_num_threads(args.n_threads) # constrain SEARCH to CPU budget
+
+ for nprobe in nprobe_sweep:
+ g = run_search(
+ gpu_searcher, queries, args.top_k, truth, nprobe, args.single_queries, args.repeat
+ )
+ c = run_search(
+ cpu_searcher, queries, args.top_k, truth, nprobe, args.single_queries, args.repeat
+ )
+ row = {"nprobe": nprobe, "gpu": g, "cpu": c}
+ result["rows"].append(row)
+ lat = c["single_ms"] / g["single_ms"] if g["single_ms"] else float("nan")
+ tpt = g["throughput_qps"] / c["throughput_qps"] if c["throughput_qps"] else float("nan")
+ print(
+ f" nprobe={nprobe:<4} | "
+ f"GPU {g['single_ms']:7.3f}ms {g['throughput_qps']:>10,.0f}q/s r{g['recall']:.3f} | "
+ f"CPU {c['single_ms']:7.3f}ms {c['throughput_qps']:>10,.0f}q/s r{c['recall']:.3f} | "
+ f"speedup {lat:5.1f}x lat {tpt:6.1f}x tpt"
+ )
+
+ del gpu_searcher, cpu_searcher
+ gc.collect()
+ torch.cuda.empty_cache()
+
+ return result
+
+
+def print_summary(rows: list[dict[str, Any]], args) -> None:
+ print(f"\n\n{'#' * 84}")
+ print("# SUMMARY: flashlib_ivf (GPU) vs ivf (FAISS, CPU) - matched nlist, nprobe sweep")
+ print(f"# CPU baseline used {args.n_threads} thread(s); metric=cosine; top_k={args.top_k}")
+ print(f"{'#' * 84}")
+
+ rcol = f"R@{args.top_k}"
+ hdr = (
+ f"{'nprobe':>6} | {'GPU ms':>8} {'GPU q/s':>11} {rcol:>6} | "
+ f"{'CPU ms':>8} {'CPU q/s':>11} {rcol:>6} | {'lat x':>6} {'tpt x':>6}"
+ )
+ for r in rows:
+ gb, cb = r["flashlib_ivf_build"], r["ivf_build"]
+ print(f"\n{'-' * len(hdr)}")
+ print(
+ f"Corpus {r['n_db']:,} x {args.dim}d | nlist={r['nlist']} | "
+ f"build: GPU {gb['build_time']:.1f}s ({gb['index_size_mb']:.0f}MB) vs "
+ f"CPU {cb['build_time']:.1f}s ({cb['index_size_mb']:.0f}MB)"
+ )
+ print("-" * len(hdr))
+ print(hdr)
+ print("-" * len(hdr))
+ for row in r["rows"]:
+ g, c = row["gpu"], row["cpu"]
+ lat = c["single_ms"] / g["single_ms"] if g["single_ms"] else float("nan")
+ tpt = g["throughput_qps"] / c["throughput_qps"] if c["throughput_qps"] else float("nan")
+ print(
+ f"{row['nprobe']:>6} | {g['single_ms']:>8.3f} {g['throughput_qps']:>11,.0f} "
+ f"{g['recall']:>6.3f} | {c['single_ms']:>8.3f} {c['throughput_qps']:>11,.0f} "
+ f"{c['recall']:>6.3f} | {lat:>6.1f} {tpt:>6.1f}"
+ )
+
+
+def main() -> None:
+ p = argparse.ArgumentParser(
+ description="FlashLib IVF (GPU) vs FAISS IVF (CPU) comparison for LEANN.",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ p.add_argument("--sizes", type=int, nargs="+", default=[100_000, 1_000_000])
+ p.add_argument("--dim", type=int, default=768)
+ p.add_argument("--queries", type=int, default=1000)
+ p.add_argument("--top-k", type=int, default=10)
+ p.add_argument("--nlist", type=int, default=0, help="IVF partitions (0 = auto ~4*sqrt(N))")
+ p.add_argument("--nprobe-sweep", type=int, nargs="+", default=[1, 4, 8, 16, 32, 64])
+ p.add_argument(
+ "--cpu-threads",
+ type=int,
+ default=0,
+ help="FAISS CPU search threads (0 = all cores, capped at 32).",
+ )
+ p.add_argument("--cluster-std", type=float, default=0.1, help="Cluster spread (lower=tighter)")
+ p.add_argument("--single-queries", type=int, default=200)
+ p.add_argument("--repeat", type=int, default=5)
+ p.add_argument("--seed", type=int, default=42)
+ args = p.parse_args()
+
+ try:
+ import torch
+ except ImportError:
+ _fail("PyTorch is required (with CUDA).")
+ if not torch.cuda.is_available():
+ _fail("FlashLib IVF is GPU-only, but no CUDA GPU is available.")
+ try:
+ import faiss
+ import flashlib
+ except ImportError as e:
+ _fail(f"Missing dependency: {e}. Need 'flashlib' and 'faiss-cpu'.")
+
+ from leann.registry import BACKEND_REGISTRY, autodiscover_backends
+
+ autodiscover_backends()
+ for need in ("ivf", "flashlib_ivf"):
+ if need not in BACKEND_REGISTRY:
+ _fail(
+ f"Backend '{need}' not registered. Install it: "
+ f"pip install -e packages/leann-backend-{need.replace('_', '-')}"
+ )
+
+ all_cores = os.cpu_count() or 1
+ args.n_threads = args.cpu_threads if args.cpu_threads > 0 else min(all_cores, 32)
+
+ print("FlashLib IVF (GPU) vs FAISS IVF (CPU) comparison for LEANN")
+ print(f"GPU: {torch.cuda.get_device_name(0)} | CPU cores available: {all_cores}")
+ print(
+ f"flashlib {flashlib.__version__} | faiss {faiss.__version__} | torch {torch.__version__}"
+ )
+ print(
+ f"Config: dim={args.dim}, queries={args.queries}, top_k={args.top_k}, "
+ f"FAISS search threads={args.n_threads} (build uses up to 64), "
+ f"nprobe_sweep={args.nprobe_sweep}"
+ )
+
+ rows = [run_size(n, args) for n in args.sizes]
+ print_summary(rows, args)
+ print("\nDone.")
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ print("\nInterrupted.")
+ sys.exit(130)
+ finally:
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(0)
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
new file mode 100644
index 0000000..2030b8e
--- /dev/null
+++ b/docs/CHANGELOG.md
@@ -0,0 +1,20 @@
+# Changelog
+
+Append-only log of major changes to LEANN (new features, breaking changes, important
+fixes). Newest entries at the bottom.
+
+## 2026-06-02: GPU FlashLib IVF backend (`flashlib_ivf`)
+
+- Add `leann-backend-flashlib-ivf`, a GPU IVF-Flat (inverted file) approximate-NN
+ backend built on FlashLib (`flash_ivf_flat`, Triton/CuteDSL) — the GPU counterpart
+ of the FAISS `ivf` backend. Registered as backend name `flashlib_ivf`; install via
+ `uv sync --extra flashlib-ivf` or `pip install leann-backend-flashlib-ivf`. Shares
+ the `nlist`/`nprobe` recall knobs with the `ivf` backend, so the two are drop-in
+ comparable. Requires a CUDA GPU at build (k-means) and search.
+- Add `benchmarks/flashlib_ivf_vs_faiss_ivf.py`: head-to-head `flashlib_ivf` (GPU) vs
+ `ivf` (FAISS, CPU) at matched `nlist` across an `nprobe` sweep (build time,
+ single-query latency, batched throughput, recall@k vs exact ground truth). On an
+ NVIDIA H200 at 1M x 768 vectors (nlist=4096, 8 CPU threads): ~13x faster build and,
+ at nprobe=32, ~6.5x lower single-query latency / ~75x higher batched throughput at
+ comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).
+- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.
diff --git a/docs/flashlib_backend_guide.md b/docs/flashlib_backend_guide.md
index f903af9..ed5a155 100644
--- a/docs/flashlib_backend_guide.md
+++ b/docs/flashlib_backend_guide.md
@@ -31,6 +31,7 @@ distances, indices = index.kneighbors(torch.randn(10_000, 128, device="cuda"), n
| `diskann` | Larger-than-memory datasets | On-disk graph | CPU |
| `ivf` | Incremental add/remove without rebuild | Full vectors (FAISS) | CPU |
| **`flashlib`** | **High-throughput search on a CUDA GPU** | Full vectors (`.npy`) | **CUDA GPU** |
+| **`flashlib_ivf`** | **GPU IVF-Flat (approximate) — the GPU counterpart of `ivf`** | Full vectors (`.pt`) | **CUDA GPU** |
Use FlashLib when you already have a GPU and want fast IVF-Flat search. It stores
the full float32 vectors rather than a pruned graph, so it trades LEANN's storage
@@ -108,6 +109,66 @@ constraint), and `nprobe` is derived from the search `complexity` knob.
| `distance_metric` (build) | `"mips"` | `mips`, `cosine`, or `l2`. |
| `nprobe` (search) | derived from `complexity` | Partitions probed per query — the recall knob (higher = better recall, slower). |
+## FlashLib IVF backend (`flashlib_ivf`)
+
+The **`flashlib_ivf`** backend (`leann-backend-flashlib-ivf`) is the GPU counterpart
+of LEANN's FAISS `ivf` backend: an IVF-Flat (inverted file) index that
+coarse-quantizes the corpus into `nlist` cells with k-means and, at search time,
+scans only the `nprobe` nearest cells — entirely on CUDA tensors via FlashLib's
+`flash_ivf_flat`. At a fixed `(nlist, nprobe)` the GPU and CPU IVF probe nearly the
+same candidate set, so recall is comparable; the only difference is GPU vs CPU
+kernels. `nprobe` is the recall knob (defaults to `min(complexity, nlist)`).
+
+```bash
+uv sync --extra flashlib-ivf # or: pip install leann-backend-flashlib-ivf
+```
+
+```python
+from leann import LeannBuilder, LeannSearcher
+
+builder = LeannBuilder(backend_name="flashlib_ivf", nlist=4096, distance_metric="cosine")
+builder.add_text("LEANN recomputes embeddings on the fly to cut storage by ~97%.")
+builder.build_index("demo.leann")
+
+searcher = LeannSearcher("demo.leann")
+results = searcher.search("How does LEANN save storage?", top_k=10, complexity=32) # nprobe=32
+```
+
+**How it works:** the builder trains the coarse quantizer on the GPU and persists the
+built index tensors with `torch.save` (`<index>.flashlib_ivf.pt`) plus an id map
+(`<index>.flashlib_ivf_id_map.json`); the searcher reloads them onto the GPU once (no
+k-means re-train). A CUDA GPU is required at **both** build (k-means) and search time.
+FlashLib IVF ranks by squared L2, so `mips`/`cosine` L2-normalize the vectors (squared-L2
+ranking then matches inner-product/cosine).
+
+### Speed — IVF (GPU) vs IVF (CPU)
+
+```bash
+python benchmarks/flashlib_ivf_vs_faiss_ivf.py \
+ --sizes 100000 1000000 --nprobe-sweep 1 8 32 128 --cpu-threads 8
+```
+
+`flashlib_ivf` (GPU) vs the FAISS `ivf` backend (CPU) at the **same `nlist`**, sweeping
+`nprobe` (NVIDIA H200, `faiss-cpu` at 8 threads, 768-dim, top-k=10). GPU latency stays
+~flat while CPU latency grows linearly with `nprobe`, so the GPU lead widens exactly as
+you raise recall:
+
+| Corpus | nprobe | GPU lat | CPU lat | GPU q/s | CPU q/s | Recall (GPU/CPU) | Speedup (lat / tpt) |
+|--------|--------|---------|---------|---------|---------|------------------|---------------------|
+| 1M | 8 | 0.45 ms | 1.14 ms | 107k | 5.9k | 0.340 / 0.321 | 2.6× / 18× |
+| 1M | 32 | 0.46 ms | 3.00 ms | 141k | 1.9k | 0.400 / 0.350 | 6.5× / 75× |
+| 1M | 128 | 0.55 ms | 9.91 ms | 95k | 0.6k | 0.539 / 0.423 | 18× / 159× |
+
+Build (1M, `nlist=4096`): GPU **10.6 s** vs FAISS CPU **140.7 s** — a **13× faster
+build** (GPU k-means vs CPU k-means training).
+
+Honest caveats: at very low `nprobe` (e.g. 1) single-query GPU latency (~0.44 ms) is
+*higher* than CPU, because the per-query work is tiny and GPU kernel-launch overhead
+dominates; the GPU advantage grows with `nprobe` (higher recall), batch size, and corpus
+size. The absolute recall above is low because the synthetic mixture-of-Gaussians corpus
+has more clusters than `nlist`; on real embeddings recall is far higher — this benchmark
+isolates the GPU-vs-CPU *relative* comparison at matched `(nlist, nprobe)`.
+
## Notes & limitations
- GPU-only at search time; building an index works on a CPU-only machine.
diff --git a/packages/leann-backend-flashlib-ivf/README.md b/packages/leann-backend-flashlib-ivf/README.md
new file mode 100644
index 0000000..79fb38a
--- /dev/null
+++ b/packages/leann-backend-flashlib-ivf/README.md
@@ -0,0 +1,71 @@
+# leann-backend-flashlib-ivf
+
+GPU-accelerated [FlashLib](https://github.com/FlashML-org/flashlib) IVF-Flat
+(inverted file) backend for LEANN - the GPU counterpart of the FAISS
+[`leann-backend-ivf`](../leann-backend-ivf) backend.
+
+FlashLib is a GPU library of classical ML operators built on Triton / CuteDSL.
+Its IVF-Flat index coarse-quantizes the corpus into `nlist` cells and, at search
+time, scans only the `nprobe` nearest cells - entirely on CUDA tensors. At a fixed
+`(nlist, nprobe)` it probes the same candidate set as a reference IVF-Flat
+(FAISS / cuVS), so recall is comparable; the difference is GPU vs CPU kernels.
+
+This is registered as the `flashlib_ivf` backend, distinct from the exact GPU
+k-NN `flashlib` backend (which does brute-force `NearestNeighbors`, not IVF).
+
+## Requirements
+
+- A CUDA GPU (required at **build** time for k-means training and at **search** time).
+- `pip install flashlib` and `torch`.
+
+## Install
+
+```bash
+# from a LEANN checkout
+uv sync --extra flashlib-ivf
+# or
+pip install leann-backend-flashlib-ivf
+```
+
+## Usage
+
+```python
+from leann import LeannBuilder, LeannSearcher
+
+builder = LeannBuilder(backend_name="flashlib_ivf", nlist=1024, distance_metric="cosine")
+builder.add_text("LEANN recomputes embeddings to save storage.")
+builder.build_index("demo.leann")
+
+searcher = LeannSearcher("demo.leann")
+print(searcher.search("How does LEANN save storage?", top_k=3, complexity=32)) # nprobe=min(32, nlist)
+```
+
+Or from the example apps:
+
+```bash
+python -m apps.document_rag --query "What are the main techniques LEANN explores?" \
+ --backend-name flashlib_ivf
+```
+
+## How it works
+
+The built IVF-Flat index is a small set of torch tensors (centroids,
+cell-contiguous data, row ids, CSR offsets), so this backend persists it with
+`torch.save` (`<index>.flashlib_ivf.pt`) plus an id map
+(`<index>.flashlib_ivf_id_map.json`) and reloads it onto the GPU at searcher
+start-up - no k-means re-train.
+
+FlashLib's only distance metric is squared L2. For `mips` / `cosine` the vectors
+are L2-normalized at build and query time, on which squared-L2 ranking is
+equivalent to inner-product / cosine ranking.
+
+## Parameters
+
+| kwarg | default | meaning |
+|-------|---------|---------|
+| `nlist` | `1024` | number of IVF partitions / coarse centroids (clamped to corpus size) |
+| `nprobe` (build) | `16` | default partitions probed per query (recall knob) |
+| `niter` | `20` | Lloyd k-means iterations for the coarse quantizer |
+| `seed` | `0` | RNG seed (deterministic build) |
+| `distance_metric` | `"mips"` | `mips`, `cosine`, or `l2` |
+| `complexity` (search) | `64` | sets `nprobe = min(complexity, nlist)` when `nprobe` is not given |
diff --git a/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.py b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.py
new file mode 100644
index 0000000..25fdeaa
--- /dev/null
+++ b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/__init__.py
@@ -0,0 +1,13 @@
+"""LEANN FlashLib IVF backend: GPU-accelerated IVF-Flat (Triton/CuteDSL) approximate-NN search."""
+
+from .flashlib_ivf_backend import (
+ FlashlibIVFBackend,
+ FlashlibIVFBuilder,
+ FlashlibIVFSearcher,
+)
+
+__all__ = [
+ "FlashlibIVFBackend",
+ "FlashlibIVFBuilder",
+ "FlashlibIVFSearcher",
+]
diff --git a/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.py b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.py
new file mode 100644
index 0000000..71eff52
--- /dev/null
+++ b/packages/leann-backend-flashlib-ivf/leann_backend_flashlib_ivf/flashlib_ivf_backend.py
@@ -0,0 +1,266 @@
+"""
+FlashLib IVF backend: GPU-accelerated IVF-Flat (inverted file) ANN search.
+
+FlashLib (https://github.com/FlashML-org/flashlib) is a GPU library of classical
+ML primitives. This backend uses its IVF-Flat index, which runs an *approximate*
+nearest-neighbor search entirely on CUDA tensors:
+
+ from flashlib import flash_ivf_flat_build, flash_ivf_flat_search
+ index = flash_ivf_flat_build(db_cuda, nlist, nprobe=..., niter=...) # (M, D) CUDA
+ vals, ids = flash_ivf_flat_search(index, queries_cuda, k, nprobe=...)
+
+This is the GPU counterpart of the FAISS ``ivf`` backend (FAISS ``IndexIVFFlat``):
+both coarse-quantize the corpus into ``nlist`` cells and, at search time, scan only
+the ``nprobe`` nearest cells. At a fixed ``(nlist, nprobe)`` FlashLib probes the same
+candidate set as a reference IVF-Flat, so recall is comparable; the difference is GPU
+vs CPU kernels.
+
+The built index is a small set of torch tensors (centroids, cell-contiguous data,
+row ids, CSR offsets), so we persist it with ``torch.save`` (``<index>.flashlib_ivf.pt``)
+plus an id map (``<index>.flashlib_ivf_id_map.json``) and reload it onto the GPU at
+searcher start-up (no k-means re-train).
+
+FlashLib ranks by squared L2. For ``mips`` / ``cosine`` we L2-normalize both the
+database and query vectors, on which squared-L2 ranking is equivalent to
+inner-product / cosine ranking.
+
+Requires a CUDA GPU at both build (k-means training) and search time.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Optional
+
+import numpy as np
+from leann.interface import (
+ LeannBackendBuilderInterface,
+ LeannBackendFactoryInterface,
+ LeannBackendSearcherInterface,
+)
+from leann.registry import register_backend
+from leann.searcher_base import BaseSearcher
+
+logger = logging.getLogger(__name__)
+
+INDEX_SUFFIX = "flashlib_ivf.pt"
+ID_MAP_SUFFIX = "flashlib_ivf_id_map.json"
+
+# IvfFlatIndex dataclass fields, split by how they serialize.
+_TENSOR_FIELDS = ("centroids", "data", "ids", "list_offsets")
+_SCALAR_FIELDS = ("metric", "D", "Dp", "nlist", "nprobe", "max_list_len")
+
+
+def _import_flashlib():
+ try:
+ import torch # noqa: F401
+ from flashlib import (
+ IvfFlatIndex,
+ flash_ivf_flat_build,
+ flash_ivf_flat_search,
+ )
+ except ImportError as e:
+ raise ImportError(
+ "The FlashLib IVF backend requires 'flashlib' and 'torch' with CUDA. "
+ "Install with: pip install flashlib (a CUDA GPU is required at build and search time)."
+ ) from e
+ return IvfFlatIndex, flash_ivf_flat_build, flash_ivf_flat_search
+
+
+def _normalize_l2(data: np.ndarray) -> np.ndarray:
+ norms = np.linalg.norm(data, axis=1, keepdims=True)
+ norms[norms == 0] = 1
+ return data / norms
+
+
+def _needs_normalize(distance_metric: str) -> bool:
+ return distance_metric.lower() in ("mips", "cosine")
+
+
+def _index_path(index_dir: Path, index_prefix: str) -> Path:
+ return index_dir / f"{index_prefix}.{INDEX_SUFFIX}"
+
+
+def _id_map_path(index_dir: Path, index_prefix: str) -> Path:
+ return index_dir / f"{index_prefix}.{ID_MAP_SUFFIX}"
+
+
+def _save_id_map(index_dir: Path, index_prefix: str, ids: list[str]) -> None:
+ with open(_id_map_path(index_dir, index_prefix), "w", encoding="utf-8") as f:
+ json.dump({"ids": ids}, f)
+
+
+def _load_id_map(index_dir: Path, index_prefix: str) -> list[str]:
+ path = _id_map_path(index_dir, index_prefix)
+ if not path.exists():
+ raise FileNotFoundError(f"FlashLib IVF id map not found at {path}")
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)["ids"]
+
+
+def _save_index(index, path: Path) -> None:
+ import torch
+
+ state: dict[str, Any] = {f: getattr(index, f).detach().cpu() for f in _TENSOR_FIELDS}
+ for f in _SCALAR_FIELDS:
+ state[f] = getattr(index, f)
+ torch.save(state, str(path))
+
+
+def _load_index(path: Path, device: str = "cuda"):
+ import torch
+
+ IvfFlatIndex, _, _ = _import_flashlib()
+ state = torch.load(str(path), map_location=device, weights_only=False)
+ kwargs: dict[str, Any] = {f: state[f].to(device) for f in _TENSOR_FIELDS}
+ for f in _SCALAR_FIELDS:
+ kwargs[f] = state[f]
+ return IvfFlatIndex(**kwargs)
+
+
+@register_backend("flashlib_ivf")
+class FlashlibIVFBackend(LeannBackendFactoryInterface):
+ @staticmethod
+ def builder(**kwargs) -> LeannBackendBuilderInterface:
+ return FlashlibIVFBuilder(**kwargs)
+
+ @staticmethod
+ def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
+ return FlashlibIVFSearcher(index_path, **kwargs)
+
+
+class FlashlibIVFBuilder(LeannBackendBuilderInterface):
+ def __init__(self, **kwargs):
+ self.build_params = kwargs.copy()
+ self.distance_metric = self.build_params.setdefault("distance_metric", "mips")
+ self.nlist = self.build_params.setdefault("nlist", 1024)
+ self.nprobe = self.build_params.setdefault("nprobe", 16)
+ self.niter = self.build_params.setdefault("niter", 20)
+ self.seed = self.build_params.setdefault("seed", 0)
+ self.dimensions = self.build_params.get("dimensions")
+
+ def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:
+ import torch
+
+ _, flash_ivf_flat_build, _ = _import_flashlib()
+ if not torch.cuda.is_available():
+ raise RuntimeError(
+ "FlashLib IVF backend requires a CUDA GPU at build time (k-means training), "
+ "but none is available."
+ )
+
+ path = Path(index_path)
+ index_dir = path.parent
+ index_prefix = path.stem
+ index_dir.mkdir(parents=True, exist_ok=True)
+
+ if data.dtype != np.float32:
+ data = data.astype(np.float32)
+ data = np.ascontiguousarray(data)
+ if _needs_normalize(self.distance_metric):
+ data = _normalize_l2(data)
+
+ n = data.shape[0]
+ nlist = int(min(self.nlist, n)) if n > 0 else int(self.nlist)
+
+ db = torch.from_numpy(data).cuda()
+ index = flash_ivf_flat_build(
+ db,
+ nlist,
+ metric="l2",
+ nprobe=int(self.nprobe),
+ niter=int(self.niter),
+ seed=int(self.seed),
+ )
+ _save_index(index, _index_path(index_dir, index_prefix))
+ _save_id_map(index_dir, index_prefix, list(ids))
+ logger.info(
+ "FlashLib IVF build: %d vectors (dim=%d, metric=%s, nlist=%d, nprobe=%d) at %s",
+ n,
+ data.shape[1],
+ self.distance_metric,
+ nlist,
+ self.nprobe,
+ _index_path(index_dir, index_prefix),
+ )
+
+
+class FlashlibIVFSearcher(BaseSearcher):
+ def __init__(self, index_path: str, **kwargs):
+ # Reuse the HNSW embedding server (if present) to embed queries, exactly like
+ # the other non-recompute backends; falls back to direct model loading.
+ super().__init__(
+ index_path,
+ backend_module_name="leann_backend_hnsw.hnsw_embedding_server",
+ **kwargs,
+ )
+ backend_kwargs = self.meta.get("backend_kwargs", {})
+ self.distance_metric = backend_kwargs.get("distance_metric", "mips").lower()
+
+ index_prefix = self.index_path.stem
+ index_file = _index_path(self.index_dir, index_prefix)
+ if not index_file.exists():
+ raise FileNotFoundError(f"FlashLib IVF index file not found at {index_file}")
+
+ import torch
+
+ if not torch.cuda.is_available():
+ raise RuntimeError(
+ "FlashLib IVF backend requires a CUDA GPU at search time, but none is available."
+ )
+
+ self._ids = _load_id_map(self.index_dir, index_prefix)
+ self._index = _load_index(index_file, device="cuda")
+ self._nlist = int(self._index.nlist)
+ self._ntotal = int(self._index.data.shape[0])
+ logger.info(
+ "FlashLib IVF searcher ready: %d vectors, nlist=%d, metric=%s",
+ self._ntotal,
+ self._nlist,
+ self.distance_metric,
+ )
+
+ def search(
+ self,
+ query: np.ndarray,
+ top_k: int,
+ complexity: int = 64,
+ nprobe: Optional[int] = None,
+ **kwargs,
+ ) -> dict[str, Any]:
+ import torch
+
+ _, _, flash_ivf_flat_search = _import_flashlib()
+ if query.dtype != np.float32:
+ query = query.astype(np.float32)
+ if _needs_normalize(self.distance_metric):
+ query = _normalize_l2(query)
+
+ # complexity is the recall knob shared with the FAISS ivf backend.
+ nprobe = nprobe or min(complexity, self._nlist)
+ k = min(top_k, self._ntotal)
+ q = torch.from_numpy(np.ascontiguousarray(query)).cuda()
+ distances, indices = flash_ivf_flat_search(self._index, q, k, nprobe=int(nprobe))
+ distances_np = distances.detach().cpu().numpy().astype(np.float32)
+ indices_np = indices.detach().cpu().numpy()
+
+ def map_label(i: int) -> str:
+ # flash_ivf_flat_search pads short candidate lists with -1.
+ return self._ids[i] if i >= 0 else "-1"
+
+ string_labels = [[map_label(int(i)) for i in row] for row in indices_np]
+ return {"labels": string_labels, "distances": distances_np}
+
+ def compute_query_embedding(
+ self,
+ query: str,
+ use_server_if_available: bool = True,
+ zmq_port: Optional[int] = None,
+ query_template: Optional[str] = None,
+ ) -> np.ndarray:
+ return super().compute_query_embedding(
+ query,
+ use_server_if_available=use_server_if_available,
+ zmq_port=zmq_port,
+ query_template=query_template,
+ )
diff --git a/packages/leann-backend-flashlib-ivf/pyproject.toml b/packages/leann-backend-flashlib-ivf/pyproject.toml
new file mode 100644
index 0000000..0437b70
--- /dev/null
+++ b/packages/leann-backend-flashlib-ivf/pyproject.toml
@@ -0,0 +1,20 @@
+[project]
+name = "leann-backend-flashlib-ivf"
+version = "0.3.6"
+description = "GPU-accelerated FlashLib IVF-Flat (Triton/CuteDSL) approximate-NN backend for LEANN."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+ "leann-core>=0.3.6",
+ "numpy>=1.20.0",
+ "flashlib",
+ "torch",
+]
+
+[project.optional-dependencies]
+# Optional: use the HNSW embedding server for query embedding (same as other backends).
+query-server = ["leann-backend-hnsw>=0.3.6", "pyzmq>=23.0.0", "msgpack>=1.0.0"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["leann_backend_flashlib_ivf*"]
diff --git a/pyproject.toml b/pyproject.toml
index 482e1f9..f96b96d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,11 +66,16 @@ diskann = [
"leann-backend-diskann",
]
-# GPU-accelerated FlashLib (IVFFlat on Triton/CuteDSL) backend. Requires a CUDA GPU.
+# GPU-accelerated FlashLib (exact k-NN on Triton/CuteDSL) backend. Requires a CUDA GPU.
flashlib = [
"leann-backend-flashlib",
]
+# GPU-accelerated FlashLib IVF-Flat (approximate-NN) backend. Requires a CUDA GPU.
+flashlib-ivf = [
+ "leann-backend-flashlib-ivf",
+]
+
# Add a new optional dependency group for document processing
documents = [
"beautifulsoup4>=4.13.0", # For HTML parsing
@@ -94,6 +99,7 @@ leann-core = { path = "packages/leann-core", editable = true }
leann-backend-diskann = { path = "packages/leann-backend-diskann", editable = true }
leann-backend-hnsw = { path = "packages/leann-backend-hnsw", editable = true }
leann-backend-flashlib = { path = "packages/leann-backend-flashlib", editable = true }
+leann-backend-flashlib-ivf = { path = "packages/leann-backend-flashlib-ivf", editable = true }
astchunk = { path = "packages/astchunk-leann", editable = true }
[dependency-groups]
diff --git a/tests/test_flashlib_ivf_backend.py b/tests/test_flashlib_ivf_backend.py
new file mode 100644
index 0000000..d5e04a6
--- /dev/null
+++ b/tests/test_flashlib_ivf_backend.py
@@ -0,0 +1,125 @@
+"""
+Correctness tests for the FlashLib IVF backend (``flashlib_ivf``).
+
+Registry-level (no embedding model needed): builds a small clustered corpus through
+the real LEANN backend builders/searchers and checks that ``flashlib_ivf`` (GPU)
+registers, persists/reloads its index, and returns recall comparable to the FAISS
+``ivf`` backend and to exact ground truth at a matched ``(nlist, nprobe)``.
+
+Requires a CUDA GPU + ``flashlib`` (skipped otherwise, e.g. in CI).
+"""
+
+import json
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+
+def _cuda_or_skip():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("flashlib")
+ if not torch.cuda.is_available():
+ pytest.skip("FlashLib IVF backend requires a CUDA GPU")
+
+
+def _registry():
+ from leann.registry import BACKEND_REGISTRY, autodiscover_backends
+
+ autodiscover_backends()
+ return BACKEND_REGISTRY
+
+
+def _clustered(n: int, dim: int, seed: int) -> np.ndarray:
+ rng = np.random.default_rng(seed)
+ n_clusters = 64
+ centers = rng.standard_normal((n_clusters, dim), dtype=np.float32)
+ centers /= np.linalg.norm(centers, axis=1, keepdims=True)
+ assign = rng.integers(0, n_clusters, size=n)
+ db = centers[assign] + 0.05 * rng.standard_normal((n, dim)).astype(np.float32)
+ db /= np.linalg.norm(db, axis=1, keepdims=True)
+ return np.ascontiguousarray(db.astype(np.float32))
+
+
+def _exact_gt(db: np.ndarray, q: np.ndarray, k: int) -> np.ndarray:
+ scores = q @ db.T
+ return np.argsort(-scores, axis=1)[:, :k]
+
+
+def _recall(found: np.ndarray, truth: np.ndarray) -> float:
+ k = truth.shape[1]
+ return float(np.mean([len(set(found[i]) & set(truth[i])) / k for i in range(len(truth))]))
+
+
+def _write_meta(index_path: str, backend: str, dim: int) -> None:
+ Path(f"{index_path}.meta.json").write_text(
+ json.dumps(
+ {
+ "version": "1.0",
+ "backend_name": backend,
+ "embedding_model": "synthetic",
+ "dimensions": dim,
+ "backend_kwargs": {"distance_metric": "cosine"},
+ "embedding_mode": "sentence-transformers",
+ "passage_sources": [],
+ }
+ )
+ )
+
+
+def _labels_to_int(out: dict) -> np.ndarray:
+ return np.array(
+ [[int(x) if x.lstrip("-").isdigit() else -1 for x in row] for row in out["labels"]],
+ dtype=np.int64,
+ )
+
+
+def test_flashlib_ivf_recall_parity_with_faiss_ivf():
+ _cuda_or_skip()
+ reg = _registry()
+ assert "flashlib_ivf" in reg, "flashlib_ivf backend not registered"
+ if "ivf" not in reg:
+ pytest.skip("faiss ivf backend not installed")
+
+ dim, n, k, nlist, nprobe = 64, 20_000, 10, 128, 16
+ db = _clustered(n, dim, seed=0)
+ queries = db[:100].copy()
+ ids = [str(i) for i in range(n)]
+ truth = _exact_gt(db, queries, k)
+
+ kw = {"dimensions": dim, "distance_metric": "cosine", "nlist": nlist, "nprobe": nprobe}
+ with tempfile.TemporaryDirectory() as tmp:
+ gpu_path = str(Path(tmp) / "g.leann")
+ cpu_path = str(Path(tmp) / "c.leann")
+
+ reg["flashlib_ivf"].builder(**kw).build(db, ids, gpu_path, **kw)
+ reg["ivf"].builder(**kw).build(db, ids, cpu_path, **kw)
+
+ # Persistence: the GPU index + id map are written to disk.
+ assert (Path(tmp) / "g.flashlib_ivf.pt").exists()
+ assert (Path(tmp) / "g.flashlib_ivf_id_map.json").exists()
+
+ _write_meta(gpu_path, "flashlib_ivf", dim)
+ _write_meta(cpu_path, "ivf", dim)
+
+ gpu = reg["flashlib_ivf"].searcher(gpu_path, enable_warmup=False, use_daemon=False)
+ cpu = reg["ivf"].searcher(cpu_path, enable_warmup=False, use_daemon=False)
+
+ gpu_found = _labels_to_int(
+ gpu.search(queries, top_k=k, nprobe=nprobe, recompute_embeddings=False)
+ )
+ cpu_found = _labels_to_int(
+ cpu.search(queries, top_k=k, nprobe=nprobe, recompute_embeddings=False)
+ )
+
+ gpu_recall = _recall(gpu_found, truth)
+ cpu_recall = _recall(cpu_found, truth)
+
+ # At matched (nlist, nprobe) both IVF backends probe ~the same candidate set, so
+ # recall should be high and comparable (GPU is often a touch higher: independent
+ # coarse-quantizer training, not "more exact").
+ assert gpu_recall > 0.7, f"flashlib_ivf recall too low: {gpu_recall:.3f}"
+ assert abs(gpu_recall - cpu_recall) < 0.15, (
+ f"recall parity off: gpu {gpu_recall:.3f} vs cpu {cpu_recall:.3f}"
+ )