https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(index): add IVF+PQ compression flag for the FAISS build (#155)
Shubham Srivastavaaba824b3172c
diff --git a/embed/src/pixelrag_embed/index.py b/embed/src/pixelrag_embed/index.pyindex 019767e..b6e5c64 100644--- a/embed/src/pixelrag_embed/index.py+++ b/embed/src/pixelrag_embed/index.py@@ -148,8 +148,10 @@ def build_ivf(train_sample: int = 500_000,metric: str = "ip",gpu_id: int = -1,+ pq_m: int = 0,+ pq_nbits: int = 8,):- """Build FAISS IVFFlat index.+ """Build a FAISS IVF index — IVFFlat by default, IVFPQ when pq_m > 0.Args:nlist: number of IVF clusters (default 4096, good for ~30M vectors)@@ -157,6 +159,10 @@ def build_ivf(train_sample: number of vectors to sample for K-means trainingmetric: 'ip' (inner product / cosine for L2-normalized vectors) or 'l2'gpu_id: GPU to use for training (-1 = CPU only)+ pq_m: PQ sub-quantizers. 0 keeps the uncompressed IVFFlat index; >0 builds+ an IVFPQ index that stores pq_m bytes/vector (at nbits=8) instead of+ dim*4 — ~128x smaller for dim=2048, pq_m=64. Must divide dim evenly.+ pq_nbits: bits per PQ sub-quantizer (4 or 8), only used when pq_m > 0."""import faiss@@ -195,7 +201,16 @@ def build_ivf(train_data = embeddings[train_indices]quantizer = faiss.IndexFlatIP(dim) if metric == "ip" else faiss.IndexFlatL2(dim)- index = faiss.IndexIVFFlat(quantizer, dim, nlist, metric_type)+ if pq_m > 0:+ if dim % pq_m != 0:+ raise ValueError(+ f"--pq-m ({pq_m}) must divide the embedding dim ({dim}) evenly"+ )+ # Pass metric_type explicitly: faiss.IndexIVFPQ defaults to METRIC_L2,+ # which is wrong for the IP-normalized embeddings this index uses.+ index = faiss.IndexIVFPQ(quantizer, dim, nlist, pq_m, pq_nbits, metric_type)+ else:+ index = faiss.IndexIVFFlat(quantizer, dim, nlist, metric_type)if gpu_id >= 0:# GPU-accelerated training: move CPU index to GPU, train, move back@@ -250,6 +265,9 @@ def build_ivf("nlist": nlist,"nprobe": nprobe,"metric": metric,+ "index_type": "ivfpq" if pq_m > 0 else "ivfflat",+ "pq_m": pq_m,+ "pq_nbits": pq_nbits if pq_m > 0 else None,"index_file": index_path,"metadata_file": metadata_path,}@@ -419,6 +437,20 @@ def main():p_build.add_argument("--nlist", type=int, default=4096, help="Number of IVF clusters (default: 4096)")+ p_build.add_argument(+ "--pq-m",+ type=int,+ default=0,+ help="PQ sub-quantizers for IVFPQ compression (0 = uncompressed IVFFlat). "+ "Must divide the embedding dim evenly; e.g. 64 for dim 2048 → 64 B/vector.",+ )+ p_build.add_argument(+ "--pq-nbits",+ type=int,+ default=8,+ choices=[4, 8],+ help="Bits per PQ sub-quantizer (default: 8). Only used when --pq-m > 0.",+ )p_build.add_argument("--nprobe",type=int,@@ -515,6 +547,8 @@ def main():train_sample=args.train_sample,metric=args.metric,gpu_id=args.gpu_id,+ pq_m=args.pq_m,+ pq_nbits=args.pq_nbits,)elif args.command == "test":test_search(args.index_dir, nprobe=args.nprobe, k=args.k)diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex 5c3943f..c55702c 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -451,6 +451,10 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:nlist,)cmd += ["--nlist", str(nlist)]+ if index_cfg.get("pq_m"):+ cmd += ["--pq-m", str(index_cfg["pq_m"])]+ if index_cfg.get("pq_nbits"):+ cmd += ["--pq-nbits", str(index_cfg["pq_nbits"])]subprocess.run(cmd, check=True)logger.info("Index built at %s", output)diff --git a/tests/test_ivfpq_index.py b/tests/test_ivfpq_index.pynew file mode 100644index 0000000..1be647f--- /dev/null+++ b/tests/test_ivfpq_index.py@@ -0,0 +1,71 @@+"""IVFPQ compression path for the FAISS index build (issue #153).++`build_ivf(pq_m>0)` must produce a trained, searchable IndexIVFPQ that keeps the+inner-product metric — `faiss.IndexIVFPQ` defaults to L2, which would silently+wreck ranking on these L2-normalized embeddings.+"""++import json+import sys++import numpy as np+import pytest++faiss = pytest.importorskip("faiss")+sys.path.insert(0, "embed/src")+from pixelrag_embed.index import build_ivf+++def _write_shard(emb_dir, n=1024, dim=32):+ rng = np.random.default_rng(0)+ emb = rng.standard_normal((n, dim)).astype(np.float32)+ emb /= np.linalg.norm(emb, axis=1, keepdims=True) # L2-normalized (cosine/IP)+ np.savez(+ emb_dir / "shard_000.npz",+ embeddings=emb,+ article_ids=np.arange(n, dtype=np.int64),+ tile_indices=np.zeros(n, dtype=np.int32),+ chunk_indices=np.arange(n, dtype=np.int32),+ y_offsets=np.zeros(n, dtype=np.int32),+ tile_heights=np.full(n, 100, dtype=np.int32),+ )+ return emb+++def test_ivfpq_index_is_built_trained_and_searchable(tmp_path):+ emb_dir = tmp_path / "emb"+ emb_dir.mkdir()+ emb = _write_shard(emb_dir, dim=32)+ out = tmp_path / "out"++ build_ivf(str(emb_dir), str(out), nlist=8, nprobe=8, pq_m=8, pq_nbits=4)++ index = faiss.read_index(str(out / "index.faiss"))+ assert isinstance(index, faiss.IndexIVFPQ)+ # The bug in the issue's own sketch: without the metric arg this would be L2.+ assert index.metric_type == faiss.METRIC_INNER_PRODUCT+ assert index.ntotal == emb.shape[0]+ _, ids = index.search(emb[:1], 5)+ assert ids.shape == (1, 5) and (ids[0] >= 0).all()++ summary = json.loads((out / "summary.json").read_text())+ assert summary["index_type"] == "ivfpq"+ assert summary["pq_m"] == 8 and summary["pq_nbits"] == 4+++def test_pq_m_must_divide_dim(tmp_path):+ emb_dir = tmp_path / "emb"+ emb_dir.mkdir()+ _write_shard(emb_dir, dim=32)+ with pytest.raises(ValueError, match="divide"):+ build_ivf(str(emb_dir), str(tmp_path / "out"), nlist=8, pq_m=7)+++def test_default_stays_ivfflat(tmp_path):+ emb_dir = tmp_path / "emb"+ emb_dir.mkdir()+ _write_shard(emb_dir, dim=32)+ out = tmp_path / "out"+ build_ivf(str(emb_dir), str(out), nlist=8)+ index = faiss.read_index(str(out / "index.faiss"))+ assert isinstance(index, faiss.IndexIVFFlat)