https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
test(serve): repair department-filter tests left stale by the backend refactor (#162)
Shubham Srivastava2a282416cce8
diff --git a/tests/test_department_filter.py b/tests/test_department_filter.pyindex 4da4186..08eb25e 100644--- a/tests/test_department_filter.py+++ b/tests/test_department_filter.py@@ -1,4 +1,10 @@-"""Department metadata (articles.json) + FAISS IDSelector pre-filter."""+"""Build side of the department feature: directory layout -> department name.++The serve side lives in test_department_search.py (API errors) and+test_serve_backends.py (the article_ids pre-filter, both backends). Keeping+them apart matters: this module needs no heavy extras, so gating it on faiss+meant CI — which installs none — skipped these cases too.+"""import pytest@@ -48,83 +54,3 @@ def test_web_url_outside_root_and_empty_root(tmp_path):# ---------------------------------------------------------------------------# Serve side: IDSelector filter# -----------------------------------------------------------------------------faiss = pytest.importorskip("faiss")-np = pytest.importorskip("numpy")-api = pytest.importorskip("pixelrag_serve.api")--DIM = 8-# 3 articles × 2 vectors; article 0, 2 → hr; article 1 → ketoan-ARTICLE_IDS = np.array([0, 0, 1, 1, 2, 2], dtype=np.int64)-HR_ROWS = {0, 1, 4, 5}---def _vectors() -> np.ndarray:- rng = np.random.default_rng(7)- v = rng.standard_normal((6, DIM)).astype(np.float32)- return v / np.linalg.norm(v, axis=1, keepdims=True)---def _setup_state(index):- api._state.clear()- api._state.update(- {- "index": index,- "metadata": {"article_ids": ARTICLE_IDS},- "dept_to_aids": {"hr": np.array([0, 2]), "ketoan": np.array([1])},- "dept_positions": {},- }- )---def test_department_positions_maps_articles_to_rows():- index = faiss.IndexFlatIP(DIM)- index.add(_vectors())- _setup_state(index)- assert set(api._department_positions("hr").tolist()) == HR_ROWS- assert set(api._department_positions("ketoan").tolist()) == {2, 3}---def test_flat_index_filter_restricts_hits():- vecs = _vectors()- index = faiss.IndexFlatIP(DIM)- index.add(vecs)- _setup_state(index)- params = api._department_search_params("hr", nprobe=1)- _, indices = index.search(vecs[:1], 6, params=params)- hits = {int(i) for i in indices[0] if i != -1}- assert hits == HR_ROWS # every hr row returned, nothing else---def test_ivf_index_filter_restricts_hits():- vecs = _vectors()- quantizer = faiss.IndexFlatIP(DIM)- index = faiss.IndexIVFFlat(quantizer, DIM, 1, faiss.METRIC_INNER_PRODUCT)- index.train(vecs)- index.add(vecs)- index.nprobe = 1- _setup_state(index)- params = api._department_search_params("ketoan", nprobe=1)- _, indices = index.search(vecs[2:3], 6, params=params)- hits = {int(i) for i in indices[0] if i != -1}- assert hits == {2, 3}---def test_unknown_department_raises_404():- index = faiss.IndexFlatIP(DIM)- index.add(_vectors())- _setup_state(index)- with pytest.raises(api.HTTPException) as exc:- api._department_positions("phong-khong-ton-tai")- assert exc.value.status_code == 404- assert "hr" in exc.value.detail # lists available departments---def test_index_without_department_metadata_raises_400():- index = faiss.IndexFlatIP(DIM)- index.add(_vectors())- _setup_state(index)- api._state["dept_to_aids"] = {}- with pytest.raises(api.HTTPException) as exc:- api._department_positions("hr")- assert exc.value.status_code == 400diff --git a/tests/test_department_search.py b/tests/test_department_search.pynew file mode 100644index 0000000..70ceb0f--- /dev/null+++ b/tests/test_department_search.py@@ -0,0 +1,39 @@+"""API-level errors for the `department` search filter.++`_department_article_ids` is the only place these two client errors are+produced, and both are reachable from the public /search endpoint. The pre-+filter's search behaviour is covered against both backends in+test_serve_backends.py; this file needs no index, only the state dict.+"""++import pytest++api = pytest.importorskip("pixelrag_serve.api", reason="serve extra not installed")++DEPTS = {"hr": [0, 2], "ketoan": [1]}++[email protected](autouse=True)+def _state():+ api._state.clear()+ api._state.update({"dept_to_aids": dict(DEPTS)})+ yield+ api._state.clear()+++def test_known_department_returns_its_article_ids():+ assert list(api._department_article_ids("hr")) == [0, 2]+++def test_unknown_department_raises_404():+ with pytest.raises(api.HTTPException) as exc:+ api._department_article_ids("phong-khong-ton-tai")+ assert exc.value.status_code == 404+ assert "hr" in exc.value.detail # lists what is available+++def test_index_without_department_metadata_raises_400():+ api._state["dept_to_aids"] = {}+ with pytest.raises(api.HTTPException) as exc:+ api._department_article_ids("hr")+ assert exc.value.status_code == 400diff --git a/tests/test_serve_backends.py b/tests/test_serve_backends.pyindex 7b68488..dac367e 100644--- a/tests/test_serve_backends.py+++ b/tests/test_serve_backends.py@@ -3,7 +3,9 @@Both backends are exercised against the same tiny synthetic corpus: 12vectors across 4 articles, two departments. The suite verifies the sharedraw_search contract — hit shape, ranking, the article_ids pre-filter (the-department feature), min_tile_height handling, and reconstruct().+department feature), min_tile_height handling, and reconstruct(). FAISS runs+twice, flat and IVF: `raw_search` takes a different filter path for an IndexIVF+(`SearchParametersIVF`) than for a flat index, and production indexes are IVF.These run only when the heavy optional deps are installed (`[serve]` /`[qdrant]` extras); CI's core-only install skips them. Qdrant runs fully@@ -21,6 +23,7 @@ from pixelrag_serve.backends import FaissBackendDIM = 8N_VECTORS = 12+NLIST = 2 # IVF lists for the ivf_faiss_backend fixture# vector i belongs to article i // 3 (articles 0..3); departments: 0,1 -> "eng", 2,3 -> "hr"ARTICLE_OF = [i // 3 for i in range(N_VECTORS)]ENG_ARTICLES = np.asarray([0, 1])@@ -41,11 +44,8 @@ def _make_vectors():VECTORS = _make_vectors()[email protected](scope="module")-def faiss_backend(tmp_path_factory):- index_dir = tmp_path_factory.mktemp("faiss_index")- index = faiss.IndexFlatIP(DIM)- index.add(VECTORS)+def _faiss_backend_for(index, index_dir):+ """Write the on-disk layout FaissBackend expects and open it."""faiss.write_index(index, str(index_dir / "index.faiss"))np.savez(index_dir / "metadata.npz",@@ -59,6 +59,30 @@ def faiss_backend(tmp_path_factory):return FaissBackend(str(index_dir), {"dimension": DIM})[email protected](scope="module")+def faiss_backend(tmp_path_factory):+ index = faiss.IndexFlatIP(DIM)+ index.add(VECTORS)+ return _faiss_backend_for(index, tmp_path_factory.mktemp("faiss_index"))++[email protected](scope="module")+def ivf_faiss_backend(tmp_path_factory):+ """IVF index: production shape, and a different filter path.++ `raw_search` builds `SearchParametersIVF(sel=..., nprobe=...)` for an+ IndexIVF and plain `SearchParameters` otherwise, so a flat index alone+ never exercises the branch real deployments run. nprobe = nlist keeps+ results exact, so the shared contract assertions still hold.+ """+ quantizer = faiss.IndexFlatIP(DIM)+ index = faiss.IndexIVFFlat(quantizer, DIM, NLIST, faiss.METRIC_INNER_PRODUCT)+ index.train(VECTORS)+ index.add(VECTORS)+ index.nprobe = NLIST+ return _faiss_backend_for(index, tmp_path_factory.mktemp("ivf_index"))++@pytest.fixture(scope="module")def qdrant_backend():pytest.importorskip("qdrant_client", reason="qdrant extra not installed")@@ -98,7 +122,7 @@ def _backends(request):return request.getfixturevalue(request.param)[email protected](params=["faiss_backend", "qdrant_backend"])[email protected](params=["faiss_backend", "ivf_faiss_backend", "qdrant_backend"])def backend(request):return _backends(request)