https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
fix(index): regenerate explicitly incomplete render caches (#176)
Adarsha8ebf9500a22
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex ace3d67..85efb2a 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -25,6 +25,9 @@ def _needs_render(tiles_dir: Path, idx: int, doc) -> bool:document's pixels with another document's metadata. On mismatch the staledirectory is removed and re-rendered; corrupt manifests (e.g. a crashmid-write) are likewise re-rendered instead of poisoning later stages.+ Explicitly incomplete captures are removed too, including derived chunks+ that would otherwise make the chunk stage skip regeneration. A missing+ ``complete`` field is legacy/unknown and does not invalidate the cache."""tile_dir = tiles_dir / f"{idx}.png.tiles"manifest_path = tile_dir / "tiles.json"@@ -38,8 +41,14 @@ def _needs_render(tiles_dir: Path, idx: int, doc) -> bool:logger.warning(" Corrupt manifest in %s — re-rendering", tile_dir.name)recorded = Noneif recorded == expected:- return False- if recorded is not None:+ if manifest.get("complete") is not False:+ return False+ logger.warning(" Incomplete capture in %s — re-rendering", tile_dir.name)+ # Do not render over a partially removed cache: surviving chunks could+ # make the chunk stage reuse obsolete data. Propagate cleanup failures.+ shutil.rmtree(tile_dir)+ return True+ elif recorded is not None:logger.warning(" %s was rendered from %r but position %d now holds %r — ""re-rendering (source set changed between runs; use --force "diff --git a/tests/test_incremental_rerun.py b/tests/test_incremental_rerun.pyindex adb2c28..d3fe078 100644--- a/tests/test_incremental_rerun.py+++ b/tests/test_incremental_rerun.py@@ -8,7 +8,11 @@ stamp loop silently pair one document's pixels with another's metadata."""import json+from pathlib import Path+from unittest.mock import Mock+import pytest+from PIL import Imagefrom pixelrag_index.pipelines import _needs_renderfrom pixelrag_index.sources.base import Document@@ -17,6 +21,8 @@ def _make_tile_dir(tiles_dir, idx, manifest):tile_dir = tiles_dir / f"{idx}.png.tiles"tile_dir.mkdir(parents=True)(tile_dir / "tiles.json").write_text(json.dumps(manifest))+ for name in manifest.get("tiles", []):+ Image.new("RGB", (32, 32), "red").save(tile_dir / name)return tile_dir@@ -25,22 +31,74 @@ def test_missing_dir_needs_render(tmp_path):assert _needs_render(tmp_path, 0, doc) is True-def test_matching_source_is_reused(tmp_path):- _make_tile_dir(tmp_path, 0, {"source": "/src/a.md", "tiles": ["tile_0000.png"]})[email protected]("completion", [{"complete": True}, {}])+def test_matching_source_is_reused(tmp_path, completion):+ tile_dir = _make_tile_dir(+ tmp_path,+ 0,+ {"source": "/src/a.md", "tiles": ["tile_0000.png"], **completion},+ )doc = Document(id="a", path="/src/a.md")assert _needs_render(tmp_path, 0, doc) is False+ assert (tile_dir / "tile_0000.png").is_file()++[email protected]("identity_field", ["source", "url"])+def test_incomplete_matching_capture_is_removed(tmp_path, identity_field):+ doc = Document(id="a", url="https://example.com/a")+ tile_dir = _make_tile_dir(+ tmp_path,+ 0,+ {identity_field: doc.url, "tiles": ["tile_0000.png"], "complete": False},+ )+ assert (tile_dir / "tile_0000.png").is_file()+ assert _needs_render(tmp_path, 0, doc) is True+ assert not tile_dir.exists()def test_legacy_url_field_is_reused(tmp_path):# Dirs stamped before the `source` field existed recorded the render URL.- _make_tile_dir(tmp_path, 0, {"url": "https://example.com/x", "tiles": []})+ _make_tile_dir(+ tmp_path, 0, {"url": "https://example.com/x", "tiles": ["tile_0000.png"]}+ )doc = Document(id="x", url="https://example.com/x")assert _needs_render(tmp_path, 0, doc) is False+def test_incomplete_capture_cleanup_failure_stops_build(tmp_path, monkeypatch):+ from pixelrag_index import pipelines+ from pixelrag_render import render++ doc = Document(id="a", url="https://example.com/a")+ tile_dir = _make_tile_dir(+ tmp_path / "tiles",+ 0,+ {"source": doc.url, "tiles": ["tile_0000.png"], "complete": False},+ )++ def fail_cleanup(path, ignore_errors=False):+ assert path == tile_dir+ if not ignore_errors:+ raise PermissionError("cache is not writable")++ renderer = Mock()+ stages = Mock()+ monkeypatch.setattr(pipelines.shutil, "rmtree", fail_cleanup)+ monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))+ monkeypatch.setattr(render, "render_urls", renderer)+ monkeypatch.setattr(pipelines.subprocess, "run", stages)++ with pytest.raises(PermissionError, match="cache is not writable"):+ pipelines.build({"source": {"type": "web"}, "output": str(tmp_path)})+ renderer.assert_not_called()+ stages.assert_not_called()++def test_shifted_position_forces_rerender_and_removes_stale_dir(tmp_path):# Built over [a, c] -> dir 1 holds c. User adds b: position 1 is now b.- stale = _make_tile_dir(tmp_path, 1, {"source": "/src/c.md", "tiles": []})+ stale = _make_tile_dir(+ tmp_path, 1, {"source": "/src/c.md", "tiles": ["tile_0000.png"]}+ )doc_b = Document(id="b", path="/src/b.md")assert _needs_render(tmp_path, 1, doc_b) is Trueassert not stale.exists(), "stale dir must be removed before re-render"@@ -64,3 +122,65 @@ def test_corrupt_manifest_skips_article_in_chunk_stage(tmp_path):article_dir.mkdir()(article_dir / "tiles.json").write_text('{"tiles": ["tile_0000.png"') # truncatedassert chunk_article(str(article_dir)) is None+++def test_build_regenerates_incomplete_capture_and_chunks(tmp_path, monkeypatch):+ from pixelrag_embed.chunk import chunk_article+ from pixelrag_index import pipelines+ from pixelrag_render import render++ doc = Document(id="a", url="https://example.com/a")+ config = {"source": {"type": "web"}, "output": str(tmp_path)}+ tile_dir = tmp_path / "tiles" / "0.png.tiles"+ chunk_results = []++ def capture(urls, output_dir, *, stems, **kwargs):+ assert urls == [doc.url]+ assert stems == ["0"]+ # Regeneration must remove the old manifest, tiles AND derived chunks.+ assert not tile_dir.exists()+ complete = renderer.call_count > 1+ names = ["tile_0000.png"] if complete else ["tile_0000.png", "tile_0001.png"]+ _make_tile_dir(+ Path(output_dir),+ 0,+ {+ "url": doc.url,+ "page_height": 32 if complete else 96,+ "tile_height": 32,+ "viewport_width": 32,+ "tiles": names,+ "complete": complete,+ },+ )+ return [tile_dir]++ def run_stage(cmd, *, check):+ assert check is True+ if cmd[2] == "pixelrag_embed.chunk":+ chunk_results.append(chunk_article(str(tile_dir)))+ # Embedding and index services are outside this regression's scope.++ renderer = Mock(side_effect=capture)+ monkeypatch.setattr(render, "render_urls", renderer)+ monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))+ monkeypatch.setattr(pipelines.subprocess, "run", run_stage)++ pipelines.build(config)+ assert renderer.call_count == 1+ assert chunk_results[-1] is not None+ assert (tile_dir / "chunk_0001_00.png").is_file()++ pipelines.build(config)+ assert renderer.call_count == 2+ assert chunk_results[-1] is not None, "chunk_article must not reuse old chunks"+ assert not (tile_dir / "tile_0001.png").exists()+ assert not (tile_dir / "chunk_0001_00.png").exists()+ chunks = json.loads((tile_dir / "chunks.json").read_text())+ assert [c["file"] for c in chunks["chunks"]] == ["chunk_0000_00.png"]+ assert chunks["article_id"] == 0+ assert json.loads((tile_dir / "tiles.json").read_text())["complete"] is True++ pipelines.build(config)+ assert renderer.call_count == 2+ assert chunk_results[-1] is None