Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(index): regenerate explicitly incomplete render caches (#176)

Adarsha8ebf9500a22
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py
index ace3d67..85efb2a 100644
--- a/index/src/pixelrag_index/pipelines.py
+++ b/index/src/pixelrag_index/pipelines.py
@@ -25,6 +25,9 @@ def _needs_render(tiles_dir: Path, idx: int, doc) -> bool:
document's pixels with another document's metadata. On mismatch the stale
directory is removed and re-rendered; corrupt manifests (e.g. a crash
mid-write) are likewise re-rendered instead of poisoning later stages.
+ Explicitly incomplete captures are removed too, including derived chunks
+ that would otherwise make the chunk stage skip regeneration. A missing
+ ``complete`` field is legacy/unknown and does not invalidate the cache.
"""
tile_dir = tiles_dir / f"{idx}.png.tiles"
manifest_path = tile_dir / "tiles.json"
@@ -38,8 +41,14 @@ def _needs_render(tiles_dir: Path, idx: int, doc) -> bool:
logger.warning(" Corrupt manifest in %s — re-rendering", tile_dir.name)
recorded = None
if recorded == expected:
- return False
- if recorded is not None:
+ if manifest.get("complete") is not False:
+ return False
+ logger.warning(" Incomplete capture in %s — re-rendering", tile_dir.name)
+ # Do not render over a partially removed cache: surviving chunks could
+ # make the chunk stage reuse obsolete data. Propagate cleanup failures.
+ shutil.rmtree(tile_dir)
+ return True
+ elif recorded is not None:
logger.warning(
" %s was rendered from %r but position %d now holds %r — "
"re-rendering (source set changed between runs; use --force "
diff --git a/tests/test_incremental_rerun.py b/tests/test_incremental_rerun.py
index adb2c28..d3fe078 100644
--- a/tests/test_incremental_rerun.py
+++ b/tests/test_incremental_rerun.py
@@ -8,7 +8,11 @@ stamp loop silently pair one document's pixels with another's metadata.
"""
import json
+from pathlib import Path
+from unittest.mock import Mock
+import pytest
+from PIL import Image
from pixelrag_index.pipelines import _needs_render
from pixelrag_index.sources.base import Document
@@ -17,6 +21,8 @@ def _make_tile_dir(tiles_dir, idx, manifest):
tile_dir = tiles_dir / f"{idx}.png.tiles"
tile_dir.mkdir(parents=True)
(tile_dir / "tiles.json").write_text(json.dumps(manifest))
+ for name in manifest.get("tiles", []):
+ Image.new("RGB", (32, 32), "red").save(tile_dir / name)
return tile_dir
@@ -25,22 +31,74 @@ def test_missing_dir_needs_render(tmp_path):
assert _needs_render(tmp_path, 0, doc) is True
-def test_matching_source_is_reused(tmp_path):
- _make_tile_dir(tmp_path, 0, {"source": "/src/a.md", "tiles": ["tile_0000.png"]})
[email protected]("completion", [{"complete": True}, {}])
+def test_matching_source_is_reused(tmp_path, completion):
+ tile_dir = _make_tile_dir(
+ tmp_path,
+ 0,
+ {"source": "/src/a.md", "tiles": ["tile_0000.png"], **completion},
+ )
doc = Document(id="a", path="/src/a.md")
assert _needs_render(tmp_path, 0, doc) is False
+ assert (tile_dir / "tile_0000.png").is_file()
+
+
[email protected]("identity_field", ["source", "url"])
+def test_incomplete_matching_capture_is_removed(tmp_path, identity_field):
+ doc = Document(id="a", url="https://example.com/a")
+ tile_dir = _make_tile_dir(
+ tmp_path,
+ 0,
+ {identity_field: doc.url, "tiles": ["tile_0000.png"], "complete": False},
+ )
+ assert (tile_dir / "tile_0000.png").is_file()
+ assert _needs_render(tmp_path, 0, doc) is True
+ assert not tile_dir.exists()
def test_legacy_url_field_is_reused(tmp_path):
# Dirs stamped before the `source` field existed recorded the render URL.
- _make_tile_dir(tmp_path, 0, {"url": "https://example.com/x", "tiles": []})
+ _make_tile_dir(
+ tmp_path, 0, {"url": "https://example.com/x", "tiles": ["tile_0000.png"]}
+ )
doc = Document(id="x", url="https://example.com/x")
assert _needs_render(tmp_path, 0, doc) is False
+def test_incomplete_capture_cleanup_failure_stops_build(tmp_path, monkeypatch):
+ from pixelrag_index import pipelines
+ from pixelrag_render import render
+
+ doc = Document(id="a", url="https://example.com/a")
+ tile_dir = _make_tile_dir(
+ tmp_path / "tiles",
+ 0,
+ {"source": doc.url, "tiles": ["tile_0000.png"], "complete": False},
+ )
+
+ def fail_cleanup(path, ignore_errors=False):
+ assert path == tile_dir
+ if not ignore_errors:
+ raise PermissionError("cache is not writable")
+
+ renderer = Mock()
+ stages = Mock()
+ monkeypatch.setattr(pipelines.shutil, "rmtree", fail_cleanup)
+ monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))
+ monkeypatch.setattr(render, "render_urls", renderer)
+ monkeypatch.setattr(pipelines.subprocess, "run", stages)
+
+ with pytest.raises(PermissionError, match="cache is not writable"):
+ pipelines.build({"source": {"type": "web"}, "output": str(tmp_path)})
+ renderer.assert_not_called()
+ stages.assert_not_called()
+
+
def test_shifted_position_forces_rerender_and_removes_stale_dir(tmp_path):
# Built over [a, c] -> dir 1 holds c. User adds b: position 1 is now b.
- stale = _make_tile_dir(tmp_path, 1, {"source": "/src/c.md", "tiles": []})
+ stale = _make_tile_dir(
+ tmp_path, 1, {"source": "/src/c.md", "tiles": ["tile_0000.png"]}
+ )
doc_b = Document(id="b", path="/src/b.md")
assert _needs_render(tmp_path, 1, doc_b) is True
assert not stale.exists(), "stale dir must be removed before re-render"
@@ -64,3 +122,65 @@ def test_corrupt_manifest_skips_article_in_chunk_stage(tmp_path):
article_dir.mkdir()
(article_dir / "tiles.json").write_text('{"tiles": ["tile_0000.png"') # truncated
assert chunk_article(str(article_dir)) is None
+
+
+def test_build_regenerates_incomplete_capture_and_chunks(tmp_path, monkeypatch):
+ from pixelrag_embed.chunk import chunk_article
+ from pixelrag_index import pipelines
+ from pixelrag_render import render
+
+ doc = Document(id="a", url="https://example.com/a")
+ config = {"source": {"type": "web"}, "output": str(tmp_path)}
+ tile_dir = tmp_path / "tiles" / "0.png.tiles"
+ chunk_results = []
+
+ def capture(urls, output_dir, *, stems, **kwargs):
+ assert urls == [doc.url]
+ assert stems == ["0"]
+ # Regeneration must remove the old manifest, tiles AND derived chunks.
+ assert not tile_dir.exists()
+ complete = renderer.call_count > 1
+ names = ["tile_0000.png"] if complete else ["tile_0000.png", "tile_0001.png"]
+ _make_tile_dir(
+ Path(output_dir),
+ 0,
+ {
+ "url": doc.url,
+ "page_height": 32 if complete else 96,
+ "tile_height": 32,
+ "viewport_width": 32,
+ "tiles": names,
+ "complete": complete,
+ },
+ )
+ return [tile_dir]
+
+ def run_stage(cmd, *, check):
+ assert check is True
+ if cmd[2] == "pixelrag_embed.chunk":
+ chunk_results.append(chunk_article(str(tile_dir)))
+ # Embedding and index services are outside this regression's scope.
+
+ renderer = Mock(side_effect=capture)
+ monkeypatch.setattr(render, "render_urls", renderer)
+ monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))
+ monkeypatch.setattr(pipelines.subprocess, "run", run_stage)
+
+ pipelines.build(config)
+ assert renderer.call_count == 1
+ assert chunk_results[-1] is not None
+ assert (tile_dir / "chunk_0001_00.png").is_file()
+
+ pipelines.build(config)
+ assert renderer.call_count == 2
+ assert chunk_results[-1] is not None, "chunk_article must not reuse old chunks"
+ assert not (tile_dir / "tile_0001.png").exists()
+ assert not (tile_dir / "chunk_0001_00.png").exists()
+ chunks = json.loads((tile_dir / "chunks.json").read_text())
+ assert [c["file"] for c in chunks["chunks"]] == ["chunk_0000_00.png"]
+ assert chunks["article_id"] == 0
+ assert json.loads((tile_dir / "tiles.json").read_text())["complete"] is True
+
+ pipelines.build(config)
+ assert renderer.call_count == 2
+ assert chunk_results[-1] is None