Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

6.9 KB
Raw Blame
"""Incremental re-run safety: stale tile dirs must be re-rendered, not relabeled.

Position indices are assigned by source enumeration order, so they shift when
the source set changes between runs (a file added, removed, or renamed).
_needs_render must detect that an existing {idx}.png.tiles directory was
rendered from a *different* document and re-render it instead of letting the
stamp loop silently pair one document's pixels with another's metadata.
"""

import json
from pathlib import Path
from unittest.mock import Mock

import pytest
from PIL import Image
from pixelrag_index.pipelines import _needs_render
from pixelrag_index.sources.base import Document


def _make_tile_dir(tiles_dir, idx, manifest):
    tile_dir = tiles_dir / f"{idx}.png.tiles"
    tile_dir.mkdir(parents=True)
    (tile_dir / "tiles.json").write_text(json.dumps(manifest))
    for name in manifest.get("tiles", []):
        Image.new("RGB", (32, 32), "red").save(tile_dir / name)
    return tile_dir


def test_missing_dir_needs_render(tmp_path):
    doc = Document(id="a", path="/src/a.md")
    assert _needs_render(tmp_path, 0, doc) is True


@pytest.mark.parametrize("completion", [{"complete": True}, {}])
def test_matching_source_is_reused(tmp_path, completion):
    tile_dir = _make_tile_dir(
        tmp_path,
        0,
        {"source": "/src/a.md", "tiles": ["tile_0000.png"], **completion},
    )
    doc = Document(id="a", path="/src/a.md")
    assert _needs_render(tmp_path, 0, doc) is False
    assert (tile_dir / "tile_0000.png").is_file()


@pytest.mark.parametrize("identity_field", ["source", "url"])
def test_incomplete_matching_capture_is_removed(tmp_path, identity_field):
    doc = Document(id="a", url="https://example.com/a")
    tile_dir = _make_tile_dir(
        tmp_path,
        0,
        {identity_field: doc.url, "tiles": ["tile_0000.png"], "complete": False},
    )
    assert (tile_dir / "tile_0000.png").is_file()
    assert _needs_render(tmp_path, 0, doc) is True
    assert not tile_dir.exists()


def test_legacy_url_field_is_reused(tmp_path):
    # Dirs stamped before the `source` field existed recorded the render URL.
    _make_tile_dir(
        tmp_path, 0, {"url": "https://example.com/x", "tiles": ["tile_0000.png"]}
    )
    doc = Document(id="x", url="https://example.com/x")
    assert _needs_render(tmp_path, 0, doc) is False


def test_incomplete_capture_cleanup_failure_stops_build(tmp_path, monkeypatch):
    from pixelrag_index import pipelines
    from pixelrag_render import render

    doc = Document(id="a", url="https://example.com/a")
    tile_dir = _make_tile_dir(
        tmp_path / "tiles",
        0,
        {"source": doc.url, "tiles": ["tile_0000.png"], "complete": False},
    )

    def fail_cleanup(path, ignore_errors=False):
        assert path == tile_dir
        if not ignore_errors:
            raise PermissionError("cache is not writable")

    renderer = Mock()
    stages = Mock()
    monkeypatch.setattr(pipelines.shutil, "rmtree", fail_cleanup)
    monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))
    monkeypatch.setattr(render, "render_urls", renderer)
    monkeypatch.setattr(pipelines.subprocess, "run", stages)

    with pytest.raises(PermissionError, match="cache is not writable"):
        pipelines.build({"source": {"type": "web"}, "output": str(tmp_path)})
    renderer.assert_not_called()
    stages.assert_not_called()


def test_shifted_position_forces_rerender_and_removes_stale_dir(tmp_path):
    # Built over [a, c] -> dir 1 holds c. User adds b: position 1 is now b.
    stale = _make_tile_dir(
        tmp_path, 1, {"source": "/src/c.md", "tiles": ["tile_0000.png"]}
    )
    doc_b = Document(id="b", path="/src/b.md")
    assert _needs_render(tmp_path, 1, doc_b) is True
    assert not stale.exists(), "stale dir must be removed before re-render"


def test_corrupt_manifest_forces_rerender(tmp_path):
    tile_dir = tmp_path / "0.png.tiles"
    tile_dir.mkdir()
    (tile_dir / "tiles.json").write_text('{"source": "/src/a.md", "til')  # truncated
    doc = Document(id="a", path="/src/a.md")
    assert _needs_render(tmp_path, 0, doc) is True
    assert not tile_dir.exists()


def test_corrupt_manifest_skips_article_in_chunk_stage(tmp_path):
    # A truncated tiles.json must not crash the chunk stage (it used to raise
    # an uncaught JSONDecodeError and fail the whole build via check=True).
    from pixelrag_embed.chunk import chunk_article

    article_dir = tmp_path / "0.png.tiles"
    article_dir.mkdir()
    (article_dir / "tiles.json").write_text('{"tiles": ["tile_0000.png"')  # truncated
    assert chunk_article(str(article_dir)) is None


def test_build_regenerates_incomplete_capture_and_chunks(tmp_path, monkeypatch):
    from pixelrag_embed.chunk import chunk_article
    from pixelrag_index import pipelines
    from pixelrag_render import render

    doc = Document(id="a", url="https://example.com/a")
    config = {"source": {"type": "web"}, "output": str(tmp_path)}
    tile_dir = tmp_path / "tiles" / "0.png.tiles"
    chunk_results = []

    def capture(urls, output_dir, *, stems, **kwargs):
        assert urls == [doc.url]
        assert stems == ["0"]
        # Regeneration must remove the old manifest, tiles AND derived chunks.
        assert not tile_dir.exists()
        complete = renderer.call_count > 1
        names = ["tile_0000.png"] if complete else ["tile_0000.png", "tile_0001.png"]
        _make_tile_dir(
            Path(output_dir),
            0,
            {
                "url": doc.url,
                "page_height": 32 if complete else 96,
                "tile_height": 32,
                "viewport_width": 32,
                "tiles": names,
                "complete": complete,
            },
        )
        return [tile_dir]

    def run_stage(cmd, *, check):
        assert check is True
        if cmd[2] == "pixelrag_embed.chunk":
            chunk_results.append(chunk_article(str(tile_dir)))
        # Embedding and index services are outside this regression's scope.

    renderer = Mock(side_effect=capture)
    monkeypatch.setattr(render, "render_urls", renderer)
    monkeypatch.setattr(pipelines, "make_source", lambda config: iter([doc]))
    monkeypatch.setattr(pipelines.subprocess, "run", run_stage)

    pipelines.build(config)
    assert renderer.call_count == 1
    assert chunk_results[-1] is not None
    assert (tile_dir / "chunk_0001_00.png").is_file()

    pipelines.build(config)
    assert renderer.call_count == 2
    assert chunk_results[-1] is not None, "chunk_article must not reuse old chunks"
    assert not (tile_dir / "tile_0001.png").exists()
    assert not (tile_dir / "chunk_0001_00.png").exists()
    chunks = json.loads((tile_dir / "chunks.json").read_text())
    assert [c["file"] for c in chunks["chunks"]] == ["chunk_0000_00.png"]
    assert chunks["article_id"] == 0
    assert json.loads((tile_dir / "tiles.json").read_text())["complete"] is True

    pipelines.build(config)
    assert renderer.call_count == 2
    assert chunk_results[-1] is None