Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

8.7 KB
Raw Blame
"""On-demand tile rendering for pixelrag-serve.

When a tile image is not on disk, render the source page from a kiwix ZIM (served
over HTTP by ``kiwix-serve``), slice it into the same 1024px chunks the index was
built on, and return the requested chunk. This removes the dependency on a
materialized multi-TB ``tiles/`` corpus -- only the pages that retrieval actually
returns get rendered, lazily, and then cached on disk.

Pixel-validated against the original pipeline's tiles (mean |diff| ~2/255 on every
referenced chunk), using the exact build config: viewport_width=875, tile_height=8192,
and the shared ``pixelrag_embed.chunk`` slicer (1024px chunks).
"""

import atexit
import glob
import os
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib.request
from urllib.parse import quote

from pixelrag_render.chrome import sandbox_args

_render_lock = threading.Lock()  # one Chrome render at a time per process
# Hard timeout (seconds) for a single page render subprocess.
_RENDER_TIMEOUT = float(os.environ.get("PIXELRAG_RENDER_TIMEOUT", "120"))
# How long a caller will wait for its turn at _render_lock before giving up.
# Renders are serialized process-wide, so without a bound a caller can queue
# behind another request's whole render budget -- 16 renders * _RENDER_TIMEOUT in
# the worst case -- while holding a thread from the event loop's executor. Giving
# up returns no image rather than no response: the hit is still served, and once
# some other request's render fills the cache the chunk is available from /tile.
# Against the persistent Chrome a render is ~1-2s (the 120s above is a safety
# ceiling), so this absorbs a deep queue and only sheds under real contention.
_RENDER_QUEUE_TIMEOUT = float(os.environ.get("PIXELRAG_RENDER_QUEUE_TIMEOUT", "30"))


class OnDemandTiles:
    def __init__(
        self,
        kiwix_url: str,
        book: str,
        cache_dir: str,
        viewport_width: int = 875,
        tile_height: int = 8192,
    ):
        self.kiwix_url = kiwix_url.rstrip("/")
        self.book = book
        self.cache_dir = cache_dir
        self.viewport_width = viewport_width
        self.tile_height = tile_height
        os.makedirs(cache_dir, exist_ok=True)
        # Persistent headless Chrome, reused across renders via --cdp-url. Starting a fresh
        # Chrome per page costs a cold start (seconds locally, tens of seconds on a cold/NFS
        # box); reusing one browser cuts per-page render from ~tens of s to ~1-2s.
        self._chrome_proc = None
        self._cdp_url = None
        self._chrome_udd = None
        self._chrome_lock = threading.Lock()
        atexit.register(self._kill_chrome)

    def _kill_chrome(self) -> None:
        if self._chrome_proc is not None:
            try:
                self._chrome_proc.kill()
            except Exception:
                pass
            self._chrome_proc = None
        self._cdp_url = None
        if self._chrome_udd:
            shutil.rmtree(self._chrome_udd, ignore_errors=True)
            self._chrome_udd = None

    def _ensure_chrome(self) -> str:
        """Start (or restart) the persistent headless Chrome; return its CDP base URL."""
        with self._chrome_lock:
            if (
                self._chrome_proc is not None
                and self._chrome_proc.poll() is None
                and self._cdp_url
            ):
                return self._cdp_url
            self._kill_chrome()
            from pixelrag_render.chrome import find_chrome

            chrome = find_chrome()
            s = socket.socket()
            s.bind(("127.0.0.1", 0))
            port = s.getsockname()[1]
            s.close()
            self._chrome_udd = os.path.join(self.cache_dir, f".chrome_{port}")
            self._chrome_proc = subprocess.Popen(
                [
                    chrome,
                    f"--remote-debugging-port={port}",
                    "--headless=new",
                    "--disable-gpu",
                    *sandbox_args(),
                    f"--user-data-dir={self._chrome_udd}",
                    "about:blank",
                ],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
            )
            base = f"http://127.0.0.1:{port}"
            deadline = time.time() + 60
            while time.time() < deadline:
                try:
                    urllib.request.urlopen(base + "/json/version", timeout=2)
                    self._cdp_url = base
                    return base
                except Exception:
                    if self._chrome_proc.poll() is not None:
                        break
                    time.sleep(0.5)
            self._kill_chrome()
            raise RuntimeError("on-demand Chrome failed to start")

    def _article_dir(self, article_id: int) -> str:
        return os.path.join(self.cache_dir, f"{article_id}.png.tiles")

    def _chunk_file(self, article_id: int, tile_index: int, chunk_index: int) -> str:
        chunk_name = f"chunk_{tile_index:04d}_{chunk_index:02d}.png"
        return os.path.join(self._article_dir(article_id), chunk_name)

    def cached_chunk_path(self, article_id: int, tile_index: int, chunk_index: int):
        """Path to an already-rendered chunk, or None. Never renders.

        Lets a caller tell a cheap cache hit from a page render before
        committing to one.
        """
        cpath = self._chunk_file(article_id, tile_index, chunk_index)
        return cpath if os.path.exists(cpath) else None

    def chunk_path(
        self, article_id: int, title: str, tile_index: int, chunk_index: int
    ):
        """Path to chunk_{ti}_{ci}.png, rendering+chunking the page on a cache miss."""
        cached = self.cached_chunk_path(article_id, tile_index, chunk_index)
        if cached:
            return cached
        cpath = self._chunk_file(article_id, tile_index, chunk_index)
        if not title:
            return None
        if not _render_lock.acquire(timeout=_RENDER_QUEUE_TIMEOUT):
            return None  # queue too deep; serve the hit without this image
        try:
            if os.path.exists(cpath):  # filled while we waited for the lock
                return cpath
            try:
                self._render_and_chunk(article_id, title)
            except Exception:
                return None
        finally:
            _render_lock.release()
        return cpath if os.path.exists(cpath) else None

    def _render_and_chunk(self, article_id: int, title: str) -> None:
        from pixelrag_embed.chunk import chunk_article

        url = f"{self.kiwix_url}/content/{self.book}/{quote(title, safe='')}"
        staging = os.path.join(self.cache_dir, f".render_{article_id}")
        shutil.rmtree(staging, ignore_errors=True)
        os.makedirs(staging, exist_ok=True)
        # Render in a SEPARATE PROCESS via the pixelshot CLI, attached to the persistent
        # Chrome with --cdp-url (renders in a fresh tab, no per-page cold start). The
        # subprocess is still needed: render_url internally uses asyncio.run() +
        # multiprocessing.Pool (fork), which deadlocks if called a 2nd time in this
        # long-lived serve process — a fresh subprocess per render avoids that.
        cdp_url = self._ensure_chrome()
        try:
            subprocess.run(
                [
                    sys.executable,
                    "-m",
                    "pixelrag_render.render",
                    url,
                    "--output",
                    staging,
                    "--viewport-width",
                    str(self.viewport_width),
                    "--tile-height",
                    str(self.tile_height),
                    "--cdp-url",
                    cdp_url,
                    "--workers",
                    "1",
                ],
                check=True,
                timeout=_RENDER_TIMEOUT,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
            )
        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
            # The shared Chrome may be wedged/dead — drop it so the next render restarts it.
            self._kill_chrome()
            shutil.rmtree(staging, ignore_errors=True)
            return
        dirs = glob.glob(os.path.join(staging, "*.png.tiles"))
        if not dirs:
            shutil.rmtree(staging, ignore_errors=True)
            return
        rendered = dirs[0]  # <sanitized-url>.png.tiles/ (has tiles.json)
        chunk_article(rendered)  # writes chunk_XXXX_YY.png + chunks.json
        dest = self._article_dir(article_id)
        shutil.rmtree(dest, ignore_errors=True)
        os.replace(rendered, dest)  # atomic; commit only after chunking succeeds
        shutil.rmtree(staging, ignore_errors=True)