Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

13.4 KB
Raw Blame
"""One-shot capture strategy.

Each article gets a FRESH headless_shell process: launch → navigate → capture → kill.
No stale state, no IPC race conditions across articles.  The per-process launch
overhead (~0.3–1 s) is hidden by running many processes concurrently.

Key difference from cdp_phased / cdp_sequential:
- Those strategies keep N Chrome processes alive and navigate them repeatedly.
  ForceRedraw IPC latency grows with concurrency because the OS must schedule
  144+ threads (48 Chrome workers × 3 threads each).
- One-shot processes live only for one article.  Each ForceRedraw runs in an
  otherwise idle process, so IPC overhead stays low regardless of concurrency.

Port allocation: each slot in [base_port, base_port+n_workers) is used by at
most one live process.  A semaphore gates concurrent slots so we never try to
bind two processes to the same port.
"""

from __future__ import annotations

import asyncio
import base64
import json
import os
import signal
import subprocess
import time
import urllib.request
from dataclasses import dataclass, field

from .base import ArticleCapture, TileCapture, article_url
from .connection import CHROME_ARGS, pick_page_ws_url

TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875

# How often to poll the /json endpoint while waiting for Chrome to start.
# 100 ms gives ~0.2–0.5 s connect time vs. the 1 s sleep in launch_websocket.
_POLL_INTERVAL = 0.1
_POLL_ATTEMPTS = 60  # 60 × 100 ms = 6 s max before giving up

WAIT_FONTS_IMGS = """new Promise(resolve => {
    const waitEagerImgs = Promise.all(
        Array.from(document.images)
            .filter(i => !i.complete && i.loading !== 'lazy')
            .map(i => new Promise(r => {
                i.addEventListener('load', r, {once: true});
                i.addEventListener('error', r, {once: true});
            }))
    );
    const timeout = new Promise(r => setTimeout(r, 2000));
    Promise.race([
        Promise.all([document.fonts.ready, waitEagerImgs]),
        timeout
    ]).then(() => {
        requestAnimationFrame(() => {
            requestAnimationFrame(() => {
                document.documentElement.style.scrollBehavior = 'auto';
                const sh = document.documentElement.scrollHeight;
                const body = document.body;
                resolve(body ? Math.min(sh, Math.max(Math.ceil(body.getBoundingClientRect().bottom), 1)) : sh);
            });
        });
    });
})"""


async def _launch_oneshot(
    chrome_path: str,
    port: int,
    headless_shell: bool,
    extra_args: list[str] | None,
):
    """Launch a fresh Chrome/headless_shell process and connect via websocket.

    Returns (WebsocketConnection, proc).  Polls every _POLL_INTERVAL seconds
    (much faster than the 1 s sleep in launch_websocket).
    """
    import websockets

    from .connection import WebsocketConnection

    args = [chrome_path, f"--remote-debugging-port={port}"]
    if not headless_shell:
        args.append("--headless=new")
    args += CHROME_ARGS
    if extra_args:
        args += extra_args
    args += ["about:blank"]

    proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

    last_exc: Exception | None = None
    for _ in range(_POLL_ATTEMPTS):
        await asyncio.sleep(_POLL_INTERVAL)
        try:
            data = urllib.request.urlopen(
                f"http://localhost:{port}/json", timeout=2
            ).read()
            targets = json.loads(data)
            ws = await websockets.connect(
                pick_page_ws_url(targets),
                open_timeout=5,
                max_size=50 * 1024 * 1024,
            )
            return WebsocketConnection(ws, proc), proc
        except Exception as e:
            last_exc = e

    proc.kill()
    raise ConnectionError(
        f"Failed to connect to Chrome on port {port} after "
        f"{_POLL_ATTEMPTS * _POLL_INTERVAL:.1f}s: {last_exc}"
    )


def _kill_proc(proc) -> None:
    """Best-effort kill + wait for a subprocess."""
    try:
        proc.send_signal(signal.SIGTERM)
    except Exception:
        pass
    try:
        proc.wait(timeout=3)
    except Exception:
        try:
            proc.kill()
        except Exception:
            pass


@dataclass
class CDPOneShotStrategy:
    """One-shot: fresh headless_shell process per article.

    n_workers controls how many articles are captured concurrently (each in its
    own short-lived Chrome process).  The semaphore gates port slot reuse so
    that slot i is never used by two live processes simultaneously.
    """

    chrome_path: str
    n_workers: int
    fmt: str = "jpeg"
    quality: int = 85
    from_surface: bool = True
    headless_shell: bool = True
    nav_timeout_ms: int = 2000
    tile_height: int = TILE_HEIGHT
    extra_chrome_args: list = None

    _base_port: int = 9500  # separate range from persistent-pool strategies
    _sem: asyncio.Semaphore = field(default=None, init=False, repr=False)

    # Track (port_slot → proc) so teardown can clean up stragglers.
    _live_procs: dict = field(default_factory=dict, init=False, repr=False)
    _procs_lock: asyncio.Lock = field(default=None, init=False, repr=False)

    @property
    def name(self) -> str:
        hs = " HS" if self.headless_shell else ""
        th = f" h{self.tile_height}" if self.tile_height != TILE_HEIGHT else ""
        return f"{self.n_workers}w {self.fmt} oneshot{hs}{th}"

    async def setup(self) -> None:
        self._sem = asyncio.Semaphore(self.n_workers)
        self._live_procs = {}
        self._procs_lock = asyncio.Lock()
        if self.fmt == "raw":
            os.makedirs("/dev/shm/pixelrag_bench", exist_ok=True)

    async def teardown(self) -> None:
        # Kill any processes that leaked (e.g. due to exceptions).
        async with self._procs_lock:
            procs = list(self._live_procs.values())
            self._live_procs.clear()
        for proc in procs:
            _kill_proc(proc)

    async def capture_articles(self, articles: list[dict]) -> list[ArticleCapture]:
        article_index = {a["path"]: i for i, a in enumerate(articles)}
        all_results: list[ArticleCapture | None] = [None] * len(articles)

        # Assign each article a port slot: slot_index = article_index % n_workers.
        # The semaphore ensures at most n_workers concurrent processes, and each
        # slot is only used by one process at a time.
        async def capture_task(article: dict):
            idx = article_index[article["path"]]
            slot = idx % self.n_workers
            port = self._base_port + slot
            ac = await self._capture_one(article, slot, port)
            all_results[idx] = ac

        await asyncio.gather(*[capture_task(a) for a in articles])
        return [r for r in all_results if r is not None]

    async def _capture_one(self, article: dict, slot: int, port: int) -> ArticleCapture:
        ac = ArticleCapture(article_path=article["path"])
        th = self.tile_height

        # Wait for this port slot to be free.
        t_sem = time.monotonic()
        async with self._sem:
            ac.sem_wait_ms = (time.monotonic() - t_sem) * 1000

            # Launch a fresh process on this port slot.
            try:
                conn, proc = await _launch_oneshot(
                    self.chrome_path,
                    port,
                    headless_shell=self.headless_shell,
                    extra_args=self.extra_chrome_args,
                )
            except Exception as e:
                ac.errors.append(f"launch: {e}")
                return ac

            async with self._procs_lock:
                self._live_procs[slot] = proc

            try:
                await self._capture_article(conn, article, ac, th)
            finally:
                # Close websocket and kill process regardless of outcome.
                try:
                    await conn.close()
                except Exception:
                    _kill_proc(proc)
                async with self._procs_lock:
                    self._live_procs.pop(slot, None)

        return ac

    async def _capture_article(
        self, conn, article: dict, ac: ArticleCapture, th: int
    ) -> None:
        """Navigate, wait for render, capture all tiles.  Mutates ac in place."""
        # === Configure viewport ===
        try:
            await conn.cdp("Page.enable")
            await conn.cdp(
                "Emulation.setDeviceMetricsOverride",
                {
                    "width": VIEWPORT_WIDTH,
                    "height": th,
                    "deviceScaleFactor": 1,
                    "mobile": False,
                },
            )
        except Exception as e:
            ac.errors.append(f"setup cdp: {e}")
            return

        # === Navigate ===
        t_nav = time.monotonic()
        target_url = article_url(article)

        # Use Page.frameStoppedLoading: reliable with --in-process-gpu
        # (Page.frameNavigated has a Chrome bug where it's sometimes not fired).
        nav_event_fut = asyncio.ensure_future(
            conn.wait_for_event("Page.frameStoppedLoading", timeout=30.0)
        )
        try:
            await conn.cdp("Page.navigate", {"url": target_url})
        except Exception as e:
            nav_event_fut.cancel()
            ac.errors.append(f"nav: {e}")
            return

        try:
            await nav_event_fut
        except asyncio.TimeoutError:
            ac.errors.append("nav: frameStoppedLoading timeout (30s)")
            return
        except Exception as e:
            ac.errors.append(f"nav: frameStoppedLoading wait error: {e}")
            return

        # === Wait for fonts + images, measure page height ===
        try:
            wait_expr = WAIT_FONTS_IMGS.replace(
                "setTimeout(r, 2000)", f"setTimeout(r, {self.nav_timeout_ms})"
            )
            r = await conn.cdp(
                "Runtime.evaluate",
                {
                    "expression": wait_expr,
                    "awaitPromise": True,
                    "returnByValue": True,
                },
            )
            page_h = r["result"]["result"]["value"]
        except Exception:
            page_h = th

        if page_h <= 0:
            page_h = th

        nav_ms = (time.monotonic() - t_nav) * 1000
        ac.total_nav_ms = nav_ms
        ac.page_height = page_h
        n_tiles = max(1, (page_h + th - 1) // th)
        ac.n_tiles_expected = n_tiles

        # === Capture tiles ===
        for t in range(n_tiles):
            clip_y = t * th
            clip_h = min(th, page_h - clip_y)
            if clip_h <= 28:
                break

            # Scroll into position and wait for viewport images.
            try:
                await conn.cdp(
                    "Runtime.evaluate",
                    {
                        "expression": f"""new Promise(resolve => {{
                        window.scrollTo(0, {clip_y});
                        requestAnimationFrame(() => requestAnimationFrame(() => {{
                            const imgs = Array.from(document.images).filter(i => {{
                                if (i.complete) return false;
                                const r = i.getBoundingClientRect();
                                return r.bottom > 0 && r.top < window.innerHeight;
                            }});
                            if (imgs.length === 0) return resolve();
                            const timeout = new Promise(r => setTimeout(r, 500));
                            const loaded = Promise.all(imgs.map(i => new Promise(r => {{
                                i.addEventListener('load', r, {{once: true}});
                                i.addEventListener('error', r, {{once: true}});
                            }})));
                            Promise.race([loaded, timeout]).then(resolve);
                        }}));
                    }})""",
                        "awaitPromise": True,
                    },
                )
            except Exception:
                pass

            params = {
                "fromSurface": self.from_surface,
                "optimizeForSpeed": True,
                "clip": {
                    "x": 0,
                    "y": clip_y,
                    "width": VIEWPORT_WIDTH,
                    "height": clip_h,
                    "scale": 1,
                },
            }

            raw_path = None
            if self.fmt == "raw":
                raw_path = (
                    f"/dev/shm/pixelrag_bench"
                    f"/os_{article['path'].replace('/', '_')}_{t}.raw"
                )
                params["rawFilePath"] = raw_path
            else:
                params["format"] = self.fmt
                if self.fmt == "jpeg":
                    params["quality"] = self.quality

            t0 = time.monotonic()
            try:
                r = await conn.cdp("Page.captureScreenshot", params)
            except Exception as e:
                ac.errors.append(f"tile {t}: {e}")
                continue
            shot_ms = (time.monotonic() - t0) * 1000
            ac.total_shot_ms += shot_ms

            if "error" in r:
                ac.errors.append(f"tile {t}: {r['error']}")
                continue

            tc = TileCapture(
                shot_ms=shot_ms,
                nav_ms=nav_ms if t == 0 else 0.0,
                tile_index=t,
                clip_y=clip_y,
                clip_h=clip_h,
            )
            if self.fmt == "raw":
                tc.raw_file_path = raw_path
            else:
                tc.image_bytes = base64.b64decode(r["result"]["data"])
            ac.tiles.append(tc)