https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(cdp): --cdp-url to attach to an already-running browser (supersedes #55) (#76)
Zhifei Li8d5c607f9e88
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex 2e03223..df09fc6 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -25,6 +25,7 @@ import ioimport jsonimport loggingimport shutil+import osimport signalimport subprocessimport tempfile@@ -83,6 +84,73 @@ async def _connect_cdp(port: int, retries: int = 5, delay: float = 1.0):raise ConnectionError(f"Failed to connect to Chrome on port {port}")+def _http_base_from_cdp_url(cdp_url: str) -> str:+ """Normalize a ``--cdp-url`` value to an http DevTools base ``http://host:port``.++ Accepts ``http://host:port`` (any path is ignored), ``ws://host:port/...``+ (scheme swapped to http, path dropped), or a bare ``host:port``.+ """+ from urllib.parse import urlparse++ p = urlparse(cdp_url if "//" in cdp_url else f"//{cdp_url}")+ netloc = p.netloc or p.path+ if not netloc:+ raise ValueError(f"Invalid --cdp-url: {cdp_url!r}")+ return f"http://{netloc}"+++async def _connect_ws(ws_url: str):+ """Open a CDP websocket to an explicit ws URL (browser- or page-level)."""+ import websockets++ return await websockets.connect(ws_url, open_timeout=10, max_size=50 * 1024 * 1024)+++def _fetch_json(url: str, cdp_url: str, timeout: float = 5):+ """GET ``url`` and parse JSON, mapping connection failures to a clear error.++ ``cdp_url`` is the user-facing endpoint, used only for the message so a bad+ or unreachable ``--cdp-url`` surfaces an actionable error instead of a raw+ URLError traceback.+ """+ try:+ data = urllib.request.urlopen(url, timeout=timeout).read()+ return json.loads(data)+ except Exception as e:+ raise RuntimeError(f"Could not reach CDP endpoint at {cdp_url}: {e}") from e+++def _browser_ws_url(http_base: str, cdp_url: str) -> str:+ """Fetch the browser-level CDP websocket URL from ``/json/version``."""+ info = _fetch_json(f"{http_base}/json/version", cdp_url)+ try:+ return info["webSocketDebuggerUrl"]+ except (KeyError, TypeError) as e:+ raise RuntimeError(+ f"Could not reach CDP endpoint at {cdp_url}: "+ f"unexpected /json/version response (no webSocketDebuggerUrl)"+ ) from e+++async def _page_ws_url_for_target(+ http_base: str, target_id: str, cdp_url: str, retries: int = 5, delay: float = 0.5+) -> str:+ """Resolve the page-level websocket URL for a freshly created ``targetId``.++ A freshly created target can momentarily be absent from ``/json``, so poll a+ few times (mirroring ``_connect_cdp``'s retry) before giving up. The blocking+ HTTP fetch runs in a thread so it doesn't block the event loop.+ """+ for attempt in range(retries):+ targets = await asyncio.to_thread(_fetch_json, f"{http_base}/json", cdp_url)+ for t in targets:+ if t.get("id") == target_id:+ return t["webSocketDebuggerUrl"]+ if attempt < retries - 1:+ await asyncio.sleep(delay)+ raise RuntimeError(f"Created target {target_id} not found in /json list")++async def _cdp_send(ws, msg_id_ref: list, method: str, params: dict | None = None):"""Send a CDP command and wait for its response."""msg_id_ref[0] += 1@@ -300,6 +368,81 @@ async def capture_url(return len(tiles)+async def _setup_page(+ ws, msg_id_ref: list, viewport_w: int, tile_height: int, wait_network_idle: bool+):+ """Enable the CDP domains and fix the viewport for a page ws before capture."""+ await _cdp_send(ws, msg_id_ref, "Page.enable")+ if wait_network_idle:+ # PerformanceObserver (used by the idle wait) needs no CDP domain, but+ # enabling Network keeps resource timing reliable across navigations.+ await _cdp_send(ws, msg_id_ref, "Network.enable")+ await _cdp_send(+ ws,+ msg_id_ref,+ "Emulation.setDeviceMetricsOverride",+ {+ "width": viewport_w,+ "height": tile_height,+ "deviceScaleFactor": 1,+ "mobile": False,+ },+ )+++async def _drain_queue(+ ws,+ msg_id_ref: list,+ work_queue: asyncio.Queue,+ output_dir: Path,+ tile_height: int,+ quality: int,+ viewport_w: int,+ image_format: str,+ from_surface: bool,+ wait_network_idle: bool,+ worker_id: int,+ stats: dict,+ results: list,+):+ """Pull URLs off the queue and capture each through ``ws`` until it's empty.++ Shared by the launch (``_worker``) and attach (``_attached_worker``) paths —+ they differ only in how ``ws`` is obtained, not in how work is processed.+ """+ while True:+ try:+ item = work_queue.get_nowait()+ except asyncio.QueueEmpty:+ break++ url = item["url"]+ stem = item["stem"]+ tile_dir = output_dir / f"{stem}.png.tiles"++ t0 = time.monotonic()+ try:+ n_tiles = await capture_url(+ ws,+ msg_id_ref,+ url,+ tile_dir,+ tile_h=tile_height,+ quality=quality,+ viewport_w=viewport_w,+ image_format=image_format,+ from_surface=from_surface,+ wait_network_idle=wait_network_idle,+ )+ stats["done"] += 1+ elapsed = time.monotonic() - t0+ logger.info("[w%d] %s → %d tiles (%.1fs)", worker_id, url, n_tiles, elapsed)+ results.append(tile_dir)+ except Exception as e:+ stats["failed"] += 1+ logger.warning("[w%d] FAIL %s: %s", worker_id, url, str(e)[:200])++async def _worker(chrome_path: str,port: int,@@ -342,57 +485,22 @@ async def _worker(ws = await _connect_cdp(port)msg_id_ref = [0]- await _cdp_send(ws, msg_id_ref, "Page.enable")- if wait_network_idle:- # PerformanceObserver (used by the idle wait) needs no CDP domain, but- # enabling Network keeps resource timing reliable across navigations.- await _cdp_send(ws, msg_id_ref, "Network.enable")- await _cdp_send(+ await _setup_page(ws, msg_id_ref, viewport_w, tile_height, wait_network_idle)+ await _drain_queue(ws,msg_id_ref,- "Emulation.setDeviceMetricsOverride",- {- "width": viewport_w,- "height": tile_height,- "deviceScaleFactor": 1,- "mobile": False,- },+ work_queue,+ output_dir,+ tile_height,+ quality,+ viewport_w,+ image_format,+ from_surface,+ wait_network_idle,+ worker_id,+ stats,+ results,)-- while True:- try:- item = work_queue.get_nowait()- except asyncio.QueueEmpty:- break-- url = item["url"]- stem = item["stem"]- tile_dir = output_dir / f"{stem}.png.tiles"-- t0 = time.monotonic()- try:- n_tiles = await capture_url(- ws,- msg_id_ref,- url,- tile_dir,- tile_h=tile_height,- quality=quality,- viewport_w=viewport_w,- image_format=image_format,- from_surface=from_surface,- wait_network_idle=wait_network_idle,- )- stats["done"] += 1- elapsed = time.monotonic() - t0- logger.info(- "[w%d] %s → %d tiles (%.1fs)", worker_id, url, n_tiles, elapsed- )- results.append(tile_dir)- except Exception as e:- stats["failed"] += 1- logger.warning("[w%d] FAIL %s: %s", worker_id, url, str(e)[:200])-await ws.close()finally:proc.send_signal(signal.SIGTERM)@@ -478,6 +586,126 @@ async def _run_batch(return results+async def _attached_worker(+ http_base: str,+ browser_ws_url: str,+ cdp_url: str,+ work_queue: asyncio.Queue,+ output_dir: Path,+ tile_height: int,+ quality: int,+ viewport_w: int,+ image_format: str,+ from_surface: bool,+ wait_network_idle: bool,+ worker_id: int,+ stats: dict,+ results: list,+):+ """Async worker that attaches to an already-running browser over CDP.++ Mirrors ``_worker`` but, instead of launching a throwaway ``--headless``+ process, creates its own fresh tab (target) in the existing browser, drives+ only that tab, and closes only that tab on teardown. The browser's profile+ — cookies, logins — is whatever the running instance has, so authenticated+ pages render. Never touches the user's other tabs; never kills the browser.+ """+ browser_ws = await _connect_ws(browser_ws_url)+ bmsg = [0]+ target_id = None+ try:+ created = await _cdp_send(+ browser_ws, bmsg, "Target.createTarget", {"url": "about:blank"}+ )+ target_id = created["targetId"]+ ws = await _connect_ws(+ await _page_ws_url_for_target(http_base, target_id, cdp_url)+ )+ msg_id_ref = [0]++ await _setup_page(ws, msg_id_ref, viewport_w, tile_height, wait_network_idle)+ await _drain_queue(+ ws,+ msg_id_ref,+ work_queue,+ output_dir,+ tile_height,+ quality,+ viewport_w,+ image_format,+ from_surface,+ wait_network_idle,+ worker_id,+ stats,+ results,+ )+ await ws.close()+ finally:+ # Close only the tab we created; leave the browser and its other tabs alone.+ if target_id is not None:+ try:+ await _cdp_send(+ browser_ws, bmsg, "Target.closeTarget", {"targetId": target_id}+ )+ except Exception:+ pass+ await browser_ws.close()+++async def _run_batch_attached(+ urls: list[str],+ output_dir: Path,+ num_workers: int,+ tile_height: int,+ quality: int,+ viewport_w: int,+ image_format: str,+ from_surface: bool,+ wait_network_idle: bool,+ stems: list[str] | None,+ cdp_url: str,+) -> list[Path]:+ http_base = _http_base_from_cdp_url(cdp_url)+ browser_ws_url = _browser_ws_url(http_base, cdp_url)++ work_queue: asyncio.Queue = asyncio.Queue()+ stem_list = _derive_stems(urls, stems)+ for url, stem in zip(urls, stem_list):+ work_queue.put_nowait({"url": url, "stem": stem})++ stats = {"done": 0, "failed": 0}+ results: list[Path] = []++ # One fresh tab per worker against the single shared browser — no extra+ # processes, no interference with the user's existing tabs.+ actual_workers = min(num_workers, len(urls))+ workers = [+ _attached_worker(+ http_base,+ browser_ws_url,+ cdp_url,+ work_queue,+ output_dir,+ tile_height,+ quality,+ viewport_w,+ image_format,+ from_surface,+ wait_network_idle,+ wid,+ stats,+ results,+ )+ for wid in range(actual_workers)+ ]+ await asyncio.gather(*workers, return_exceptions=True)++ logger.info(+ "Batch complete (attached): done=%d failed=%d", stats["done"], stats["failed"]+ )+ return results++def render_urls(urls: list[str],output_dir: str | Path,@@ -492,6 +720,7 @@ def render_urls(wait_network_idle: bool = False,turbo: bool | None = None,chrome_path: str | None = None,+ cdp_url: str | None = None,) -> list[Path]:"""Render URLs to tiled images via CDP.@@ -517,7 +746,15 @@ def render_urls(force. Turbo only applies to the default capture profile(jpeg, default viewport, fromSurface, no network-idle wait);other options always use the standard path.- chrome_path: Path to Chrome binary. Auto-detected if None.+ chrome_path: Path to Chrome binary. Auto-detected if None. Ignored when+ ``cdp_url`` is set (no browser is launched).+ cdp_url: DevTools endpoint of an already-running browser (e.g.+ ``http://127.0.0.1:9222``). When set (or via the+ ``PIXELSHOT_CDP_URL`` env var), pixelshot attaches to that+ browser and renders in a fresh tab using its existing+ session (cookies/logins) instead of launching a throwaway+ headless instance. Forces the standard path (no turbo) and+ needs no local Chrome binary.Returns:List of Path objects for created tile directories.@@ -528,6 +765,25 @@ def render_urls(if not urls:return []+ cdp_url = cdp_url or os.environ.get("PIXELSHOT_CDP_URL")+ if cdp_url:+ logger.info("Attaching to existing browser at %s", cdp_url)+ return asyncio.run(+ _run_batch_attached(+ urls,+ output_dir,+ workers,+ tile_height,+ quality,+ viewport_width,+ image_format,+ from_surface,+ wait_network_idle,+ stems,+ cdp_url,+ )+ )+chrome = chrome_path or _find_chrome()# Turbo only covers fast_cdp's capture profile; anything else → standard path.diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.pyindex 35a4753..554765d 100644--- a/render/src/pixelrag_render/render.py+++ b/render/src/pixelrag_render/render.py@@ -8,6 +8,7 @@ Entry point:import argparseimport logging+import osimport shutilimport sysfrom pathlib import Path@@ -290,6 +291,16 @@ def main() -> None:default=200,help="DPI for PDF rendering (default: 200).",)+ parser.add_argument(+ "--cdp-url",+ default=os.environ.get("PIXELSHOT_CDP_URL"),+ metavar="URL",+ help="Attach to an already-running Chrome/Brave DevTools endpoint "+ "(e.g. http://127.0.0.1:9222) instead of launching a throwaway headless "+ "browser. Renders each input in a fresh tab using that browser's existing "+ "session (cookies/logins) — so authenticated pages work — then closes only "+ "that tab. Needs no local Chrome binary. Env: PIXELSHOT_CDP_URL.",+ )args = parser.parse_args()output_dir = Path(args.output)@@ -322,6 +333,7 @@ def main() -> None:viewport_width=args.viewport_width,workers=args.workers,wait_network_idle=args.wait_network_idle,+ cdp_url=args.cdp_url,)results.extend(tile_dirs)@@ -344,6 +356,7 @@ def main() -> None:viewport_width=args.viewport_width,workers=1,wait_network_idle=args.wait_network_idle,+ cdp_url=args.cdp_url,)elif suffix in {".png", ".jpg", ".jpeg", ".webp"}:tile_dirs = render_file(fpath, output_dir)diff --git a/tests/repro_bug1.py b/tests/repro_bug1.pydeleted file mode 100644index 4758466..0000000--- a/tests/repro_bug1.py+++ /dev/null@@ -1,54 +0,0 @@-#!/usr/bin/env python3-"""-repro_bug1.py — Minimal reproduction of the int() crash on non-numeric local IDs.--Reproduce WITHOUT needing a full PixelRAG installation:- python repro_bug1.py--Expected on upstream v0.2.1 (before fix):- ValueError: invalid literal for int() with base 10: 'my_photo'--Expected after fix:- PASS — article_entries has 2 entries with display titles.-"""--# Simulate the upstream article list from the pipeline (strings from filename stems)-articles = [- {"id": "my_photo", "url": "", "path": "/tmp/my_photo.jpg", "metadata": {}},- {"id": "chart_btcusdt", "url": "", "path": "/tmp/chart.png", "metadata": {}},-]--print("Testing upstream behavior (BEFORE fix):")-print("=" * 50)-try:- max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0- print(f" max_idx = {max_idx}")- article_entries = [{"title": "", "url": ""}] * max_idx- for a in articles:- idx = int(a["id"]) # <-- crashes here on non-numeric IDs- article_entries[idx] = {"title": str(idx), "url": a.get("url", "")}- print(" PASS (unexpected — upstream has numeric IDs in this path)")-except ValueError as e:- print(f" FAIL (expected on upstream): {e}")--print()-print("Testing FIXED behavior (enumerate-based):")-print("=" * 50)-article_entries = []-for enum_idx, a in enumerate(articles):- title = a.get("metadata", {}).get("title", "")- if not title and a.get("url"):- title = a["url"].split("/")[-1]- if not title:- title = a.get("id", str(enum_idx)) # filename stem as fallback title- url = a.get("url", "") or a.get("path", "")- article_entries.append({"title": title, "url": url})--for i, entry in enumerate(article_entries):- print(f" [{i}] title={entry['title']!r:20s} url={entry['url']!r}")--assert len(article_entries) == 2-assert article_entries[0]["title"] == "my_photo"-assert article_entries[1]["title"] == "chart_btcusdt"-print()-print("PASS — all assertions OK")diff --git a/tests/test_cdp_attach.py b/tests/test_cdp_attach.pynew file mode 100644index 0000000..f5ae596--- /dev/null+++ b/tests/test_cdp_attach.py@@ -0,0 +1,171 @@+"""Tests for the --cdp-url attach-to-existing-browser path.++These run on a core install (no chrome, no browser): they exercise URL+normalization and the routing logic that decides between attaching to a running+browser and launching a throwaway one — without ever opening a browser.+"""++import json+import sys+from pathlib import Path++import pytest++from pixelrag_render.backends import cdp++_BIN = Path(sys.executable).parent+++def test_http_base_normalization():+ f = cdp._http_base_from_cdp_url+ assert f("http://127.0.0.1:9222") == "http://127.0.0.1:9222"+ assert f("http://127.0.0.1:9222/json/version") == "http://127.0.0.1:9222"+ assert f("127.0.0.1:9222") == "http://127.0.0.1:9222"+ assert f("ws://localhost:9222/devtools/browser/abc") == "http://localhost:9222"+++def test_cdp_url_routes_to_attach_without_launching(monkeypatch, tmp_path):+ """With cdp_url set, render_urls must take the attach path and never call+ _find_chrome (i.e. never try to launch/auto-install a browser)."""+ called = {}++ async def fake_attached(urls, output_dir, *a, **kw):+ called["attached"] = list(urls)+ return [Path(output_dir) / "x.png.tiles"]++ def boom():+ raise AssertionError("_find_chrome must not run on the attach path")++ monkeypatch.setattr(cdp, "_run_batch_attached", fake_attached)+ monkeypatch.setattr(cdp, "_find_chrome", boom)++ out = cdp.render_urls(+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9222"+ )+ assert called["attached"] == ["https://example.com"]+ assert out and out[0].name == "x.png.tiles"+++def test_env_var_fallback_routes_to_attach(monkeypatch, tmp_path):+ called = {}++ async def fake_attached(urls, output_dir, *a, **kw):+ called["hit"] = True+ return []++ monkeypatch.setattr(cdp, "_run_batch_attached", fake_attached)+ monkeypatch.setattr(cdp, "_find_chrome", lambda: pytest.fail("should not launch"))+ monkeypatch.setenv("PIXELSHOT_CDP_URL", "http://127.0.0.1:9222")++ cdp.render_urls(["https://example.com"], tmp_path)+ assert called.get("hit") is True+++def test_default_path_still_resolves_chrome(monkeypatch, tmp_path):+ """No cdp_url (and no env) → the launch path runs find_chrome as before."""+ monkeypatch.delenv("PIXELSHOT_CDP_URL", raising=False)+ sentinel = RuntimeError("find_chrome reached")++ def boom():+ raise sentinel++ monkeypatch.setattr(cdp, "_find_chrome", boom)+ with pytest.raises(RuntimeError, match="find_chrome reached"):+ cdp.render_urls(["https://example.com"], tmp_path)+++def test_attach_creates_and_closes_only_its_own_target(monkeypatch, tmp_path):+ """The attach path must create its own target, and on teardown close ONLY+ that target — never a pre-existing one, and never close/kill the browser.++ Mocks the websocket/CDP layer so no browser is needed: a fake ws records+ every CDP method sent on the browser-level connection.+ """+ browser_methods = [] # (method, params) sent on the browser ws++ class FakeBrowserWS:+ async def send(self, raw):+ msg = json.loads(raw)+ browser_methods.append((msg["method"], msg.get("params", {})))+ method = msg["method"]+ result = {}+ if method == "Target.createTarget":+ result = {"targetId": "OUR-TARGET-123"}+ self._reply = {"id": msg["id"], "result": result}++ async def recv(self):+ return json.dumps(self._reply)++ async def close(self):+ pass++ class FakePageWS:+ async def send(self, raw):+ msg = json.loads(raw)+ self._reply = {"id": msg["id"], "result": {}}++ async def recv(self):+ return json.dumps(self._reply)++ async def close(self):+ pass++ async def fake_connect_ws(ws_url):+ return FakeBrowserWS() if ws_url == "BROWSER_WS" else FakePageWS()++ # Pre-existing target plus the one we create — only ours must be closed.+ def fake_fetch_json(url, cdp_url, timeout=5):+ if url.endswith("/json/version"):+ return {"webSocketDebuggerUrl": "BROWSER_WS"}+ return [+ {"id": "PREEXISTING-999", "webSocketDebuggerUrl": "ws://other"},+ {"id": "OUR-TARGET-123", "webSocketDebuggerUrl": "PAGE_WS"},+ ]++ async def fake_capture_url(*a, **kw):+ return 1++ monkeypatch.setattr(cdp, "_connect_ws", fake_connect_ws)+ monkeypatch.setattr(cdp, "_fetch_json", fake_fetch_json)+ monkeypatch.setattr(cdp, "capture_url", fake_capture_url)++ out = cdp.render_urls(+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9222"+ )+ assert out and out[0].name.endswith(".png.tiles")++ methods = [m for m, _ in browser_methods]+ assert "Target.createTarget" in methods+ assert "Target.closeTarget" in methods+ # Never closed/killed the browser.+ assert "Browser.close" not in methods++ # closeTarget targeted ONLY our own created target, never the pre-existing one.+ closed = [p["targetId"] for m, p in browser_methods if m == "Target.closeTarget"]+ assert closed == ["OUR-TARGET-123"]+++def test_attach_bad_cdp_url_raises_clean_error(monkeypatch, tmp_path):+ """An unreachable/bad endpoint surfaces a clear RuntimeError, not a raw+ URLError/KeyError traceback."""+ import urllib.error++ def boom(url, timeout=5):+ raise urllib.error.URLError("connection refused")++ monkeypatch.setattr(cdp.urllib.request, "urlopen", boom)++ with pytest.raises(RuntimeError, match="Could not reach CDP endpoint at"):+ cdp.render_urls(+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9999"+ )+++def test_cli_help_exposes_cdp_url():+ import subprocess++ r = subprocess.run(+ [str(_BIN / "pixelshot"), "--help"], capture_output=True, text=True+ )+ assert r.returncode == 0+ assert "--cdp-url" in r.stdout