Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(cdp): --cdp-url to attach to an already-running browser (supersedes #55) (#76)

Zhifei Li8d5c607f9e88
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index 2e03223..df09fc6 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -25,6 +25,7 @@ import io
import json
import logging
import shutil
+import os
import signal
import subprocess
import tempfile
@@ -83,6 +84,73 @@ async def _connect_cdp(port: int, retries: int = 5, delay: float = 1.0):
raise ConnectionError(f"Failed to connect to Chrome on port {port}")
+def _http_base_from_cdp_url(cdp_url: str) -> str:
+ """Normalize a ``--cdp-url`` value to an http DevTools base ``http://host:port``.
+
+ Accepts ``http://host:port`` (any path is ignored), ``ws://host:port/...``
+ (scheme swapped to http, path dropped), or a bare ``host:port``.
+ """
+ from urllib.parse import urlparse
+
+ p = urlparse(cdp_url if "//" in cdp_url else f"//{cdp_url}")
+ netloc = p.netloc or p.path
+ if not netloc:
+ raise ValueError(f"Invalid --cdp-url: {cdp_url!r}")
+ return f"http://{netloc}"
+
+
+async def _connect_ws(ws_url: str):
+ """Open a CDP websocket to an explicit ws URL (browser- or page-level)."""
+ import websockets
+
+ return await websockets.connect(ws_url, open_timeout=10, max_size=50 * 1024 * 1024)
+
+
+def _fetch_json(url: str, cdp_url: str, timeout: float = 5):
+ """GET ``url`` and parse JSON, mapping connection failures to a clear error.
+
+ ``cdp_url`` is the user-facing endpoint, used only for the message so a bad
+ or unreachable ``--cdp-url`` surfaces an actionable error instead of a raw
+ URLError traceback.
+ """
+ try:
+ data = urllib.request.urlopen(url, timeout=timeout).read()
+ return json.loads(data)
+ except Exception as e:
+ raise RuntimeError(f"Could not reach CDP endpoint at {cdp_url}: {e}") from e
+
+
+def _browser_ws_url(http_base: str, cdp_url: str) -> str:
+ """Fetch the browser-level CDP websocket URL from ``/json/version``."""
+ info = _fetch_json(f"{http_base}/json/version", cdp_url)
+ try:
+ return info["webSocketDebuggerUrl"]
+ except (KeyError, TypeError) as e:
+ raise RuntimeError(
+ f"Could not reach CDP endpoint at {cdp_url}: "
+ f"unexpected /json/version response (no webSocketDebuggerUrl)"
+ ) from e
+
+
+async def _page_ws_url_for_target(
+ http_base: str, target_id: str, cdp_url: str, retries: int = 5, delay: float = 0.5
+) -> str:
+ """Resolve the page-level websocket URL for a freshly created ``targetId``.
+
+ A freshly created target can momentarily be absent from ``/json``, so poll a
+ few times (mirroring ``_connect_cdp``'s retry) before giving up. The blocking
+ HTTP fetch runs in a thread so it doesn't block the event loop.
+ """
+ for attempt in range(retries):
+ targets = await asyncio.to_thread(_fetch_json, f"{http_base}/json", cdp_url)
+ for t in targets:
+ if t.get("id") == target_id:
+ return t["webSocketDebuggerUrl"]
+ if attempt < retries - 1:
+ await asyncio.sleep(delay)
+ raise RuntimeError(f"Created target {target_id} not found in /json list")
+
+
async def _cdp_send(ws, msg_id_ref: list, method: str, params: dict | None = None):
"""Send a CDP command and wait for its response."""
msg_id_ref[0] += 1
@@ -300,6 +368,81 @@ async def capture_url(
return len(tiles)
+async def _setup_page(
+ ws, msg_id_ref: list, viewport_w: int, tile_height: int, wait_network_idle: bool
+):
+ """Enable the CDP domains and fix the viewport for a page ws before capture."""
+ await _cdp_send(ws, msg_id_ref, "Page.enable")
+ if wait_network_idle:
+ # PerformanceObserver (used by the idle wait) needs no CDP domain, but
+ # enabling Network keeps resource timing reliable across navigations.
+ await _cdp_send(ws, msg_id_ref, "Network.enable")
+ await _cdp_send(
+ ws,
+ msg_id_ref,
+ "Emulation.setDeviceMetricsOverride",
+ {
+ "width": viewport_w,
+ "height": tile_height,
+ "deviceScaleFactor": 1,
+ "mobile": False,
+ },
+ )
+
+
+async def _drain_queue(
+ ws,
+ msg_id_ref: list,
+ work_queue: asyncio.Queue,
+ output_dir: Path,
+ tile_height: int,
+ quality: int,
+ viewport_w: int,
+ image_format: str,
+ from_surface: bool,
+ wait_network_idle: bool,
+ worker_id: int,
+ stats: dict,
+ results: list,
+):
+ """Pull URLs off the queue and capture each through ``ws`` until it's empty.
+
+ Shared by the launch (``_worker``) and attach (``_attached_worker``) paths —
+ they differ only in how ``ws`` is obtained, not in how work is processed.
+ """
+ while True:
+ try:
+ item = work_queue.get_nowait()
+ except asyncio.QueueEmpty:
+ break
+
+ url = item["url"]
+ stem = item["stem"]
+ tile_dir = output_dir / f"{stem}.png.tiles"
+
+ t0 = time.monotonic()
+ try:
+ n_tiles = await capture_url(
+ ws,
+ msg_id_ref,
+ url,
+ tile_dir,
+ tile_h=tile_height,
+ quality=quality,
+ viewport_w=viewport_w,
+ image_format=image_format,
+ from_surface=from_surface,
+ wait_network_idle=wait_network_idle,
+ )
+ stats["done"] += 1
+ elapsed = time.monotonic() - t0
+ logger.info("[w%d] %s → %d tiles (%.1fs)", worker_id, url, n_tiles, elapsed)
+ results.append(tile_dir)
+ except Exception as e:
+ stats["failed"] += 1
+ logger.warning("[w%d] FAIL %s: %s", worker_id, url, str(e)[:200])
+
+
async def _worker(
chrome_path: str,
port: int,
@@ -342,57 +485,22 @@ async def _worker(
ws = await _connect_cdp(port)
msg_id_ref = [0]
- await _cdp_send(ws, msg_id_ref, "Page.enable")
- if wait_network_idle:
- # PerformanceObserver (used by the idle wait) needs no CDP domain, but
- # enabling Network keeps resource timing reliable across navigations.
- await _cdp_send(ws, msg_id_ref, "Network.enable")
- await _cdp_send(
+ await _setup_page(ws, msg_id_ref, viewport_w, tile_height, wait_network_idle)
+ await _drain_queue(
ws,
msg_id_ref,
- "Emulation.setDeviceMetricsOverride",
- {
- "width": viewport_w,
- "height": tile_height,
- "deviceScaleFactor": 1,
- "mobile": False,
- },
+ work_queue,
+ output_dir,
+ tile_height,
+ quality,
+ viewport_w,
+ image_format,
+ from_surface,
+ wait_network_idle,
+ worker_id,
+ stats,
+ results,
)
-
- while True:
- try:
- item = work_queue.get_nowait()
- except asyncio.QueueEmpty:
- break
-
- url = item["url"]
- stem = item["stem"]
- tile_dir = output_dir / f"{stem}.png.tiles"
-
- t0 = time.monotonic()
- try:
- n_tiles = await capture_url(
- ws,
- msg_id_ref,
- url,
- tile_dir,
- tile_h=tile_height,
- quality=quality,
- viewport_w=viewport_w,
- image_format=image_format,
- from_surface=from_surface,
- wait_network_idle=wait_network_idle,
- )
- stats["done"] += 1
- elapsed = time.monotonic() - t0
- logger.info(
- "[w%d] %s → %d tiles (%.1fs)", worker_id, url, n_tiles, elapsed
- )
- results.append(tile_dir)
- except Exception as e:
- stats["failed"] += 1
- logger.warning("[w%d] FAIL %s: %s", worker_id, url, str(e)[:200])
-
await ws.close()
finally:
proc.send_signal(signal.SIGTERM)
@@ -478,6 +586,126 @@ async def _run_batch(
return results
+async def _attached_worker(
+ http_base: str,
+ browser_ws_url: str,
+ cdp_url: str,
+ work_queue: asyncio.Queue,
+ output_dir: Path,
+ tile_height: int,
+ quality: int,
+ viewport_w: int,
+ image_format: str,
+ from_surface: bool,
+ wait_network_idle: bool,
+ worker_id: int,
+ stats: dict,
+ results: list,
+):
+ """Async worker that attaches to an already-running browser over CDP.
+
+ Mirrors ``_worker`` but, instead of launching a throwaway ``--headless``
+ process, creates its own fresh tab (target) in the existing browser, drives
+ only that tab, and closes only that tab on teardown. The browser's profile
+ — cookies, logins — is whatever the running instance has, so authenticated
+ pages render. Never touches the user's other tabs; never kills the browser.
+ """
+ browser_ws = await _connect_ws(browser_ws_url)
+ bmsg = [0]
+ target_id = None
+ try:
+ created = await _cdp_send(
+ browser_ws, bmsg, "Target.createTarget", {"url": "about:blank"}
+ )
+ target_id = created["targetId"]
+ ws = await _connect_ws(
+ await _page_ws_url_for_target(http_base, target_id, cdp_url)
+ )
+ msg_id_ref = [0]
+
+ await _setup_page(ws, msg_id_ref, viewport_w, tile_height, wait_network_idle)
+ await _drain_queue(
+ ws,
+ msg_id_ref,
+ work_queue,
+ output_dir,
+ tile_height,
+ quality,
+ viewport_w,
+ image_format,
+ from_surface,
+ wait_network_idle,
+ worker_id,
+ stats,
+ results,
+ )
+ await ws.close()
+ finally:
+ # Close only the tab we created; leave the browser and its other tabs alone.
+ if target_id is not None:
+ try:
+ await _cdp_send(
+ browser_ws, bmsg, "Target.closeTarget", {"targetId": target_id}
+ )
+ except Exception:
+ pass
+ await browser_ws.close()
+
+
+async def _run_batch_attached(
+ urls: list[str],
+ output_dir: Path,
+ num_workers: int,
+ tile_height: int,
+ quality: int,
+ viewport_w: int,
+ image_format: str,
+ from_surface: bool,
+ wait_network_idle: bool,
+ stems: list[str] | None,
+ cdp_url: str,
+) -> list[Path]:
+ http_base = _http_base_from_cdp_url(cdp_url)
+ browser_ws_url = _browser_ws_url(http_base, cdp_url)
+
+ work_queue: asyncio.Queue = asyncio.Queue()
+ stem_list = _derive_stems(urls, stems)
+ for url, stem in zip(urls, stem_list):
+ work_queue.put_nowait({"url": url, "stem": stem})
+
+ stats = {"done": 0, "failed": 0}
+ results: list[Path] = []
+
+ # One fresh tab per worker against the single shared browser — no extra
+ # processes, no interference with the user's existing tabs.
+ actual_workers = min(num_workers, len(urls))
+ workers = [
+ _attached_worker(
+ http_base,
+ browser_ws_url,
+ cdp_url,
+ work_queue,
+ output_dir,
+ tile_height,
+ quality,
+ viewport_w,
+ image_format,
+ from_surface,
+ wait_network_idle,
+ wid,
+ stats,
+ results,
+ )
+ for wid in range(actual_workers)
+ ]
+ await asyncio.gather(*workers, return_exceptions=True)
+
+ logger.info(
+ "Batch complete (attached): done=%d failed=%d", stats["done"], stats["failed"]
+ )
+ return results
+
+
def render_urls(
urls: list[str],
output_dir: str | Path,
@@ -492,6 +720,7 @@ def render_urls(
wait_network_idle: bool = False,
turbo: bool | None = None,
chrome_path: str | None = None,
+ cdp_url: str | None = None,
) -> list[Path]:
"""Render URLs to tiled images via CDP.
@@ -517,7 +746,15 @@ def render_urls(
force. Turbo only applies to the default capture profile
(jpeg, default viewport, fromSurface, no network-idle wait);
other options always use the standard path.
- chrome_path: Path to Chrome binary. Auto-detected if None.
+ chrome_path: Path to Chrome binary. Auto-detected if None. Ignored when
+ ``cdp_url`` is set (no browser is launched).
+ cdp_url: DevTools endpoint of an already-running browser (e.g.
+ ``http://127.0.0.1:9222``). When set (or via the
+ ``PIXELSHOT_CDP_URL`` env var), pixelshot attaches to that
+ browser and renders in a fresh tab using its existing
+ session (cookies/logins) instead of launching a throwaway
+ headless instance. Forces the standard path (no turbo) and
+ needs no local Chrome binary.
Returns:
List of Path objects for created tile directories.
@@ -528,6 +765,25 @@ def render_urls(
if not urls:
return []
+ cdp_url = cdp_url or os.environ.get("PIXELSHOT_CDP_URL")
+ if cdp_url:
+ logger.info("Attaching to existing browser at %s", cdp_url)
+ return asyncio.run(
+ _run_batch_attached(
+ urls,
+ output_dir,
+ workers,
+ tile_height,
+ quality,
+ viewport_width,
+ image_format,
+ from_surface,
+ wait_network_idle,
+ stems,
+ cdp_url,
+ )
+ )
+
chrome = chrome_path or _find_chrome()
# Turbo only covers fast_cdp's capture profile; anything else → standard path.
diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.py
index 35a4753..554765d 100644
--- a/render/src/pixelrag_render/render.py
+++ b/render/src/pixelrag_render/render.py
@@ -8,6 +8,7 @@ Entry point:
import argparse
import logging
+import os
import shutil
import sys
from pathlib import Path
@@ -290,6 +291,16 @@ def main() -> None:
default=200,
help="DPI for PDF rendering (default: 200).",
)
+ parser.add_argument(
+ "--cdp-url",
+ default=os.environ.get("PIXELSHOT_CDP_URL"),
+ metavar="URL",
+ help="Attach to an already-running Chrome/Brave DevTools endpoint "
+ "(e.g. http://127.0.0.1:9222) instead of launching a throwaway headless "
+ "browser. Renders each input in a fresh tab using that browser's existing "
+ "session (cookies/logins) — so authenticated pages work — then closes only "
+ "that tab. Needs no local Chrome binary. Env: PIXELSHOT_CDP_URL.",
+ )
args = parser.parse_args()
output_dir = Path(args.output)
@@ -322,6 +333,7 @@ def main() -> None:
viewport_width=args.viewport_width,
workers=args.workers,
wait_network_idle=args.wait_network_idle,
+ cdp_url=args.cdp_url,
)
results.extend(tile_dirs)
@@ -344,6 +356,7 @@ def main() -> None:
viewport_width=args.viewport_width,
workers=1,
wait_network_idle=args.wait_network_idle,
+ cdp_url=args.cdp_url,
)
elif suffix in {".png", ".jpg", ".jpeg", ".webp"}:
tile_dirs = render_file(fpath, output_dir)
diff --git a/tests/repro_bug1.py b/tests/repro_bug1.py
deleted file mode 100644
index 4758466..0000000
--- a/tests/repro_bug1.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-"""
-repro_bug1.py — Minimal reproduction of the int() crash on non-numeric local IDs.
-
-Reproduce WITHOUT needing a full PixelRAG installation:
- python repro_bug1.py
-
-Expected on upstream v0.2.1 (before fix):
- ValueError: invalid literal for int() with base 10: 'my_photo'
-
-Expected after fix:
- PASS — article_entries has 2 entries with display titles.
-"""
-
-# Simulate the upstream article list from the pipeline (strings from filename stems)
-articles = [
- {"id": "my_photo", "url": "", "path": "/tmp/my_photo.jpg", "metadata": {}},
- {"id": "chart_btcusdt", "url": "", "path": "/tmp/chart.png", "metadata": {}},
-]
-
-print("Testing upstream behavior (BEFORE fix):")
-print("=" * 50)
-try:
- max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0
- print(f" max_idx = {max_idx}")
- article_entries = [{"title": "", "url": ""}] * max_idx
- for a in articles:
- idx = int(a["id"]) # <-- crashes here on non-numeric IDs
- article_entries[idx] = {"title": str(idx), "url": a.get("url", "")}
- print(" PASS (unexpected — upstream has numeric IDs in this path)")
-except ValueError as e:
- print(f" FAIL (expected on upstream): {e}")
-
-print()
-print("Testing FIXED behavior (enumerate-based):")
-print("=" * 50)
-article_entries = []
-for enum_idx, a in enumerate(articles):
- title = a.get("metadata", {}).get("title", "")
- if not title and a.get("url"):
- title = a["url"].split("/")[-1]
- if not title:
- title = a.get("id", str(enum_idx)) # filename stem as fallback title
- url = a.get("url", "") or a.get("path", "")
- article_entries.append({"title": title, "url": url})
-
-for i, entry in enumerate(article_entries):
- print(f" [{i}] title={entry['title']!r:20s} url={entry['url']!r}")
-
-assert len(article_entries) == 2
-assert article_entries[0]["title"] == "my_photo"
-assert article_entries[1]["title"] == "chart_btcusdt"
-print()
-print("PASS — all assertions OK")
diff --git a/tests/test_cdp_attach.py b/tests/test_cdp_attach.py
new file mode 100644
index 0000000..f5ae596
--- /dev/null
+++ b/tests/test_cdp_attach.py
@@ -0,0 +1,171 @@
+"""Tests for the --cdp-url attach-to-existing-browser path.
+
+These run on a core install (no chrome, no browser): they exercise URL
+normalization and the routing logic that decides between attaching to a running
+browser and launching a throwaway one — without ever opening a browser.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+from pixelrag_render.backends import cdp
+
+_BIN = Path(sys.executable).parent
+
+
+def test_http_base_normalization():
+ f = cdp._http_base_from_cdp_url
+ assert f("http://127.0.0.1:9222") == "http://127.0.0.1:9222"
+ assert f("http://127.0.0.1:9222/json/version") == "http://127.0.0.1:9222"
+ assert f("127.0.0.1:9222") == "http://127.0.0.1:9222"
+ assert f("ws://localhost:9222/devtools/browser/abc") == "http://localhost:9222"
+
+
+def test_cdp_url_routes_to_attach_without_launching(monkeypatch, tmp_path):
+ """With cdp_url set, render_urls must take the attach path and never call
+ _find_chrome (i.e. never try to launch/auto-install a browser)."""
+ called = {}
+
+ async def fake_attached(urls, output_dir, *a, **kw):
+ called["attached"] = list(urls)
+ return [Path(output_dir) / "x.png.tiles"]
+
+ def boom():
+ raise AssertionError("_find_chrome must not run on the attach path")
+
+ monkeypatch.setattr(cdp, "_run_batch_attached", fake_attached)
+ monkeypatch.setattr(cdp, "_find_chrome", boom)
+
+ out = cdp.render_urls(
+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9222"
+ )
+ assert called["attached"] == ["https://example.com"]
+ assert out and out[0].name == "x.png.tiles"
+
+
+def test_env_var_fallback_routes_to_attach(monkeypatch, tmp_path):
+ called = {}
+
+ async def fake_attached(urls, output_dir, *a, **kw):
+ called["hit"] = True
+ return []
+
+ monkeypatch.setattr(cdp, "_run_batch_attached", fake_attached)
+ monkeypatch.setattr(cdp, "_find_chrome", lambda: pytest.fail("should not launch"))
+ monkeypatch.setenv("PIXELSHOT_CDP_URL", "http://127.0.0.1:9222")
+
+ cdp.render_urls(["https://example.com"], tmp_path)
+ assert called.get("hit") is True
+
+
+def test_default_path_still_resolves_chrome(monkeypatch, tmp_path):
+ """No cdp_url (and no env) → the launch path runs find_chrome as before."""
+ monkeypatch.delenv("PIXELSHOT_CDP_URL", raising=False)
+ sentinel = RuntimeError("find_chrome reached")
+
+ def boom():
+ raise sentinel
+
+ monkeypatch.setattr(cdp, "_find_chrome", boom)
+ with pytest.raises(RuntimeError, match="find_chrome reached"):
+ cdp.render_urls(["https://example.com"], tmp_path)
+
+
+def test_attach_creates_and_closes_only_its_own_target(monkeypatch, tmp_path):
+ """The attach path must create its own target, and on teardown close ONLY
+ that target — never a pre-existing one, and never close/kill the browser.
+
+ Mocks the websocket/CDP layer so no browser is needed: a fake ws records
+ every CDP method sent on the browser-level connection.
+ """
+ browser_methods = [] # (method, params) sent on the browser ws
+
+ class FakeBrowserWS:
+ async def send(self, raw):
+ msg = json.loads(raw)
+ browser_methods.append((msg["method"], msg.get("params", {})))
+ method = msg["method"]
+ result = {}
+ if method == "Target.createTarget":
+ result = {"targetId": "OUR-TARGET-123"}
+ self._reply = {"id": msg["id"], "result": result}
+
+ async def recv(self):
+ return json.dumps(self._reply)
+
+ async def close(self):
+ pass
+
+ class FakePageWS:
+ async def send(self, raw):
+ msg = json.loads(raw)
+ self._reply = {"id": msg["id"], "result": {}}
+
+ async def recv(self):
+ return json.dumps(self._reply)
+
+ async def close(self):
+ pass
+
+ async def fake_connect_ws(ws_url):
+ return FakeBrowserWS() if ws_url == "BROWSER_WS" else FakePageWS()
+
+ # Pre-existing target plus the one we create — only ours must be closed.
+ def fake_fetch_json(url, cdp_url, timeout=5):
+ if url.endswith("/json/version"):
+ return {"webSocketDebuggerUrl": "BROWSER_WS"}
+ return [
+ {"id": "PREEXISTING-999", "webSocketDebuggerUrl": "ws://other"},
+ {"id": "OUR-TARGET-123", "webSocketDebuggerUrl": "PAGE_WS"},
+ ]
+
+ async def fake_capture_url(*a, **kw):
+ return 1
+
+ monkeypatch.setattr(cdp, "_connect_ws", fake_connect_ws)
+ monkeypatch.setattr(cdp, "_fetch_json", fake_fetch_json)
+ monkeypatch.setattr(cdp, "capture_url", fake_capture_url)
+
+ out = cdp.render_urls(
+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9222"
+ )
+ assert out and out[0].name.endswith(".png.tiles")
+
+ methods = [m for m, _ in browser_methods]
+ assert "Target.createTarget" in methods
+ assert "Target.closeTarget" in methods
+ # Never closed/killed the browser.
+ assert "Browser.close" not in methods
+
+ # closeTarget targeted ONLY our own created target, never the pre-existing one.
+ closed = [p["targetId"] for m, p in browser_methods if m == "Target.closeTarget"]
+ assert closed == ["OUR-TARGET-123"]
+
+
+def test_attach_bad_cdp_url_raises_clean_error(monkeypatch, tmp_path):
+ """An unreachable/bad endpoint surfaces a clear RuntimeError, not a raw
+ URLError/KeyError traceback."""
+ import urllib.error
+
+ def boom(url, timeout=5):
+ raise urllib.error.URLError("connection refused")
+
+ monkeypatch.setattr(cdp.urllib.request, "urlopen", boom)
+
+ with pytest.raises(RuntimeError, match="Could not reach CDP endpoint at"):
+ cdp.render_urls(
+ ["https://example.com"], tmp_path, cdp_url="http://127.0.0.1:9999"
+ )
+
+
+def test_cli_help_exposes_cdp_url():
+ import subprocess
+
+ r = subprocess.run(
+ [str(_BIN / "pixelshot"), "--help"], capture_output=True, text=True
+ )
+ assert r.returncode == 0
+ assert "--cdp-url" in r.stdout