Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

Fix CDP capture hang when Chrome exposes component-extension targets (#56)

MrTHROS1119c84e2ece
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index 6f9f65c..f421161 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -65,8 +65,12 @@ async def _connect_cdp(port: int, retries: int = 5, delay: float = 1.0):
f"http://localhost:{port}/json", timeout=3
).read()
targets = json.loads(data)
+ # Pick a real page target — Chrome's built-in component extensions
+ # (Cast/Media Router) expose background_page targets that show up
+ # first in /json but never render navigations, hanging CDP capture.
+ pages = [t for t in targets if t.get("type") == "page"] or targets
ws = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"],
+ pages[0]["webSocketDebuggerUrl"],
open_timeout=10,
max_size=50 * 1024 * 1024,
)
diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.py
index bf4d38d..b2fa74c 100644
--- a/render/src/pixelrag_render/backends/fast_cdp.py
+++ b/render/src/pixelrag_render/backends/fast_cdp.py
@@ -133,8 +133,12 @@ async def _launch_chrome(chrome_path: str, port: int) -> tuple:
f"http://localhost:{port}/json", timeout=3
).read()
targets = json.loads(data)
+ # Pick a real page target — Chrome's built-in component extensions
+ # (Cast/Media Router) expose background_page targets that show up
+ # first in /json but never render navigations, hanging CDP capture.
+ pages = [t for t in targets if t.get("type") == "page"] or targets
ws = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"],
+ pages[0]["webSocketDebuggerUrl"],
open_timeout=10,
max_size=50 * 1024 * 1024,
)
diff --git a/render/src/pixelrag_render/strategies/cdp_multitab.py b/render/src/pixelrag_render/strategies/cdp_multitab.py
index 75a2df0..49c41a8 100644
--- a/render/src/pixelrag_render/strategies/cdp_multitab.py
+++ b/render/src/pixelrag_render/strategies/cdp_multitab.py
@@ -22,7 +22,7 @@ import urllib.request
from dataclasses import dataclass
from .base import article_url, ArticleCapture, TileCapture
-from .connection import WebsocketConnection, CHROME_ARGS
+from .connection import WebsocketConnection, CHROME_ARGS, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
@@ -95,7 +95,7 @@ class CDPMultiTabStrategy:
port = self._base_port + pi
args = [self.chrome_path, f"--remote-debugging-port={port}"]
if not self.headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args += CHROME_ARGS + ["--in-process-gpu", "about:blank"]
proc = subprocess.Popen(
@@ -118,7 +118,7 @@ class CDPMultiTabStrategy:
raise ConnectionError(f"Chrome port {port}")
ws0 = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"],
+ pick_page_ws_url(targets),
open_timeout=10,
max_size=50 * 1024 * 1024,
)
diff --git a/render/src/pixelrag_render/strategies/cdp_oneshot.py b/render/src/pixelrag_render/strategies/cdp_oneshot.py
index 8cdefc3..71992df 100644
--- a/render/src/pixelrag_render/strategies/cdp_oneshot.py
+++ b/render/src/pixelrag_render/strategies/cdp_oneshot.py
@@ -29,7 +29,7 @@ import urllib.request
from dataclasses import dataclass, field
from .base import article_url, ArticleCapture, TileCapture
-from .connection import CHROME_ARGS
+from .connection import CHROME_ARGS, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
@@ -81,7 +81,7 @@ async def _launch_oneshot(
args = [chrome_path, f"--remote-debugging-port={port}"]
if not headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args += CHROME_ARGS
if extra_args:
args += extra_args
@@ -98,7 +98,7 @@ async def _launch_oneshot(
).read()
targets = json.loads(data)
ws = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"],
+ pick_page_ws_url(targets),
open_timeout=5,
max_size=50 * 1024 * 1024,
)
diff --git a/render/src/pixelrag_render/strategies/cdp_overlap.py b/render/src/pixelrag_render/strategies/cdp_overlap.py
index a09aa26..3470829 100644
--- a/render/src/pixelrag_render/strategies/cdp_overlap.py
+++ b/render/src/pixelrag_render/strategies/cdp_overlap.py
@@ -22,7 +22,7 @@ import urllib.request
from dataclasses import dataclass
from .base import article_url, ArticleCapture, TileCapture
-from .connection import WebsocketConnection
+from .connection import WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
@@ -59,7 +59,7 @@ async def _launch_two_tabs(chrome_path: str, port: int, headless_shell: bool = F
args = [chrome_path, f"--remote-debugging-port={port}"]
if not headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args += [
"--no-sandbox",
"--disable-dev-shm-usage",
@@ -78,7 +78,7 @@ async def _launch_two_tabs(chrome_path: str, port: int, headless_shell: bool = F
f"http://localhost:{port}/json", timeout=3
).read()
targets = json.loads(data)
- ws_url_a = targets[0]["webSocketDebuggerUrl"]
+ ws_url_a = pick_page_ws_url(targets)
break
except Exception:
if attempt == 9:
diff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
index dfa9f55..0ad88d6 100644
--- a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
@@ -21,7 +21,7 @@ import urllib.request
from dataclasses import dataclass
from .base import article_url, ArticleCapture, TileCapture
-from .connection import WebsocketConnection
+from .connection import WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
@@ -78,7 +78,7 @@ async def launch_two_tab(
args = [chrome_path, f"--remote-debugging-port={port}"]
if not headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args += [
"--no-sandbox",
"--disable-dev-shm-usage",
@@ -102,8 +102,9 @@ async def launch_two_tab(
proc.kill()
raise ConnectionError(f"Chrome port {port}")
+ ws_url_a = pick_page_ws_url(targets)
ws_a = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"], open_timeout=10, max_size=50 * 1024 * 1024
+ ws_url_a, open_timeout=10, max_size=50 * 1024 * 1024
)
tab_a = WebsocketConnection(ws_a, proc)
@@ -118,7 +119,7 @@ async def launch_two_tab(
break
if not ws_url_b:
for t in targets2:
- if t["webSocketDebuggerUrl"] != targets[0]["webSocketDebuggerUrl"]:
+ if t["webSocketDebuggerUrl"] != ws_url_a:
ws_url_b = t["webSocketDebuggerUrl"]
break
diff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py b/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py
index ee8afa0..87cb056 100644
--- a/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py
+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py
@@ -19,7 +19,7 @@ import time
from dataclasses import dataclass
from .base import ArticleCapture, TileCapture, article_url
-from .connection import WebsocketConnection
+from .connection import WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
@@ -84,7 +84,7 @@ class CDPPipelinedTabsStrategy:
"--disable-dev-shm-usage",
]
if not self.headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args.append("about:blank")
proc = subprocess.Popen(
args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
@@ -102,7 +102,7 @@ class CDPPipelinedTabsStrategy:
data = urllib.request.urlopen(f"http://localhost:{port}/json").read()
targets = json.loads(data)
ws_a = await __import__("websockets").connect(
- targets[0]["webSocketDebuggerUrl"],
+ pick_page_ws_url(targets),
open_timeout=10,
max_size=50 * 1024 * 1024,
)
diff --git a/render/src/pixelrag_render/strategies/connection.py b/render/src/pixelrag_render/strategies/connection.py
index aa5449e..e096017 100644
--- a/render/src/pixelrag_render/strategies/connection.py
+++ b/render/src/pixelrag_render/strategies/connection.py
@@ -17,6 +17,20 @@ CHROME_ARGS = [
]
+def pick_page_ws_url(targets: list[dict]) -> str:
+ """Return the webSocketDebuggerUrl for a ``type: page`` target.
+
+ Chrome's ``/json`` endpoint may list ``background_page`` targets (from
+ built-in extensions like Cast / Media Router) before real page targets.
+ Connecting to a ``background_page`` hangs because it ignores
+ ``Page.navigate``. Filter for ``type == "page"`` first; fall back to
+ the unfiltered list so pre-existing behaviour is preserved when Chrome
+ reports no page targets at all.
+ """
+ pages = [t for t in targets if t.get("type") == "page"] or targets
+ return pages[0]["webSocketDebuggerUrl"]
+
+
class WebsocketConnection:
"""Direct websocket CDP connection."""
@@ -189,7 +203,7 @@ async def launch_websocket(
args = [chrome_path, f"--remote-debugging-port={port}"]
if not headless_shell:
- args.append("--headless")
+ args.append("--headless=new")
args += CHROME_ARGS
if extra_args:
args += extra_args
@@ -220,7 +234,7 @@ async def launch_websocket(
).read()
targets = json.loads(data)
ws = await websockets.connect(
- targets[0]["webSocketDebuggerUrl"],
+ pick_page_ws_url(targets),
open_timeout=10,
max_size=50 * 1024 * 1024,
)