https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
Fix CDP capture hang when Chrome exposes component-extension targets (#56)
MrTHROS1119c84e2ece
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex 6f9f65c..f421161 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -65,8 +65,12 @@ async def _connect_cdp(port: int, retries: int = 5, delay: float = 1.0):f"http://localhost:{port}/json", timeout=3).read()targets = json.loads(data)+ # Pick a real page target — Chrome's built-in component extensions+ # (Cast/Media Router) expose background_page targets that show up+ # first in /json but never render navigations, hanging CDP capture.+ pages = [t for t in targets if t.get("type") == "page"] or targetsws = await websockets.connect(- targets[0]["webSocketDebuggerUrl"],+ pages[0]["webSocketDebuggerUrl"],open_timeout=10,max_size=50 * 1024 * 1024,)diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.pyindex bf4d38d..b2fa74c 100644--- a/render/src/pixelrag_render/backends/fast_cdp.py+++ b/render/src/pixelrag_render/backends/fast_cdp.py@@ -133,8 +133,12 @@ async def _launch_chrome(chrome_path: str, port: int) -> tuple:f"http://localhost:{port}/json", timeout=3).read()targets = json.loads(data)+ # Pick a real page target — Chrome's built-in component extensions+ # (Cast/Media Router) expose background_page targets that show up+ # first in /json but never render navigations, hanging CDP capture.+ pages = [t for t in targets if t.get("type") == "page"] or targetsws = await websockets.connect(- targets[0]["webSocketDebuggerUrl"],+ pages[0]["webSocketDebuggerUrl"],open_timeout=10,max_size=50 * 1024 * 1024,)diff --git a/render/src/pixelrag_render/strategies/cdp_multitab.py b/render/src/pixelrag_render/strategies/cdp_multitab.pyindex 75a2df0..49c41a8 100644--- a/render/src/pixelrag_render/strategies/cdp_multitab.py+++ b/render/src/pixelrag_render/strategies/cdp_multitab.py@@ -22,7 +22,7 @@ import urllib.requestfrom dataclasses import dataclassfrom .base import article_url, ArticleCapture, TileCapture-from .connection import WebsocketConnection, CHROME_ARGS+from .connection import WebsocketConnection, CHROME_ARGS, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875@@ -95,7 +95,7 @@ class CDPMultiTabStrategy:port = self._base_port + piargs = [self.chrome_path, f"--remote-debugging-port={port}"]if not self.headless_shell:- args.append("--headless")+ args.append("--headless=new")args += CHROME_ARGS + ["--in-process-gpu", "about:blank"]proc = subprocess.Popen(@@ -118,7 +118,7 @@ class CDPMultiTabStrategy:raise ConnectionError(f"Chrome port {port}")ws0 = await websockets.connect(- targets[0]["webSocketDebuggerUrl"],+ pick_page_ws_url(targets),open_timeout=10,max_size=50 * 1024 * 1024,)diff --git a/render/src/pixelrag_render/strategies/cdp_oneshot.py b/render/src/pixelrag_render/strategies/cdp_oneshot.pyindex 8cdefc3..71992df 100644--- a/render/src/pixelrag_render/strategies/cdp_oneshot.py+++ b/render/src/pixelrag_render/strategies/cdp_oneshot.py@@ -29,7 +29,7 @@ import urllib.requestfrom dataclasses import dataclass, fieldfrom .base import article_url, ArticleCapture, TileCapture-from .connection import CHROME_ARGS+from .connection import CHROME_ARGS, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875@@ -81,7 +81,7 @@ async def _launch_oneshot(args = [chrome_path, f"--remote-debugging-port={port}"]if not headless_shell:- args.append("--headless")+ args.append("--headless=new")args += CHROME_ARGSif extra_args:args += extra_args@@ -98,7 +98,7 @@ async def _launch_oneshot().read()targets = json.loads(data)ws = await websockets.connect(- targets[0]["webSocketDebuggerUrl"],+ pick_page_ws_url(targets),open_timeout=5,max_size=50 * 1024 * 1024,)diff --git a/render/src/pixelrag_render/strategies/cdp_overlap.py b/render/src/pixelrag_render/strategies/cdp_overlap.pyindex a09aa26..3470829 100644--- a/render/src/pixelrag_render/strategies/cdp_overlap.py+++ b/render/src/pixelrag_render/strategies/cdp_overlap.py@@ -22,7 +22,7 @@ import urllib.requestfrom dataclasses import dataclassfrom .base import article_url, ArticleCapture, TileCapture-from .connection import WebsocketConnection+from .connection import WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875@@ -59,7 +59,7 @@ async def _launch_two_tabs(chrome_path: str, port: int, headless_shell: bool = Fargs = [chrome_path, f"--remote-debugging-port={port}"]if not headless_shell:- args.append("--headless")+ args.append("--headless=new")args += ["--no-sandbox","--disable-dev-shm-usage",@@ -78,7 +78,7 @@ async def _launch_two_tabs(chrome_path: str, port: int, headless_shell: bool = Ff"http://localhost:{port}/json", timeout=3).read()targets = json.loads(data)- ws_url_a = targets[0]["webSocketDebuggerUrl"]+ ws_url_a = pick_page_ws_url(targets)breakexcept Exception:if attempt == 9:diff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.pyindex dfa9f55..0ad88d6 100644--- a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py@@ -21,7 +21,7 @@ import urllib.requestfrom dataclasses import dataclassfrom .base import article_url, ArticleCapture, TileCapture-from .connection import WebsocketConnection+from .connection import WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875@@ -78,7 +78,7 @@ async def launch_two_tab(args = [chrome_path, f"--remote-debugging-port={port}"]if not headless_shell:- args.append("--headless")+ args.append("--headless=new")args += ["--no-sandbox","--disable-dev-shm-usage",@@ -102,8 +102,9 @@ async def launch_two_tab(proc.kill()raise ConnectionError(f"Chrome port {port}")+ ws_url_a = pick_page_ws_url(targets)ws_a = await websockets.connect(- targets[0]["webSocketDebuggerUrl"], open_timeout=10, max_size=50 * 1024 * 1024+ ws_url_a, open_timeout=10, max_size=50 * 1024 * 1024)tab_a = WebsocketConnection(ws_a, proc)@@ -118,7 +119,7 @@ async def launch_two_tab(breakif not ws_url_b:for t in targets2:- if t["webSocketDebuggerUrl"] != targets[0]["webSocketDebuggerUrl"]:+ if t["webSocketDebuggerUrl"] != ws_url_a:ws_url_b = t["webSocketDebuggerUrl"]breakdiff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py b/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.pyindex ee8afa0..87cb056 100644--- a/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_tabs.py@@ -19,7 +19,7 @@ import timefrom dataclasses import dataclassfrom .base import ArticleCapture, TileCapture, article_url-from .connection import WebsocketConnection+from .connection import WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875@@ -84,7 +84,7 @@ class CDPPipelinedTabsStrategy:"--disable-dev-shm-usage",]if not self.headless_shell:- args.append("--headless")+ args.append("--headless=new")args.append("about:blank")proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL@@ -102,7 +102,7 @@ class CDPPipelinedTabsStrategy:data = urllib.request.urlopen(f"http://localhost:{port}/json").read()targets = json.loads(data)ws_a = await __import__("websockets").connect(- targets[0]["webSocketDebuggerUrl"],+ pick_page_ws_url(targets),open_timeout=10,max_size=50 * 1024 * 1024,)diff --git a/render/src/pixelrag_render/strategies/connection.py b/render/src/pixelrag_render/strategies/connection.pyindex aa5449e..e096017 100644--- a/render/src/pixelrag_render/strategies/connection.py+++ b/render/src/pixelrag_render/strategies/connection.py@@ -17,6 +17,20 @@ CHROME_ARGS = []+def pick_page_ws_url(targets: list[dict]) -> str:+ """Return the webSocketDebuggerUrl for a ``type: page`` target.++ Chrome's ``/json`` endpoint may list ``background_page`` targets (from+ built-in extensions like Cast / Media Router) before real page targets.+ Connecting to a ``background_page`` hangs because it ignores+ ``Page.navigate``. Filter for ``type == "page"`` first; fall back to+ the unfiltered list so pre-existing behaviour is preserved when Chrome+ reports no page targets at all.+ """+ pages = [t for t in targets if t.get("type") == "page"] or targets+ return pages[0]["webSocketDebuggerUrl"]++class WebsocketConnection:"""Direct websocket CDP connection."""@@ -189,7 +203,7 @@ async def launch_websocket(args = [chrome_path, f"--remote-debugging-port={port}"]if not headless_shell:- args.append("--headless")+ args.append("--headless=new")args += CHROME_ARGSif extra_args:args += extra_args@@ -220,7 +234,7 @@ async def launch_websocket().read()targets = json.loads(data)ws = await websockets.connect(- targets[0]["webSocketDebuggerUrl"],+ pick_page_ws_url(targets),open_timeout=10,max_size=50 * 1024 * 1024,)