https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(render): --extract-text flag for hybrid output (tiles + text.md) (#106)
Aafaq Rashid86102d01191a
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex 434285e..14b6018 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -414,6 +414,7 @@ async def capture_url(image_format: str = "jpeg",from_surface: bool = True,wait_network_idle: bool = False,+ extract_text: bool = False,) -> int:"""Capture a URL as tiled images via direct CDP websocket.@@ -534,6 +535,24 @@ async def capture_url(with open(tile_dir / "tiles.json", "w") as f:json.dump(manifest, f)+ # Extract page text alongside tiles (hybrid output mode)+ if extract_text:+ try:+ text_result = await _cdp_send(+ ws,+ msg_id_ref,+ "Runtime.evaluate",+ {"expression": "document.body.innerText", "returnByValue": True},+ )+ page_text = text_result.get("result", {}).get("value", "")+ if page_text:+ with open(tile_dir / "text.md", "w") as f:+ f.write(f"# {url}\n\n{page_text}\n")+ except Exception:+ # Best-effort: the tiles are the primary output and are already+ # written, so a failed text probe must not fail the capture.+ logger.warning("%s: page text extraction failed", url, exc_info=True)+return len(tiles)@@ -571,6 +590,7 @@ async def _drain_queue(image_format: str,from_surface: bool,wait_network_idle: bool,+ extract_text: bool,worker_id: int,stats: dict,results: list,@@ -603,6 +623,7 @@ async def _drain_queue(image_format=image_format,from_surface=from_surface,wait_network_idle=wait_network_idle,+ extract_text=extract_text,)stats["done"] += 1elapsed = time.monotonic() - t0@@ -624,6 +645,7 @@ async def _worker(image_format: str,from_surface: bool,wait_network_idle: bool,+ extract_text: bool,worker_id: int,stats: dict,results: list,@@ -667,6 +689,7 @@ async def _worker(image_format,from_surface,wait_network_idle,+ extract_text,worker_id,stats,results,@@ -719,6 +742,7 @@ async def _run_batch(image_format: str,from_surface: bool,wait_network_idle: bool,+ extract_text: bool,stems: list[str] | None,chrome_path: str,) -> list[Path]:@@ -744,6 +768,7 @@ async def _run_batch(image_format,from_surface,wait_network_idle,+ extract_text,wid,stats,results,@@ -768,6 +793,7 @@ async def _attached_worker(image_format: str,from_surface: bool,wait_network_idle: bool,+ extract_text: bool,worker_id: int,stats: dict,results: list,@@ -805,6 +831,7 @@ async def _attached_worker(image_format,from_surface,wait_network_idle,+ extract_text,worker_id,stats,results,@@ -832,6 +859,7 @@ async def _run_batch_attached(image_format: str,from_surface: bool,wait_network_idle: bool,+ extract_text: bool,stems: list[str] | None,cdp_url: str,) -> list[Path]:@@ -862,6 +890,7 @@ async def _run_batch_attached(image_format,from_surface,wait_network_idle,+ extract_text,wid,stats,results,@@ -888,6 +917,7 @@ def render_urls(image_format: str = "jpeg",from_surface: bool = True,wait_network_idle: bool = False,+ extract_text: bool = False,turbo: bool | None = None,chrome_path: str | None = None,cdp_url: str | None = None,@@ -949,6 +979,7 @@ def render_urls(image_format,from_surface,wait_network_idle,+ extract_text,stems,cdp_url,)@@ -964,8 +995,16 @@ def render_urls(image_format != "jpeg"or viewport_width != VIEWPORT_Wor wait_network_idle+ or extract_textor not from_surface):+ if extract_text:+ # Same trade as wait_network_idle below: say so rather than+ # letting --extract-text silently produce no text.md.+ logger.info(+ "extract_text is set: using the standard capture path "+ "(the turbo path does not implement text extraction yet)"+ )if wait_network_idle:# Be loud about the trade: users with a turbo-capable Chrome would# otherwise silently lose ~half their capture throughput.@@ -1016,6 +1055,7 @@ def render_urls(image_format,from_surface,wait_network_idle,+ extract_text,stems,chrome,)diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.pyindex 05ba473..c4bb7fc 100644--- a/render/src/pixelrag_render/render.py+++ b/render/src/pixelrag_render/render.py@@ -306,6 +306,13 @@ def main() -> None:"session (cookies/logins) — so authenticated pages work — then closes only ""that tab. Needs no local Chrome binary. Env: PIXELSHOT_CDP_URL.",)+ parser.add_argument(+ "--extract-text",+ action="store_true",+ help="Extract page text alongside tiles (hybrid output). Saves a text.md "+ "file in each tile directory with the page's innerText. Useful for "+ "reducing LLM token usage on text-heavy pages.",+ )args = parser.parse_args()output_dir = Path(args.output)@@ -345,6 +352,7 @@ def main() -> None:viewport_width=args.viewport_width,workers=args.workers,wait_network_idle=args.wait_network_idle,+ extract_text=args.extract_text,cdp_url=args.cdp_url,)results.extend(tile_dirs)diff --git a/tests/test_extract_text.py b/tests/test_extract_text.pynew file mode 100644index 0000000..99a3fa3--- /dev/null+++ b/tests/test_extract_text.py@@ -0,0 +1,61 @@+"""Tests for --extract-text hybrid output (issue #93)."""++from pathlib import Path++from pixelrag_render import render_file+++def test_extract_text_produces_text_md(tmp_path):+ """With extract_text=True, a text.md file should be created alongside tiles."""+ html = tmp_path / "page.html"+ html.write_text(+ "<html><body>"+ "<h1>Hello World</h1>"+ "<p>This is a test paragraph with important content.</p>"+ "<ul><li>Item one</li><li>Item two</li></ul>"+ "</body></html>"+ )+ out = tmp_path / "tiles"++ dirs = render_file(html, out, extract_text=True)++ assert dirs, "render_file returned no tile directories"+ tile_dir = Path(dirs[0])+ text_file = tile_dir / "text.md"+ assert text_file.exists(), f"text.md not created in {tile_dir}"++ content = text_file.read_text()+ assert "Hello World" in content+ assert "test paragraph" in content+ assert "Item one" in content+++def test_no_extract_text_by_default(tmp_path):+ """Without extract_text, no text.md should be created."""+ html = tmp_path / "page.html"+ html.write_text("<html><body><p>Some text</p></body></html>")+ out = tmp_path / "tiles"++ dirs = render_file(html, out)++ tile_dir = Path(dirs[0])+ assert not (tile_dir / "text.md").exists()+++def test_extract_text_wins_over_the_turbo_path(tmp_path):+ """A turbo-capable Chrome must not silently swallow --extract-text.++ The turbo backend (fast_cdp) has no text extraction, so requesting it has+ to send the capture down the standard path — the same trade already made+ for wait_network_idle. Without that, anyone whose Chrome is turbo-capable+ (which is what `pixelshot install-chrome` produces) gets tiles and no+ text.md, with nothing said about it.+ """+ html = tmp_path / "page.html"+ html.write_text("<html><body><p>Turbo should not eat this</p></body></html>")++ dirs = render_file(html, tmp_path / "tiles", extract_text=True, turbo=True)++ text_file = Path(dirs[0]) / "text.md"+ assert text_file.exists(), "turbo path silently dropped the text extraction"+ assert "Turbo should not eat this" in text_file.read_text()