Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(render): --extract-text flag for hybrid output (tiles + text.md) (#106)

Aafaq Rashid86102d01191a
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index 434285e..14b6018 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -414,6 +414,7 @@ async def capture_url(
image_format: str = "jpeg",
from_surface: bool = True,
wait_network_idle: bool = False,
+ extract_text: bool = False,
) -> int:
"""Capture a URL as tiled images via direct CDP websocket.
@@ -534,6 +535,24 @@ async def capture_url(
with open(tile_dir / "tiles.json", "w") as f:
json.dump(manifest, f)
+ # Extract page text alongside tiles (hybrid output mode)
+ if extract_text:
+ try:
+ text_result = await _cdp_send(
+ ws,
+ msg_id_ref,
+ "Runtime.evaluate",
+ {"expression": "document.body.innerText", "returnByValue": True},
+ )
+ page_text = text_result.get("result", {}).get("value", "")
+ if page_text:
+ with open(tile_dir / "text.md", "w") as f:
+ f.write(f"# {url}\n\n{page_text}\n")
+ except Exception:
+ # Best-effort: the tiles are the primary output and are already
+ # written, so a failed text probe must not fail the capture.
+ logger.warning("%s: page text extraction failed", url, exc_info=True)
+
return len(tiles)
@@ -571,6 +590,7 @@ async def _drain_queue(
image_format: str,
from_surface: bool,
wait_network_idle: bool,
+ extract_text: bool,
worker_id: int,
stats: dict,
results: list,
@@ -603,6 +623,7 @@ async def _drain_queue(
image_format=image_format,
from_surface=from_surface,
wait_network_idle=wait_network_idle,
+ extract_text=extract_text,
)
stats["done"] += 1
elapsed = time.monotonic() - t0
@@ -624,6 +645,7 @@ async def _worker(
image_format: str,
from_surface: bool,
wait_network_idle: bool,
+ extract_text: bool,
worker_id: int,
stats: dict,
results: list,
@@ -667,6 +689,7 @@ async def _worker(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
worker_id,
stats,
results,
@@ -719,6 +742,7 @@ async def _run_batch(
image_format: str,
from_surface: bool,
wait_network_idle: bool,
+ extract_text: bool,
stems: list[str] | None,
chrome_path: str,
) -> list[Path]:
@@ -744,6 +768,7 @@ async def _run_batch(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
wid,
stats,
results,
@@ -768,6 +793,7 @@ async def _attached_worker(
image_format: str,
from_surface: bool,
wait_network_idle: bool,
+ extract_text: bool,
worker_id: int,
stats: dict,
results: list,
@@ -805,6 +831,7 @@ async def _attached_worker(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
worker_id,
stats,
results,
@@ -832,6 +859,7 @@ async def _run_batch_attached(
image_format: str,
from_surface: bool,
wait_network_idle: bool,
+ extract_text: bool,
stems: list[str] | None,
cdp_url: str,
) -> list[Path]:
@@ -862,6 +890,7 @@ async def _run_batch_attached(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
wid,
stats,
results,
@@ -888,6 +917,7 @@ def render_urls(
image_format: str = "jpeg",
from_surface: bool = True,
wait_network_idle: bool = False,
+ extract_text: bool = False,
turbo: bool | None = None,
chrome_path: str | None = None,
cdp_url: str | None = None,
@@ -949,6 +979,7 @@ def render_urls(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
stems,
cdp_url,
)
@@ -964,8 +995,16 @@ def render_urls(
image_format != "jpeg"
or viewport_width != VIEWPORT_W
or wait_network_idle
+ or extract_text
or not from_surface
):
+ if extract_text:
+ # Same trade as wait_network_idle below: say so rather than
+ # letting --extract-text silently produce no text.md.
+ logger.info(
+ "extract_text is set: using the standard capture path "
+ "(the turbo path does not implement text extraction yet)"
+ )
if wait_network_idle:
# Be loud about the trade: users with a turbo-capable Chrome would
# otherwise silently lose ~half their capture throughput.
@@ -1016,6 +1055,7 @@ def render_urls(
image_format,
from_surface,
wait_network_idle,
+ extract_text,
stems,
chrome,
)
diff --git a/render/src/pixelrag_render/render.py b/render/src/pixelrag_render/render.py
index 05ba473..c4bb7fc 100644
--- a/render/src/pixelrag_render/render.py
+++ b/render/src/pixelrag_render/render.py
@@ -306,6 +306,13 @@ def main() -> None:
"session (cookies/logins) — so authenticated pages work — then closes only "
"that tab. Needs no local Chrome binary. Env: PIXELSHOT_CDP_URL.",
)
+ parser.add_argument(
+ "--extract-text",
+ action="store_true",
+ help="Extract page text alongside tiles (hybrid output). Saves a text.md "
+ "file in each tile directory with the page's innerText. Useful for "
+ "reducing LLM token usage on text-heavy pages.",
+ )
args = parser.parse_args()
output_dir = Path(args.output)
@@ -345,6 +352,7 @@ def main() -> None:
viewport_width=args.viewport_width,
workers=args.workers,
wait_network_idle=args.wait_network_idle,
+ extract_text=args.extract_text,
cdp_url=args.cdp_url,
)
results.extend(tile_dirs)
diff --git a/tests/test_extract_text.py b/tests/test_extract_text.py
new file mode 100644
index 0000000..99a3fa3
--- /dev/null
+++ b/tests/test_extract_text.py
@@ -0,0 +1,61 @@
+"""Tests for --extract-text hybrid output (issue #93)."""
+
+from pathlib import Path
+
+from pixelrag_render import render_file
+
+
+def test_extract_text_produces_text_md(tmp_path):
+ """With extract_text=True, a text.md file should be created alongside tiles."""
+ html = tmp_path / "page.html"
+ html.write_text(
+ "<html><body>"
+ "<h1>Hello World</h1>"
+ "<p>This is a test paragraph with important content.</p>"
+ "<ul><li>Item one</li><li>Item two</li></ul>"
+ "</body></html>"
+ )
+ out = tmp_path / "tiles"
+
+ dirs = render_file(html, out, extract_text=True)
+
+ assert dirs, "render_file returned no tile directories"
+ tile_dir = Path(dirs[0])
+ text_file = tile_dir / "text.md"
+ assert text_file.exists(), f"text.md not created in {tile_dir}"
+
+ content = text_file.read_text()
+ assert "Hello World" in content
+ assert "test paragraph" in content
+ assert "Item one" in content
+
+
+def test_no_extract_text_by_default(tmp_path):
+ """Without extract_text, no text.md should be created."""
+ html = tmp_path / "page.html"
+ html.write_text("<html><body><p>Some text</p></body></html>")
+ out = tmp_path / "tiles"
+
+ dirs = render_file(html, out)
+
+ tile_dir = Path(dirs[0])
+ assert not (tile_dir / "text.md").exists()
+
+
+def test_extract_text_wins_over_the_turbo_path(tmp_path):
+ """A turbo-capable Chrome must not silently swallow --extract-text.
+
+ The turbo backend (fast_cdp) has no text extraction, so requesting it has
+ to send the capture down the standard path — the same trade already made
+ for wait_network_idle. Without that, anyone whose Chrome is turbo-capable
+ (which is what `pixelshot install-chrome` produces) gets tiles and no
+ text.md, with nothing said about it.
+ """
+ html = tmp_path / "page.html"
+ html.write_text("<html><body><p>Turbo should not eat this</p></body></html>")
+
+ dirs = render_file(html, tmp_path / "tiles", extract_text=True, turbo=True)
+
+ text_file = Path(dirs[0]) / "text.md"
+ assert text_file.exists(), "turbo path silently dropped the text extraction"
+ assert "Turbo should not eat this" in text_file.read_text()