https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
"""Public API for pixelshot.
Renders documents (URLs, PDFs, local HTML/image files) to image tiles.
Entry point:
pixelshot <inputs> --output ./tiles --backend cdp --workers 4
"""
import argparse
import logging
import os
import shutil
import sys
from pathlib import Path
from typing import Optional
logger = logging.getLogger("pixelrag_render.render")
def render_url(
url: str,
output_dir: str | Path,
backend: str = "cdp",
*,
tile_height: int = 8192,
quality: int = 85,
viewport_width: int = 875,
workers: int = 1,
**kwargs,
) -> list[Path]:
"""Render a URL to tiled JPEG images.
Args:
url: URL to capture (http:// or https:// or file://).
output_dir: Directory to write tile subdirectories into.
backend: Rendering backend. Only ``"cdp"`` is implemented.
tile_height: Maximum tile height in pixels (default 8192).
quality: JPEG quality 1-100 (default 85).
viewport_width: Browser viewport width in pixels (default 875).
workers: Number of parallel browser processes (default 1).
**kwargs: Additional keyword arguments forwarded to the backend.
Returns:
List of Path objects pointing to created tile directories.
"""
return render_urls(
[url],
output_dir,
backend=backend,
tile_height=tile_height,
quality=quality,
viewport_width=viewport_width,
workers=workers,
**kwargs,
)
def render_urls(
urls: list[str],
output_dir: str | Path,
backend: str = "cdp",
*,
stems: list[str] | None = None,
tile_height: int = 8192,
quality: int = 85,
viewport_width: int = 875,
workers: int = 4,
**kwargs,
) -> list[Path]:
"""Render a list of URLs to tiled JPEG images.
Args:
urls: URLs to capture.
output_dir: Directory to write tile subdirectories into.
backend: Rendering backend. Only ``"cdp"`` is implemented.
stems: Optional list of output directory stems (one per URL).
If provided, tiles are written to ``{output_dir}/{stem}.png.tiles/``
instead of deriving names from URLs. Useful for assigning
sequential IDs (e.g. ``["0", "1", "2"]``).
tile_height: Maximum tile height in pixels (default 8192).
quality: JPEG quality 1-100 (default 85).
viewport_width: Browser viewport width in pixels (default 875).
workers: Number of parallel browser processes (default 4).
**kwargs: Additional keyword arguments forwarded to the backend.
Returns:
List of Path objects pointing to created tile directories.
"""
if backend in ("cdp", "websocket"): # "websocket" kept as a back-compat alias
from .backends.cdp import render_urls as _render_urls
else:
raise ValueError(
f"Unknown backend: {backend!r}. Choose 'cdp'."
" The cdp backend auto-selects a turbo capture path when a turbo-capable"
" Chrome is present."
)
return _render_urls(
urls,
output_dir,
stems=stems,
tile_height=tile_height,
quality=quality,
viewport_width=viewport_width,
workers=workers,
**kwargs,
)
def render_pdf(
path: str | Path,
output_dir: str | Path,
*,
dpi: int = 200,
pages: Optional[list[int]] = None,
quality: int = 85,
stem: str | None = None,
) -> list[Path]:
"""Render a PDF file to tiled JPEG images.
Args:
path: Path to the PDF file.
output_dir: Directory to write the tile subdirectory into.
dpi: Rendering resolution (default 200 ≈ 1650×2200 for A4).
pages: 1-based list of page numbers to render. ``None`` renders all.
quality: JPEG quality 1-100 (default 85).
stem: Override for the tile directory name (default: PDF filename stem).
Returns:
List containing the tile directory Path on success.
"""
from .backends.pdf import render_pdf as _render_pdf
return _render_pdf(
path, output_dir, dpi=dpi, pages=pages, quality=quality, stem=stem
)
def render_file(
path: str | Path,
output_dir: str | Path,
backend: str = "cdp",
**kwargs,
) -> list[Path]:
"""Auto-detect file type and render to tiled JPEG images.
Dispatch rules:
- ``.pdf`` → ``render_pdf()``
- ``.html`` / ``.htm`` → ``render_url(file://...)``
- ``.png`` / ``.jpg`` / ``.jpeg`` / ``.webp`` → copy into output_dir as-is
- ``http://`` or ``https://`` prefix → ``render_url()``
Args:
path: Path to a local file, or a URL string.
output_dir: Directory to write tile subdirectories into.
backend: Browser backend for HTML/URL rendering (default ``"cdp"``).
**kwargs: Forwarded to the underlying render function.
Returns:
List of Path objects pointing to created tile directories or copied files.
"""
path_str = str(path)
output_dir = Path(output_dir)
# URL strings
if path_str.startswith("http://") or path_str.startswith("https://"):
return render_url(path_str, output_dir, backend=backend, **kwargs)
p = Path(path)
suffix = p.suffix.lower()
if suffix == ".pdf":
return render_pdf(p, output_dir, **kwargs)
if suffix in {".html", ".htm"}:
file_url = p.resolve().as_uri()
return render_url(file_url, output_dir, backend=backend, **kwargs)
if suffix in {".png", ".jpg", ".jpeg", ".webp"}:
output_dir.mkdir(parents=True, exist_ok=True)
dest = output_dir / p.name
shutil.copy2(str(p), str(dest))
logger.info("Copied image: %s → %s", p, dest)
return [dest]
raise ValueError(
f"Cannot auto-detect render method for {path!r}. "
"Supported: .pdf, .html, .htm, .png, .jpg, .jpeg, .webp, http://, https://"
)
def main() -> None:
"""CLI entry point: pixelshot.
Usage examples::
# Single URL, default CDP backend
pixelshot https://example.com --output ./tiles
# Multiple inputs
pixelshot https://a.com https://b.com --output ./tiles
# PDF
pixelshot report.pdf --output ./tiles
# Local HTML
pixelshot index.html --output ./tiles
# URL file
pixelshot urls.txt --output ./tiles
# Chrome management (folded from the former `pixelrag-chrome`)
pixelshot install-chrome # download the patched headless Chrome
pixelshot which-chrome # print the active Chrome binary path
"""
# Chrome management subcommands — dispatch before building the render parser.
if len(sys.argv) > 1 and sys.argv[1] in ("install-chrome", "which-chrome"):
from pixelrag_render import chrome
if sys.argv[1] == "install-chrome":
chrome.install_chrome()
else:
try:
print(chrome.find_chrome(auto_install=False))
except FileNotFoundError as e:
print(str(e), file=sys.stderr)
sys.exit(1)
return
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
parser = argparse.ArgumentParser(
prog="pixelshot",
description="Render documents (URLs, PDFs, HTML files) to tiled JPEG images.",
)
parser.add_argument(
"inputs",
nargs="+",
metavar="INPUT",
help="URLs or file paths to render.",
)
parser.add_argument(
"--output",
"-o",
default="./tiles",
metavar="DIR",
help="Output directory for tile subdirectories (default: ./tiles).",
)
parser.add_argument(
"--backend",
choices=["cdp"],
default="cdp",
help="Browser backend for URL/HTML rendering (default: cdp).",
)
parser.add_argument(
"--workers",
"-w",
type=int,
default=4,
help="Number of parallel browser processes (default: 4).",
)
parser.add_argument(
"--tile-height",
type=int,
default=8192,
help="Maximum tile height in pixels (default: 8192).",
)
parser.add_argument(
"--quality",
type=int,
default=85,
help="JPEG quality 1-100 (default: 85).",
)
parser.add_argument(
"--viewport-width",
type=int,
default=875,
help="Browser viewport width in pixels (default: 875).",
)
parser.add_argument(
"--wait-network-idle",
action="store_true",
help="After the page's load event, also wait until at most 2 network "
"requests have been in flight for ~500ms (networkidle2) before "
"capturing, capped at 12s. Helps JS/SPA pages that fetch content "
"after load; tolerates persistent analytics/long-poll connections. "
"Off by default; the index pipeline's `web` source and the "
"pixelbrowse skill enable it.",
)
parser.add_argument(
"--dpi",
type=int,
default=200,
help="DPI for PDF rendering (default: 200).",
)
parser.add_argument(
"--cdp-url",
default=os.environ.get("PIXELSHOT_CDP_URL"),
metavar="URL",
help="Attach to an already-running Chrome/Brave DevTools endpoint "
"(e.g. http://127.0.0.1:9222) instead of launching a throwaway headless "
"browser. Renders each input in a fresh tab using that browser's existing "
"session (cookies/logins) — so authenticated pages work — then closes only "
"that tab. Needs no local Chrome binary. Env: PIXELSHOT_CDP_URL.",
)
parser.add_argument(
"--extract-text",
action="store_true",
help="Extract page text alongside tiles (hybrid output). Saves a text.md "
"file in each tile directory with the page's innerText. Useful for "
"reducing LLM token usage on text-heavy pages.",
)
args = parser.parse_args()
output_dir = Path(args.output)
urls = []
files = []
for inp in args.inputs:
if inp.lower().endswith(".txt"):
try:
with open(inp, encoding="utf-8") as f:
for line in f:
url = line.strip()
if url:
urls.append(url)
except FileNotFoundError:
parser.error(f"URL file not found: {inp}")
elif inp.startswith("http://") or inp.startswith("https://"):
urls.append(inp)
else:
files.append(Path(inp))
results: list[Path] = []
if urls:
logger.info(
"Rendering %d URL(s) with backend=%s workers=%d",
len(urls),
args.backend,
args.workers,
)
tile_dirs = render_urls(
urls,
output_dir,
backend=args.backend,
tile_height=args.tile_height,
quality=args.quality,
viewport_width=args.viewport_width,
workers=args.workers,
wait_network_idle=args.wait_network_idle,
extract_text=args.extract_text,
cdp_url=args.cdp_url,
)
results.extend(tile_dirs)
# Handle files individually (they may need different backends)
for fpath in files:
suffix = fpath.suffix.lower()
try:
if suffix == ".pdf":
tile_dirs = render_pdf(
fpath, output_dir, dpi=args.dpi, quality=args.quality
)
elif suffix in {".html", ".htm"}:
file_url = fpath.resolve().as_uri()
tile_dirs = render_url(
file_url,
output_dir,
backend=args.backend,
tile_height=args.tile_height,
quality=args.quality,
viewport_width=args.viewport_width,
workers=1,
wait_network_idle=args.wait_network_idle,
extract_text=args.extract_text,
cdp_url=args.cdp_url,
)
elif suffix in {".png", ".jpg", ".jpeg", ".webp"}:
tile_dirs = render_file(fpath, output_dir)
else:
logger.warning("Unsupported file type: %s — skipping", fpath)
continue
results.extend(tile_dirs)
except Exception as e:
logger.error("Failed to render %s: %s", fpath, e)
if results:
logger.info("Done. %d output(s):", len(results))
for r in results:
print(r)
else:
logger.warning("No outputs produced.")
sys.exit(1)
if __name__ == "__main__":
main()