https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
#!/usr/bin/env python3
"""
Screenshot benchmark with correctness verification.
Bench is a clean measurement harness. It takes a strategy object, runs it,
verifies results against cached GT, and dumps config + results.
Strategies live in pixelrag_render.strategies — bench does NOT know about
specific strategy implementations or naming conventions.
Usage (programmatic):
from pixelrag_render.strategies import CDPSequentialStrategy
from pixelrag_render.bench.bench_throughput import Bench
bench = Bench(zim_path="...", chrome_path="...", output_dir="./results")
strategy = CDPSequentialStrategy(chrome_path=..., n_workers=32, fmt="raw")
result = await bench.run(strategy)
"""
from __future__ import annotations
import hashlib
import io
import json
import os
import random
import struct
import tempfile
import time
from pathlib import Path
import numpy as np
from PIL import Image
from pixelrag_render.strategies.base import TileCapture
from pixelrag_render.strategies.cdp_sequential import (
VIEWPORT_WIDTH,
CDPSequentialStrategy,
)
CORRECT_THRESHOLD = 99.0
JPEG_MAX_MEAN_DIFF = 5.0
LOSSLESS_MAX_MEAN_DIFF = 3.0
# ---------------------------------------------------------------------------
# Article preparation
# ---------------------------------------------------------------------------
def prepare_articles(
zim_path: str, n: int, seed: int = 42, kiwix_url: str | None = None
) -> list[dict]:
"""Sample articles from ZIM.
kiwix_url can be:
- None: write HTML to temp files (file:// mode)
- "http://host:port": single kiwix-serve instance
- "http://host:9461,http://host:9462,...": multiple instances (round-robin)
"""
from urllib.parse import quote
from libzim.reader import Archive
archive = Archive(zim_path)
# Support multiple kiwix URLs (comma-separated)
kiwix_urls = kiwix_url.split(",") if kiwix_url else []
# Detect book_name from first URL or ZIM filename
if kiwix_urls:
# Extract book_name from URL: http://host:port/content/{book_name}/...
# For symlinks like wiki_1.zim, book_name = wiki_1
# We need to figure out the right book_name for each URL
pass
book_name = Path(zim_path).stem
rng = random.Random(seed)
articles = []
tried = 0
while len(articles) < n and tried < n * 20:
idx = rng.randint(0, archive.all_entry_count - 1)
tried += 1
try:
e = archive._get_entry_by_id(idx)
if e.is_redirect or e.path.startswith("-/") or len(e.path) <= 2:
continue
entry = archive.get_entry_by_path(e.path)
item = entry.get_item()
if "html" not in item.mimetype:
continue
html = bytes(item.content).decode("utf-8")
if 'http-equiv="refresh"' in html.lower() or len(html) < 300:
continue
if kiwix_urls:
safe = "/:@!$&'()*+,;="
# Round-robin across kiwix instances
base = kiwix_urls[len(articles) % len(kiwix_urls)]
# Detect book_name from the symlink/ZIM each instance serves
parts = base.rstrip("/").rsplit(":", 1)
port = int(parts[1]) if len(parts) > 1 else 9454
# Each instance may have different book_name (wiki_1, wiki_2, etc.)
bname = f"wiki_{port - 9460}" if port > 9460 else book_name
url = f"{base}/content/{bname}/{quote(e.path, safe=safe)}"
articles.append({"path": e.path, "file": url})
else:
tmp = tempfile.NamedTemporaryFile(
suffix=".html", delete=False, dir="/tmp", prefix="bench_"
)
tmp.write(html.encode())
tmp.close()
articles.append({"path": e.path, "file": tmp.name})
except Exception:
continue
return articles
def cleanup_articles(articles: list[dict]):
for a in articles:
if a["file"].startswith("http"):
continue
try:
os.unlink(a["file"])
except OSError:
pass
# ---------------------------------------------------------------------------
# Ground truth (cached)
# ---------------------------------------------------------------------------
def gt_cache_key(articles: list[dict], seed: int) -> str:
paths = sorted(a["path"] for a in articles)
content = f"seed={seed}\n" + "\n".join(paths)
return hashlib.sha256(content.encode()).hexdigest()[:16]
async def generate_ground_truth(
articles: list[dict],
chrome_path: str,
cache_dir: Path,
seed: int,
timeout_ms: int = 5000,
) -> dict[str, list[Path]]:
cache_key = gt_cache_key(articles, seed)
manifest_path = cache_dir / f"gt_{cache_key}.json"
if manifest_path.exists():
manifest = json.loads(manifest_path.read_text())
all_exist = all(Path(p).exists() for paths in manifest.values() for p in paths)
if all_exist:
result = {k: [Path(p) for p in v] for k, v in manifest.items()}
total = sum(len(v) for v in result.values())
print(
f"Ground truth cache hit: {len(result)} articles, {total} tiles",
flush=True,
)
return result
cache_dir.mkdir(parents=True, exist_ok=True)
strategy = _make_gt_strategy(chrome_path, timeout_ms)
await strategy.setup()
try:
results = await strategy.capture_articles(articles)
finally:
await strategy.teardown()
ground_truth = {}
for ac in results:
tile_paths = []
for tc in ac.tiles:
tile_path = (
cache_dir
/ f"gt_{cache_key}_{ac.article_path.replace('/', '_')}_{tc.tile_index:02d}.png"
)
if tc.image_bytes:
tile_path.write_bytes(tc.image_bytes)
tile_paths.append(tile_path)
ground_truth[ac.article_path] = tile_paths
manifest = {k: [str(p) for p in v] for k, v in ground_truth.items()}
manifest_path.write_text(json.dumps(manifest))
total = sum(len(v) for v in ground_truth.values())
print(
f"Ground truth generated: {len(ground_truth)} articles, {total} tiles",
flush=True,
)
return ground_truth
def _make_gt_strategy(chrome_path: str, timeout_ms: int):
"""GT uses the most conservative strategy: 1 worker, PNG, long timeout.
Uses port 9222 to avoid TIME_WAIT conflicts with test strategies (9300+).
"""
s = CDPSequentialStrategy(
chrome_path=chrome_path, n_workers=1, fmt="png", from_surface=True
)
s._base_port = 9222
return s
def validate_gt(ground_truth: dict[str, list[Path]]) -> tuple[int, int, list[str]]:
"""Validate GT tiles are non-degenerate (not blank, not tiny, readable).
Returns (ok, bad, bad_examples).
"""
ok = 0
bad = 0
examples = []
for article_path, tile_paths in ground_truth.items():
for tp in tile_paths:
try:
img = Image.open(tp)
arr = np.array(img)
if arr.std() < 1.0:
bad += 1
if len(examples) < 10:
examples.append(
f"{article_path} {tp.name}: blank (std={arr.std():.1f})"
)
continue
ok += 1
except Exception as e:
bad += 1
if len(examples) < 10:
examples.append(f"{article_path} {tp.name}: {e}")
return ok, bad, examples
# ---------------------------------------------------------------------------
# Decode + verify (NOT timed)
# ---------------------------------------------------------------------------
def decode_tile(tc: TileCapture) -> Image.Image | None:
try:
if tc.raw_file_path and os.path.exists(tc.raw_file_path):
data = open(tc.raw_file_path, "rb").read()
w, h, rb = struct.unpack_from("<III", data, 0)
img = Image.frombuffer(
"RGBA", (w, h), data[12:], "raw", "BGRA", rb, 1
).convert("RGB")
return img
elif tc.image_bytes:
return Image.open(io.BytesIO(tc.image_bytes)).convert("RGB")
except Exception:
return None
return None
def verify_tile(
captured: Image.Image, gt_path: Path, is_lossy: bool
) -> tuple[bool, float]:
gt = Image.open(gt_path).convert("RGB")
cap_arr = np.array(captured, dtype=np.float32)
gt_arr = np.array(gt, dtype=np.float32)
if cap_arr.shape != gt_arr.shape:
return False, 999.0
diff = np.abs(cap_arr - gt_arr)
mean_diff = float(diff.mean())
threshold = JPEG_MAX_MEAN_DIFF if is_lossy else LOSSLESS_MAX_MEAN_DIFF
return mean_diff <= threshold, mean_diff
# ---------------------------------------------------------------------------
# Run one strategy: time capture, then verify separately
# ---------------------------------------------------------------------------
async def run_and_verify(strategy, articles, ground_truth) -> dict:
await strategy.setup()
t0 = time.monotonic()
try:
article_captures = await strategy.capture_articles(articles)
finally:
wall_s = time.monotonic() - t0
await strategy.teardown()
# --- UNTIMED: decode + verify ---
tiles_ok = 0
tiles_bad = 0
tiles_total = 0
total_shot_ms = 0.0
total_nav_ms = 0.0
total_pixels = 0
total_height_px = 0
per_tile_shot_ms = []
per_tile_nav_ms = []
bad_examples = []
is_lossy = strategy.fmt in ("jpeg",)
for ac in article_captures:
gt_tiles = ground_truth.get(ac.article_path, [])
total_height_px += ac.page_height
total_shot_ms += ac.total_shot_ms
total_nav_ms += ac.total_nav_ms
for tc in ac.tiles:
tiles_total += 1
total_pixels += VIEWPORT_WIDTH * tc.clip_h
per_tile_shot_ms.append(tc.shot_ms)
if tc.nav_ms > 0:
per_tile_nav_ms.append(tc.nav_ms)
if tc.tile_index >= len(gt_tiles):
tiles_bad += 1
bad_examples.append(f"{ac.article_path} tile {tc.tile_index}: no GT")
continue
img = decode_tile(tc)
if img is None:
tiles_bad += 1
bad_examples.append(
f"{ac.article_path} tile {tc.tile_index}: decode failed"
)
continue
ok, mean_diff = verify_tile(img, gt_tiles[tc.tile_index], is_lossy)
if ok:
tiles_ok += 1
else:
tiles_bad += 1
if len(bad_examples) < 10:
bad_examples.append(
f"{ac.article_path} tile {tc.tile_index}: mean_diff={mean_diff:.2f}"
)
for ac in article_captures:
for tc in ac.tiles:
if tc.raw_file_path:
try:
os.unlink(tc.raw_file_path)
except OSError:
pass
correct_pct = tiles_ok / tiles_total * 100 if tiles_total > 0 else 0
tps = tiles_total / wall_s if wall_s > 0 else 0
ms_per_tile = total_shot_ms / tiles_total if tiles_total > 0 else 0
articles_per_s = len(article_captures) / wall_s if wall_s > 0 else 0
mpix_per_s = (total_pixels / 1_000_000) / wall_s if wall_s > 0 else 0
shot_share = (
total_shot_ms / (total_shot_ms + total_nav_ms)
if (total_shot_ms + total_nav_ms) > 0
else 0
)
# Latency percentiles
sorted_shots = sorted(per_tile_shot_ms) if per_tile_shot_ms else [0]
sorted_navs = sorted(per_tile_nav_ms) if per_tile_nav_ms else [0]
def percentile(arr, p):
idx = int(len(arr) * p / 100)
return arr[min(idx, len(arr) - 1)]
return {
"name": strategy.name,
"tiles_total": tiles_total,
"tiles_ok": tiles_ok,
"tiles_bad": tiles_bad,
"correct_pct": correct_pct,
"wall_s": wall_s,
"tiles_per_s": tps,
"ms_per_tile": ms_per_tile,
"articles_per_s": articles_per_s,
"mpix_per_s": mpix_per_s,
"height_kpx_per_s": (total_height_px / 1000) / wall_s if wall_s > 0 else 0,
"shot_pct": shot_share * 100,
"bad_examples": bad_examples,
# Latency distribution
"shot_min": sorted_shots[0],
"shot_p50": percentile(sorted_shots, 50),
"shot_p95": percentile(sorted_shots, 95),
"shot_p99": percentile(sorted_shots, 99),
"shot_max": sorted_shots[-1],
"nav_avg": sum(sorted_navs) / len(sorted_navs),
"nav_p95": percentile(sorted_navs, 95),
}
# ---------------------------------------------------------------------------
# Bench: clean harness that takes any CaptureStrategy
# ---------------------------------------------------------------------------
class Bench:
"""Benchmark harness. Measures throughput/latency and verifies correctness.
Usage:
bench = Bench(zim_path="...", chrome_path="...", output_dir="./results")
result = await bench.run(strategy, articles=200, seed=42)
"""
def __init__(
self,
zim_path: str,
chrome_path: str,
output_dir: str = "./bench_results",
kiwix_url: str | None = None,
gt_timeout_ms: int = 5000,
):
self.zim_path = zim_path
self.chrome_path = chrome_path
self.output_dir = Path(output_dir)
self.kiwix_url = kiwix_url
self.gt_timeout_ms = gt_timeout_ms
self._articles: list[dict] | None = None
self._gt: dict[str, list[Path]] | None = None
def prepare(self, n_articles: int = 200, seed: int = 42) -> list[dict]:
if self._articles is None:
self._articles = prepare_articles(
self.zim_path, n_articles, seed, kiwix_url=self.kiwix_url
)
return self._articles
async def ensure_gt(
self, n_articles: int = 200, seed: int = 42
) -> dict[str, list[Path]]:
if self._gt is not None:
return self._gt
articles = self.prepare(n_articles, seed)
gt_dir = self.output_dir / "ground_truth"
self._gt = await generate_ground_truth(
articles, self.chrome_path, gt_dir, seed, timeout_ms=self.gt_timeout_ms
)
ok, bad, examples = validate_gt(self._gt)
total = ok + bad
print(f"GT validation: {ok}/{total} OK, {bad} bad", flush=True)
if examples:
for ex in examples:
print(f" GT BAD: {ex}", flush=True)
if bad > 0:
pct = ok / total * 100 if total else 0
if pct < CORRECT_THRESHOLD:
raise RuntimeError(
f"GT itself is only {pct:.1f}% valid ({bad} bad tiles). "
f"Fix image loading or increase gt_timeout_ms."
)
return self._gt
async def run(self, strategy, n_articles: int = 200, seed: int = 42) -> dict:
articles = self.prepare(n_articles, seed)
gt = await self.ensure_gt(n_articles, seed)
result = await run_and_verify(strategy, articles, gt)
exp = self._build_experiment(strategy, result, n_articles, seed)
try:
self._dump_experiment(exp)
except Exception as e:
print(f"Warning: failed to save experiment: {e}", flush=True)
return result
def _build_experiment(
self, strategy, result: dict, n_articles: int, seed: int
) -> dict:
config = {
"strategy_class": type(strategy).__name__,
"strategy_name": strategy.name,
"n_workers": getattr(strategy, "n_workers", None),
"fmt": strategy.fmt,
"launcher": getattr(strategy, "launcher", None),
"from_surface": getattr(strategy, "from_surface", None),
"chrome_path": getattr(strategy, "chrome_path", None),
"n_articles": n_articles,
"seed": seed,
"zim_path": self.zim_path,
"kiwix_url": self.kiwix_url,
"gt_timeout_ms": self.gt_timeout_ms,
}
return {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
"config": config,
"results": {k: v for k, v in result.items() if k != "bad_examples"},
"bad_examples": result.get("bad_examples", []),
}
def _dump_experiment(self, exp: dict):
exp_dir = self.output_dir / "experiments"
exp_dir.mkdir(parents=True, exist_ok=True)
ts = exp["timestamp"].replace(":", "")
name = exp["config"]["strategy_name"].replace(" ", "_").replace("/", "-")
path = exp_dir / f"{ts}_{name}.json"
path.write_text(json.dumps(exp, indent=2, default=str))
print(f"Experiment saved: {path}", flush=True)
# ---------------------------------------------------------------------------
# Display
# ---------------------------------------------------------------------------
def format_result_line(r: dict) -> str:
status = "PASS" if r["correct_pct"] >= CORRECT_THRESHOLD else "FAIL"
ok = f"{r['tiles_ok']}/{r['tiles_total']}"
return (
f" {r['name']:<25} {ok:>7} {r['correct_pct']:>5.1f}% "
f"{r['tiles_per_s']:>6.1f} {r['ms_per_tile']:>5.0f} "
f"{r['shot_pct']:>4.0f}% {status}"
)
def print_results(results: list[dict]):
for r in results:
print(format_result_line(r), flush=True)
if r["bad_examples"]:
for ex in r["bad_examples"][:3]:
print(f" BAD: {ex}", flush=True)