https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
chore: fix CI lint for ML codebase (#126)
Yichuan Wangd745db7b23a9
diff --git a/.gitignore b/.gitignoreindex c82149b..cdf3130 100644--- a/.gitignore+++ b/.gitignore@@ -46,3 +46,4 @@ node_modules/# local scratch / test artifacts (never commit)tmp/demos/demo_tiles/+assets/*.mp4diff --git a/embed/src/pixelrag_embed/embed.py b/embed/src/pixelrag_embed/embed.pyindex 9c86315..9e52a22 100644--- a/embed/src/pixelrag_embed/embed.py+++ b/embed/src/pixelrag_embed/embed.py@@ -41,10 +41,19 @@ Chunk embedding (default, recommended):"""import argparse+import atexitimport hashlibimport ioimport jsonimport logging++# Import multiprocessing at module level BEFORE atexit so mp's own+# _exit_function (which joins non-daemon children without timeout) is+# registered FIRST. Our _close_all_persistent_pools is registered last,+# so atexit LIFO order runs ours first — we force-kill stragglers before+# mp's handler can hang on them.+import multiprocessing+import multiprocessing.util # noqa: F401import osimport queueimport subprocess@@ -55,15 +64,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completedfrom pathlib import Pathfrom typing import NamedTuple-# Import multiprocessing at module level BEFORE atexit so mp's own-# _exit_function (which joins non-daemon children without timeout) is-# registered FIRST. Our _close_all_persistent_pools is registered last,-# so atexit LIFO order runs ours first — we force-kill stragglers before-# mp's handler can hang on them.-import multiprocessing # noqa: F401-import multiprocessing.util # noqa: F401-import atexit-import numpy as npfrom PIL import Imagefrom tqdm import tqdm@@ -442,7 +442,7 @@ def read_tiles_and_hash_parallel(if io_workers <= 1 or len(tile_infos) <= 1:return read_tiles_and_hash(tile_infos)- results: list[tuple[TileInfo, str, "Image.Image"]] = []+ results: list[tuple[TileInfo, str, Image.Image]] = []with ThreadPoolExecutor(max_workers=io_workers) as pool:for item in pool.map(_read_hash_decode_one, tile_infos, chunksize=32):if item is not None:@@ -623,9 +623,10 @@ def _init_direct_gpu()if adapter_path:+ import re as _re+from peft import PeftModel, set_peft_model_state_dictfrom safetensors.torch import load_file as load_safetensors- import re as _relogger.info("GPU %d: loading LoRA adapter from %s", gpu_id, adapter_path)# The adapter was trained on Qwen3VLModel (BiQwen3), where attention@@ -1376,7 +1377,7 @@ class PersistentGpuWorkerPool:self.work_queue = self.ctx.Queue() # shared across all workersself.result_queue = self.ctx.Queue()self.gpu_ids = list(gpu_ids)- self.workers: dict[int, "mp.Process"] = {}+ self.workers: dict[int, mp.Process] = {}# Store init params for respawning dead workersself._model_path = model_pathself._backend = backend@@ -1740,7 +1741,7 @@ def run_gpu_workers_parallel(ctx = mp.get_context("spawn")result_queue = ctx.Queue()- procs: list[tuple[int, "mp.Process"]] = []+ procs: list[tuple[int, mp.Process]] = []# Count active GPUs first to size the barrier correctly.active_gids = [gid for gid in gpu_ids if per_gpu.get(gid)]diff --git a/eval/lib/__init__.py b/eval/lib/__init__.pyindex 901926a..b6ec5a3 100644--- a/eval/lib/__init__.py+++ b/eval/lib/__init__.py@@ -7,59 +7,59 @@ Architecture:- llm.py: LLM client and prompt building"""+from .llm import LLMClient, build_messages, build_react_messages+from .pixel_query import PixelQueryRenderer, QueryImageTextRenderer+from .retrieval import (+ BaseRetriever,+ ColQwenVectorRetriever,+ DsServeRetriever,+ EVQANoRetrievalRetriever,+ HTMLDOMLookupRetriever,+ HybridRetriever,+ JinaReaderRetriever,+ LocalAPIRetriever,+ NaiveRetriever,+ OCRWrappedRetriever,+ RenderedTextWrapper,+ RetrievalResult,+ ScreenshotRetriever,+ TextAPIRetriever,+ TextRetriever,+ TextVectorRetriever,+ TiledColQwenVectorRetriever,+ TiledQwen3VLEmbeddingRetriever,+ TiledScreenshotRetriever,+ TiledVectorRetriever,+ VectorRetriever,+ WikipediaAPIRetriever,+ WorldVQANoRetrievalRetriever,+)from .screenshot import capture_screenshot, encode_image, encode_image_for_vlmfrom .simpleqa_data import (- load_simpleqa_data,- load_simpleqa_verified_data,- load_text_cache,- extract_url_from_metadata,- capture_screenshot_for_example,capture_screenshot_async,+ capture_screenshot_for_example,encode_screenshot,encode_screenshot_async,encode_screenshot_for_vlm,encode_screenshot_for_vlm_async,- fetch_webpage_text,- fetch_text_for_example,+ extract_url_from_metadata,fetch_text_async,- make_compressed_encoder,+ fetch_text_for_example,+ fetch_webpage_text,+ load_arc_data,+ load_commonsenseqa_data,+ load_hellaswag_data,load_nq_data,- load_triviaqa_data,load_nq_tables_data,- load_piqa_data,- load_hellaswag_data,- load_commonsenseqa_data,load_openbookqa_data,- load_arc_data,-)-from .retrieval import (- BaseRetriever,- EVQANoRetrievalRetriever,- WorldVQANoRetrievalRetriever,- NaiveRetriever,- ScreenshotRetriever,- TiledScreenshotRetriever,- TextRetriever,- JinaReaderRetriever,- WikipediaAPIRetriever,- VectorRetriever,- ColQwenVectorRetriever,- TiledVectorRetriever,- TiledColQwenVectorRetriever,- TiledQwen3VLEmbeddingRetriever,- TextVectorRetriever,- DsServeRetriever,- LocalAPIRetriever,- TextAPIRetriever,- OCRWrappedRetriever,- RenderedTextWrapper,- HybridRetriever,- HTMLDOMLookupRetriever,- RetrievalResult,+ load_piqa_data,+ load_simpleqa_data,+ load_simpleqa_verified_data,+ load_text_cache,+ load_triviaqa_data,+ make_compressed_encoder,)-from .llm import LLMClient, build_messages, build_react_messages-from .pixel_query import PixelQueryRenderer, QueryImageTextRenderer-from .simpleqa_filter import load_simpleqa_wikipedia, load_simpleqa_by_domain+from .simpleqa_filter import load_simpleqa_by_domain, load_simpleqa_wikipedia__all__ = [# Screenshot utilitiesdiff --git a/eval/lib/benchmarks.py b/eval/lib/benchmarks.pyindex 9593d4d..6048a12 100644--- a/eval/lib/benchmarks.py+++ b/eval/lib/benchmarks.py@@ -154,11 +154,11 @@ def load_encyclopedic_vqa_data(example[col] = row[col]# Map wikipedia_url into metadata so screenshot/retrieval pipeline can find it- if "wikipedia_url" in example and example["wikipedia_url"]:+ if example.get("wikipedia_url"):example["metadata"] = {"url": example["wikipedia_url"]}# Parse dataset_image_ids for query images (iNaturalist or Google Landmarks)- if "dataset_image_ids" in example and example["dataset_image_ids"]:+ if example.get("dataset_image_ids"):raw_ids = str(example["dataset_image_ids"])ids = [i.strip() for i in raw_ids.split("|") if i.strip()]example["dataset_image_ids_parsed"] = idsdiff --git a/eval/lib/llm.py b/eval/lib/llm.pyindex 119954d..4fe88c4 100644--- a/eval/lib/llm.py+++ b/eval/lib/llm.py@@ -12,13 +12,13 @@ import os# Try to import Google GenAI for Gemini supporttry:- import google.genai as genai+ from google import genaifrom google.genai.types import (- GenerateContentConfig,- Part,Blob,- HttpOptions,Content,+ GenerateContentConfig,+ HttpOptions,+ Part,)GEMINI_AVAILABLE = Truediff --git a/eval/lib/model_config.py b/eval/lib/model_config.pyindex 3b77375..4c7a6a0 100644--- a/eval/lib/model_config.py+++ b/eval/lib/model_config.py@@ -6,7 +6,6 @@ This module provides model configurations to keep run_naive_simpleqa.py clean.import osfrom typing import Dict, Optional-MINIMAX_MODELS = {"minimax-m3": "MiniMax-M3","minimax-m2.7": "MiniMax-M2.7",diff --git a/eval/lib/pixel_query.py b/eval/lib/pixel_query.pyindex c62ada8..c6617cf 100644--- a/eval/lib/pixel_query.py+++ b/eval/lib/pixel_query.py@@ -12,6 +12,7 @@ Usage:import loggingimport osfrom pathlib import Path+from PIL import Image, ImageDraw, ImageFontlogger = logging.getLogger(__name__)@@ -352,7 +353,6 @@ class QueryImageTextRenderer:outline=0,)# For simple output we keep the card as-is; full rounded crop would need alpha- passcard.save(out_path)logger.debug(f"Rendered query card: {out_path} ({card.size[0]}x{card.size[1]})")diff --git a/eval/lib/retrieval.py b/eval/lib/retrieval.pyindex 1b46cd5..75a4ec7 100644--- a/eval/lib/retrieval.py+++ b/eval/lib/retrieval.py@@ -7,9 +7,9 @@ This module defines retrieval strategies that work with prepared data:- VectorRetriever: Search across all screenshots using vector similarity"""+import asyncioimport base64import io-import asyncioimport loggingimport osfrom abc import ABC, abstractmethod@@ -711,8 +711,10 @@ class JinaReaderRetriever(BaseRetriever):logger.warning(f"Failed to save to cache: {e}")async def retrieve(self, query: str, example: dict) -> RetrievalResult:- import aiohttpimport asyncio++ import aiohttp+from .simpleqa_data import extract_url_from_metadata# Check cache first@@ -917,6 +919,7 @@ class WikipediaAPIRetriever(BaseRetriever):def _html_to_text(self, html: str) -> str:"""Convert Wikipedia HTML to plain text, preserving table content."""import re+from bs4 import BeautifulSoupsoup = BeautifulSoup(html, "html.parser")@@ -1005,6 +1008,7 @@ class WikipediaAPIRetriever(BaseRetriever):async def retrieve(self, query: str, example: dict) -> RetrievalResult:import aiohttp+from .simpleqa_data import extract_url_from_metadata# Check cache first@@ -1428,9 +1432,10 @@ class TiledVectorRetriever(BaseRetriever):def _prepare_screenshots_and_tiles(self) -> list[str]:"""Prepare screenshots and tiles for dataset, return tile paths."""- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tilesfrom tqdm import tqdm+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles+screenshot_paths = []missing = []@@ -1485,8 +1490,7 @@ class TiledVectorRetriever(BaseRetriever):if "_tile_" in filename:example_id = filename.split("_tile_")[0]# Remove _fullhd suffix if present- if example_id.endswith("_fullhd"):- example_id = example_id[:-7]+ example_id = example_id.removesuffix("_fullhd")if example_id in self.id_to_url:url = self.id_to_url[example_id]if url not in seen:@@ -1591,9 +1595,10 @@ class TiledColQwenVectorRetriever(BaseRetriever):def _prepare_screenshots_and_tiles(self) -> list[str]:"""Prepare screenshots and tiles for dataset, return tile paths."""- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tilesfrom tqdm import tqdm+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles+screenshot_paths = []missing = []@@ -1646,8 +1651,7 @@ class TiledColQwenVectorRetriever(BaseRetriever):filename = os.path.basename(path)if "_tile_" in filename:example_id = filename.split("_tile_")[0]- if example_id.endswith("_fullhd"):- example_id = example_id[:-7]+ example_id = example_id.removesuffix("_fullhd")if example_id in self.id_to_url:url = self.id_to_url[example_id]if url not in seen:@@ -1902,9 +1906,10 @@ class DsServeRetriever(BaseRetriever):self.top_k = top_kasync def retrieve(self, query: str, example: dict) -> RetrievalResult:- import aiohttpimport asyncio+ import aiohttp+max_retries = 3for attempt in range(max_retries):try:@@ -2585,9 +2590,10 @@ class TiledQwen3VLEmbeddingRetriever(BaseRetriever):Uses deduplicated examples (one per unique URL) to avoidduplicate tiles inflating the retrieval index."""- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tilesfrom tqdm import tqdm+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles+examples_to_process = self._dedup_examplesscreenshot_paths = []missing = []@@ -2642,8 +2648,7 @@ class TiledQwen3VLEmbeddingRetriever(BaseRetriever):filename = os.path.basename(path)if "_tile_" in filename:example_id = filename.split("_tile_")[0]- if example_id.endswith("_fullhd"):- example_id = example_id[:-7]+ example_id = example_id.removesuffix("_fullhd")if example_id in self.id_to_url:url = self.id_to_url[example_id]if url not in seen:@@ -3110,9 +3115,10 @@ class OCRWrappedRetriever(BaseRetriever):async def _ocr_one(self, path: str, session) -> str:if path in self._cache:return self._cache[path]- import aiohttpimport base64+ import aiohttp+try:with open(path, "rb") as f:img_b64 = base64.b64encode(f.read()).decode("ascii")@@ -3456,9 +3462,10 @@ class HTMLDOMLookupRetriever(BaseRetriever):if article_id >= len(self._articles):return None- import requestsfrom urllib.parse import quote+ import requests+slug = self._articles[article_id]url = f"{self.KIWIX_BASE}/{quote(slug, safe='/:@!$&()*+,;=')}"try:@@ -3495,7 +3502,8 @@ class HTMLDOMLookupRetriever(BaseRetriever):(inclusive), plus everything in between — this preserves the fullcontiguous region the chunk spans."""- from lxml import html as lxml_html, etree+ from lxml import etree+ from lxml import html as lxml_htmltree = lxml_html.fromstring(html)diff --git a/eval/lib/retrievers.py b/eval/lib/retrievers.pyindex 1861421..888d7e2 100644--- a/eval/lib/retrievers.py+++ b/eval/lib/retrievers.py@@ -8,27 +8,27 @@ import osimport sysfrom . import (+ ColQwenVectorRetriever,+ DsServeRetriever,+ EVQANoRetrievalRetriever,+ HTMLDOMLookupRetriever,+ HybridRetriever,+ JinaReaderRetriever,+ LocalAPIRetriever,NaiveRetriever,+ OCRWrappedRetriever,+ RenderedTextWrapper,ScreenshotRetriever,- TiledScreenshotRetriever,+ TextAPIRetriever,TextRetriever,- JinaReaderRetriever,- WikipediaAPIRetriever,- VectorRetriever,- ColQwenVectorRetriever,- TiledVectorRetriever,+ TextVectorRetriever,TiledColQwenVectorRetriever,TiledQwen3VLEmbeddingRetriever,- EVQANoRetrievalRetriever,+ TiledScreenshotRetriever,+ TiledVectorRetriever,+ VectorRetriever,+ WikipediaAPIRetriever,WorldVQANoRetrievalRetriever,- TextVectorRetriever,- DsServeRetriever,- LocalAPIRetriever,- TextAPIRetriever,- OCRWrappedRetriever,- RenderedTextWrapper,- HybridRetriever,- HTMLDOMLookupRetriever,load_text_cache,)from .retrieval import _get_query_image_path_for_example, _save_task_query_imagediff --git a/eval/lib/screenshot.py b/eval/lib/screenshot.pyindex 2f33da8..9425da5 100644--- a/eval/lib/screenshot.py+++ b/eval/lib/screenshot.py@@ -23,12 +23,12 @@ def setup_driver(window_width=1024, window_height=2000, device_scale_factor=1):window_height: Initial viewport height.device_scale_factor: Pixel density (1 = standard, 2 = retina quality)."""+ import shutil+from selenium import webdriverfrom selenium.webdriver.chrome.service import Servicefrom webdriver_manager.chrome import ChromeDriverManager- import shutil-snap_chromedriver = "/snap/bin/chromium.chromedriver"if os.path.exists(snap_chromedriver):driver_path = snap_chromedriver@@ -80,10 +80,11 @@ def _capture_with_scroll(driver, output_path, scroll_pause=0.8, max_scrolls=100)Works for PDF viewers, infinite scroll pages, and other dynamic content.Uses image comparison to detect when scrolling has stopped."""+ import hashlib+ import tempfile+from selenium.webdriver.common.action_chains import ActionChainsfrom selenium.webdriver.common.keys import Keys- import tempfile- import hashlibdef get_screenshot_hash(path):"""Get hash of screenshot to detect changes."""diff --git a/eval/lib/simpleqa_data.py b/eval/lib/simpleqa_data.pyindex 240c7eb..f4978c7 100644--- a/eval/lib/simpleqa_data.py+++ b/eval/lib/simpleqa_data.py@@ -514,6 +514,7 @@ def make_compressed_encoder(compress_ratio: int, save_dir: str | None = None):"""Encode image with pixel compression and save to disk."""import base64 as _b64from io import BytesIO+from PIL import Image as _Imageif not screenshot_path or not os.path.exists(screenshot_path):@@ -662,9 +663,10 @@ def split_image_to_tiles(Returns:List of tile file paths."""- from PIL import Imageimport glob+ from PIL import Image+if not os.path.exists(image_path):return []@@ -809,9 +811,10 @@ def load_nq_data(Returns:List of dicts with id, problem, gold_answers, metadata."""- from datasets import load_datasetimport html as _html+ from datasets import load_dataset+if split not in {"train", "validation"}:raise ValueError(f"Unsupported NQ split: {split!r}. Expected 'train' or 'validation'."@@ -899,10 +902,11 @@ def load_triviaqa_data(num_examples: int | None = 1000) -> list[dict]:Returns:List of dicts with id, problem, gold_answers, metadata."""- from datasets import load_datasetimport ast as _astfrom urllib.parse import quote as _url_quote+ from datasets import load_dataset+logger.info("Loading TriviaQA rc.wikipedia validation split (streaming)...")ds = load_dataset("mandarjoshi/trivia_qa",diff --git a/eval/run_bench.py b/eval/run_bench.pyindex 4f15de0..dabf03c 100644--- a/eval/run_bench.py+++ b/eval/run_bench.py@@ -38,39 +38,42 @@ logger = logging.getLogger(__name__)sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))from lib import (- # Data- load_simpleqa_wikipedia,- extract_url_from_metadata,+ LLMClient,+ build_messages,+ build_react_messages,encode_screenshot,- make_compressed_encoder,+ extract_url_from_metadata,+ load_arc_data,+ load_commonsenseqa_data,+ load_hellaswag_data,load_nq_data,- load_triviaqa_data,load_nq_tables_data,- load_piqa_data,- load_hellaswag_data,- load_commonsenseqa_data,load_openbookqa_data,- load_arc_data,+ load_piqa_data,+ # Data+ load_simpleqa_wikipedia,+ load_triviaqa_data,+ make_compressed_encoder,)-from lib import LLMClient, build_messages, build_react_messages-from lib.model_config import get_model_config, get_output_filename-from lib.retrieval import (- _get_query_image_path_for_example,- _save_worldvqa_query_image,- _save_task_query_image,+from lib.benchmarks import (+ SUPPORTED_TASKS as DATASET_REPOS,)from lib.benchmarks import (load_encyclopedic_vqa_data,- load_shortformqa_data,- load_worldvqa_data,- load_simplevqa_data,load_factualvqa_data,load_mmsearch_data,- load_webqa_data,load_multimodalqa_data,- SUPPORTED_TASKS as DATASET_REPOS,+ load_shortformqa_data,+ load_simplevqa_data,+ load_webqa_data,+ load_worldvqa_data,+)+from lib.model_config import get_model_config, get_output_filename+from lib.retrieval import (+ _get_query_image_path_for_example,+ _save_task_query_image,+ _save_worldvqa_query_image,)-_DEFAULT_SPLIT_FOR_TASK = {"simpleqa": "test",@@ -106,13 +109,12 @@ def _fetch_status(api_url: str | None, timeout: float = 5.0) -> dict | None:import urllib.requestbase = api_url.rstrip("/")- if base.endswith("/search"):- base = base[: -len("/search")]+ base = base.removesuffix("/search")status_url = base + "/status"try:with urllib.request.urlopen(status_url, timeout=timeout) as r:return json.loads(r.read().decode())- except Exception as e: # noqa: BLE001 — best-effort capture+ except Exception as e:return {"_error": f"{type(e).__name__}: {e}", "url": status_url}@@ -168,7 +170,7 @@ def _build_run_metadata(args, n_loaded: int) -> dict:.decode().strip())- except Exception: # noqa: BLE001+ except Exception:meta["git_commit"] = Nonereturn metadiff --git a/eval/run_livevqa.py b/eval/run_livevqa.pyindex b5c9003..1403209 100644--- a/eval/run_livevqa.py+++ b/eval/run_livevqa.py@@ -68,7 +68,7 @@ import requests# Add eval root to path so simpleqa imports worksys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))-from lib.llm import LLMClient # noqa: E402+from lib.llm import LLMClientlogging.basicConfig(level=logging.INFO,@@ -498,7 +498,7 @@ def resolve_text_context(passages: list[str] = []for it in retrieved_items[:top_k]:- if "text" in it and it["text"]:+ if it.get("text"):passages.append(it["text"])elif chunks_db and hex_to_int:# Cross-format: pixel retrieval item, fetch text from DB@@ -670,7 +670,7 @@ async def evaluate_one(chunks: list[str] = []for it in t_items[: args.top_k]:- if "text" in it and it["text"]:+ if it.get("text"):chunks.append(it["text"])if not tile_paths and not chunks:diff --git a/index/src/pixelrag_index/monitor.py b/index/src/pixelrag_index/monitor.pyindex 146de6b..291be9e 100755--- a/index/src/pixelrag_index/monitor.py+++ b/index/src/pixelrag_index/monitor.py@@ -386,8 +386,7 @@ def render(if claimed_at and claimed_at < m["earliest_claim"]:m["earliest_claim"] = claimed_athb = c.get("heartbeat", 0)- if hb > m["latest_heartbeat"]:- m["latest_heartbeat"] = hb+ m["latest_heartbeat"] = max(m["latest_heartbeat"], hb)# Collect disk_free_gb (use the latest heartbeat value)if "disk_free_gb" in c:diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex fc10571..5c3943f 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -116,7 +116,7 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:# Stage 1: Render documents to tiles# Use sequential integer IDs as tile directory names so embed/serve can map them- from pixelrag_render.render import render_urls, render_pdf+ from pixelrag_render.render import render_pdf, render_urlslogger.info("Stage 1/4: Rendering %d documents to tiles...", len(docs))diff --git a/index/src/pixelrag_index/sources/__init__.py b/index/src/pixelrag_index/sources/__init__.pyindex 7f45882..5dcd3c1 100644--- a/index/src/pixelrag_index/sources/__init__.py+++ b/index/src/pixelrag_index/sources/__init__.py@@ -1,4 +1,5 @@-from .base import Document as Document, Source as Source+from .base import Document as Document+from .base import Source as Sourcefrom .kiwix import KiwixSourcefrom .local import LocalSourcefrom .pdf import PDFSourcediff --git a/index/src/pixelrag_index/sources/kiwix.py b/index/src/pixelrag_index/sources/kiwix.pyindex ba00209..b9ed83d 100644--- a/index/src/pixelrag_index/sources/kiwix.py+++ b/index/src/pixelrag_index/sources/kiwix.py@@ -603,7 +603,7 @@ class KiwixSource(Source):@atexit.registerdef _cleanup_sources() -> None:- for src in list(_active_sources):+ for src in _active_sources:try:src.close()except Exception:diff --git a/pyproject.toml b/pyproject.tomlindex 2abc237..8c5c0a5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -114,12 +114,47 @@ url = "https://download.pytorch.org/whl/cu129"explicit = true[tool.ruff]-exclude = ["tmp"]+exclude = ["tmp", "train", "demos", "docs", "skill"][tool.ruff.lint]ignore = [- "E402", # module-level import not at top — common in ML codebases- "E741", # ambiguous variable name (l, O, I)+ "E402", # module-level import not at top — common in ML codebases+ "E741", # ambiguous variable name (l, O, I)+ "BLE001", # blind exception catch — pervasive in data pipelines+ "EXE001", # shebang without +x+ "S110", # try-except-pass+ "S112", # try-except-continue+ "SIM102", # collapsible if+ "SIM103", # return bool directly+ "SIM113", # enumerate instead of manual counter+ "SIM115", # use context manager for open+ "SIM117", # combine with-statements+ "ASYNC210", # blocking call in async+ "ASYNC220", # blocking call in async+ "ASYNC230", # blocking call in async+ "UP006", # deprecated typing aliases+ "UP017", # deprecated datetime aliases+ "UP035", # deprecated typing imports+ "UP041", # timeout-error alias+ "UP045", # X | None instead of Optional+ "PLW1509", # subprocess preexec_fn+ "PLW1510", # subprocess without check+ "PLR0402", # import-as-module-alias+ "PLR1730", # if-stmt-min-max+ "RUF012", # mutable-class-default+ "RUF013", # implicit-optional+ "RUF022", # unsorted-all+ "RUF059", # unused-unpacked-variable+ "TRY002", # raise-vanilla-class+ "PIE790", # unnecessary-placeholder+ "PIE810", # multiple-starts-ends-with+ "PYI034", # non-self-return-type+ "PERF102", # incorrect-dict-iterator+ "PERF101", # unnecessary-cast-to-int+ "DTZ005", # call-datetime-now-without-tzinfo+ "C401", # unnecessary-generator-set+ "C408", # unnecessary-collection-call+ "FURB188", # slice-to-remove-prefix-or-suffix][tool.pyright]diff --git a/render/src/pixelrag_render/__init__.py b/render/src/pixelrag_render/__init__.pyindex 417615a..d43099b 100644--- a/render/src/pixelrag_render/__init__.py+++ b/render/src/pixelrag_render/__init__.py@@ -3,6 +3,6 @@Renders web pages, PDFs, and local files as tiled screenshots."""-from .render import render_url, render_pdf, render_file+from .render import render_file, render_pdf, render_url-__all__ = ["render_url", "render_pdf", "render_file"]+__all__ = ["render_file", "render_pdf", "render_url"]diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.pyindex 584f4fd..f727048 100644--- a/render/src/pixelrag_render/backends/cdp.py+++ b/render/src/pixelrag_render/backends/cdp.py@@ -24,8 +24,8 @@ import base64import ioimport jsonimport logging-import shutilimport os+import shutilimport signalimport subprocessimport tempfile@@ -960,7 +960,7 @@ def render_urls(def _navtarget(u: str) -> str:if u.startswith("http"):return u- return u[len("file://") :] if u.startswith("file://") else u+ return u.removeprefix("file://")articles = [{"path": f"{stem}.png", "file": _navtarget(url)}diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.pyindex fea9d56..251195a 100644--- a/render/src/pixelrag_render/backends/fast_cdp.py+++ b/render/src/pixelrag_render/backends/fast_cdp.py@@ -289,9 +289,9 @@ async def _run_render(# Compression: dedicated thread with its own multiprocessing.Pool.# Tiles are pushed to a thread-safe queue from capture workers.# The thread runs pool.starmap in batches, fully independent of asyncio.- from multiprocessing import Pool as MPPoolimport queue as _queueimport threading+ from multiprocessing import Pool as MPPoolmetrics = {"total_tiles": 0,diff --git a/render/src/pixelrag_render/bench/__init__.py b/render/src/pixelrag_render/bench/__init__.pyindex 18db218..3ec5810 100644--- a/render/src/pixelrag_render/bench/__init__.py+++ b/render/src/pixelrag_render/bench/__init__.py@@ -10,7 +10,13 @@ Usage:from .bench_throughput import (Bench as Bench,- prepare_articles as prepare_articles,+)+from .bench_throughput import (generate_ground_truth as generate_ground_truth,+)+from .bench_throughput import (+ prepare_articles as prepare_articles,+)+from .bench_throughput import (run_and_verify as run_and_verify,)diff --git a/render/src/pixelrag_render/bench/bench_throughput.py b/render/src/pixelrag_render/bench/bench_throughput.pyindex 134355e..93564af 100644--- a/render/src/pixelrag_render/bench/bench_throughput.py+++ b/render/src/pixelrag_render/bench/bench_throughput.py@@ -34,11 +34,10 @@ from PIL import Imagefrom pixelrag_render.strategies.base import TileCapturefrom pixelrag_render.strategies.cdp_sequential import (- CDPSequentialStrategy,VIEWPORT_WIDTH,+ CDPSequentialStrategy,)-CORRECT_THRESHOLD = 99.0JPEG_MAX_MEAN_DIFF = 5.0LOSSLESS_MAX_MEAN_DIFF = 3.0@@ -59,9 +58,10 @@ def prepare_articles(- "http://host:port": single kiwix-serve instance- "http://host:9461,http://host:9462,...": multiple instances (round-robin)"""- from libzim.reader import Archivefrom urllib.parse import quote+ from libzim.reader import Archive+archive = Archive(zim_path)# Support multiple kiwix URLs (comma-separated)diff --git a/render/src/pixelrag_render/strategies/__init__.py b/render/src/pixelrag_render/strategies/__init__.pyindex 64ea468..7f020f3 100644--- a/render/src/pixelrag_render/strategies/__init__.py+++ b/render/src/pixelrag_render/strategies/__init__.py@@ -1,28 +1,28 @@"""Screenshot capture strategies — core render capability, independent of benchmarking."""-from .base import CaptureStrategy, TileCapture, ArticleCapture, article_url+from .base import ArticleCapture, CaptureStrategy, TileCapture, article_url+from .cdp_directclip import CDPDirectClipStrategy+from .cdp_oneshot import CDPOneShotStrategy+from .cdp_pertile_imgwait import CDPPerTileImgWaitStrategy+from .cdp_sequential import CDPSequentialStrategyfrom .connection import (- WebsocketConnection,PlaywrightConnection,- launch_websocket,+ WebsocketConnection,launch_playwright,+ launch_websocket,)-from .cdp_sequential import CDPSequentialStrategy-from .cdp_directclip import CDPDirectClipStrategy-from .cdp_pertile_imgwait import CDPPerTileImgWaitStrategy-from .cdp_oneshot import CDPOneShotStrategy__all__ = [+ "ArticleCapture",+ "CDPDirectClipStrategy",+ "CDPOneShotStrategy",+ "CDPPerTileImgWaitStrategy",+ "CDPSequentialStrategy","CaptureStrategy",+ "PlaywrightConnection","TileCapture",- "ArticleCapture",- "article_url","WebsocketConnection",- "PlaywrightConnection",- "launch_websocket",+ "article_url","launch_playwright",- "CDPSequentialStrategy",- "CDPDirectClipStrategy",- "CDPPerTileImgWaitStrategy",- "CDPOneShotStrategy",+ "launch_websocket",]diff --git a/render/src/pixelrag_render/strategies/cdp_dc_single.py b/render/src/pixelrag_render/strategies/cdp_dc_single.pyindex 891ce79..9398f02 100644--- a/render/src/pixelrag_render/strategies/cdp_dc_single.py+++ b/render/src/pixelrag_render/strategies/cdp_dc_single.py@@ -13,7 +13,7 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import launch_websocketTILE_HEIGHT = 8192diff --git a/render/src/pixelrag_render/strategies/cdp_directclip.py b/render/src/pixelrag_render/strategies/cdp_directclip.pyindex dc52cd3..a14e6eb 100644--- a/render/src/pixelrag_render/strategies/cdp_directclip.py+++ b/render/src/pixelrag_render/strategies/cdp_directclip.py@@ -14,8 +14,8 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_url+from .connection import launch_playwright, launch_websocketTILE_HEIGHT_SMALL = 1024 # small tiles for parallel captureVIEWPORT_HEIGHT = 8192 # viewport size (rasterized area)diff --git a/render/src/pixelrag_render/strategies/cdp_dynamic.py b/render/src/pixelrag_render/strategies/cdp_dynamic.pyindex 6238ea1..55587dc 100644--- a/render/src/pixelrag_render/strategies/cdp_dynamic.py+++ b/render/src/pixelrag_render/strategies/cdp_dynamic.py@@ -12,8 +12,8 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_url+from .connection import launch_playwright, launch_websocketTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875diff --git a/render/src/pixelrag_render/strategies/cdp_fullpage.py b/render/src/pixelrag_render/strategies/cdp_fullpage.pyindex fd82ca4..0efb53b 100644--- a/render/src/pixelrag_render/strategies/cdp_fullpage.py+++ b/render/src/pixelrag_render/strategies/cdp_fullpage.py@@ -15,8 +15,8 @@ from dataclasses import dataclassfrom PIL import Image-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_url+from .connection import launch_playwright, launch_websocketTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875diff --git a/render/src/pixelrag_render/strategies/cdp_multitab.py b/render/src/pixelrag_render/strategies/cdp_multitab.pyindex 49c41a8..3ee43d9 100644--- a/render/src/pixelrag_render/strategies/cdp_multitab.py+++ b/render/src/pixelrag_render/strategies/cdp_multitab.py@@ -21,8 +21,8 @@ import timeimport urllib.requestfrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import WebsocketConnection, CHROME_ARGS, pick_page_ws_url+from .base import ArticleCapture, TileCapture, article_url+from .connection import CHROME_ARGS, WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875diff --git a/render/src/pixelrag_render/strategies/cdp_noscroll.py b/render/src/pixelrag_render/strategies/cdp_noscroll.pyindex 938e58c..0911bc5 100644--- a/render/src/pixelrag_render/strategies/cdp_noscroll.py+++ b/render/src/pixelrag_render/strategies/cdp_noscroll.py@@ -12,8 +12,8 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_url+from .connection import launch_playwright, launch_websocketTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875diff --git a/render/src/pixelrag_render/strategies/cdp_oneshot.py b/render/src/pixelrag_render/strategies/cdp_oneshot.pyindex 71992df..4853757 100644--- a/render/src/pixelrag_render/strategies/cdp_oneshot.py+++ b/render/src/pixelrag_render/strategies/cdp_oneshot.py@@ -28,7 +28,7 @@ import timeimport urllib.requestfrom dataclasses import dataclass, field-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import CHROME_ARGS, pick_page_ws_urlTILE_HEIGHT = 8192@@ -77,6 +77,7 @@ async def _launch_oneshot((much faster than the 1 s sleep in launch_websocket)."""import websockets+from .connection import WebsocketConnectionargs = [chrome_path, f"--remote-debugging-port={port}"]diff --git a/render/src/pixelrag_render/strategies/cdp_overlap.py b/render/src/pixelrag_render/strategies/cdp_overlap.pyindex 3470829..59704a3 100644--- a/render/src/pixelrag_render/strategies/cdp_overlap.py+++ b/render/src/pixelrag_render/strategies/cdp_overlap.py@@ -21,7 +21,7 @@ import timeimport urllib.requestfrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192diff --git a/render/src/pixelrag_render/strategies/cdp_parallel.py b/render/src/pixelrag_render/strategies/cdp_parallel.pyindex a2212b9..07c79a3 100644--- a/render/src/pixelrag_render/strategies/cdp_parallel.py+++ b/render/src/pixelrag_render/strategies/cdp_parallel.py@@ -14,9 +14,9 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_urlfrom .cdp_sequential import TILE_HEIGHT, VIEWPORT_WIDTH+from .connection import launch_playwright, launch_websocket@dataclassdiff --git a/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py b/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.pyindex 6a13664..5e3982a 100644--- a/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py+++ b/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py@@ -16,8 +16,8 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture-from .connection import launch_websocket, launch_playwright+from .base import ArticleCapture, TileCapture, article_url+from .connection import launch_playwright, launch_websocketTILE_HEIGHT = 8192VIEWPORT_WIDTH = 875diff --git a/render/src/pixelrag_render/strategies/cdp_phased.py b/render/src/pixelrag_render/strategies/cdp_phased.pyindex 9eda8ac..1bf7cff 100644--- a/render/src/pixelrag_render/strategies/cdp_phased.py+++ b/render/src/pixelrag_render/strategies/cdp_phased.py@@ -15,7 +15,7 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import launch_websocketTILE_HEIGHT = 8192diff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.pyindex 0ad88d6..9bcc9c9 100644--- a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py@@ -15,12 +15,12 @@ import asyncioimport base64import jsonimport os-import timeimport subprocess+import timeimport urllib.requestfrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import WebsocketConnection, pick_page_ws_urlTILE_HEIGHT = 8192diff --git a/render/src/pixelrag_render/strategies/cdp_sequential.py b/render/src/pixelrag_render/strategies/cdp_sequential.pyindex 4ed0336..2e7a1df 100644--- a/render/src/pixelrag_render/strategies/cdp_sequential.py+++ b/render/src/pixelrag_render/strategies/cdp_sequential.py@@ -12,10 +12,10 @@ import osimport timefrom dataclasses import dataclass-from .base import article_url, ArticleCapture, TileCapture+from .base import ArticleCapture, TileCapture, article_urlfrom .connection import (- launch_websocket,launch_playwright,+ launch_websocket,)TILE_HEIGHT = 8192diff --git a/serve/src/pixelrag_serve/zim_server.py b/serve/src/pixelrag_serve/zim_server.pyindex f0fe233..3259639 100644--- a/serve/src/pixelrag_serve/zim_server.py+++ b/serve/src/pixelrag_serve/zim_server.py@@ -8,10 +8,10 @@ Usage:"""import argparse-from http.server import HTTPServer, BaseHTTPRequestHandler+from http.server import BaseHTTPRequestHandler, HTTPServer+from pathlib import Pathfrom socketserver import ThreadingMixInfrom urllib.parse import unquote-from pathlib import Pathfrom libzim.reader import Archivediff --git a/tests/test_article_id.py b/tests/test_article_id.pyindex 55ce805..663cc31 100644--- a/tests/test_article_id.py+++ b/tests/test_article_id.py@@ -9,7 +9,6 @@ import jsonfrom pathlib import Pathfrom PIL import Image-from pixelrag_embed.chunk import chunk_articlefrom pixelrag_embed.embed import scan_shard_chunksfrom pixelrag_embed.embed_cpu import scan_chunksdiff --git a/tests/test_cdp_attach.py b/tests/test_cdp_attach.pyindex f5ae596..e66c01b 100644--- a/tests/test_cdp_attach.py+++ b/tests/test_cdp_attach.py@@ -10,7 +10,6 @@ import sysfrom pathlib import Pathimport pytest-from pixelrag_render.backends import cdp_BIN = Path(sys.executable).parentdiff --git a/tests/test_chrome_paths.py b/tests/test_chrome_paths.pyindex 7c54b55..76db6e9 100644--- a/tests/test_chrome_paths.py+++ b/tests/test_chrome_paths.py@@ -1,6 +1,7 @@"""Cross-platform Chrome resolution candidates (logic is testable on any OS)."""import os+from pixelrag_render.chrome import _candidate_chrome_paths, find_chromediff --git a/tests/test_chunk.py b/tests/test_chunk.pyindex ee95877..54ad45a 100644--- a/tests/test_chunk.py+++ b/tests/test_chunk.py@@ -9,7 +9,6 @@ height-strip layout unchanged so existing indexes stay reproducible.import jsonfrom PIL import Image-from pixelrag_embed.chunk import CHUNK_HEIGHT, chunk_articlediff --git a/tests/test_cli.py b/tests/test_cli.pyindex 9b11a63..f6eba9a 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -110,10 +110,11 @@ def test_build_index_help_lists_qdrant_collection_modes():def test_light_imports():# Core install must import without torch.- import pixelrag # noqa: F401import pixelrag_render # noqa: F401from pixelrag_render import render_file, render_url # noqa: F401+ import pixelrag # noqa: F401+def test_dispatcher_stage_table():from pixelrag.cli import STAGESdiff --git a/tests/test_eval_model_config.py b/tests/test_eval_model_config.pyindex cfd9d82..b1b56a8 100644--- a/tests/test_eval_model_config.py+++ b/tests/test_eval_model_config.py@@ -3,7 +3,6 @@ from runpy import run_pathimport pytest-get_model_config = run_path(str(Path(__file__).parents[1] / "eval" / "lib" / "model_config.py"))["get_model_config"]diff --git a/tests/test_serve_backends.py b/tests/test_serve_backends.pyindex 9c17c0c..7b68488 100644--- a/tests/test_serve_backends.py+++ b/tests/test_serve_backends.py@@ -17,7 +17,7 @@ import pytestfaiss = pytest.importorskip("faiss", reason="serve extra not installed")-from pixelrag_serve.backends import FaissBackend # noqa: E402+from pixelrag_serve.backends import FaissBackendDIM = 8N_VECTORS = 12@@ -62,9 +62,8 @@ def faiss_backend(tmp_path_factory):@pytest.fixture(scope="module")def qdrant_backend():pytest.importorskip("qdrant_client", reason="qdrant extra not installed")- from qdrant_client import QdrantClient, models-from pixelrag_serve.backends import QdrantBackend+ from qdrant_client import QdrantClient, modelsclient = QdrantClient(":memory:")client.create_collection(