Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

chore: fix CI lint for ML codebase (#126)

Yichuan Wangd745db7b23a9
diff --git a/.gitignore b/.gitignore
index c82149b..cdf3130 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,4 @@ node_modules/
# local scratch / test artifacts (never commit)
tmp/
demos/demo_tiles/
+assets/*.mp4
diff --git a/embed/src/pixelrag_embed/embed.py b/embed/src/pixelrag_embed/embed.py
index 9c86315..9e52a22 100644
--- a/embed/src/pixelrag_embed/embed.py
+++ b/embed/src/pixelrag_embed/embed.py
@@ -41,10 +41,19 @@ Chunk embedding (default, recommended):
"""
import argparse
+import atexit
import hashlib
import io
import json
import logging
+
+# Import multiprocessing at module level BEFORE atexit so mp's own
+# _exit_function (which joins non-daemon children without timeout) is
+# registered FIRST. Our _close_all_persistent_pools is registered last,
+# so atexit LIFO order runs ours first — we force-kill stragglers before
+# mp's handler can hang on them.
+import multiprocessing
+import multiprocessing.util # noqa: F401
import os
import queue
import subprocess
@@ -55,15 +64,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import NamedTuple
-# Import multiprocessing at module level BEFORE atexit so mp's own
-# _exit_function (which joins non-daemon children without timeout) is
-# registered FIRST. Our _close_all_persistent_pools is registered last,
-# so atexit LIFO order runs ours first — we force-kill stragglers before
-# mp's handler can hang on them.
-import multiprocessing # noqa: F401
-import multiprocessing.util # noqa: F401
-import atexit
-
import numpy as np
from PIL import Image
from tqdm import tqdm
@@ -442,7 +442,7 @@ def read_tiles_and_hash_parallel(
if io_workers <= 1 or len(tile_infos) <= 1:
return read_tiles_and_hash(tile_infos)
- results: list[tuple[TileInfo, str, "Image.Image"]] = []
+ results: list[tuple[TileInfo, str, Image.Image]] = []
with ThreadPoolExecutor(max_workers=io_workers) as pool:
for item in pool.map(_read_hash_decode_one, tile_infos, chunksize=32):
if item is not None:
@@ -623,9 +623,10 @@ def _init_direct_gpu(
)
if adapter_path:
+ import re as _re
+
from peft import PeftModel, set_peft_model_state_dict
from safetensors.torch import load_file as load_safetensors
- import re as _re
logger.info("GPU %d: loading LoRA adapter from %s", gpu_id, adapter_path)
# The adapter was trained on Qwen3VLModel (BiQwen3), where attention
@@ -1376,7 +1377,7 @@ class PersistentGpuWorkerPool:
self.work_queue = self.ctx.Queue() # shared across all workers
self.result_queue = self.ctx.Queue()
self.gpu_ids = list(gpu_ids)
- self.workers: dict[int, "mp.Process"] = {}
+ self.workers: dict[int, mp.Process] = {}
# Store init params for respawning dead workers
self._model_path = model_path
self._backend = backend
@@ -1740,7 +1741,7 @@ def run_gpu_workers_parallel(
ctx = mp.get_context("spawn")
result_queue = ctx.Queue()
- procs: list[tuple[int, "mp.Process"]] = []
+ procs: list[tuple[int, mp.Process]] = []
# Count active GPUs first to size the barrier correctly.
active_gids = [gid for gid in gpu_ids if per_gpu.get(gid)]
diff --git a/eval/lib/__init__.py b/eval/lib/__init__.py
index 901926a..b6ec5a3 100644
--- a/eval/lib/__init__.py
+++ b/eval/lib/__init__.py
@@ -7,59 +7,59 @@ Architecture:
- llm.py: LLM client and prompt building
"""
+from .llm import LLMClient, build_messages, build_react_messages
+from .pixel_query import PixelQueryRenderer, QueryImageTextRenderer
+from .retrieval import (
+ BaseRetriever,
+ ColQwenVectorRetriever,
+ DsServeRetriever,
+ EVQANoRetrievalRetriever,
+ HTMLDOMLookupRetriever,
+ HybridRetriever,
+ JinaReaderRetriever,
+ LocalAPIRetriever,
+ NaiveRetriever,
+ OCRWrappedRetriever,
+ RenderedTextWrapper,
+ RetrievalResult,
+ ScreenshotRetriever,
+ TextAPIRetriever,
+ TextRetriever,
+ TextVectorRetriever,
+ TiledColQwenVectorRetriever,
+ TiledQwen3VLEmbeddingRetriever,
+ TiledScreenshotRetriever,
+ TiledVectorRetriever,
+ VectorRetriever,
+ WikipediaAPIRetriever,
+ WorldVQANoRetrievalRetriever,
+)
from .screenshot import capture_screenshot, encode_image, encode_image_for_vlm
from .simpleqa_data import (
- load_simpleqa_data,
- load_simpleqa_verified_data,
- load_text_cache,
- extract_url_from_metadata,
- capture_screenshot_for_example,
capture_screenshot_async,
+ capture_screenshot_for_example,
encode_screenshot,
encode_screenshot_async,
encode_screenshot_for_vlm,
encode_screenshot_for_vlm_async,
- fetch_webpage_text,
- fetch_text_for_example,
+ extract_url_from_metadata,
fetch_text_async,
- make_compressed_encoder,
+ fetch_text_for_example,
+ fetch_webpage_text,
+ load_arc_data,
+ load_commonsenseqa_data,
+ load_hellaswag_data,
load_nq_data,
- load_triviaqa_data,
load_nq_tables_data,
- load_piqa_data,
- load_hellaswag_data,
- load_commonsenseqa_data,
load_openbookqa_data,
- load_arc_data,
-)
-from .retrieval import (
- BaseRetriever,
- EVQANoRetrievalRetriever,
- WorldVQANoRetrievalRetriever,
- NaiveRetriever,
- ScreenshotRetriever,
- TiledScreenshotRetriever,
- TextRetriever,
- JinaReaderRetriever,
- WikipediaAPIRetriever,
- VectorRetriever,
- ColQwenVectorRetriever,
- TiledVectorRetriever,
- TiledColQwenVectorRetriever,
- TiledQwen3VLEmbeddingRetriever,
- TextVectorRetriever,
- DsServeRetriever,
- LocalAPIRetriever,
- TextAPIRetriever,
- OCRWrappedRetriever,
- RenderedTextWrapper,
- HybridRetriever,
- HTMLDOMLookupRetriever,
- RetrievalResult,
+ load_piqa_data,
+ load_simpleqa_data,
+ load_simpleqa_verified_data,
+ load_text_cache,
+ load_triviaqa_data,
+ make_compressed_encoder,
)
-from .llm import LLMClient, build_messages, build_react_messages
-from .pixel_query import PixelQueryRenderer, QueryImageTextRenderer
-from .simpleqa_filter import load_simpleqa_wikipedia, load_simpleqa_by_domain
+from .simpleqa_filter import load_simpleqa_by_domain, load_simpleqa_wikipedia
__all__ = [
# Screenshot utilities
diff --git a/eval/lib/benchmarks.py b/eval/lib/benchmarks.py
index 9593d4d..6048a12 100644
--- a/eval/lib/benchmarks.py
+++ b/eval/lib/benchmarks.py
@@ -154,11 +154,11 @@ def load_encyclopedic_vqa_data(
example[col] = row[col]
# Map wikipedia_url into metadata so screenshot/retrieval pipeline can find it
- if "wikipedia_url" in example and example["wikipedia_url"]:
+ if example.get("wikipedia_url"):
example["metadata"] = {"url": example["wikipedia_url"]}
# Parse dataset_image_ids for query images (iNaturalist or Google Landmarks)
- if "dataset_image_ids" in example and example["dataset_image_ids"]:
+ if example.get("dataset_image_ids"):
raw_ids = str(example["dataset_image_ids"])
ids = [i.strip() for i in raw_ids.split("|") if i.strip()]
example["dataset_image_ids_parsed"] = ids
diff --git a/eval/lib/llm.py b/eval/lib/llm.py
index 119954d..4fe88c4 100644
--- a/eval/lib/llm.py
+++ b/eval/lib/llm.py
@@ -12,13 +12,13 @@ import os
# Try to import Google GenAI for Gemini support
try:
- import google.genai as genai
+ from google import genai
from google.genai.types import (
- GenerateContentConfig,
- Part,
Blob,
- HttpOptions,
Content,
+ GenerateContentConfig,
+ HttpOptions,
+ Part,
)
GEMINI_AVAILABLE = True
diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py
index 3b77375..4c7a6a0 100644
--- a/eval/lib/model_config.py
+++ b/eval/lib/model_config.py
@@ -6,7 +6,6 @@ This module provides model configurations to keep run_naive_simpleqa.py clean.
import os
from typing import Dict, Optional
-
MINIMAX_MODELS = {
"minimax-m3": "MiniMax-M3",
"minimax-m2.7": "MiniMax-M2.7",
diff --git a/eval/lib/pixel_query.py b/eval/lib/pixel_query.py
index c62ada8..c6617cf 100644
--- a/eval/lib/pixel_query.py
+++ b/eval/lib/pixel_query.py
@@ -12,6 +12,7 @@ Usage:
import logging
import os
from pathlib import Path
+
from PIL import Image, ImageDraw, ImageFont
logger = logging.getLogger(__name__)
@@ -352,7 +353,6 @@ class QueryImageTextRenderer:
outline=0,
)
# For simple output we keep the card as-is; full rounded crop would need alpha
- pass
card.save(out_path)
logger.debug(f"Rendered query card: {out_path} ({card.size[0]}x{card.size[1]})")
diff --git a/eval/lib/retrieval.py b/eval/lib/retrieval.py
index 1b46cd5..75a4ec7 100644
--- a/eval/lib/retrieval.py
+++ b/eval/lib/retrieval.py
@@ -7,9 +7,9 @@ This module defines retrieval strategies that work with prepared data:
- VectorRetriever: Search across all screenshots using vector similarity
"""
+import asyncio
import base64
import io
-import asyncio
import logging
import os
from abc import ABC, abstractmethod
@@ -711,8 +711,10 @@ class JinaReaderRetriever(BaseRetriever):
logger.warning(f"Failed to save to cache: {e}")
async def retrieve(self, query: str, example: dict) -> RetrievalResult:
- import aiohttp
import asyncio
+
+ import aiohttp
+
from .simpleqa_data import extract_url_from_metadata
# Check cache first
@@ -917,6 +919,7 @@ class WikipediaAPIRetriever(BaseRetriever):
def _html_to_text(self, html: str) -> str:
"""Convert Wikipedia HTML to plain text, preserving table content."""
import re
+
from bs4 import BeautifulSoup
soup = BeautifulSoup(html, "html.parser")
@@ -1005,6 +1008,7 @@ class WikipediaAPIRetriever(BaseRetriever):
async def retrieve(self, query: str, example: dict) -> RetrievalResult:
import aiohttp
+
from .simpleqa_data import extract_url_from_metadata
# Check cache first
@@ -1428,9 +1432,10 @@ class TiledVectorRetriever(BaseRetriever):
def _prepare_screenshots_and_tiles(self) -> list[str]:
"""Prepare screenshots and tiles for dataset, return tile paths."""
- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
from tqdm import tqdm
+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
+
screenshot_paths = []
missing = []
@@ -1485,8 +1490,7 @@ class TiledVectorRetriever(BaseRetriever):
if "_tile_" in filename:
example_id = filename.split("_tile_")[0]
# Remove _fullhd suffix if present
- if example_id.endswith("_fullhd"):
- example_id = example_id[:-7]
+ example_id = example_id.removesuffix("_fullhd")
if example_id in self.id_to_url:
url = self.id_to_url[example_id]
if url not in seen:
@@ -1591,9 +1595,10 @@ class TiledColQwenVectorRetriever(BaseRetriever):
def _prepare_screenshots_and_tiles(self) -> list[str]:
"""Prepare screenshots and tiles for dataset, return tile paths."""
- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
from tqdm import tqdm
+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
+
screenshot_paths = []
missing = []
@@ -1646,8 +1651,7 @@ class TiledColQwenVectorRetriever(BaseRetriever):
filename = os.path.basename(path)
if "_tile_" in filename:
example_id = filename.split("_tile_")[0]
- if example_id.endswith("_fullhd"):
- example_id = example_id[:-7]
+ example_id = example_id.removesuffix("_fullhd")
if example_id in self.id_to_url:
url = self.id_to_url[example_id]
if url not in seen:
@@ -1902,9 +1906,10 @@ class DsServeRetriever(BaseRetriever):
self.top_k = top_k
async def retrieve(self, query: str, example: dict) -> RetrievalResult:
- import aiohttp
import asyncio
+ import aiohttp
+
max_retries = 3
for attempt in range(max_retries):
try:
@@ -2585,9 +2590,10 @@ class TiledQwen3VLEmbeddingRetriever(BaseRetriever):
Uses deduplicated examples (one per unique URL) to avoid
duplicate tiles inflating the retrieval index.
"""
- from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
from tqdm import tqdm
+ from .simpleqa_data import capture_screenshot_for_example, split_image_to_tiles
+
examples_to_process = self._dedup_examples
screenshot_paths = []
missing = []
@@ -2642,8 +2648,7 @@ class TiledQwen3VLEmbeddingRetriever(BaseRetriever):
filename = os.path.basename(path)
if "_tile_" in filename:
example_id = filename.split("_tile_")[0]
- if example_id.endswith("_fullhd"):
- example_id = example_id[:-7]
+ example_id = example_id.removesuffix("_fullhd")
if example_id in self.id_to_url:
url = self.id_to_url[example_id]
if url not in seen:
@@ -3110,9 +3115,10 @@ class OCRWrappedRetriever(BaseRetriever):
async def _ocr_one(self, path: str, session) -> str:
if path in self._cache:
return self._cache[path]
- import aiohttp
import base64
+ import aiohttp
+
try:
with open(path, "rb") as f:
img_b64 = base64.b64encode(f.read()).decode("ascii")
@@ -3456,9 +3462,10 @@ class HTMLDOMLookupRetriever(BaseRetriever):
if article_id >= len(self._articles):
return None
- import requests
from urllib.parse import quote
+ import requests
+
slug = self._articles[article_id]
url = f"{self.KIWIX_BASE}/{quote(slug, safe='/:@!$&()*+,;=')}"
try:
@@ -3495,7 +3502,8 @@ class HTMLDOMLookupRetriever(BaseRetriever):
(inclusive), plus everything in between — this preserves the full
contiguous region the chunk spans.
"""
- from lxml import html as lxml_html, etree
+ from lxml import etree
+ from lxml import html as lxml_html
tree = lxml_html.fromstring(html)
diff --git a/eval/lib/retrievers.py b/eval/lib/retrievers.py
index 1861421..888d7e2 100644
--- a/eval/lib/retrievers.py
+++ b/eval/lib/retrievers.py
@@ -8,27 +8,27 @@ import os
import sys
from . import (
+ ColQwenVectorRetriever,
+ DsServeRetriever,
+ EVQANoRetrievalRetriever,
+ HTMLDOMLookupRetriever,
+ HybridRetriever,
+ JinaReaderRetriever,
+ LocalAPIRetriever,
NaiveRetriever,
+ OCRWrappedRetriever,
+ RenderedTextWrapper,
ScreenshotRetriever,
- TiledScreenshotRetriever,
+ TextAPIRetriever,
TextRetriever,
- JinaReaderRetriever,
- WikipediaAPIRetriever,
- VectorRetriever,
- ColQwenVectorRetriever,
- TiledVectorRetriever,
+ TextVectorRetriever,
TiledColQwenVectorRetriever,
TiledQwen3VLEmbeddingRetriever,
- EVQANoRetrievalRetriever,
+ TiledScreenshotRetriever,
+ TiledVectorRetriever,
+ VectorRetriever,
+ WikipediaAPIRetriever,
WorldVQANoRetrievalRetriever,
- TextVectorRetriever,
- DsServeRetriever,
- LocalAPIRetriever,
- TextAPIRetriever,
- OCRWrappedRetriever,
- RenderedTextWrapper,
- HybridRetriever,
- HTMLDOMLookupRetriever,
load_text_cache,
)
from .retrieval import _get_query_image_path_for_example, _save_task_query_image
diff --git a/eval/lib/screenshot.py b/eval/lib/screenshot.py
index 2f33da8..9425da5 100644
--- a/eval/lib/screenshot.py
+++ b/eval/lib/screenshot.py
@@ -23,12 +23,12 @@ def setup_driver(window_width=1024, window_height=2000, device_scale_factor=1):
window_height: Initial viewport height.
device_scale_factor: Pixel density (1 = standard, 2 = retina quality).
"""
+ import shutil
+
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
- import shutil
-
snap_chromedriver = "/snap/bin/chromium.chromedriver"
if os.path.exists(snap_chromedriver):
driver_path = snap_chromedriver
@@ -80,10 +80,11 @@ def _capture_with_scroll(driver, output_path, scroll_pause=0.8, max_scrolls=100)
Works for PDF viewers, infinite scroll pages, and other dynamic content.
Uses image comparison to detect when scrolling has stopped.
"""
+ import hashlib
+ import tempfile
+
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
- import tempfile
- import hashlib
def get_screenshot_hash(path):
"""Get hash of screenshot to detect changes."""
diff --git a/eval/lib/simpleqa_data.py b/eval/lib/simpleqa_data.py
index 240c7eb..f4978c7 100644
--- a/eval/lib/simpleqa_data.py
+++ b/eval/lib/simpleqa_data.py
@@ -514,6 +514,7 @@ def make_compressed_encoder(compress_ratio: int, save_dir: str | None = None):
"""Encode image with pixel compression and save to disk."""
import base64 as _b64
from io import BytesIO
+
from PIL import Image as _Image
if not screenshot_path or not os.path.exists(screenshot_path):
@@ -662,9 +663,10 @@ def split_image_to_tiles(
Returns:
List of tile file paths.
"""
- from PIL import Image
import glob
+ from PIL import Image
+
if not os.path.exists(image_path):
return []
@@ -809,9 +811,10 @@ def load_nq_data(
Returns:
List of dicts with id, problem, gold_answers, metadata.
"""
- from datasets import load_dataset
import html as _html
+ from datasets import load_dataset
+
if split not in {"train", "validation"}:
raise ValueError(
f"Unsupported NQ split: {split!r}. Expected 'train' or 'validation'."
@@ -899,10 +902,11 @@ def load_triviaqa_data(num_examples: int | None = 1000) -> list[dict]:
Returns:
List of dicts with id, problem, gold_answers, metadata.
"""
- from datasets import load_dataset
import ast as _ast
from urllib.parse import quote as _url_quote
+ from datasets import load_dataset
+
logger.info("Loading TriviaQA rc.wikipedia validation split (streaming)...")
ds = load_dataset(
"mandarjoshi/trivia_qa",
diff --git a/eval/run_bench.py b/eval/run_bench.py
index 4f15de0..dabf03c 100644
--- a/eval/run_bench.py
+++ b/eval/run_bench.py
@@ -38,39 +38,42 @@ logger = logging.getLogger(__name__)
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from lib import (
- # Data
- load_simpleqa_wikipedia,
- extract_url_from_metadata,
+ LLMClient,
+ build_messages,
+ build_react_messages,
encode_screenshot,
- make_compressed_encoder,
+ extract_url_from_metadata,
+ load_arc_data,
+ load_commonsenseqa_data,
+ load_hellaswag_data,
load_nq_data,
- load_triviaqa_data,
load_nq_tables_data,
- load_piqa_data,
- load_hellaswag_data,
- load_commonsenseqa_data,
load_openbookqa_data,
- load_arc_data,
+ load_piqa_data,
+ # Data
+ load_simpleqa_wikipedia,
+ load_triviaqa_data,
+ make_compressed_encoder,
)
-from lib import LLMClient, build_messages, build_react_messages
-from lib.model_config import get_model_config, get_output_filename
-from lib.retrieval import (
- _get_query_image_path_for_example,
- _save_worldvqa_query_image,
- _save_task_query_image,
+from lib.benchmarks import (
+ SUPPORTED_TASKS as DATASET_REPOS,
)
from lib.benchmarks import (
load_encyclopedic_vqa_data,
- load_shortformqa_data,
- load_worldvqa_data,
- load_simplevqa_data,
load_factualvqa_data,
load_mmsearch_data,
- load_webqa_data,
load_multimodalqa_data,
- SUPPORTED_TASKS as DATASET_REPOS,
+ load_shortformqa_data,
+ load_simplevqa_data,
+ load_webqa_data,
+ load_worldvqa_data,
+)
+from lib.model_config import get_model_config, get_output_filename
+from lib.retrieval import (
+ _get_query_image_path_for_example,
+ _save_task_query_image,
+ _save_worldvqa_query_image,
)
-
_DEFAULT_SPLIT_FOR_TASK = {
"simpleqa": "test",
@@ -106,13 +109,12 @@ def _fetch_status(api_url: str | None, timeout: float = 5.0) -> dict | None:
import urllib.request
base = api_url.rstrip("/")
- if base.endswith("/search"):
- base = base[: -len("/search")]
+ base = base.removesuffix("/search")
status_url = base + "/status"
try:
with urllib.request.urlopen(status_url, timeout=timeout) as r:
return json.loads(r.read().decode())
- except Exception as e: # noqa: BLE001 — best-effort capture
+ except Exception as e:
return {"_error": f"{type(e).__name__}: {e}", "url": status_url}
@@ -168,7 +170,7 @@ def _build_run_metadata(args, n_loaded: int) -> dict:
.decode()
.strip()
)
- except Exception: # noqa: BLE001
+ except Exception:
meta["git_commit"] = None
return meta
diff --git a/eval/run_livevqa.py b/eval/run_livevqa.py
index b5c9003..1403209 100644
--- a/eval/run_livevqa.py
+++ b/eval/run_livevqa.py
@@ -68,7 +68,7 @@ import requests
# Add eval root to path so simpleqa imports work
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from lib.llm import LLMClient # noqa: E402
+from lib.llm import LLMClient
logging.basicConfig(
level=logging.INFO,
@@ -498,7 +498,7 @@ def resolve_text_context(
passages: list[str] = []
for it in retrieved_items[:top_k]:
- if "text" in it and it["text"]:
+ if it.get("text"):
passages.append(it["text"])
elif chunks_db and hex_to_int:
# Cross-format: pixel retrieval item, fetch text from DB
@@ -670,7 +670,7 @@ async def evaluate_one(
chunks: list[str] = []
for it in t_items[: args.top_k]:
- if "text" in it and it["text"]:
+ if it.get("text"):
chunks.append(it["text"])
if not tile_paths and not chunks:
diff --git a/index/src/pixelrag_index/monitor.py b/index/src/pixelrag_index/monitor.py
index 146de6b..291be9e 100755
--- a/index/src/pixelrag_index/monitor.py
+++ b/index/src/pixelrag_index/monitor.py
@@ -386,8 +386,7 @@ def render(
if claimed_at and claimed_at < m["earliest_claim"]:
m["earliest_claim"] = claimed_at
hb = c.get("heartbeat", 0)
- if hb > m["latest_heartbeat"]:
- m["latest_heartbeat"] = hb
+ m["latest_heartbeat"] = max(m["latest_heartbeat"], hb)
# Collect disk_free_gb (use the latest heartbeat value)
if "disk_free_gb" in c:
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py
index fc10571..5c3943f 100644
--- a/index/src/pixelrag_index/pipelines.py
+++ b/index/src/pixelrag_index/pipelines.py
@@ -116,7 +116,7 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
# Stage 1: Render documents to tiles
# Use sequential integer IDs as tile directory names so embed/serve can map them
- from pixelrag_render.render import render_urls, render_pdf
+ from pixelrag_render.render import render_pdf, render_urls
logger.info("Stage 1/4: Rendering %d documents to tiles...", len(docs))
diff --git a/index/src/pixelrag_index/sources/__init__.py b/index/src/pixelrag_index/sources/__init__.py
index 7f45882..5dcd3c1 100644
--- a/index/src/pixelrag_index/sources/__init__.py
+++ b/index/src/pixelrag_index/sources/__init__.py
@@ -1,4 +1,5 @@
-from .base import Document as Document, Source as Source
+from .base import Document as Document
+from .base import Source as Source
from .kiwix import KiwixSource
from .local import LocalSource
from .pdf import PDFSource
diff --git a/index/src/pixelrag_index/sources/kiwix.py b/index/src/pixelrag_index/sources/kiwix.py
index ba00209..b9ed83d 100644
--- a/index/src/pixelrag_index/sources/kiwix.py
+++ b/index/src/pixelrag_index/sources/kiwix.py
@@ -603,7 +603,7 @@ class KiwixSource(Source):
@atexit.register
def _cleanup_sources() -> None:
- for src in list(_active_sources):
+ for src in _active_sources:
try:
src.close()
except Exception:
diff --git a/pyproject.toml b/pyproject.toml
index 2abc237..8c5c0a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -114,12 +114,47 @@ url = "https://download.pytorch.org/whl/cu129"
explicit = true
[tool.ruff]
-exclude = ["tmp"]
+exclude = ["tmp", "train", "demos", "docs", "skill"]
[tool.ruff.lint]
ignore = [
- "E402", # module-level import not at top — common in ML codebases
- "E741", # ambiguous variable name (l, O, I)
+ "E402", # module-level import not at top — common in ML codebases
+ "E741", # ambiguous variable name (l, O, I)
+ "BLE001", # blind exception catch — pervasive in data pipelines
+ "EXE001", # shebang without +x
+ "S110", # try-except-pass
+ "S112", # try-except-continue
+ "SIM102", # collapsible if
+ "SIM103", # return bool directly
+ "SIM113", # enumerate instead of manual counter
+ "SIM115", # use context manager for open
+ "SIM117", # combine with-statements
+ "ASYNC210", # blocking call in async
+ "ASYNC220", # blocking call in async
+ "ASYNC230", # blocking call in async
+ "UP006", # deprecated typing aliases
+ "UP017", # deprecated datetime aliases
+ "UP035", # deprecated typing imports
+ "UP041", # timeout-error alias
+ "UP045", # X | None instead of Optional
+ "PLW1509", # subprocess preexec_fn
+ "PLW1510", # subprocess without check
+ "PLR0402", # import-as-module-alias
+ "PLR1730", # if-stmt-min-max
+ "RUF012", # mutable-class-default
+ "RUF013", # implicit-optional
+ "RUF022", # unsorted-all
+ "RUF059", # unused-unpacked-variable
+ "TRY002", # raise-vanilla-class
+ "PIE790", # unnecessary-placeholder
+ "PIE810", # multiple-starts-ends-with
+ "PYI034", # non-self-return-type
+ "PERF102", # incorrect-dict-iterator
+ "PERF101", # unnecessary-cast-to-int
+ "DTZ005", # call-datetime-now-without-tzinfo
+ "C401", # unnecessary-generator-set
+ "C408", # unnecessary-collection-call
+ "FURB188", # slice-to-remove-prefix-or-suffix
]
[tool.pyright]
diff --git a/render/src/pixelrag_render/__init__.py b/render/src/pixelrag_render/__init__.py
index 417615a..d43099b 100644
--- a/render/src/pixelrag_render/__init__.py
+++ b/render/src/pixelrag_render/__init__.py
@@ -3,6 +3,6 @@
Renders web pages, PDFs, and local files as tiled screenshots.
"""
-from .render import render_url, render_pdf, render_file
+from .render import render_file, render_pdf, render_url
-__all__ = ["render_url", "render_pdf", "render_file"]
+__all__ = ["render_file", "render_pdf", "render_url"]
diff --git a/render/src/pixelrag_render/backends/cdp.py b/render/src/pixelrag_render/backends/cdp.py
index 584f4fd..f727048 100644
--- a/render/src/pixelrag_render/backends/cdp.py
+++ b/render/src/pixelrag_render/backends/cdp.py
@@ -24,8 +24,8 @@ import base64
import io
import json
import logging
-import shutil
import os
+import shutil
import signal
import subprocess
import tempfile
@@ -960,7 +960,7 @@ def render_urls(
def _navtarget(u: str) -> str:
if u.startswith("http"):
return u
- return u[len("file://") :] if u.startswith("file://") else u
+ return u.removeprefix("file://")
articles = [
{"path": f"{stem}.png", "file": _navtarget(url)}
diff --git a/render/src/pixelrag_render/backends/fast_cdp.py b/render/src/pixelrag_render/backends/fast_cdp.py
index fea9d56..251195a 100644
--- a/render/src/pixelrag_render/backends/fast_cdp.py
+++ b/render/src/pixelrag_render/backends/fast_cdp.py
@@ -289,9 +289,9 @@ async def _run_render(
# Compression: dedicated thread with its own multiprocessing.Pool.
# Tiles are pushed to a thread-safe queue from capture workers.
# The thread runs pool.starmap in batches, fully independent of asyncio.
- from multiprocessing import Pool as MPPool
import queue as _queue
import threading
+ from multiprocessing import Pool as MPPool
metrics = {
"total_tiles": 0,
diff --git a/render/src/pixelrag_render/bench/__init__.py b/render/src/pixelrag_render/bench/__init__.py
index 18db218..3ec5810 100644
--- a/render/src/pixelrag_render/bench/__init__.py
+++ b/render/src/pixelrag_render/bench/__init__.py
@@ -10,7 +10,13 @@ Usage:
from .bench_throughput import (
Bench as Bench,
- prepare_articles as prepare_articles,
+)
+from .bench_throughput import (
generate_ground_truth as generate_ground_truth,
+)
+from .bench_throughput import (
+ prepare_articles as prepare_articles,
+)
+from .bench_throughput import (
run_and_verify as run_and_verify,
)
diff --git a/render/src/pixelrag_render/bench/bench_throughput.py b/render/src/pixelrag_render/bench/bench_throughput.py
index 134355e..93564af 100644
--- a/render/src/pixelrag_render/bench/bench_throughput.py
+++ b/render/src/pixelrag_render/bench/bench_throughput.py
@@ -34,11 +34,10 @@ from PIL import Image
from pixelrag_render.strategies.base import TileCapture
from pixelrag_render.strategies.cdp_sequential import (
- CDPSequentialStrategy,
VIEWPORT_WIDTH,
+ CDPSequentialStrategy,
)
-
CORRECT_THRESHOLD = 99.0
JPEG_MAX_MEAN_DIFF = 5.0
LOSSLESS_MAX_MEAN_DIFF = 3.0
@@ -59,9 +58,10 @@ def prepare_articles(
- "http://host:port": single kiwix-serve instance
- "http://host:9461,http://host:9462,...": multiple instances (round-robin)
"""
- from libzim.reader import Archive
from urllib.parse import quote
+ from libzim.reader import Archive
+
archive = Archive(zim_path)
# Support multiple kiwix URLs (comma-separated)
diff --git a/render/src/pixelrag_render/strategies/__init__.py b/render/src/pixelrag_render/strategies/__init__.py
index 64ea468..7f020f3 100644
--- a/render/src/pixelrag_render/strategies/__init__.py
+++ b/render/src/pixelrag_render/strategies/__init__.py
@@ -1,28 +1,28 @@
"""Screenshot capture strategies — core render capability, independent of benchmarking."""
-from .base import CaptureStrategy, TileCapture, ArticleCapture, article_url
+from .base import ArticleCapture, CaptureStrategy, TileCapture, article_url
+from .cdp_directclip import CDPDirectClipStrategy
+from .cdp_oneshot import CDPOneShotStrategy
+from .cdp_pertile_imgwait import CDPPerTileImgWaitStrategy
+from .cdp_sequential import CDPSequentialStrategy
from .connection import (
- WebsocketConnection,
PlaywrightConnection,
- launch_websocket,
+ WebsocketConnection,
launch_playwright,
+ launch_websocket,
)
-from .cdp_sequential import CDPSequentialStrategy
-from .cdp_directclip import CDPDirectClipStrategy
-from .cdp_pertile_imgwait import CDPPerTileImgWaitStrategy
-from .cdp_oneshot import CDPOneShotStrategy
__all__ = [
+ "ArticleCapture",
+ "CDPDirectClipStrategy",
+ "CDPOneShotStrategy",
+ "CDPPerTileImgWaitStrategy",
+ "CDPSequentialStrategy",
"CaptureStrategy",
+ "PlaywrightConnection",
"TileCapture",
- "ArticleCapture",
- "article_url",
"WebsocketConnection",
- "PlaywrightConnection",
- "launch_websocket",
+ "article_url",
"launch_playwright",
- "CDPSequentialStrategy",
- "CDPDirectClipStrategy",
- "CDPPerTileImgWaitStrategy",
- "CDPOneShotStrategy",
+ "launch_websocket",
]
diff --git a/render/src/pixelrag_render/strategies/cdp_dc_single.py b/render/src/pixelrag_render/strategies/cdp_dc_single.py
index 891ce79..9398f02 100644
--- a/render/src/pixelrag_render/strategies/cdp_dc_single.py
+++ b/render/src/pixelrag_render/strategies/cdp_dc_single.py
@@ -13,7 +13,7 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import launch_websocket
TILE_HEIGHT = 8192
diff --git a/render/src/pixelrag_render/strategies/cdp_directclip.py b/render/src/pixelrag_render/strategies/cdp_directclip.py
index dc52cd3..a14e6eb 100644
--- a/render/src/pixelrag_render/strategies/cdp_directclip.py
+++ b/render/src/pixelrag_render/strategies/cdp_directclip.py
@@ -14,8 +14,8 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import launch_playwright, launch_websocket
TILE_HEIGHT_SMALL = 1024 # small tiles for parallel capture
VIEWPORT_HEIGHT = 8192 # viewport size (rasterized area)
diff --git a/render/src/pixelrag_render/strategies/cdp_dynamic.py b/render/src/pixelrag_render/strategies/cdp_dynamic.py
index 6238ea1..55587dc 100644
--- a/render/src/pixelrag_render/strategies/cdp_dynamic.py
+++ b/render/src/pixelrag_render/strategies/cdp_dynamic.py
@@ -12,8 +12,8 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import launch_playwright, launch_websocket
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
diff --git a/render/src/pixelrag_render/strategies/cdp_fullpage.py b/render/src/pixelrag_render/strategies/cdp_fullpage.py
index fd82ca4..0efb53b 100644
--- a/render/src/pixelrag_render/strategies/cdp_fullpage.py
+++ b/render/src/pixelrag_render/strategies/cdp_fullpage.py
@@ -15,8 +15,8 @@ from dataclasses import dataclass
from PIL import Image
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import launch_playwright, launch_websocket
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
diff --git a/render/src/pixelrag_render/strategies/cdp_multitab.py b/render/src/pixelrag_render/strategies/cdp_multitab.py
index 49c41a8..3ee43d9 100644
--- a/render/src/pixelrag_render/strategies/cdp_multitab.py
+++ b/render/src/pixelrag_render/strategies/cdp_multitab.py
@@ -21,8 +21,8 @@ import time
import urllib.request
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import WebsocketConnection, CHROME_ARGS, pick_page_ws_url
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import CHROME_ARGS, WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
diff --git a/render/src/pixelrag_render/strategies/cdp_noscroll.py b/render/src/pixelrag_render/strategies/cdp_noscroll.py
index 938e58c..0911bc5 100644
--- a/render/src/pixelrag_render/strategies/cdp_noscroll.py
+++ b/render/src/pixelrag_render/strategies/cdp_noscroll.py
@@ -12,8 +12,8 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import launch_playwright, launch_websocket
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
diff --git a/render/src/pixelrag_render/strategies/cdp_oneshot.py b/render/src/pixelrag_render/strategies/cdp_oneshot.py
index 71992df..4853757 100644
--- a/render/src/pixelrag_render/strategies/cdp_oneshot.py
+++ b/render/src/pixelrag_render/strategies/cdp_oneshot.py
@@ -28,7 +28,7 @@ import time
import urllib.request
from dataclasses import dataclass, field
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import CHROME_ARGS, pick_page_ws_url
TILE_HEIGHT = 8192
@@ -77,6 +77,7 @@ async def _launch_oneshot(
(much faster than the 1 s sleep in launch_websocket).
"""
import websockets
+
from .connection import WebsocketConnection
args = [chrome_path, f"--remote-debugging-port={port}"]
diff --git a/render/src/pixelrag_render/strategies/cdp_overlap.py b/render/src/pixelrag_render/strategies/cdp_overlap.py
index 3470829..59704a3 100644
--- a/render/src/pixelrag_render/strategies/cdp_overlap.py
+++ b/render/src/pixelrag_render/strategies/cdp_overlap.py
@@ -21,7 +21,7 @@ import time
import urllib.request
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
diff --git a/render/src/pixelrag_render/strategies/cdp_parallel.py b/render/src/pixelrag_render/strategies/cdp_parallel.py
index a2212b9..07c79a3 100644
--- a/render/src/pixelrag_render/strategies/cdp_parallel.py
+++ b/render/src/pixelrag_render/strategies/cdp_parallel.py
@@ -14,9 +14,9 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
from .cdp_sequential import TILE_HEIGHT, VIEWPORT_WIDTH
+from .connection import launch_playwright, launch_websocket
@dataclass
diff --git a/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py b/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py
index 6a13664..5e3982a 100644
--- a/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py
+++ b/render/src/pixelrag_render/strategies/cdp_pertile_imgwait.py
@@ -16,8 +16,8 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
-from .connection import launch_websocket, launch_playwright
+from .base import ArticleCapture, TileCapture, article_url
+from .connection import launch_playwright, launch_websocket
TILE_HEIGHT = 8192
VIEWPORT_WIDTH = 875
diff --git a/render/src/pixelrag_render/strategies/cdp_phased.py b/render/src/pixelrag_render/strategies/cdp_phased.py
index 9eda8ac..1bf7cff 100644
--- a/render/src/pixelrag_render/strategies/cdp_phased.py
+++ b/render/src/pixelrag_render/strategies/cdp_phased.py
@@ -15,7 +15,7 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import launch_websocket
TILE_HEIGHT = 8192
diff --git a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
index 0ad88d6..9bcc9c9 100644
--- a/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
+++ b/render/src/pixelrag_render/strategies/cdp_pipelined_dc.py
@@ -15,12 +15,12 @@ import asyncio
import base64
import json
import os
-import time
import subprocess
+import time
import urllib.request
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import WebsocketConnection, pick_page_ws_url
TILE_HEIGHT = 8192
diff --git a/render/src/pixelrag_render/strategies/cdp_sequential.py b/render/src/pixelrag_render/strategies/cdp_sequential.py
index 4ed0336..2e7a1df 100644
--- a/render/src/pixelrag_render/strategies/cdp_sequential.py
+++ b/render/src/pixelrag_render/strategies/cdp_sequential.py
@@ -12,10 +12,10 @@ import os
import time
from dataclasses import dataclass
-from .base import article_url, ArticleCapture, TileCapture
+from .base import ArticleCapture, TileCapture, article_url
from .connection import (
- launch_websocket,
launch_playwright,
+ launch_websocket,
)
TILE_HEIGHT = 8192
diff --git a/serve/src/pixelrag_serve/zim_server.py b/serve/src/pixelrag_serve/zim_server.py
index f0fe233..3259639 100644
--- a/serve/src/pixelrag_serve/zim_server.py
+++ b/serve/src/pixelrag_serve/zim_server.py
@@ -8,10 +8,10 @@ Usage:
"""
import argparse
-from http.server import HTTPServer, BaseHTTPRequestHandler
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
from socketserver import ThreadingMixIn
from urllib.parse import unquote
-from pathlib import Path
from libzim.reader import Archive
diff --git a/tests/test_article_id.py b/tests/test_article_id.py
index 55ce805..663cc31 100644
--- a/tests/test_article_id.py
+++ b/tests/test_article_id.py
@@ -9,7 +9,6 @@ import json
from pathlib import Path
from PIL import Image
-
from pixelrag_embed.chunk import chunk_article
from pixelrag_embed.embed import scan_shard_chunks
from pixelrag_embed.embed_cpu import scan_chunks
diff --git a/tests/test_cdp_attach.py b/tests/test_cdp_attach.py
index f5ae596..e66c01b 100644
--- a/tests/test_cdp_attach.py
+++ b/tests/test_cdp_attach.py
@@ -10,7 +10,6 @@ import sys
from pathlib import Path
import pytest
-
from pixelrag_render.backends import cdp
_BIN = Path(sys.executable).parent
diff --git a/tests/test_chrome_paths.py b/tests/test_chrome_paths.py
index 7c54b55..76db6e9 100644
--- a/tests/test_chrome_paths.py
+++ b/tests/test_chrome_paths.py
@@ -1,6 +1,7 @@
"""Cross-platform Chrome resolution candidates (logic is testable on any OS)."""
import os
+
from pixelrag_render.chrome import _candidate_chrome_paths, find_chrome
diff --git a/tests/test_chunk.py b/tests/test_chunk.py
index ee95877..54ad45a 100644
--- a/tests/test_chunk.py
+++ b/tests/test_chunk.py
@@ -9,7 +9,6 @@ height-strip layout unchanged so existing indexes stay reproducible.
import json
from PIL import Image
-
from pixelrag_embed.chunk import CHUNK_HEIGHT, chunk_article
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 9b11a63..f6eba9a 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -110,10 +110,11 @@ def test_build_index_help_lists_qdrant_collection_modes():
def test_light_imports():
# Core install must import without torch.
- import pixelrag # noqa: F401
import pixelrag_render # noqa: F401
from pixelrag_render import render_file, render_url # noqa: F401
+ import pixelrag # noqa: F401
+
def test_dispatcher_stage_table():
from pixelrag.cli import STAGES
diff --git a/tests/test_eval_model_config.py b/tests/test_eval_model_config.py
index cfd9d82..b1b56a8 100644
--- a/tests/test_eval_model_config.py
+++ b/tests/test_eval_model_config.py
@@ -3,7 +3,6 @@ from runpy import run_path
import pytest
-
get_model_config = run_path(
str(Path(__file__).parents[1] / "eval" / "lib" / "model_config.py")
)["get_model_config"]
diff --git a/tests/test_serve_backends.py b/tests/test_serve_backends.py
index 9c17c0c..7b68488 100644
--- a/tests/test_serve_backends.py
+++ b/tests/test_serve_backends.py
@@ -17,7 +17,7 @@ import pytest
faiss = pytest.importorskip("faiss", reason="serve extra not installed")
-from pixelrag_serve.backends import FaissBackend # noqa: E402
+from pixelrag_serve.backends import FaissBackend
DIM = 8
N_VECTORS = 12
@@ -62,9 +62,8 @@ def faiss_backend(tmp_path_factory):
@pytest.fixture(scope="module")
def qdrant_backend():
pytest.importorskip("qdrant_client", reason="qdrant extra not installed")
- from qdrant_client import QdrantClient, models
-
from pixelrag_serve.backends import QdrantBackend
+ from qdrant_client import QdrantClient, models
client = QdrantClient(":memory:")
client.create_collection(