https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
fix(embed): resize wide PDF chunks instead of skipping them (#51)
Yichuan Wangbe0a9412821b
diff --git a/embed/src/pixelrag_embed/embed.py b/embed/src/pixelrag_embed/embed.pyindex 1936241..eea8ce7 100644--- a/embed/src/pixelrag_embed/embed.py+++ b/embed/src/pixelrag_embed/embed.py@@ -111,6 +111,7 @@ def resolve_gpu_ids(gpu_ids_arg: str) -> list[int]:_RESIZE_FACTOR = 28 # Qwen3-VL patch alignment+_MAX_CHUNK_WIDTH = 875 # web viewport width; wider images (e.g. PDF tiles) are resizeddef _smart_resize_pil(img: "Image.Image", max_pixels: int) -> "Image.Image":@@ -127,6 +128,21 @@ def _smart_resize_pil(img: "Image.Image", max_pixels: int) -> "Image.Image":return img.resize((new_w, new_h), Image.LANCZOS)+def _clamp_width_pil(img: "Image.Image", max_width: int = _MAX_CHUNK_WIDTH) -> "Image.Image":+ """Resize image so width <= max_width, preserving aspect ratio.++ Dimensions are rounded to multiples of 28 (Qwen3-VL patch alignment).+ Used for PDF tiles that render wider than the web viewport.+ """+ w, h = img.size+ if w <= max_width:+ return img+ scale = max_width / w+ new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)+ new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)+ return img.resize((new_w, new_h), Image.LANCZOS)++def save_npz(path: str, compressed: bool, **arrays) -> None:"""Save npz with optional compression."""if compressed:@@ -881,7 +897,7 @@ def _embed_tile_infos_with_engine(unique_queue: queue.Queue = queue.Queue(maxsize=batch_size * 2)producer_error: list[Exception | None] = [None]- skipped_oversized = [0] # mutable counter for threads+ resized_wide = [0] # mutable counter for threadsdef _io_producer() -> None:"""Read tiles, chunk tall images, hash chunks, skip decode for duplicates."""@@ -920,15 +936,14 @@ def _embed_tile_infos_with_engine(cw / max(ch, 1),)continue- if cw > 875:- logger.warning(- "Skipping oversized chunk %s ci=%d width %d > 875",- img_path,- ci,- cw,+ if cw > _MAX_CHUNK_WIDTH:+ chunk_img = _clamp_width_pil(chunk_img, _MAX_CHUNK_WIDTH)+ cw, ch = chunk_img.size+ resized_wide[0] += 1+ logger.debug(+ "Resized wide chunk %s ci=%d to %dx%d",+ img_path, ci, cw, ch,)- skipped_oversized[0] += 1- continue# Dedup key: use file path for pre-chunked files (avoids# expensive tobytes + MD5 on ~2.7MB raw pixels per chunk)if isinstance(ti, ChunkInfo):@@ -1056,15 +1071,16 @@ def _embed_tile_infos_with_engine(deduped = len(tile_hashes) - unique_total100.0 * deduped / len(tile_hashes) if tile_hashes else 0.0- n_skipped = skipped_oversized[0]+ n_resized = resized_wide[0]logger.info(- "GPU %d: %d images, %d unique (%d deduped, %d skipped>875), embedded %d, pipeline %.2fs (%.1f chunks/s)"+ "GPU %d: %d images, %d unique (%d deduped, %d resized>%d), embedded %d, pipeline %.2fs (%.1f chunks/s)"" | embed=%.2fs (%d batches) queue_wait=%.2fs first_item=%.3fs",gpu_id,len(tile_hashes),unique_total,deduped,- n_skipped,+ n_resized,+ _MAX_CHUNK_WIDTH,embedded,pipeline_s,embedded / pipeline_s if pipeline_s > 0 else 0,diff --git a/embed/src/pixelrag_embed/embed_cpu.py b/embed/src/pixelrag_embed/embed_cpu.pyindex 15965f3..e782e4e 100644--- a/embed/src/pixelrag_embed/embed_cpu.py+++ b/embed/src/pixelrag_embed/embed_cpu.py@@ -1,14 +1,17 @@#!/usr/bin/env python3-"""CPU embedding: embed tile chunks using transformers on CPU.+"""Local embedding: embed tile chunks using transformers on CPU or Apple MPS.-Slower than GPU backends (vLLM/sglang) but works without CUDA.-Suitable for small-scale demos and testing.+Works without CUDA — suitable for macOS (Apple Silicon), small-scale demos,+and testing.Usage:+ # CPU (any platform)python -m pixelrag_embed.embed_cpu \- --shard-dir ./tiles \- --output-dir ./embeddings \- --model Qwen/Qwen3-VL-Embedding-2B+ --shard-dir ./tiles --output-dir ./embeddings++ # Apple Silicon GPU (macOS)+ python -m pixelrag_embed.embed_cpu \+ --shard-dir ./tiles --output-dir ./embeddings --device mps"""import argparse@@ -22,10 +25,37 @@ from PIL import Imagefrom tqdm import tqdmlogging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")-logger = logging.getLogger("embed_cpu")+logger = logging.getLogger("embed_local")Image.MAX_IMAGE_PIXELS = None+_RESIZE_FACTOR = 28+_MAX_CHUNK_WIDTH = 875+++def _clamp_width(img: Image.Image, max_width: int = _MAX_CHUNK_WIDTH) -> Image.Image:+ """Resize so width <= max_width, preserving aspect ratio (28px alignment)."""+ w, h = img.size+ if w <= max_width:+ return img+ scale = max_width / w+ new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)+ new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)+ return img.resize((new_w, new_h), Image.LANCZOS)+++def _resolve_device(device: str) -> str:+ """Resolve device string, auto-detecting MPS on macOS."""+ import torch++ if device == "auto":+ if torch.cuda.is_available():+ return "cuda"+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():+ return "mps"+ return "cpu"+ return device+def scan_chunks(shard_dir: str) -> list[dict]:"""Scan for chunk images in a shard directory.@@ -38,7 +68,6 @@ def scan_chunks(shard_dir: str) -> list[dict]:for entry in sorted(shard.iterdir()):if not entry.is_dir():continue- # Support both flat (*.png.tiles/) and nested (sub_shard/*/**.png.tiles/)tile_dirs = []if entry.name.endswith(".png.tiles"):tile_dirs = [entry]@@ -97,21 +126,29 @@ def scan_chunks(shard_dir: str) -> list[dict]:def embed_items(- items: list[dict], model_name: str, instruction: str = ""+ items: list[dict],+ model_name: str,+ device: str = "cpu",+ instruction: str = "",) -> np.ndarray:- """Embed a list of image items using transformers on CPU."""+ """Embed image items using transformers on the given device."""import torchfrom transformers import AutoProcessor, Qwen3VLForConditionalGeneration- logger.info("Loading model %s on CPU...", model_name)+ device = _resolve_device(device)+ dtype = torch.float32 if device == "cpu" else torch.float16++ logger.info("Loading model %s on %s (%s)...", model_name, device, dtype)processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)model = Qwen3VLForConditionalGeneration.from_pretrained(model_name,trust_remote_code=True,- dtype=torch.float32,+ torch_dtype=dtype,attn_implementation="sdpa",).eval()- logger.info("Model loaded")+ if device != "cpu":+ model = model.to(device)+ logger.info("Model loaded on %s", device)dim = model.config.text_config.hidden_sizeembeddings = np.zeros((len(items), dim), dtype=np.float16)@@ -120,6 +157,7 @@ def embed_items(for i, item in enumerate(tqdm(items, desc="Embedding")):img = Image.open(item["path"]).convert("RGB")+ img = _clamp_width(img)messages = [{@@ -135,17 +173,17 @@ def embed_items(messages, tokenize=False, add_generation_prompt=True)inputs = processor(text=[text], images=[img], return_tensors="pt", padding=True)+ if device != "cpu":+ inputs = {k: v.to(device) if hasattr(v, "to") else v for k, v in inputs.items()}with torch.no_grad():outputs = model(**inputs, output_hidden_states=True)last_hidden = outputs.hidden_states[-1]- # Last token poolingseq_lens = inputs["attention_mask"].sum(dim=1)last_idx = seq_lens - 1pooled = last_hidden[0, last_idx[0]]- # L2 normalizepooled = pooled / pooled.norm()- embeddings[i] = pooled.numpy().astype(np.float16)+ embeddings[i] = pooled.cpu().numpy().astype(np.float16)if (i + 1) % 10 == 0:logger.info("Embedded %d/%d", i + 1, len(items))@@ -154,12 +192,18 @@ def embed_items(def main():- parser = argparse.ArgumentParser(description="CPU embedding for tile chunks")+ parser = argparse.ArgumentParser(description="Local embedding (CPU / MPS / CUDA)")parser.add_argument("--shard-dir", required=True, help="Directory with *.png.tiles/ subdirs")parser.add_argument("--output-dir", required=True, help="Output directory for .npz")parser.add_argument("--model", default="Qwen/Qwen3-VL-Embedding-2B")+ parser.add_argument(+ "--device",+ default="auto",+ choices=["auto", "cpu", "mps", "cuda"],+ help="Device (default: auto-detect)",+ )parser.add_argument("--instruction", default="", help="Instruction prefix for queries")@@ -178,9 +222,8 @@ def main():items = items[: args.limit]else:logger.info("Found %d chunks to embed", len(items))- embeddings = embed_items(items, args.model, args.instruction)+ embeddings = embed_items(items, args.model, device=args.device, instruction=args.instruction)- # Save NPZ in same format as embed.pyoutput_path = Path(args.output_dir) / "shard_000.npz"np.savez(output_path,diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex 038d3a3..ab0fea7 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -132,8 +132,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:# Stage 3: Embed chunks to vectorslogger.info("Stage 3/4: Embedding chunks (device=%s)...", device)- if device == "cpu":- # Use CPU embedder for machines without GPU+ if device in ("cpu", "mps", "auto"):+ # Local embedder: CPU, Apple MPS, or auto-detectcmd = [sys.executable,"-m",@@ -142,6 +142,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:str(tiles_dir),"--output-dir",str(embeddings_dir),+ "--device",+ device,]if "model" in embed_cfg:cmd += ["--model", embed_cfg["model"]]diff --git a/render/src/pixelrag_render/backends/pdf.py b/render/src/pixelrag_render/backends/pdf.pyindex 87b9849..8897fcf 100644--- a/render/src/pixelrag_render/backends/pdf.py+++ b/render/src/pixelrag_render/backends/pdf.py@@ -76,6 +76,7 @@ def render_pdf(images = convert_from_path(**convert_kwargs)saved_tiles: list[str] = []+ chunks_info: list[dict] = []for idx, img in enumerate(images):# If caller provided a sparse page list, skip pages not in the listif pages is not None:@@ -87,7 +88,18 @@ def render_pdf(tile_path = tile_dir / tile_nameimg.save(str(tile_path), "JPEG", quality=quality)saved_tiles.append(tile_name)- logger.debug(" Page %d → %s (%dx%d)", idx, tile_name, *img.size)+ w, h = img.size+ # Each PDF page = one chunk (no further splitting)+ chunks_info.append({+ "tile": tile_name,+ "tile_index": idx,+ "chunk_index": 0,+ "file": tile_name,+ "y_offset": 0,+ "height": h,+ "width": w,+ })+ logger.debug(" Page %d → %s (%dx%d)", idx, tile_name, w, h)manifest = {"source": str(path),@@ -99,5 +111,19 @@ def render_pdf(with open(tile_dir / "tiles.json", "w") as f:json.dump(manifest, f)+ # Write chunks.json so the chunker skips this directory —+ # each PDF page is already a natural semantic unit.+ chunks_manifest = {+ "page_height": 0,+ "viewport_width": images[0].size[0] if images else 0,+ "tile_height": images[0].size[1] if images else 0,+ "chunk_height": images[0].size[1] if images else 0,+ "num_tiles": len(saved_tiles),+ "num_chunks": len(chunks_info),+ "chunks": chunks_info,+ }+ with open(tile_dir / "chunks.json", "w") as f:+ json.dump(chunks_manifest, f)+logger.info("PDF rendered: %d pages → %s", len(saved_tiles), tile_dir)return [tile_dir]