Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(embed): resize wide PDF chunks instead of skipping them (#51)

Yichuan Wangbe0a9412821b
diff --git a/embed/src/pixelrag_embed/embed.py b/embed/src/pixelrag_embed/embed.py
index 1936241..eea8ce7 100644
--- a/embed/src/pixelrag_embed/embed.py
+++ b/embed/src/pixelrag_embed/embed.py
@@ -111,6 +111,7 @@ def resolve_gpu_ids(gpu_ids_arg: str) -> list[int]:
_RESIZE_FACTOR = 28 # Qwen3-VL patch alignment
+_MAX_CHUNK_WIDTH = 875 # web viewport width; wider images (e.g. PDF tiles) are resized
def _smart_resize_pil(img: "Image.Image", max_pixels: int) -> "Image.Image":
@@ -127,6 +128,21 @@ def _smart_resize_pil(img: "Image.Image", max_pixels: int) -> "Image.Image":
return img.resize((new_w, new_h), Image.LANCZOS)
+def _clamp_width_pil(img: "Image.Image", max_width: int = _MAX_CHUNK_WIDTH) -> "Image.Image":
+ """Resize image so width <= max_width, preserving aspect ratio.
+
+ Dimensions are rounded to multiples of 28 (Qwen3-VL patch alignment).
+ Used for PDF tiles that render wider than the web viewport.
+ """
+ w, h = img.size
+ if w <= max_width:
+ return img
+ scale = max_width / w
+ new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
+ new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
+ return img.resize((new_w, new_h), Image.LANCZOS)
+
+
def save_npz(path: str, compressed: bool, **arrays) -> None:
"""Save npz with optional compression."""
if compressed:
@@ -881,7 +897,7 @@ def _embed_tile_infos_with_engine(
unique_queue: queue.Queue = queue.Queue(maxsize=batch_size * 2)
producer_error: list[Exception | None] = [None]
- skipped_oversized = [0] # mutable counter for threads
+ resized_wide = [0] # mutable counter for threads
def _io_producer() -> None:
"""Read tiles, chunk tall images, hash chunks, skip decode for duplicates."""
@@ -920,15 +936,14 @@ def _embed_tile_infos_with_engine(
cw / max(ch, 1),
)
continue
- if cw > 875:
- logger.warning(
- "Skipping oversized chunk %s ci=%d width %d > 875",
- img_path,
- ci,
- cw,
+ if cw > _MAX_CHUNK_WIDTH:
+ chunk_img = _clamp_width_pil(chunk_img, _MAX_CHUNK_WIDTH)
+ cw, ch = chunk_img.size
+ resized_wide[0] += 1
+ logger.debug(
+ "Resized wide chunk %s ci=%d to %dx%d",
+ img_path, ci, cw, ch,
)
- skipped_oversized[0] += 1
- continue
# Dedup key: use file path for pre-chunked files (avoids
# expensive tobytes + MD5 on ~2.7MB raw pixels per chunk)
if isinstance(ti, ChunkInfo):
@@ -1056,15 +1071,16 @@ def _embed_tile_infos_with_engine(
deduped = len(tile_hashes) - unique_total
100.0 * deduped / len(tile_hashes) if tile_hashes else 0.0
- n_skipped = skipped_oversized[0]
+ n_resized = resized_wide[0]
logger.info(
- "GPU %d: %d images, %d unique (%d deduped, %d skipped>875), embedded %d, pipeline %.2fs (%.1f chunks/s)"
+ "GPU %d: %d images, %d unique (%d deduped, %d resized>%d), embedded %d, pipeline %.2fs (%.1f chunks/s)"
" | embed=%.2fs (%d batches) queue_wait=%.2fs first_item=%.3fs",
gpu_id,
len(tile_hashes),
unique_total,
deduped,
- n_skipped,
+ n_resized,
+ _MAX_CHUNK_WIDTH,
embedded,
pipeline_s,
embedded / pipeline_s if pipeline_s > 0 else 0,
diff --git a/embed/src/pixelrag_embed/embed_cpu.py b/embed/src/pixelrag_embed/embed_cpu.py
index 15965f3..e782e4e 100644
--- a/embed/src/pixelrag_embed/embed_cpu.py
+++ b/embed/src/pixelrag_embed/embed_cpu.py
@@ -1,14 +1,17 @@
#!/usr/bin/env python3
-"""CPU embedding: embed tile chunks using transformers on CPU.
+"""Local embedding: embed tile chunks using transformers on CPU or Apple MPS.
-Slower than GPU backends (vLLM/sglang) but works without CUDA.
-Suitable for small-scale demos and testing.
+Works without CUDA — suitable for macOS (Apple Silicon), small-scale demos,
+and testing.
Usage:
+ # CPU (any platform)
python -m pixelrag_embed.embed_cpu \
- --shard-dir ./tiles \
- --output-dir ./embeddings \
- --model Qwen/Qwen3-VL-Embedding-2B
+ --shard-dir ./tiles --output-dir ./embeddings
+
+ # Apple Silicon GPU (macOS)
+ python -m pixelrag_embed.embed_cpu \
+ --shard-dir ./tiles --output-dir ./embeddings --device mps
"""
import argparse
@@ -22,10 +25,37 @@ from PIL import Image
from tqdm import tqdm
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
-logger = logging.getLogger("embed_cpu")
+logger = logging.getLogger("embed_local")
Image.MAX_IMAGE_PIXELS = None
+_RESIZE_FACTOR = 28
+_MAX_CHUNK_WIDTH = 875
+
+
+def _clamp_width(img: Image.Image, max_width: int = _MAX_CHUNK_WIDTH) -> Image.Image:
+ """Resize so width <= max_width, preserving aspect ratio (28px alignment)."""
+ w, h = img.size
+ if w <= max_width:
+ return img
+ scale = max_width / w
+ new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
+ new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
+ return img.resize((new_w, new_h), Image.LANCZOS)
+
+
+def _resolve_device(device: str) -> str:
+ """Resolve device string, auto-detecting MPS on macOS."""
+ import torch
+
+ if device == "auto":
+ if torch.cuda.is_available():
+ return "cuda"
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ return "mps"
+ return "cpu"
+ return device
+
def scan_chunks(shard_dir: str) -> list[dict]:
"""Scan for chunk images in a shard directory.
@@ -38,7 +68,6 @@ def scan_chunks(shard_dir: str) -> list[dict]:
for entry in sorted(shard.iterdir()):
if not entry.is_dir():
continue
- # Support both flat (*.png.tiles/) and nested (sub_shard/*/**.png.tiles/)
tile_dirs = []
if entry.name.endswith(".png.tiles"):
tile_dirs = [entry]
@@ -97,21 +126,29 @@ def scan_chunks(shard_dir: str) -> list[dict]:
def embed_items(
- items: list[dict], model_name: str, instruction: str = ""
+ items: list[dict],
+ model_name: str,
+ device: str = "cpu",
+ instruction: str = "",
) -> np.ndarray:
- """Embed a list of image items using transformers on CPU."""
+ """Embed image items using transformers on the given device."""
import torch
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
- logger.info("Loading model %s on CPU...", model_name)
+ device = _resolve_device(device)
+ dtype = torch.float32 if device == "cpu" else torch.float16
+
+ logger.info("Loading model %s on %s (%s)...", model_name, device, dtype)
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
model = Qwen3VLForConditionalGeneration.from_pretrained(
model_name,
trust_remote_code=True,
- dtype=torch.float32,
+ torch_dtype=dtype,
attn_implementation="sdpa",
).eval()
- logger.info("Model loaded")
+ if device != "cpu":
+ model = model.to(device)
+ logger.info("Model loaded on %s", device)
dim = model.config.text_config.hidden_size
embeddings = np.zeros((len(items), dim), dtype=np.float16)
@@ -120,6 +157,7 @@ def embed_items(
for i, item in enumerate(tqdm(items, desc="Embedding")):
img = Image.open(item["path"]).convert("RGB")
+ img = _clamp_width(img)
messages = [
{
@@ -135,17 +173,17 @@ def embed_items(
messages, tokenize=False, add_generation_prompt=True
)
inputs = processor(text=[text], images=[img], return_tensors="pt", padding=True)
+ if device != "cpu":
+ inputs = {k: v.to(device) if hasattr(v, "to") else v for k, v in inputs.items()}
with torch.no_grad():
outputs = model(**inputs, output_hidden_states=True)
last_hidden = outputs.hidden_states[-1]
- # Last token pooling
seq_lens = inputs["attention_mask"].sum(dim=1)
last_idx = seq_lens - 1
pooled = last_hidden[0, last_idx[0]]
- # L2 normalize
pooled = pooled / pooled.norm()
- embeddings[i] = pooled.numpy().astype(np.float16)
+ embeddings[i] = pooled.cpu().numpy().astype(np.float16)
if (i + 1) % 10 == 0:
logger.info("Embedded %d/%d", i + 1, len(items))
@@ -154,12 +192,18 @@ def embed_items(
def main():
- parser = argparse.ArgumentParser(description="CPU embedding for tile chunks")
+ parser = argparse.ArgumentParser(description="Local embedding (CPU / MPS / CUDA)")
parser.add_argument(
"--shard-dir", required=True, help="Directory with *.png.tiles/ subdirs"
)
parser.add_argument("--output-dir", required=True, help="Output directory for .npz")
parser.add_argument("--model", default="Qwen/Qwen3-VL-Embedding-2B")
+ parser.add_argument(
+ "--device",
+ default="auto",
+ choices=["auto", "cpu", "mps", "cuda"],
+ help="Device (default: auto-detect)",
+ )
parser.add_argument(
"--instruction", default="", help="Instruction prefix for queries"
)
@@ -178,9 +222,8 @@ def main():
items = items[: args.limit]
else:
logger.info("Found %d chunks to embed", len(items))
- embeddings = embed_items(items, args.model, args.instruction)
+ embeddings = embed_items(items, args.model, device=args.device, instruction=args.instruction)
- # Save NPZ in same format as embed.py
output_path = Path(args.output_dir) / "shard_000.npz"
np.savez(
output_path,
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py
index 038d3a3..ab0fea7 100644
--- a/index/src/pixelrag_index/pipelines.py
+++ b/index/src/pixelrag_index/pipelines.py
@@ -132,8 +132,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
# Stage 3: Embed chunks to vectors
logger.info("Stage 3/4: Embedding chunks (device=%s)...", device)
- if device == "cpu":
- # Use CPU embedder for machines without GPU
+ if device in ("cpu", "mps", "auto"):
+ # Local embedder: CPU, Apple MPS, or auto-detect
cmd = [
sys.executable,
"-m",
@@ -142,6 +142,8 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
str(tiles_dir),
"--output-dir",
str(embeddings_dir),
+ "--device",
+ device,
]
if "model" in embed_cfg:
cmd += ["--model", embed_cfg["model"]]
diff --git a/render/src/pixelrag_render/backends/pdf.py b/render/src/pixelrag_render/backends/pdf.py
index 87b9849..8897fcf 100644
--- a/render/src/pixelrag_render/backends/pdf.py
+++ b/render/src/pixelrag_render/backends/pdf.py
@@ -76,6 +76,7 @@ def render_pdf(
images = convert_from_path(**convert_kwargs)
saved_tiles: list[str] = []
+ chunks_info: list[dict] = []
for idx, img in enumerate(images):
# If caller provided a sparse page list, skip pages not in the list
if pages is not None:
@@ -87,7 +88,18 @@ def render_pdf(
tile_path = tile_dir / tile_name
img.save(str(tile_path), "JPEG", quality=quality)
saved_tiles.append(tile_name)
- logger.debug(" Page %d → %s (%dx%d)", idx, tile_name, *img.size)
+ w, h = img.size
+ # Each PDF page = one chunk (no further splitting)
+ chunks_info.append({
+ "tile": tile_name,
+ "tile_index": idx,
+ "chunk_index": 0,
+ "file": tile_name,
+ "y_offset": 0,
+ "height": h,
+ "width": w,
+ })
+ logger.debug(" Page %d → %s (%dx%d)", idx, tile_name, w, h)
manifest = {
"source": str(path),
@@ -99,5 +111,19 @@ def render_pdf(
with open(tile_dir / "tiles.json", "w") as f:
json.dump(manifest, f)
+ # Write chunks.json so the chunker skips this directory —
+ # each PDF page is already a natural semantic unit.
+ chunks_manifest = {
+ "page_height": 0,
+ "viewport_width": images[0].size[0] if images else 0,
+ "tile_height": images[0].size[1] if images else 0,
+ "chunk_height": images[0].size[1] if images else 0,
+ "num_tiles": len(saved_tiles),
+ "num_chunks": len(chunks_info),
+ "chunks": chunks_info,
+ }
+ with open(tile_dir / "chunks.json", "w") as f:
+ json.dump(chunks_manifest, f)
+
logger.info("PDF rendered: %d pages → %s", len(saved_tiles), tile_dir)
return [tile_dir]