https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
#!/usr/bin/env python3
"""Core embedding pipeline: scan tiles from a shard, embed with vLLM, write .npz.
Usage (single shard):
uv run embed_tiles.py \
--shard-dir /opt/dlami/nvme/wiki-screenshot/output_coordinated/shard_042 \
--output-dir ./output/shard_042 --gpu-ids 0
Multi-GPU:
uv run embed_tiles.py \
--shard-dir .../shard_042 --output-dir ./output/shard_042 --gpu-ids 0,1,2,3
Chunk embedding (default, recommended):
Each 8192px tile is pre-split into 1024px strips (chunks) by the tiling pipeline.
Chunks are stored alongside tiles in *.png.tiles/chunks.json.
Embedding chunks instead of full tiles reduces the visual token count ~8x,
significantly improving throughput.
uv run embed_tiles.py \
--shard-dir .../shard_042 \
--output-dir ./output/shard_042 \
--gpu-ids 0,1,2,3 \
--mode chunks # default, can be omitted
--backend sglang \
--batch-size 128
Output npz arrays per chunk:
embeddings float16 [N, D] — embedding vector
article_ids int64 [N] — Wikipedia article ID
tile_indices int32 [N] — which 8192px tile (0-based)
chunk_indices int32 [N] — which 1024px strip within tile (0-based)
y_offsets int32 [N] — Y position of chunk top edge in page (px)
tile_heights int32 [N] — actual chunk height (last chunk may be <1024px)
page_heights int32 [N] — full page height (px)
viewport_widths int32 [N] — page render width (px, capped at 875)
image_hashes S32 [N] — MD5 of source PNG (used for dedup & patching)
tile_paths S512 [N] — absolute path to chunk PNG file
shard_id int32 scalar — shard number
Lookup key: (article_id, tile_index, chunk_index) — lexsorted in output.
"""
import argparse
import atexit
import hashlib
import io
import json
import logging
# Import multiprocessing at module level BEFORE atexit so mp's own
# _exit_function (which joins non-daemon children without timeout) is
# registered FIRST. Our _close_all_persistent_pools is registered last,
# so atexit LIFO order runs ours first — we force-kill stragglers before
# mp's handler can hang on them.
import multiprocessing
import multiprocessing.util # noqa: F401
import os
import queue
import subprocess
import threading
import time
import traceback
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import NamedTuple
import numpy as np
from PIL import Image
from tqdm import tqdm
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger("embed_tiles")
logger.setLevel(logging.INFO)
_PERSISTENT_POOLS: dict[tuple, "PersistentGpuWorkerPool"] = {}
def resolve_gpu_ids(gpu_ids_arg: str) -> list[int]:
"""Resolve GPU IDs from CLI arg.
Supports:
- "all" / "auto": use all visible GPUs
- comma-separated IDs, e.g. "0,1,2,3"
"""
value = gpu_ids_arg.strip().lower()
if value in {"all", "auto"}:
visible = os.environ.get("CUDA_VISIBLE_DEVICES", "").strip()
if visible:
parts = [p.strip() for p in visible.split(",") if p.strip()]
if parts:
# Under CUDA_VISIBLE_DEVICES, local IDs are 0..N-1.
return list(range(len(parts)))
try:
out = subprocess.check_output(
["nvidia-smi", "--query-gpu=index", "--format=csv,noheader"],
text=True,
)
lines = [ln.strip() for ln in out.splitlines() if ln.strip()]
if lines:
return list(range(len(lines)))
except Exception:
logger.warning(
"Failed to detect GPUs via nvidia-smi; falling back to GPU 0"
)
return [0]
return [int(g.strip()) for g in gpu_ids_arg.split(",") if g.strip()]
_RESIZE_FACTOR = 28 # Qwen3-VL patch alignment
_MAX_CHUNK_WIDTH = 875 # web viewport width; wider images (e.g. PDF tiles) are resized
def _smart_resize_pil(img: "Image.Image", max_pixels: int) -> "Image.Image":
"""Resize image to fit within max_pixels, preserving aspect ratio.
Dimensions are rounded to multiples of 28 (Qwen3-VL patch alignment).
"""
w, h = img.size
if w * h <= max_pixels:
return img
scale = (max_pixels / (w * h)) ** 0.5
new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
return img.resize((new_w, new_h), Image.LANCZOS)
def _clamp_width_pil(
img: "Image.Image", max_width: int = _MAX_CHUNK_WIDTH
) -> "Image.Image":
"""Resize image so width <= max_width, preserving aspect ratio.
Dimensions are rounded to multiples of 28 (Qwen3-VL patch alignment).
Used for PDF tiles that render wider than the web viewport.
"""
w, h = img.size
if w <= max_width:
return img
scale = max_width / w
new_w = max(round(w * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
new_h = max(round(h * scale / _RESIZE_FACTOR) * _RESIZE_FACTOR, _RESIZE_FACTOR)
return img.resize((new_w, new_h), Image.LANCZOS)
def save_npz(path: str, compressed: bool, **arrays) -> None:
"""Save npz with optional compression."""
if compressed:
np.savez_compressed(path, **arrays)
else:
np.savez(path, **arrays)
# ---------------------------------------------------------------------------
# Redirect filtering
# ---------------------------------------------------------------------------
def load_redirect_ids(path: str) -> set[int]:
"""Load redirect article IDs from a .redirects.json file.
The file is a dict mapping article index (str) to target path (str).
We only need the keys (indices of redirect articles).
Args:
path: Path to the .redirects.json file.
Returns:
Set of article indices that are client-side redirects.
"""
with open(path, "r") as f:
redirects = json.load(f)
ids = {int(k) for k in redirects}
logger.info("Loaded %d redirect IDs from %s", len(ids), path)
return ids
# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------
class TileInfo(NamedTuple):
"""Metadata for a single tile image (or a chunk of one)."""
article_id: int
tile_index: int
tile_path: str
page_height: int
viewport_width: int
tile_height: int
chunk_index: int = 0 # 0 = whole tile or first chunk
class ChunkInfo(NamedTuple):
"""Metadata for a single chunk image (1024px strip of a tile)."""
article_id: int
tile_index: int
chunk_index: int
chunk_path: str
page_height: int
viewport_width: int
y_offset: int
chunk_height: int
def _image_path(ti: "TileInfo | ChunkInfo") -> str:
"""Return the image file path regardless of info type."""
return ti.tile_path if isinstance(ti, TileInfo) else ti.chunk_path
# ---------------------------------------------------------------------------
# Tile scanning
# ---------------------------------------------------------------------------
def scan_shard_tiles(
shard_dir: str,
skip_article_ids: set[int] | None = None,
) -> list[TileInfo]:
"""Walk a shard directory and collect all completed tiles.
Looks for ``*.png.tiles/tiles.json`` files with ``complete: true``.
Each tile PNG listed in tiles.json becomes one TileInfo entry.
Args:
shard_dir: Path to a shard directory (e.g. output_coordinated/shard_042).
skip_article_ids: Article IDs to skip (already embedded).
Returns:
Sorted list of TileInfo (by article_id, then tile_index).
"""
shard_path = Path(shard_dir)
skip = skip_article_ids or set()
tiles: list[TileInfo] = []
# Shard directories contain sub-shard dirs (shard_00000, shard_00001, ...)
# which contain article tile dirs (NNN.png.tiles/)
tile_dirs: list[Path] = []
for entry in sorted(shard_path.iterdir()):
if entry.is_dir() and entry.name.startswith("shard_"):
# Sub-shard directory
for sub_entry in sorted(entry.iterdir()):
if sub_entry.is_dir() and sub_entry.name.endswith(".png.tiles"):
tile_dirs.append(sub_entry)
elif entry.is_dir() and entry.name.endswith(".png.tiles"):
# Direct tile dir (flat shard layout)
tile_dirs.append(entry)
for tiles_dir in tile_dirs:
tiles_json = tiles_dir / "tiles.json"
if not tiles_json.exists():
continue
try:
meta = json.loads(tiles_json.read_text())
except (json.JSONDecodeError, OSError) as e:
logger.warning("Skipping %s: %s", tiles_json, e)
continue
if not meta.get("complete", False):
continue
article_id = meta.get("article_id")
if article_id is None:
dir_name = tiles_dir.name
try:
article_id = int(dir_name.split(".")[0])
except (ValueError, IndexError):
logger.warning("Cannot parse article_id from %s", dir_name)
continue
if article_id in skip:
continue
page_height = meta.get("page_height", 0)
viewport_width = meta.get("viewport_width", 875)
tile_height = meta.get("tile_height", 8192)
for idx, tile_name in enumerate(meta.get("tiles", [])):
tile_path = tiles_dir / tile_name
if tile_path.exists():
tiles.append(
TileInfo(
article_id=article_id,
tile_index=idx,
tile_path=str(tile_path),
page_height=page_height,
viewport_width=viewport_width,
tile_height=tile_height,
)
)
tiles.sort(key=lambda t: (t.article_id, t.tile_index))
logger.info(
"Scanned %s: %d tiles from %d articles",
shard_dir,
len(tiles),
len({t.article_id for t in tiles}),
)
return tiles
def scan_shard_chunks(
shard_dir: str,
skip_article_ids: set[int] | None = None,
) -> list[ChunkInfo]:
"""Walk a shard directory and collect all chunk images.
Looks for ``*.png.tiles/chunks.json`` files. Each entry in
``meta["chunks"]`` becomes one ChunkInfo.
Args:
shard_dir: Path to a shard directory.
skip_article_ids: Article IDs to skip (already embedded).
Returns:
Sorted list of ChunkInfo (by article_id, tile_index, chunk_index).
"""
shard_path = Path(shard_dir)
skip = skip_article_ids or set()
chunks: list[ChunkInfo] = []
tile_dirs: list[Path] = []
for entry in sorted(shard_path.iterdir()):
if entry.is_dir() and entry.name.startswith("shard_"):
for sub_entry in sorted(entry.iterdir()):
if sub_entry.is_dir() and sub_entry.name.endswith(".png.tiles"):
tile_dirs.append(sub_entry)
elif entry.is_dir() and entry.name.endswith(".png.tiles"):
tile_dirs.append(entry)
for tiles_dir in tile_dirs:
chunks_json = tiles_dir / "chunks.json"
if not chunks_json.exists():
continue
try:
meta = json.loads(chunks_json.read_text())
except (json.JSONDecodeError, OSError) as e:
logger.warning("Skipping %s: %s", chunks_json, e)
continue
article_id = meta.get("article_id")
if article_id is None:
# chunks.json predates the article_id contract — try the sibling
# tiles.json (CPU embedder does the same), then the directory name.
tiles_json = tiles_dir / "tiles.json"
if tiles_json.exists():
try:
article_id = json.loads(tiles_json.read_text()).get("article_id")
except (json.JSONDecodeError, OSError):
pass
if article_id is None:
dir_name = tiles_dir.name
try:
article_id = int(dir_name.split(".")[0])
except (ValueError, IndexError):
logger.warning("Cannot parse article_id from %s", dir_name)
continue
if article_id in skip:
continue
page_height = meta.get("page_height", 0)
viewport_width = meta.get("viewport_width", 875)
for chunk in meta.get("chunks", []):
chunk_path = tiles_dir / chunk["file"]
if chunk_path.exists():
chunks.append(
ChunkInfo(
article_id=article_id,
tile_index=chunk["tile_index"],
chunk_index=chunk["chunk_index"],
chunk_path=str(chunk_path),
page_height=page_height,
viewport_width=viewport_width,
y_offset=chunk["y_offset"],
chunk_height=chunk["height"],
)
)
chunks.sort(key=lambda c: (c.article_id, c.tile_index, c.chunk_index))
logger.info(
"Scanned %s: %d chunks from %d articles",
shard_dir,
len(chunks),
len({c.article_id for c in chunks}),
)
return chunks
# ---------------------------------------------------------------------------
# Tile reading and hashing (backend-agnostic)
# ---------------------------------------------------------------------------
def read_tiles_and_hash(
tile_infos: list[TileInfo],
) -> list[tuple[TileInfo, str, "Image.Image"]]:
"""Read tile files, compute MD5 hashes, and load PIL images.
Returns:
List of (tile_info, md5_hex, pil_image) tuples for successfully loaded tiles.
"""
results = []
for ti in tile_infos:
try:
raw = Path(ti.tile_path).read_bytes()
md5_hex = hashlib.md5(raw).hexdigest()
img = Image.open(io.BytesIO(raw)).convert("RGB")
results.append((ti, md5_hex, img))
except Exception as e:
logger.warning("Failed to load %s: %s", ti.tile_path, e)
return results
def _read_hash_decode_one(
ti: TileInfo,
) -> tuple[TileInfo, str, "Image.Image"] | None:
"""Read one tile, compute hash, and decode into RGB image."""
try:
raw = Path(ti.tile_path).read_bytes()
md5_hex = hashlib.md5(raw).hexdigest()
# Decode from in-memory bytes to avoid a second filesystem read.
img = Image.open(io.BytesIO(raw)).convert("RGB")
return (ti, md5_hex, img)
except Exception as e:
logger.warning("Failed to load %s: %s", ti.tile_path, e)
return None
def read_tiles_and_hash_parallel(
tile_infos: list[TileInfo],
io_workers: int = 8,
) -> list[tuple[TileInfo, str, "Image.Image"]]:
"""Parallel tile read/hash/decode using a thread pool."""
if io_workers <= 1 or len(tile_infos) <= 1:
return read_tiles_and_hash(tile_infos)
results: list[tuple[TileInfo, str, Image.Image]] = []
with ThreadPoolExecutor(max_workers=io_workers) as pool:
for item in pool.map(_read_hash_decode_one, tile_infos, chunksize=32):
if item is not None:
results.append(item)
return results
DEFAULT_INSTRUCTION = "Represent the user's input."
# Module-level instruction used by all embedding backends.
# Set via set_instruction() before starting workers to change from default.
_INSTRUCTION = DEFAULT_INSTRUCTION
def set_instruction(instruction: str) -> None:
"""Set the embedding instruction for all backends."""
global _INSTRUCTION
_INSTRUCTION = instruction
logger.info("Embedding instruction set to: %r", instruction)
def _build_chat_prompt(tokenizer, instruction: str | None = None) -> str:
"""Build the embedding prompt using the model's chat template.
Uses the official Qwen3-VL-Embedding prompt format:
system: <instruction>
user: [image]
"""
instr = instruction if instruction is not None else _INSTRUCTION
conversation = [
{"role": "system", "content": [{"type": "text", "text": instr}]},
{"role": "user", "content": [{"type": "image"}]},
]
return tokenizer.apply_chat_template(
conversation,
tokenize=False,
add_generation_prompt=True,
)
# ---------------------------------------------------------------------------
# Backend: vLLM
# ---------------------------------------------------------------------------
def _init_vllm(model_path: str, gpu_id: int, enforce_eager: bool = False):
"""Initialize vLLM embedding engine on a single GPU."""
from vllm import LLM, EngineArgs
from vllm.config import PoolerConfig
llm = LLM(
**vars(
EngineArgs(
model=model_path,
runner="pooling",
dtype="bfloat16",
trust_remote_code=True,
max_model_len=4096,
enforce_eager=enforce_eager,
pooler_config=PoolerConfig(pooling_type="LAST"),
)
)
)
return llm
def _get_tokenizer_vllm(engine):
# vLLM API changed across versions:
# - older: engine.llm_engine.tokenizer.tokenizer
# - newer: engine.get_tokenizer()
if hasattr(engine, "get_tokenizer"):
return engine.get_tokenizer()
return engine.llm_engine.tokenizer.tokenizer
def _embed_vllm(engine, prompt: str, images: list["Image.Image"]) -> list[np.ndarray]:
"""Run embedding on a batch of images via vLLM."""
inputs = [{"prompt": prompt, "multi_modal_data": {"image": img}} for img in images]
outputs = engine.embed(inputs)
# L2 normalize (vLLM pooler does not normalize internally)
embs = np.array([out.outputs.embedding for out in outputs], dtype=np.float32)
norms = np.linalg.norm(embs, axis=1, keepdims=True)
embs = embs / np.maximum(norms, 1e-12)
return [embs[i].astype(np.float16) for i in range(len(embs))]
# ---------------------------------------------------------------------------
# Backend: SGLang
# ---------------------------------------------------------------------------
def _init_sglang(model_path: str, gpu_id: int, enforce_eager: bool = False):
"""Initialize SGLang embedding engine on a single GPU.
Key settings for vision embedding throughput on H100:
- disable_cuda_graph: no decode phase in embedding → graphs waste memory
- chunked_prefill_size=16384: large prefill chunks for batched vision tokens
- max_prefill_tokens=65536: high ceiling so scheduler doesn't throttle
- keep_mm_feature_on_device=True: avoid GPU→CPU→GPU copies of ViT features
- mm_max_concurrent_calls=64: parallel image preprocessing
- max_running_requests=512: pack more embedding requests
- schedule_conservativeness=0.3: admit requests aggressively (KV freed instantly)
- mem_fraction_static=0.80: safe with cuda graphs disabled
- disable_radix_cache=True: embedding has no KV reuse
"""
from sglang.srt.entrypoints.engine import Engine
os.environ.setdefault("SGLANG_LOG_LEVEL", "error")
return Engine(
model_path=model_path,
is_embedding=True,
dtype="bfloat16",
trust_remote_code=True,
# Prefill tuning
chunked_prefill_size=16384,
max_prefill_tokens=65536,
# Memory — no CUDA graphs frees several GB for activations
mem_fraction_static=0.82,
disable_radix_cache=True,
disable_cuda_graph=True,
# Vision
keep_mm_feature_on_device=True,
mm_max_concurrent_calls=64,
# Scheduling — aggressive for embedding (no decode, KV freed instantly)
max_running_requests=512,
schedule_conservativeness=0.3,
)
def _get_tokenizer_sglang(engine):
return engine.tokenizer_manager.tokenizer
def _embed_sglang(engine, prompt: str, images: list["Image.Image"]) -> list[np.ndarray]:
"""Run embedding on a batch via SGLang engine.encode().
SGLang encode() takes prompts + image_data separately.
Image data is passed as PIL Images (supported since sglang 0.5.x).
"""
prompts = [prompt] * len(images)
outputs = engine.encode(prompts, image_data=images)
# L2 normalize (sglang encode() does not normalize internally)
embs = np.array([out["embedding"] for out in outputs], dtype=np.float32)
norms = np.linalg.norm(embs, axis=1, keepdims=True)
embs = embs / np.maximum(norms, 1e-12)
return [embs[i].astype(np.float16) for i in range(len(embs))]
# ---------------------------------------------------------------------------
# Backend: direct_gpu (transformers + GPU-accelerated preprocessing)
# ---------------------------------------------------------------------------
def _init_direct_gpu(
model_path: str,
gpu_id: int,
enforce_eager: bool = False,
adapter_path: str | None = None,
):
"""Load model + processor for direct GPU inference.
Uses Qwen3VLForConditionalGeneration (not AutoModel, which loads the
base Qwen3VLModel with uninitialized language_model weights).
GPU-accelerated image preprocessing (device='cuda' in processor) avoids
the CPU preprocessing bottleneck: 0.2s vs 12s per batch of 64.
If adapter_path is given, loads a PEFT LoRA adapter and merges it into
the base weights (merge_and_unload) so inference runs at base-model speed.
"""
import torch
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
model = Qwen3VLForConditionalGeneration.from_pretrained(
model_path,
trust_remote_code=True,
torch_dtype=torch.bfloat16,
)
if adapter_path:
import re as _re
from peft import PeftModel, set_peft_model_state_dict
from safetensors.torch import load_file as load_safetensors
logger.info("GPU %d: loading LoRA adapter from %s", gpu_id, adapter_path)
# The adapter was trained on Qwen3VLModel (BiQwen3), where attention
# layers live at language_model.layers.*. But Qwen3VLForConditional-
# Generation wraps that inside self.model, so PEFT expects an extra
# "model." prefix. Remap keys before loading.
adapter_weights_path = os.path.join(adapter_path, "adapter_model.safetensors")
raw_sd = load_safetensors(adapter_weights_path)
remapped_sd = {}
for k, v in raw_sd.items():
# base_model.model.language_model.* -> base_model.model.model.language_model.*
new_k = _re.sub(
r"^(base_model\.model\.)(language_model\.|visual\.)", r"\1model.\2", k
)
remapped_sd[new_k] = v
n_remapped = sum(
1 for ok, nk in zip(raw_sd.keys(), remapped_sd.keys()) if ok != nk
)
logger.info(
"GPU %d: remapped %d/%d adapter keys (BiQwen3 -> ConditionalGeneration)",
gpu_id,
n_remapped,
len(raw_sd),
)
model = PeftModel.from_pretrained(model, adapter_path)
set_peft_model_state_dict(model, remapped_sd)
model = model.merge_and_unload()
logger.info("GPU %d: LoRA merged into base weights", gpu_id)
model = model.cuda().eval()
# WORKAROUND: torch 2.9.x Conv3d bf16 bug (pytorch/pytorch#166122) —
# cuDNN 9.8–9.14 disabled for 3D conv, vol2col fallback is 16,400x slower for bf16.
# Fixed in cuDNN >= 9.15. Only apply fp32 workaround if needed.
cudnn_ver = torch.backends.cudnn.version()
if cudnn_ver < 91500:
_pe = model.model.visual.patch_embed
def _fp32_patch_embed(hidden_states, _pe=_pe):
conv = _pe.proj
x = hidden_states.view(
-1,
_pe.in_channels,
_pe.temporal_patch_size,
_pe.patch_size,
_pe.patch_size,
)
old_w, old_b = conv.weight.data, conv.bias.data
conv.weight.data = old_w.float()
conv.bias.data = old_b.float()
out = conv(x.float()).view(-1, _pe.embed_dim)
conv.weight.data = old_w
conv.bias.data = old_b
return out.to(torch.bfloat16)
_pe.forward = _fp32_patch_embed
logger.info(f"Applied fp32 Conv3d workaround (cuDNN {cudnn_ver} < 91500)")
else:
logger.info(f"Native bf16 Conv3d (cuDNN {cudnn_ver} >= 91500, bug fixed)")
return (model, processor)
def _get_tokenizer_direct_gpu(engine):
_, processor = engine
return processor.tokenizer
def _embed_direct_gpu(
engine, prompt: str, images: list["Image.Image"]
) -> list[np.ndarray]:
"""Embed images using direct model forward with GPU preprocessing.
The prompt arg is the chat template text (same format as sglang).
We pass device='cuda' to the processor so all image tensor ops
(resize, normalize, stack, cat) run on GPU instead of CPU.
"""
import torch
model, processor = engine
# Build per-image message format for the processor
messages_batch = [
[
{"role": "system", "content": [{"type": "text", "text": _INSTRUCTION}]},
{"role": "user", "content": [{"type": "image", "image": img}]},
]
for img in images
]
texts = [
processor.apply_chat_template(m, tokenize=False, add_generation_prompt=True)
for m in messages_batch
]
# GPU-accelerated preprocessing: device="cuda" moves resize/normalize/stack to GPU
inputs = processor(
text=texts, images=images, return_tensors="pt", padding=True, device="cuda"
)
inputs = {k: v.to("cuda") if hasattr(v, "to") else v for k, v in inputs.items()}
with torch.no_grad():
outputs = model.model(**inputs)
# Last-token pooling — use last_hidden_state (post-RMSNorm)
last_hidden = outputs.last_hidden_state # (batch, seq_len, hidden_dim)
attention_mask = inputs["attention_mask"] # (batch, seq_len)
# Find the last non-padding token for each sequence
last_token_indices = attention_mask.sum(dim=1) - 1 # (batch,)
pooled = last_hidden[
torch.arange(last_hidden.size(0), device=last_hidden.device), last_token_indices
]
# L2 normalize
pooled = torch.nn.functional.normalize(pooled, p=2, dim=-1)
return [emb.cpu().float().numpy().astype(np.float16) for emb in pooled]
# ---------------------------------------------------------------------------
# Backend dispatch
# ---------------------------------------------------------------------------
BACKENDS = {
"vllm": (_init_vllm, _get_tokenizer_vllm, _embed_vllm),
"sglang": (_init_sglang, _get_tokenizer_sglang, _embed_sglang),
"direct_gpu": (_init_direct_gpu, _get_tokenizer_direct_gpu, _embed_direct_gpu),
}
# ---------------------------------------------------------------------------
# Single-GPU worker (runs in a subprocess via torch.multiprocessing)
# ---------------------------------------------------------------------------
def gpu_worker(
gpu_id: int,
tile_infos: list[TileInfo],
model_path: str,
batch_size: int,
result_dir: str,
backend: str = "vllm",
io_workers: int = 8,
compress_npz: bool = False,
max_pixels: int | None = None,
chunk_height: int | None = None,
enforce_eager: bool = False,
init_barrier=None,
adapter_path: str | None = None,
) -> str:
"""Load model on one GPU, embed all assigned tiles, write partial .npz.
This function runs in a child process with CUDA_VISIBLE_DEVICES set.
Args:
gpu_id: Physical GPU ID.
tile_infos: Tiles assigned to this GPU.
model_path: HuggingFace model name or local path.
batch_size: Tiles per embed call.
result_dir: Directory to write partial result file.
backend: "vllm" or "sglang".
init_barrier: If provided, wait here after model load so all GPUs
finish CUDA graph capture before any starts embedding. Avoids
concurrent-capture stalls on some GPUs (observed on GPU 6/vLLM).
Returns:
Path to the partial .npz file written by this worker.
"""
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
os.environ.setdefault("SGLANG_LOG_LEVEL", "error")
init_fn, get_tok_fn, embed_fn = BACKENDS[backend]
logger.info(
"GPU %d: loading model %s via %s (%d tiles)",
gpu_id,
model_path,
backend,
len(tile_infos),
)
t0 = time.time()
# Pass adapter_path only for direct_gpu (other backends don't support it)
if adapter_path and backend == "direct_gpu":
engine = init_fn(
model_path, gpu_id, enforce_eager=enforce_eager, adapter_path=adapter_path
)
else:
engine = init_fn(model_path, gpu_id, enforce_eager=enforce_eager)
dt = time.time() - t0
logger.info("GPU %d: model loaded in %.1fs", gpu_id, dt)
# Wait for all GPUs to finish init (CUDA graph capture) before embedding.
if init_barrier is not None:
try:
init_barrier.wait(timeout=120)
logger.info("GPU %d: all workers ready, starting embed", gpu_id)
except threading.BrokenBarrierError:
logger.error(
"GPU %d: barrier broken (another GPU failed to init), continuing anyway",
gpu_id,
)
def _cleanup_engine() -> None:
shutdown = getattr(engine, "shutdown", None)
if callable(shutdown):
try:
shutdown()
except Exception as e:
logger.warning("GPU %d: engine shutdown failed: %s", gpu_id, e)
# Build prompt via chat template (official Qwen3-VL-Embedding format)
tokenizer = get_tok_fn(engine)
prompt = _build_chat_prompt(tokenizer)
logger.info("GPU %d: prompt template: %s", gpu_id, repr(prompt[:120]))
partial_path = _embed_tile_infos_with_engine(
engine=engine,
gpu_id=gpu_id,
tile_infos=tile_infos,
batch_size=batch_size,
result_dir=result_dir,
embed_fn=embed_fn,
prompt=prompt,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
chunk_height=chunk_height,
)
_cleanup_engine()
return partial_path
_MIN_CHUNK_HEIGHT = 28 # One Qwen3-VL patch; smaller chunks cause processor errors
def _chunk_image(img: "Image.Image", chunk_height: int) -> list["Image.Image"]:
"""Split a tall image into horizontal strips of chunk_height pixels.
The last chunk is merged into the previous one if it would be shorter
than _MIN_CHUNK_HEIGHT pixels.
"""
w, h = img.size
if h <= chunk_height:
return [img]
chunks = []
for y in range(0, h, chunk_height):
y_end = min(y + chunk_height, h)
# Merge tiny remainder into previous chunk
if y_end - y < _MIN_CHUNK_HEIGHT and chunks:
prev = chunks[-1]
merged = Image.new("RGB", (w, prev.size[1] + y_end - y))
merged.paste(prev, (0, 0))
merged.paste(img.crop((0, y, w, y_end)), (0, prev.size[1]))
chunks[-1] = merged
else:
chunks.append(img.crop((0, y, w, y_end)))
return chunks
def _embed_tile_infos_with_engine(
engine,
gpu_id: int,
tile_infos: list[TileInfo],
batch_size: int,
result_dir: str,
embed_fn,
prompt: str,
io_workers: int = 8,
compress_npz: bool = False,
max_pixels: int | None = None,
chunk_height: int | None = None,
task_id: str | None = None,
) -> str:
"""Embed tile infos with a pre-initialized engine function + prompt.
Uses a pipeline: background threads read files, hash, and decode only
unique tiles while the main thread feeds decoded batches to the GPU.
Duplicate tiles are hashed but never decoded.
If chunk_height is set, tall tiles are split into chunks of that height.
If max_pixels is set, images are resized in the I/O threads (overlaps GPU).
"""
tile_hashes: list[tuple[TileInfo, str]] = [] # filled by producer
unique_queue: queue.Queue = queue.Queue(maxsize=batch_size * 2)
producer_error: list[Exception | None] = [None]
resized_wide = [0] # mutable counter for threads
def _io_producer() -> None:
"""Read tiles, chunk tall images, hash chunks, skip decode for duplicates."""
seen: set[str] = set()
lock = threading.Lock()
results_lock = threading.Lock()
def _read_chunk_hash(ti):
img_path = _image_path(ti)
try:
raw = Path(img_path).read_bytes()
img = Image.open(io.BytesIO(raw)).convert("RGB")
except Exception as e:
logger.warning("Skipping unreadable image %s: %s", img_path, e)
return []
# Chunk tall images into strips (skip if already pre-chunked)
if chunk_height is not None and not isinstance(ti, ChunkInfo):
chunks = _chunk_image(img, chunk_height)
else:
chunks = [img]
out = []
for ci, chunk_img in enumerate(chunks):
if max_pixels is not None:
chunk_img = _smart_resize_pil(chunk_img, max_pixels)
cw, ch = chunk_img.size
# Skip chunks with extreme aspect ratio (sglang rejects >200)
if ch < 5 or (cw / max(ch, 1)) > 150:
logger.warning(
"Skipping bad chunk %s ci=%d size=%dx%d (aspect ratio %.1f)",
img_path,
ci,
cw,
ch,
cw / max(ch, 1),
)
continue
if cw > _MAX_CHUNK_WIDTH:
chunk_img = _clamp_width_pil(chunk_img, _MAX_CHUNK_WIDTH)
cw, ch = chunk_img.size
resized_wide[0] += 1
logger.debug(
"Resized wide chunk %s ci=%d to %dx%d",
img_path,
ci,
cw,
ch,
)
# Dedup key: use file path for pre-chunked files (avoids
# expensive tobytes + MD5 on ~2.7MB raw pixels per chunk)
if isinstance(ti, ChunkInfo):
h = img_path # unique on disk, no hash needed
else:
chunk_bytes = chunk_img.tobytes()
h = hashlib.md5(chunk_bytes).hexdigest()
if len(chunks) > 1:
if isinstance(ti, ChunkInfo):
chunk_ti = ti._replace(
chunk_index=ci,
chunk_height=chunk_img.size[1],
)
else:
chunk_ti = ti._replace(
chunk_index=ci,
tile_height=chunk_img.size[1],
)
else:
chunk_ti = ti
with lock:
is_new = h not in seen
if is_new:
seen.add(h)
out.append((chunk_ti, h, chunk_img if is_new else None))
return out
try:
with ThreadPoolExecutor(max_workers=io_workers) as pool:
for items in pool.map(_read_chunk_hash, tile_infos):
for chunk_ti, h, img in items:
with results_lock:
tile_hashes.append((chunk_ti, h))
if img is not None:
unique_queue.put((h, img))
except Exception as exc:
producer_error[0] = exc
finally:
unique_queue.put(None) # sentinel
t0 = time.time()
producer = threading.Thread(target=_io_producer, daemon=True)
producer.start()
# Consume unique decoded tiles and embed in batches (overlaps with I/O)
hash_to_embedding: dict[str, np.ndarray] = {}
batch_h: list[str] = []
batch_imgs: list = []
unique_total = 0
embedded = 0
embed_time_total = 0.0
queue_wait_total = 0.0
n_batches = 0
first_item_time = None
consecutive_failures = 0
_MAX_CONSECUTIVE_FAILURES = (
3 # after 3 consecutive CUDA failures, abort this work item
)
gpu_dead = False
while True:
tq0 = time.time()
item = unique_queue.get()
queue_wait_total += time.time() - tq0
if item is None:
break
if first_item_time is None:
first_item_time = time.time() - t0
h, img = item
batch_h.append(h)
batch_imgs.append(img)
unique_total += 1
if len(batch_h) >= batch_size:
if gpu_dead:
# Drain queue without embedding
batch_h, batch_imgs = [], []
continue
try:
te0 = time.time()
embs = embed_fn(engine, prompt, batch_imgs)
embed_time_total += time.time() - te0
n_batches += 1
consecutive_failures = 0
for hh, emb in zip(batch_h, embs):
hash_to_embedding[hh] = emb
except Exception as e:
consecutive_failures += 1
logger.error(
"GPU %d: embed failed at offset %d (%d/%d consecutive): %s",
gpu_id,
embedded,
consecutive_failures,
_MAX_CONSECUTIVE_FAILURES,
e,
)
if consecutive_failures >= _MAX_CONSECUTIVE_FAILURES:
logger.error(
"GPU %d: %d consecutive failures — marking GPU as dead",
gpu_id,
consecutive_failures,
)
gpu_dead = True
embedded += len(batch_h)
batch_h, batch_imgs = [], []
# Final partial batch
if batch_h and not gpu_dead:
try:
te0 = time.time()
embs = embed_fn(engine, prompt, batch_imgs)
embed_time_total += time.time() - te0
n_batches += 1
for hh, emb in zip(batch_h, embs):
hash_to_embedding[hh] = emb
except Exception as e:
logger.error("GPU %d: embed failed on final batch: %s", gpu_id, e)
embedded += len(batch_h)
producer.join()
pipeline_s = time.time() - t0
if producer_error[0]:
logger.error("GPU %d: I/O producer failed: %s", gpu_id, producer_error[0])
deduped = len(tile_hashes) - unique_total
100.0 * deduped / len(tile_hashes) if tile_hashes else 0.0
n_resized = resized_wide[0]
logger.info(
"GPU %d: %d images, %d unique (%d deduped, %d resized>%d), embedded %d, pipeline %.2fs (%.1f chunks/s)"
" | embed=%.2fs (%d batches) queue_wait=%.2fs first_item=%.3fs",
gpu_id,
len(tile_hashes),
unique_total,
deduped,
n_resized,
_MAX_CHUNK_WIDTH,
embedded,
pipeline_s,
embedded / pipeline_s if pipeline_s > 0 else 0,
embed_time_total,
n_batches,
queue_wait_total,
first_item_time or 0,
)
if not hash_to_embedding:
logger.warning("GPU %d: no embeddings produced", gpu_id)
if gpu_dead:
raise RuntimeError(
f"GPU {gpu_id}: CUDA dead after {_MAX_CONSECUTIVE_FAILURES} consecutive failures"
)
return ""
# Detect chunk mode from the type of items in tile_hashes
is_chunk_mode = tile_hashes and isinstance(tile_hashes[0][0], ChunkInfo)
# Phase 3: expand back to all tiles/chunks, reusing embeddings for duplicate hashes
all_embeddings = []
all_article_ids = []
all_tile_indices = []
all_chunk_indices = []
all_page_heights = []
all_viewport_widths = []
all_tile_heights = []
all_image_hashes = []
all_tile_paths = []
all_y_offsets = [] if is_chunk_mode else None
for ti, h in tile_hashes:
if h not in hash_to_embedding:
continue # embedding failed for this hash
all_embeddings.append(hash_to_embedding[h])
all_article_ids.append(ti.article_id)
all_tile_indices.append(ti.tile_index)
all_page_heights.append(ti.page_height)
all_viewport_widths.append(ti.viewport_width)
all_image_hashes.append(h)
if is_chunk_mode:
all_chunk_indices.append(ti.chunk_index)
all_tile_heights.append(ti.chunk_height)
all_tile_paths.append(ti.chunk_path)
all_y_offsets.append(ti.y_offset)
else:
all_chunk_indices.append(ti.chunk_index)
all_tile_heights.append(ti.tile_height)
all_tile_paths.append(ti.tile_path)
if not all_embeddings:
logger.warning("GPU %d: no embeddings after expansion", gpu_id)
return ""
suffix = f"_{task_id}" if task_id else ""
partial_path = os.path.join(result_dir, f"partial_gpu{gpu_id}{suffix}.npz")
t_write0 = time.time()
extra_arrays = {}
if is_chunk_mode:
extra_arrays["y_offsets"] = np.array(all_y_offsets, dtype=np.int32)
save_npz(
partial_path,
compressed=compress_npz,
embeddings=np.stack(all_embeddings),
article_ids=np.array(all_article_ids, dtype=np.int64),
tile_indices=np.array(all_tile_indices, dtype=np.int32),
chunk_indices=np.array(all_chunk_indices, dtype=np.int32),
page_heights=np.array(all_page_heights, dtype=np.int32),
viewport_widths=np.array(all_viewport_widths, dtype=np.int32),
tile_heights=np.array(all_tile_heights, dtype=np.int32),
image_hashes=np.array(all_image_hashes, dtype="S32"),
tile_paths=np.array(all_tile_paths, dtype="S512"),
**extra_arrays,
)
write_s = time.time() - t_write0
logger.info(
"GPU %d: wrote %d embeddings (%d unique) to %s [pipeline %.2fs, write %.2fs]",
gpu_id,
len(all_embeddings),
len(hash_to_embedding),
partial_path,
pipeline_s,
write_s,
)
return partial_path
def _gpu_worker_entry(
result_queue,
gpu_id: int,
tile_infos: list[TileInfo],
model_path: str,
batch_size: int,
result_dir: str,
backend: str,
io_workers: int,
compress_npz: bool,
max_pixels: int | None = None,
chunk_height: int | None = None,
enforce_eager: bool = False,
init_barrier=None,
adapter_path: str | None = None,
) -> None:
"""Run gpu_worker in a subprocess and report result via queue."""
try:
partial_path = gpu_worker(
gpu_id=gpu_id,
tile_infos=tile_infos,
model_path=model_path,
batch_size=batch_size,
result_dir=result_dir,
backend=backend,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
chunk_height=chunk_height,
enforce_eager=enforce_eager,
init_barrier=init_barrier,
adapter_path=adapter_path,
)
result_queue.put(
{
"gpu_id": gpu_id,
"partial_path": partial_path,
"error": "",
}
)
except Exception:
result_queue.put(
{
"gpu_id": gpu_id,
"partial_path": "",
"error": traceback.format_exc(),
}
)
def _gpu_worker_persistent_entry(
work_queue,
result_queue,
gpu_id: int,
model_path: str,
backend: str,
io_workers: int,
compress_npz: bool,
max_pixels: int | None = None,
chunk_height: int | None = None,
enforce_eager: bool = False,
init_barrier=None,
adapter_path: str | None = None,
) -> None:
"""Persistent GPU worker: load model once, pull work dynamically from shared queue.
Protocol:
- Work items: dict with "task_id", "tile_infos", "batch_size", "result_dir"
- Round sentinel: dict with "task_id"=None, "round_id" — worker sends round_done and continues
- Shutdown: None — worker exits
"""
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
os.environ.setdefault("SGLANG_LOG_LEVEL", "error")
init_fn, get_tok_fn, embed_fn = BACKENDS[backend]
engine = None
try:
if adapter_path and backend == "direct_gpu":
engine = init_fn(
model_path,
gpu_id,
enforce_eager=enforce_eager,
adapter_path=adapter_path,
)
else:
engine = init_fn(model_path, gpu_id, enforce_eager=enforce_eager)
if init_barrier is not None:
try:
init_barrier.wait(timeout=120)
logger.info("GPU %d: all persistent workers ready", gpu_id)
except threading.BrokenBarrierError:
logger.error("GPU %d: barrier broken, continuing anyway", gpu_id)
tokenizer = get_tok_fn(engine)
prompt = _build_chat_prompt(tokenizer)
while True:
task = work_queue.get()
if task is None:
# Shutdown signal
break
if task.get("task_id") is None:
# Round-end sentinel — signal done for this round, keep looping
result_queue.put(
{
"gpu_id": gpu_id,
"round_done": True,
"round_id": task.get("round_id"),
}
)
continue
task_id = task["task_id"]
try:
partial_path = _embed_tile_infos_with_engine(
engine=engine,
gpu_id=gpu_id,
tile_infos=task["tile_infos"],
batch_size=task["batch_size"],
result_dir=task["result_dir"],
embed_fn=embed_fn,
prompt=prompt,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
chunk_height=chunk_height,
task_id=task_id,
)
result_queue.put(
{
"task_id": task_id,
"gpu_id": gpu_id,
"partial_path": partial_path,
"error": "",
}
)
except Exception:
result_queue.put(
{
"task_id": task_id,
"gpu_id": gpu_id,
"partial_path": "",
"error": traceback.format_exc(),
}
)
except Exception:
result_queue.put(
{
"task_id": "__init__",
"gpu_id": gpu_id,
"partial_path": "",
"error": traceback.format_exc(),
}
)
finally:
if engine is not None:
shutdown = getattr(engine, "shutdown", None)
if callable(shutdown):
try:
shutdown()
except Exception:
pass
_WORK_CHUNK_SIZE = 500 # tiles per dynamic work item (good balance: ~4 batches of 128)
class PersistentGpuWorkerPool:
"""Pool of persistent one-GPU workers with dynamic work distribution.
Workers share a single work queue. Fast GPUs pull more work items,
eliminating idle time from static pre-splitting.
"""
def __init__(
self,
gpu_ids: list[int],
model_path: str,
backend: str,
io_workers: int,
compress_npz: bool,
max_pixels: int | None = None,
chunk_height: int | None = None,
enforce_eager: bool = False,
adapter_path: str | None = None,
) -> None:
import torch.multiprocessing as mp
self.ctx = mp.get_context("spawn")
self.work_queue = self.ctx.Queue() # shared across all workers
self.result_queue = self.ctx.Queue()
self.gpu_ids = list(gpu_ids)
self.workers: dict[int, mp.Process] = {}
# Store init params for respawning dead workers
self._model_path = model_path
self._backend = backend
self._io_workers = io_workers
self._compress_npz = compress_npz
self._max_pixels = max_pixels
self._chunk_height = chunk_height
self._enforce_eager = enforce_eager
self._adapter_path = adapter_path
# Store barrier as instance var to prevent GC before children unpickle
self._init_barrier = (
self.ctx.Barrier(len(gpu_ids)) if len(gpu_ids) > 1 else None
)
self._gpu_death_count: dict[int, int] = {} # gid -> number of times worker died
self._excluded_gpus: set[int] = set() # permanently excluded GPUs
for gid in gpu_ids:
p = self.ctx.Process(
target=_gpu_worker_persistent_entry,
args=(
self.work_queue,
self.result_queue,
gid,
model_path,
backend,
io_workers,
compress_npz,
max_pixels,
chunk_height,
enforce_eager,
self._init_barrier,
adapter_path,
),
daemon=False,
)
p.start()
self.workers[gid] = p
def run(
self,
tile_infos: list[TileInfo],
batch_size: int,
result_dir: str,
) -> list[str]:
"""Distribute tile_infos dynamically across workers and collect results.
Splits tile_infos into small work chunks and puts them into the shared
queue. Workers pull chunks as they finish, so fast GPUs process more.
"""
round_id = f"{time.time_ns()}"
t_round_start = time.time()
# Respawn dead workers from previous rounds (skip permanently excluded GPUs)
dead_before = [
gid
for gid, p in self.workers.items()
if not p.is_alive() and gid not in self._excluded_gpus
]
for gid in dead_before:
self._gpu_death_count[gid] = self._gpu_death_count.get(gid, 0) + 1
if self._gpu_death_count[gid] > 2:
logger.error(
"GPU %d: died %d times — permanently excluding from pool",
gid,
self._gpu_death_count[gid],
)
self._excluded_gpus.add(gid)
self.workers[gid].close()
del self.workers[gid]
continue
logger.warning(
"GPU %d: worker died (exitcode=%s), respawning (death #%d)",
gid,
self.workers[gid].exitcode,
self._gpu_death_count[gid],
)
self.workers[gid].close()
p = self.ctx.Process(
target=_gpu_worker_persistent_entry,
args=(
self.work_queue,
self.result_queue,
gid,
self._model_path,
self._backend,
self._io_workers,
self._compress_npz,
self._max_pixels,
self._chunk_height,
self._enforce_eager,
None,
self._adapter_path,
), # no barrier for respawned workers
daemon=False,
)
p.start()
self.workers[gid] = p
logger.info("GPU %d: respawned worker (pid=%d)", gid, p.pid)
if dead_before:
# Give respawned workers time to load model
logger.info(
"Waiting 30s for %d respawned workers to load model...",
len(dead_before),
)
time.sleep(30)
n_workers = len(self.workers)
if n_workers == 0:
raise RuntimeError("All persistent workers are dead, cannot run")
# Split into work chunks
work_items = []
for i in range(0, len(tile_infos), _WORK_CHUNK_SIZE):
chunk = tile_infos[i : i + _WORK_CHUNK_SIZE]
work_items.append(
{
"task_id": f"{round_id}_{len(work_items)}",
"round_id": round_id,
"tile_infos": chunk,
"batch_size": batch_size,
"result_dir": result_dir,
}
)
logger.info(
"Dynamic distribution: %d tiles -> %d work items for %d GPUs",
len(tile_infos),
len(work_items),
n_workers,
)
# Enqueue all work items (no sentinels — we count results instead)
for item in work_items:
self.work_queue.put(item)
n_work_items = len(work_items)
# Collect results until we've received one result per work item
partial_paths: list[str] = []
errors: list[str] = []
results_received = 0
gpu_work_counts: dict[int, int] = {}
gpu_consecutive_errors: dict[int, int] = {} # track per-GPU error streaks
dead_reported: set[int] = set()
stall_count = 0 # consecutive timeouts with no new results
while results_received < n_work_items:
try:
msg = self.result_queue.get(timeout=10)
stall_count = 0 # got something, reset
except queue.Empty:
stall_count += 1
# Check for dead workers
for gid, p in self.workers.items():
if gid not in dead_reported and not p.is_alive():
dead_reported.add(gid)
errors.append(f"GPU {gid}: worker died (exitcode={p.exitcode})")
logger.error(
"GPU %d: worker died (exitcode=%s), %d/%d results so far",
gid,
p.exitcode,
results_received,
n_work_items,
)
alive = [gid for gid, p in self.workers.items() if p.is_alive()]
if not alive:
remaining = n_work_items - results_received
errors.append(
f"All workers dead, {remaining} work items unprocessed"
)
break
# If a worker died and we've stalled for 60s, remaining items
# were likely in that worker's pipeline — stop waiting
if dead_reported and stall_count >= 6:
remaining = n_work_items - results_received
logger.warning(
"Stalled 60s after worker death, giving up on %d remaining items",
remaining,
)
break
continue
if msg.get("round_done"):
# Stale sentinel from a previous round — ignore
continue
# Normal work result
results_received += 1
gid = msg.get("gpu_id", -1)
gpu_work_counts[gid] = gpu_work_counts.get(gid, 0) + 1
if msg.get("error"):
gpu_consecutive_errors[gid] = gpu_consecutive_errors.get(gid, 0) + 1
if gpu_consecutive_errors[gid] >= 3 and gid not in dead_reported:
dead_reported.add(gid)
logger.error(
"GPU %d: %d consecutive task errors — treating as dead, "
"killing worker to prevent it from stealing work",
gid,
gpu_consecutive_errors[gid],
)
# Kill the worker process so it stops pulling from the shared queue
p = self.workers.get(gid)
if p and p.is_alive():
p.kill()
errors.append(
f"GPU {gid} task {msg.get('task_id')} failed:\n{msg['error']}"
)
continue
# Successful result — reset error streak for this GPU
gpu_consecutive_errors[gid] = 0
pp = msg.get("partial_path", "")
if pp:
partial_paths.append(pp)
# Log per-GPU work distribution and round summary
round_elapsed = time.time() - t_round_start
total_chunks = len(tile_infos)
throughput = total_chunks / round_elapsed if round_elapsed > 0 else 0
per_gpu_tp = throughput / n_workers if n_workers > 0 else 0
gpu_summary = ", ".join(
f"GPU {gid}: {gpu_work_counts.get(gid, 0)}" for gid in sorted(self.workers)
)
logger.info(
"Round done: %d chunks in %.1fs = %.1f chunks/s (%.1f/GPU) | %s",
total_chunks,
round_elapsed,
throughput,
per_gpu_tp,
gpu_summary,
)
if errors:
logger.error("Dynamic worker errors:\n%s", "\n".join(errors))
if not partial_paths:
raise RuntimeError("All dynamic workers failed:\n" + "\n".join(errors))
# If a GPU died and stole work, the shard is incomplete — raise so caller retries
if dead_reported:
raise RuntimeError(
f"GPU(s) {dead_reported} died during round — shard is incomplete "
f"({len(partial_paths)} partial results from {n_work_items} work items). "
f"Caller should retry without dead GPU(s)."
)
logger.warning(
"%d errors, continuing with %d partial results",
len(errors),
len(partial_paths),
)
return partial_paths
def close(self) -> None:
# Send shutdown sentinels. Workers read None from work_queue and exit.
for _ in self.workers:
try:
self.work_queue.put(None)
except Exception:
pass
# Soft-join: give each worker a short window to exit cleanly.
for _gid, p in self.workers.items():
p.join(timeout=10)
# Force-kill any stragglers so interpreter shutdown isn't blocked on
# non-daemon children. CUDA context teardown in child can hang
# indefinitely; SIGTERM/SIGKILL is safe here because results already
# landed in result_queue and partial_path files before close() runs.
for gid, p in list(self.workers.items()):
if p.is_alive():
logger.warning(
"GPU %d worker pid=%s didn't exit on sentinel, SIGTERM", gid, p.pid
)
p.terminate()
p.join(timeout=5)
if p.is_alive():
logger.error(
"GPU %d worker pid=%s ignored SIGTERM, SIGKILL", gid, p.pid
)
p.kill()
p.join(timeout=5)
# Drop queue background feeder threads so they don't block atexit.
for q in (self.work_queue, self.result_queue):
try:
q.close()
q.cancel_join_thread()
except Exception:
pass
def _close_all_persistent_pools() -> None:
for _k, pool in list(_PERSISTENT_POOLS.items()):
try:
pool.close()
except Exception:
pass
_PERSISTENT_POOLS.clear()
atexit.register(_close_all_persistent_pools)
def run_gpu_workers_parallel(
per_gpu: dict[int, list[TileInfo]] | None,
gpu_ids: list[int],
model_path: str,
batch_size: int,
result_dir: str,
backend: str,
io_workers: int,
compress_npz: bool,
reuse_workers: bool = False,
max_pixels: int | None = None,
chunk_height: int | None = None,
enforce_eager: bool = False,
tile_infos: list[TileInfo] | None = None,
adapter_path: str | None = None,
) -> list[str]:
"""Launch one non-daemonic process per GPU and collect partial .npz paths.
When reuse_workers=True, uses dynamic work distribution via a shared queue.
Pass tile_infos directly (per_gpu is ignored). Workers pull work chunks
dynamically, so fast GPUs process more.
When reuse_workers=False, uses static per_gpu assignment with one-shot processes.
"""
if reuse_workers:
# Dynamic distribution: pass all tiles, workers pull from shared queue
all_tiles = tile_infos if tile_infos is not None else []
if not all_tiles and per_gpu:
# Fallback: flatten per_gpu into single list
for tiles in per_gpu.values():
all_tiles.extend(tiles)
key = (
tuple(gpu_ids),
model_path,
backend,
io_workers,
bool(compress_npz),
max_pixels,
chunk_height,
enforce_eager,
adapter_path,
)
pool = _PERSISTENT_POOLS.get(key)
if pool is None:
pool = PersistentGpuWorkerPool(
gpu_ids=gpu_ids,
model_path=model_path,
backend=backend,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
chunk_height=chunk_height,
enforce_eager=enforce_eager,
adapter_path=adapter_path,
)
_PERSISTENT_POOLS[key] = pool
return pool.run(
tile_infos=all_tiles, batch_size=batch_size, result_dir=result_dir
)
import torch.multiprocessing as mp
ctx = mp.get_context("spawn")
result_queue = ctx.Queue()
procs: list[tuple[int, mp.Process]] = []
# Count active GPUs first to size the barrier correctly.
active_gids = [gid for gid in gpu_ids if per_gpu.get(gid)]
init_barrier = ctx.Barrier(len(active_gids)) if len(active_gids) > 1 else None
for gid in active_gids:
tiles = per_gpu[gid]
p = ctx.Process(
target=_gpu_worker_entry,
args=(
result_queue,
gid,
tiles,
model_path,
batch_size,
result_dir,
backend,
io_workers,
compress_npz,
max_pixels,
chunk_height,
enforce_eager,
init_barrier,
adapter_path,
),
daemon=False,
)
p.start()
procs.append((gid, p))
expected = len(procs)
results: dict[int, dict] = {}
while len(results) < expected:
try:
res = result_queue.get(timeout=5)
results[res["gpu_id"]] = res
except queue.Empty:
if all(not p.is_alive() for _gid, p in procs):
break
for _gid, p in procs:
p.join()
errors: list[str] = []
partial_paths: list[str] = []
for gid, p in procs:
res = results.get(gid)
if res is None:
msg = f"GPU {gid}: no result reported (exitcode={p.exitcode})"
logger.error(msg)
errors.append(msg)
continue
if res.get("error"):
msg = f"GPU {gid} failed:\n{res['error']}"
logger.error(msg)
errors.append(msg)
continue
partial_path = res.get("partial_path", "")
if partial_path:
partial_paths.append(partial_path)
if p.exitcode not in (0, None):
errors.append(f"GPU {gid}: worker exited with code {p.exitcode}")
if errors:
logger.error("Multi-GPU worker failures:\n%s", "\n".join(errors))
if not partial_paths:
raise RuntimeError("All GPU workers failed:\n" + "\n".join(errors))
logger.warning(
"%d/%d GPU workers failed, continuing with %d partial results",
len(errors),
len(procs),
len(partial_paths),
)
return partial_paths
# ---------------------------------------------------------------------------
# Multi-GPU orchestration
# ---------------------------------------------------------------------------
def embed_shard(
shard_dir: str,
output_dir: str,
gpu_ids: list[int],
model: str = "Qwen/Qwen3-VL-Embedding-2B",
batch_size: int = 512,
io_workers: int = 8,
compress_npz: bool = False,
reuse_workers: bool = False,
skip_article_ids: set[int] | None = None,
backend: str = "vllm",
max_pixels: int | None = None,
mode: str = "chunks",
chunk_height: int | None = None,
enforce_eager: bool = False,
adapter_path: str | None = None,
) -> dict:
"""Scan a shard, embed across GPUs, merge results into a single .npz.
Args:
shard_dir: Path to shard directory.
output_dir: Directory for output .npz file.
gpu_ids: List of GPU IDs to use.
model: Model name or path.
batch_size: Tiles per embed call.
skip_article_ids: Article IDs already embedded (for resume).
backend: "vllm" or "sglang".
Returns:
Dict with keys: shard_dir, output_path, num_tiles, num_articles, elapsed_s.
"""
t0 = time.time()
os.makedirs(output_dir, exist_ok=True)
# Extract shard_id from directory name
shard_name = os.path.basename(shard_dir.rstrip("/")) # e.g. "shard_042"
try:
shard_id = int(shard_name.split("_")[1])
except (ValueError, IndexError):
shard_id = 0
# Scan images (chunks or tiles depending on mode)
if mode == "chunks":
tile_infos = scan_shard_chunks(shard_dir, skip_article_ids)
unit = "chunks"
else:
tile_infos = scan_shard_tiles(shard_dir, skip_article_ids)
unit = "tiles"
if not tile_infos:
logger.warning("No %s found in %s", unit, shard_dir)
return {
"shard_dir": shard_dir,
"output_path": "",
"num_tiles": 0,
"num_articles": 0,
"elapsed_s": time.time() - t0,
}
# Incremental: load existing .npz, detect new + updated chunks via mtime
output_path = os.path.join(output_dir, f"shard_{shard_id:03d}.npz")
existing_npz = None
stale_keys: set[tuple[int, int, int]] = set() # keys to replace in existing npz
if os.path.exists(output_path):
try:
npz_mtime = os.path.getmtime(output_path)
existing_npz = np.load(output_path)
existing_keys = set(
zip(
existing_npz["article_ids"].tolist(),
existing_npz["tile_indices"].tolist(),
existing_npz["chunk_indices"].tolist(),
)
)
before = len(tile_infos)
# Partition tile_infos into: new (not in npz) + updated (in npz but file is newer)
new_infos = []
for ti in tile_infos:
key = (
ti.article_id,
ti.tile_index,
ti.chunk_index if hasattr(ti, "chunk_index") else 0,
)
if key not in existing_keys:
new_infos.append(ti)
else:
# Check mtime — if chunk file is newer than npz, re-embed it
chunk_path = (
ti.chunk_path
if hasattr(ti, "chunk_path")
else getattr(ti, "tile_path", None)
)
if chunk_path and os.path.getmtime(chunk_path) > npz_mtime:
new_infos.append(ti)
stale_keys.add(key)
tile_infos = new_infos
logger.info(
"Incremental: %d existing, %d new, %d updated (was %d total) in %s",
len(existing_keys),
len(tile_infos) - len(stale_keys),
len(stale_keys),
before,
shard_name,
)
if not tile_infos:
logger.info(
"Shard %d: all %d chunks up to date, skipping",
shard_id,
len(existing_keys),
)
return {
"shard_dir": shard_dir,
"output_path": output_path,
"num_tiles": len(existing_keys),
"num_articles": len(set(existing_npz["article_ids"].tolist())),
"elapsed_s": time.time() - t0,
}
except Exception as e:
logger.warning(
"Could not load existing %s for incremental: %s", output_path, e
)
existing_npz = None
num_gpus = len(gpu_ids)
if num_gpus == 1:
# Single GPU — run in-process (no multiprocessing overhead)
partial_path = gpu_worker(
gpu_ids[0],
tile_infos,
model,
batch_size,
output_dir,
backend,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
chunk_height=chunk_height,
enforce_eager=enforce_eager,
adapter_path=adapter_path,
)
partial_paths = [partial_path] if partial_path else []
else:
if reuse_workers:
# Dynamic distribution — workers pull from shared queue
logger.info(
"Multi-GPU dynamic: %d tiles across %d GPUs", len(tile_infos), num_gpus
)
partial_paths = run_gpu_workers_parallel(
per_gpu=None,
gpu_ids=gpu_ids,
model_path=model,
batch_size=batch_size,
result_dir=output_dir,
backend=backend,
io_workers=io_workers,
compress_npz=compress_npz,
reuse_workers=True,
max_pixels=max_pixels,
chunk_height=chunk_height,
enforce_eager=enforce_eager,
tile_infos=tile_infos,
adapter_path=adapter_path,
)
else:
# Static split — round-robin by article
article_ids = sorted(set(t.article_id for t in tile_infos))
gpu_assignment: dict[int, int] = {}
for i, aid in enumerate(article_ids):
gpu_assignment[aid] = gpu_ids[i % num_gpus]
per_gpu: dict[int, list[TileInfo]] = {gid: [] for gid in gpu_ids}
for ti in tile_infos:
per_gpu[gpu_assignment[ti.article_id]].append(ti)
for gid, tiles in per_gpu.items():
logger.info(
"GPU %d: assigned %d tiles from %d articles",
gid,
len(tiles),
len({t.article_id for t in tiles}),
)
partial_paths = run_gpu_workers_parallel(
per_gpu=per_gpu,
gpu_ids=gpu_ids,
model_path=model,
batch_size=batch_size,
result_dir=output_dir,
backend=backend,
io_workers=io_workers,
compress_npz=compress_npz,
reuse_workers=False,
max_pixels=max_pixels,
chunk_height=chunk_height,
enforce_eager=enforce_eager,
adapter_path=adapter_path,
)
# Merge partial results into final .npz
if not partial_paths:
logger.error("No partial results produced for %s", shard_dir)
return {
"shard_dir": shard_dir,
"output_path": "",
"num_tiles": 0,
"num_articles": 0,
"elapsed_s": time.time() - t0,
}
is_chunk_mode = mode == "chunks"
all_emb, all_aids, all_tidx = [], [], []
all_cidx, all_yo = [], []
all_ph, all_vw, all_th = [], [], []
all_ih, all_tp = [], []
# Prepend existing data for incremental merge (excluding stale entries)
if existing_npz is not None:
if stale_keys:
# Build mask: keep rows whose key is NOT in stale_keys
ex_aids = existing_npz["article_ids"]
ex_tidx = existing_npz["tile_indices"]
ex_cidx = existing_npz["chunk_indices"]
keep_mask = np.array(
[
(int(ex_aids[j]), int(ex_tidx[j]), int(ex_cidx[j]))
not in stale_keys
for j in range(len(ex_aids))
],
dtype=bool,
)
logger.info(
"Incremental merge: keeping %d/%d existing rows (%d stale replaced)",
keep_mask.sum(),
len(keep_mask),
len(stale_keys),
)
all_emb.append(existing_npz["embeddings"][keep_mask])
all_aids.append(ex_aids[keep_mask])
all_tidx.append(ex_tidx[keep_mask])
all_cidx.append(ex_cidx[keep_mask])
all_ph.append(existing_npz["page_heights"][keep_mask])
all_vw.append(existing_npz["viewport_widths"][keep_mask])
all_th.append(existing_npz["tile_heights"][keep_mask])
all_ih.append(existing_npz["image_hashes"][keep_mask])
all_tp.append(existing_npz["tile_paths"][keep_mask])
if is_chunk_mode and "y_offsets" in existing_npz:
all_yo.append(existing_npz["y_offsets"][keep_mask])
else:
all_emb.append(existing_npz["embeddings"])
all_aids.append(existing_npz["article_ids"])
all_tidx.append(existing_npz["tile_indices"])
all_cidx.append(existing_npz["chunk_indices"])
all_ph.append(existing_npz["page_heights"])
all_vw.append(existing_npz["viewport_widths"])
all_th.append(existing_npz["tile_heights"])
all_ih.append(existing_npz["image_hashes"])
all_tp.append(existing_npz["tile_paths"])
if is_chunk_mode and "y_offsets" in existing_npz:
all_yo.append(existing_npz["y_offsets"])
# Deduplicate partial paths: persistent workers overwrite the same
# partial_gpu{id}.npz for each work item, so the same file may appear
# multiple times in partial_paths. Reading it more than once duplicates
# every embedding in the final .npz.
seen_pp: set[str] = set()
unique_partial_paths = []
for pp in partial_paths:
rp = os.path.realpath(pp)
if rp not in seen_pp:
seen_pp.add(rp)
unique_partial_paths.append(pp)
if len(unique_partial_paths) < len(partial_paths):
logger.info(
"Deduplicated partial paths: %d -> %d unique",
len(partial_paths),
len(unique_partial_paths),
)
for pp in unique_partial_paths:
data = np.load(pp)
all_emb.append(data["embeddings"])
all_aids.append(data["article_ids"])
all_tidx.append(data["tile_indices"])
all_cidx.append(data["chunk_indices"])
all_ph.append(data["page_heights"])
all_vw.append(data["viewport_widths"])
all_th.append(data["tile_heights"])
all_ih.append(data["image_hashes"])
all_tp.append(data["tile_paths"])
if is_chunk_mode and "y_offsets" in data:
all_yo.append(data["y_offsets"])
embeddings = np.concatenate(all_emb, axis=0)
article_ids = np.concatenate(all_aids)
tile_indices = np.concatenate(all_tidx)
chunk_indices = np.concatenate(all_cidx)
page_heights = np.concatenate(all_ph)
viewport_widths = np.concatenate(all_vw)
tile_heights = np.concatenate(all_th)
image_hashes = np.concatenate(all_ih)
tile_paths = np.concatenate(all_tp)
if is_chunk_mode:
y_offsets = (
np.concatenate(all_yo)
if all_yo
else np.zeros(len(embeddings), dtype=np.int32)
)
# Sort by (article_id, tile_index, chunk_index)
sort_idx = np.lexsort((chunk_indices, tile_indices, article_ids))
else:
# Sort by (article_id, tile_index)
sort_idx = np.lexsort((tile_indices, article_ids))
embeddings = embeddings[sort_idx]
article_ids = article_ids[sort_idx]
tile_indices = tile_indices[sort_idx]
chunk_indices = chunk_indices[sort_idx]
page_heights = page_heights[sort_idx]
viewport_widths = viewport_widths[sort_idx]
tile_heights = tile_heights[sort_idx]
image_hashes = image_hashes[sort_idx]
tile_paths = tile_paths[sort_idx]
extra_arrays = {}
if is_chunk_mode:
y_offsets = y_offsets[sort_idx]
extra_arrays["y_offsets"] = y_offsets
output_path = os.path.join(output_dir, f"shard_{shard_id:03d}.npz")
save_npz(
output_path,
compressed=compress_npz,
embeddings=embeddings,
article_ids=article_ids,
tile_indices=tile_indices,
chunk_indices=chunk_indices,
page_heights=page_heights,
viewport_widths=viewport_widths,
tile_heights=tile_heights,
image_hashes=image_hashes,
tile_paths=tile_paths,
shard_id=np.int32(shard_id),
**extra_arrays,
)
# Clean up partial files
for pp in unique_partial_paths:
try:
os.remove(pp)
except OSError:
pass
elapsed = time.time() - t0
num_articles = len(set(article_ids.tolist()))
logger.info(
"Shard %d: %d embeddings (%d articles) in %.1fs -> %s",
shard_id,
len(embeddings),
num_articles,
elapsed,
output_path,
)
return {
"shard_dir": shard_dir,
"output_path": output_path,
"num_tiles": len(embeddings),
"num_articles": num_articles,
"elapsed_s": elapsed,
}
# ---------------------------------------------------------------------------
# Checkpoint for resume
# ---------------------------------------------------------------------------
def load_checkpoint(output_dir: str) -> set[int]:
"""Load set of already-embedded article IDs from checkpoint.json."""
ckpt_path = os.path.join(output_dir, "checkpoint.json")
if not os.path.exists(ckpt_path):
return set()
try:
data = json.loads(Path(ckpt_path).read_text())
return set(data.get("embedded_article_ids", []))
except Exception as e:
logger.warning("Failed to load checkpoint %s: %s", ckpt_path, e)
return set()
def save_checkpoint(output_dir: str, article_ids: set[int]) -> None:
"""Save embedded article IDs to checkpoint.json."""
ckpt_path = os.path.join(output_dir, "checkpoint.json")
data = {"embedded_article_ids": sorted(article_ids)}
Path(ckpt_path).write_text(json.dumps(data))
logger.info("Checkpoint saved: %d article IDs -> %s", len(article_ids), ckpt_path)
# ---------------------------------------------------------------------------
# Patch mode: diff hashes, re-embed only changed tiles, update npz in-place
# ---------------------------------------------------------------------------
def _hash_file(path: str) -> str:
"""Compute MD5 hex digest of a file. Returns empty string on error."""
try:
return hashlib.md5(Path(path).read_bytes()).hexdigest()
except OSError:
return ""
def diff_shard_hashes(
npz_path: str,
shard_dir: str,
hash_workers: int = 32,
) -> tuple[list[TileInfo], list[TileInfo], set[int]]:
"""Compare stored hashes in npz against current tile files on disk.
Uses a thread pool to hash files in parallel (IO-bound on NVMe).
Args:
npz_path: Path to existing shard .npz file.
shard_dir: Path to shard tile directory.
hash_workers: Number of threads for parallel hashing.
Returns:
(stale_tiles, new_tiles, removed_rows):
stale_tiles: TileInfos whose disk hash differs from stored hash.
new_tiles: TileInfos on disk but not in npz at all.
removed_rows: Row indices in npz whose tile_path no longer exists.
"""
t0 = time.time()
# Load stored index
logger.info("Loading existing npz: %s", npz_path)
data = np.load(npz_path)
stored_paths = data["tile_paths"] # S512
stored_hashes = data["image_hashes"] # S32
logger.info(
"Loaded %d stored embeddings in %.1fs", len(stored_paths), time.time() - t0
)
# Build index: tile_path -> (row_index, stored_hash)
path_to_row: dict[str, tuple[int, str]] = {}
for i in range(len(stored_paths)):
p = (
stored_paths[i].decode()
if isinstance(stored_paths[i], bytes)
else str(stored_paths[i])
)
h = (
stored_hashes[i].decode()
if isinstance(stored_hashes[i], bytes)
else str(stored_hashes[i])
)
path_to_row[p] = (i, h)
# Scan current tiles on disk
disk_tiles = scan_shard_tiles(shard_dir)
# Split into known (need hash check) vs new (not in npz)
tiles_to_hash: list[TileInfo] = []
new_tiles: list[TileInfo] = []
seen_paths: set[str] = set()
for ti in disk_tiles:
seen_paths.add(ti.tile_path)
if ti.tile_path in path_to_row:
tiles_to_hash.append(ti)
else:
new_tiles.append(ti)
# Parallel hash computation for known tiles
stale_tiles: list[TileInfo] = []
if tiles_to_hash:
logger.info(
"Hashing %d existing tiles (%d threads)...",
len(tiles_to_hash),
hash_workers,
)
t1 = time.time()
matched = 0
with ThreadPoolExecutor(max_workers=hash_workers) as pool:
futures = {
pool.submit(_hash_file, ti.tile_path): ti for ti in tiles_to_hash
}
pbar = tqdm(
as_completed(futures),
total=len(futures),
desc="Hashing tiles",
unit="tile",
)
for fut in pbar:
ti = futures[fut]
current_hash = fut.result()
_row_idx, old_hash = path_to_row[ti.tile_path]
if current_hash and current_hash != old_hash:
stale_tiles.append(ti)
else:
matched += 1
pbar.set_postfix(stale=len(stale_tiles), matched=matched)
pbar.close()
hash_rate = len(tiles_to_hash) / max(time.time() - t1, 0.001)
logger.info(
"Hashed %d tiles in %.1fs (%.0f tiles/s): %d matched, %d stale",
len(tiles_to_hash),
time.time() - t1,
hash_rate,
matched,
len(stale_tiles),
)
# Rows whose source file was deleted
removed_rows: set[int] = set()
for p, (row_idx, _h) in path_to_row.items():
if p not in seen_paths:
removed_rows.add(row_idx)
elapsed = time.time() - t0
logger.info(
"Diff complete in %.1fs: %d stale, %d new, %d removed, %d unchanged "
"(stored=%d, on_disk=%d)",
elapsed,
len(stale_tiles),
len(new_tiles),
len(removed_rows),
len(tiles_to_hash) - len(stale_tiles),
len(stored_paths),
len(disk_tiles),
)
return stale_tiles, new_tiles, removed_rows
def patch_shard(
npz_path: str,
shard_dir: str,
gpu_ids: list[int],
model: str = "Qwen/Qwen3-VL-Embedding-2B",
batch_size: int = 512,
io_workers: int = 8,
compress_npz: bool = False,
yes: bool = False,
dry_run: bool = False,
backend: str = "vllm",
max_pixels: int | None = None,
adapter_path: str | None = None,
) -> dict:
"""Patch an existing shard npz by re-embedding only changed/new tiles.
1. Diff stored hashes vs disk files
2. Show summary and ask for confirmation
3. Re-embed stale + new tiles
4. Update npz: replace stale rows, append new rows, drop removed rows
Args:
npz_path: Path to existing shard .npz.
shard_dir: Path to shard tile directory.
gpu_ids: GPU IDs to use.
model: Model name or path.
batch_size: Tiles per embed() call.
yes: Skip confirmation prompt.
dry_run: Only diff, don't embed or write.
Returns:
Dict with keys: npz_path, patched, added, removed, unchanged.
"""
stale_tiles, new_tiles, removed_rows = diff_shard_hashes(npz_path, shard_dir)
tiles_to_embed = stale_tiles + new_tiles
if not tiles_to_embed and not removed_rows:
logger.info("Nothing to patch — all hashes match")
return {
"npz_path": npz_path,
"patched": 0,
"added": 0,
"removed": 0,
"unchanged": True,
}
# Summary
print(f"\n{'=' * 60}")
print(f" Patch summary for {os.path.basename(npz_path)}")
print(f"{'=' * 60}")
print(f" Stale (hash changed): {len(stale_tiles):>8} → re-embed")
print(f" New (not in npz): {len(new_tiles):>8} → embed")
print(f" Removed (file gone): {len(removed_rows):>8} → drop")
print(f" Total to embed: {len(tiles_to_embed):>8}")
print(f"{'=' * 60}\n")
if dry_run:
logger.info("Dry run — stopping before embedding")
return {
"npz_path": npz_path,
"patched": len(stale_tiles),
"added": len(new_tiles),
"removed": len(removed_rows),
"dry_run": True,
}
if not yes:
answer = input("Proceed with patch? [y/N] ").strip().lower()
if answer not in ("y", "yes"):
logger.info("Aborted by user")
return {"npz_path": npz_path, "aborted": True}
# Load existing data
data = np.load(npz_path)
old_embeddings = data["embeddings"]
old_article_ids = data["article_ids"]
old_tile_indices = data["tile_indices"]
old_page_heights = data["page_heights"]
old_viewport_widths = data["viewport_widths"]
old_tile_heights = data["tile_heights"]
old_image_hashes = data["image_hashes"]
old_tile_paths = data["tile_paths"]
shard_id = int(data["shard_id"])
# Build stale path -> row index for replacement
path_to_row: dict[str, int] = {}
for i in range(len(old_tile_paths)):
p = (
old_tile_paths[i].decode()
if isinstance(old_tile_paths[i], bytes)
else str(old_tile_paths[i])
)
path_to_row[p] = i
stale_row_indices = {path_to_row[ti.tile_path] for ti in stale_tiles}
# Embed the changed + new tiles
t_embed = time.time()
if tiles_to_embed:
logger.info(
"Embedding %d tiles (%d stale + %d new) on GPUs %s",
len(tiles_to_embed),
len(stale_tiles),
len(new_tiles),
gpu_ids,
)
output_dir = os.path.dirname(npz_path)
num_gpus = len(gpu_ids)
if num_gpus == 1:
partial_path = gpu_worker(
gpu_ids[0],
tiles_to_embed,
model,
batch_size,
output_dir,
backend,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
adapter_path=adapter_path,
)
partial_paths = [partial_path] if partial_path else []
else:
article_ids_list = sorted(set(t.article_id for t in tiles_to_embed))
gpu_assignment = {
aid: gpu_ids[i % num_gpus] for i, aid in enumerate(article_ids_list)
}
per_gpu: dict[int, list[TileInfo]] = {gid: [] for gid in gpu_ids}
for ti in tiles_to_embed:
per_gpu[gpu_assignment[ti.article_id]].append(ti)
partial_paths = run_gpu_workers_parallel(
per_gpu=per_gpu,
gpu_ids=gpu_ids,
model_path=model,
batch_size=batch_size,
result_dir=output_dir,
backend=backend,
io_workers=io_workers,
compress_npz=compress_npz,
max_pixels=max_pixels,
adapter_path=adapter_path,
)
# Collect new embeddings
new_emb, new_aids, new_tidx = [], [], []
new_ph, new_vw, new_th = [], [], []
new_ih, new_tp = [], []
for pp in partial_paths:
d = np.load(pp)
new_emb.append(d["embeddings"])
new_aids.append(d["article_ids"])
new_tidx.append(d["tile_indices"])
new_ph.append(d["page_heights"])
new_vw.append(d["viewport_widths"])
new_th.append(d["tile_heights"])
new_ih.append(d["image_hashes"])
new_tp.append(d["tile_paths"])
try:
os.remove(pp)
except OSError:
pass
if new_emb:
fresh_embeddings = np.concatenate(new_emb, axis=0)
fresh_article_ids = np.concatenate(new_aids)
fresh_tile_indices = np.concatenate(new_tidx)
fresh_page_heights = np.concatenate(new_ph)
fresh_viewport_widths = np.concatenate(new_vw)
fresh_tile_heights = np.concatenate(new_th)
fresh_image_hashes = np.concatenate(new_ih)
fresh_tile_paths = np.concatenate(new_tp)
else:
tiles_to_embed = [] # embedding failed
# Build path index for fresh results
fresh_by_path: dict[str, int] = {}
if tiles_to_embed:
for i in range(len(fresh_tile_paths)):
p = (
fresh_tile_paths[i].decode()
if isinstance(fresh_tile_paths[i], bytes)
else str(fresh_tile_paths[i])
)
fresh_by_path[p] = i
# Assemble final arrays: keep unchanged rows, replace stale, drop removed
rows_to_drop = stale_row_indices | removed_rows
keep_mask = np.ones(len(old_embeddings), dtype=bool)
for r in rows_to_drop:
keep_mask[r] = False
parts_emb = [old_embeddings[keep_mask]]
parts_aids = [old_article_ids[keep_mask]]
parts_tidx = [old_tile_indices[keep_mask]]
parts_ph = [old_page_heights[keep_mask]]
parts_vw = [old_viewport_widths[keep_mask]]
parts_th = [old_tile_heights[keep_mask]]
parts_ih = [old_image_hashes[keep_mask]]
parts_tp = [old_tile_paths[keep_mask]]
if tiles_to_embed:
parts_emb.append(fresh_embeddings)
parts_aids.append(fresh_article_ids)
parts_tidx.append(fresh_tile_indices)
parts_ph.append(fresh_page_heights)
parts_vw.append(fresh_viewport_widths)
parts_th.append(fresh_tile_heights)
parts_ih.append(fresh_image_hashes)
parts_tp.append(fresh_tile_paths)
final_emb = np.concatenate(parts_emb, axis=0)
final_aids = np.concatenate(parts_aids)
final_tidx = np.concatenate(parts_tidx)
final_ph = np.concatenate(parts_ph)
final_vw = np.concatenate(parts_vw)
final_th = np.concatenate(parts_th)
final_ih = np.concatenate(parts_ih)
final_tp = np.concatenate(parts_tp)
# Re-sort
sort_idx = np.lexsort((final_tidx, final_aids))
final_emb = final_emb[sort_idx]
final_aids = final_aids[sort_idx]
final_tidx = final_tidx[sort_idx]
final_ph = final_ph[sort_idx]
final_vw = final_vw[sort_idx]
final_th = final_th[sort_idx]
final_ih = final_ih[sort_idx]
final_tp = final_tp[sort_idx]
save_npz(
npz_path,
compressed=compress_npz,
embeddings=final_emb,
article_ids=final_aids,
tile_indices=final_tidx,
page_heights=final_ph,
viewport_widths=final_vw,
tile_heights=final_th,
image_hashes=final_ih,
tile_paths=final_tp,
shard_id=np.int32(shard_id),
)
elapsed_total = time.time() - t_embed
logger.info(
"Patched %s in %.1fs: %d replaced, %d added, %d removed, %d total",
npz_path,
elapsed_total,
len(stale_tiles),
len(new_tiles),
len(removed_rows),
len(final_emb),
)
return {
"npz_path": npz_path,
"patched": len(stale_tiles),
"added": len(new_tiles),
"removed": len(removed_rows),
"total": len(final_emb),
"unchanged": False,
}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Embed tiles from a single shard using Qwen3-VL-Embedding-2B.",
)
parser.add_argument(
"--shard-dir",
required=True,
help="Path to shard directory (e.g. output_coordinated/shard_042)",
)
parser.add_argument(
"--output-dir", required=True, help="Output directory for .npz file"
)
parser.add_argument(
"--gpu-ids",
default="all",
help="Comma-separated GPU IDs, or 'all' (default: all)",
)
parser.add_argument(
"--model",
default="Qwen/Qwen3-VL-Embedding-2B",
help="Model name or path (default: Qwen/Qwen3-VL-Embedding-2B)",
)
parser.add_argument(
"--batch-size",
type=int,
default=128,
help="Tiles per embed() call (default: 128)",
)
parser.add_argument(
"--io-workers",
type=int,
default=8,
help="Threads per GPU process for tile read/hash/decode (default: 8)",
)
parser.add_argument(
"--compress-npz",
action="store_true",
help="Compress output npz files (smaller but slower)",
)
parser.add_argument(
"--reuse-workers",
action="store_true",
help="Reuse persistent GPU workers across multiple embed_shard calls in-process",
)
parser.add_argument(
"--backend",
choices=["vllm", "sglang", "direct_gpu"],
default="sglang",
help="Embedding backend (default: sglang)",
)
parser.add_argument(
"--resume",
action="store_true",
help="Resume from checkpoint (skip already embedded articles)",
)
parser.add_argument(
"--patch",
action="store_true",
help="Patch mode: diff hashes, re-embed only changed tiles",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="With --patch: only show diff, don't embed or write",
)
parser.add_argument(
"--yes",
"-y",
action="store_true",
help="With --patch: skip confirmation prompt",
)
parser.add_argument(
"--hash-workers",
type=int,
default=32,
help="Threads for parallel hashing in patch mode (default: 32)",
)
parser.add_argument(
"--max-pixels",
type=int,
default=None,
help="Max pixels per tile image before resize (e.g. 200000). "
"Reduces visual tokens for faster inference. None = no resize.",
)
parser.add_argument(
"--chunk-height",
type=int,
default=None,
help="Split tall tiles into chunks of this height (e.g. 1024). "
"Each chunk gets a separate embedding. None = no chunking.",
)
parser.add_argument(
"--enforce-eager",
action="store_true",
help="Disable CUDA graph capture in vLLM (fixes hangs on some GPUs).",
)
parser.add_argument(
"--redirects-json",
default=None,
help="Path to .redirects.json to skip redirect articles",
)
parser.add_argument(
"--mode",
choices=["chunks", "tiles"],
default="chunks",
help="Embedding unit: 'chunks' (1024px strips, default) or 'tiles' (full 8192px tiles)",
)
parser.add_argument(
"--instruction",
default=DEFAULT_INSTRUCTION,
help="System prompt instruction for embedding (default: %(default)r)",
)
parser.add_argument(
"--adapter",
default=None,
help="Path to PEFT LoRA adapter checkpoint directory. "
"Loaded and merged into base model weights before embedding. "
"Only supported with direct_gpu backend.",
)
return parser.parse_args()
def main():
args = parse_args()
gpu_ids = resolve_gpu_ids(args.gpu_ids)
if args.adapter:
if args.backend != "direct_gpu":
logger.info(
"--adapter requires direct_gpu backend, overriding --backend=%s",
args.backend,
)
args.backend = "direct_gpu"
if args.instruction != DEFAULT_INSTRUCTION:
set_instruction(args.instruction)
if args.patch:
# Patch mode: find existing npz, diff, re-embed changed tiles
shard_name = os.path.basename(args.shard_dir.rstrip("/"))
try:
shard_id = int(shard_name.split("_")[1])
except (ValueError, IndexError):
shard_id = 0
npz_path = os.path.join(args.output_dir, f"shard_{shard_id:03d}.npz")
if not os.path.exists(npz_path):
logger.error("Patch mode requires existing npz at %s", npz_path)
return
logger.info("Patch mode: %s vs %s", npz_path, args.shard_dir)
result = patch_shard(
npz_path=npz_path,
shard_dir=args.shard_dir,
gpu_ids=gpu_ids,
model=args.model,
batch_size=args.batch_size,
io_workers=args.io_workers,
compress_npz=args.compress_npz,
yes=args.yes,
dry_run=args.dry_run,
backend=args.backend,
max_pixels=args.max_pixels,
adapter_path=args.adapter,
)
logger.info("Patch done: %s", result)
return
# Normal mode
logger.info("Embedding shard %s on GPUs %s", args.shard_dir, gpu_ids)
skip_ids = load_checkpoint(args.output_dir) if args.resume else set()
if skip_ids:
logger.info("Resuming: skipping %d already-embedded articles", len(skip_ids))
if args.redirects_json:
redirect_ids = load_redirect_ids(args.redirects_json)
skip_ids |= redirect_ids
logger.info("After adding redirects: skipping %d total articles", len(skip_ids))
result = embed_shard(
shard_dir=args.shard_dir,
output_dir=args.output_dir,
gpu_ids=gpu_ids,
model=args.model,
batch_size=args.batch_size,
io_workers=args.io_workers,
compress_npz=args.compress_npz,
reuse_workers=args.reuse_workers,
skip_article_ids=skip_ids,
backend=args.backend,
max_pixels=args.max_pixels,
mode=args.mode,
chunk_height=args.chunk_height,
enforce_eager=args.enforce_eager,
adapter_path=args.adapter,
)
if result["num_tiles"] > 0:
# Update checkpoint with newly embedded articles
npz = np.load(result["output_path"])
new_ids = set(npz["article_ids"].tolist())
all_ids = skip_ids | new_ids
save_checkpoint(args.output_dir, all_ids)
logger.info("Done: %s", result)
if __name__ == "__main__":
main()