https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(embed): add tqdm progress bar to chunk stage, clean up embed logging (#108)
Yichuan Wang2a005dbc4e07
diff --git a/embed/src/pixelrag_embed/chunk.py b/embed/src/pixelrag_embed/chunk.pyindex 95ac0c0..85e6c0d 100644--- a/embed/src/pixelrag_embed/chunk.py+++ b/embed/src/pixelrag_embed/chunk.py@@ -36,6 +36,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completedfrom pathlib import Pathfrom PIL import Image+from tqdm import tqdmImage.MAX_IMAGE_PIXELS = None # some tiles exceed default 178M pixel limit@@ -307,21 +308,27 @@ def process_shard(# Flat structure — article dirs directly in shard_dirsub_dirs = [Path(shard_dir)]- for sub_dir in sub_dirs:- for article_dir in sorted(sub_dir.iterdir()):- if not article_dir.is_dir() or not article_dir.name.endswith(".png.tiles"):- continue- total_articles += 1-- result = chunk_article(str(article_dir), dry_run=dry_run, force=force)- if result is None:- skipped_articles += 1- continue+ all_article_dirs = [+ article_dir+ for sub_dir in sub_dirs+ for article_dir in sorted(sub_dir.iterdir())+ if article_dir.is_dir() and article_dir.name.endswith(".png.tiles")+ ]++ for article_dir in tqdm(+ all_article_dirs, desc="Chunking", disable=not all_article_dirs+ ):+ total_articles += 1++ result = chunk_article(str(article_dir), dry_run=dry_run, force=force)+ if result is None:+ skipped_articles += 1+ continue- chunked_articles += 1- total_tiles += result["num_tiles"]- total_chunks += result["num_chunks"]- total_files += result["files_written"]+ chunked_articles += 1+ total_tiles += result["num_tiles"]+ total_chunks += result["num_chunks"]+ total_files += result["files_written"]# Delete tiles after chunking the whole shardtiles_deleted = 0diff --git a/embed/src/pixelrag_embed/embed_cpu.py b/embed/src/pixelrag_embed/embed_cpu.pyindex 21e7050..dfed21a 100644--- a/embed/src/pixelrag_embed/embed_cpu.py+++ b/embed/src/pixelrag_embed/embed_cpu.py@@ -203,9 +203,6 @@ def embed_items(pooled = pooled / pooled.norm()embeddings[i] = pooled.cpu().numpy().astype(np.float16)- if (i + 1) % 10 == 0:- logger.info("Embedded %d/%d", i + 1, len(items))-return embeddings