Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(embed): add tqdm progress bar to chunk stage, clean up embed logging (#108)

Yichuan Wang2a005dbc4e07
diff --git a/embed/src/pixelrag_embed/chunk.py b/embed/src/pixelrag_embed/chunk.py
index 95ac0c0..85e6c0d 100644
--- a/embed/src/pixelrag_embed/chunk.py
+++ b/embed/src/pixelrag_embed/chunk.py
@@ -36,6 +36,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from PIL import Image
+from tqdm import tqdm
Image.MAX_IMAGE_PIXELS = None # some tiles exceed default 178M pixel limit
@@ -307,21 +308,27 @@ def process_shard(
# Flat structure — article dirs directly in shard_dir
sub_dirs = [Path(shard_dir)]
- for sub_dir in sub_dirs:
- for article_dir in sorted(sub_dir.iterdir()):
- if not article_dir.is_dir() or not article_dir.name.endswith(".png.tiles"):
- continue
- total_articles += 1
-
- result = chunk_article(str(article_dir), dry_run=dry_run, force=force)
- if result is None:
- skipped_articles += 1
- continue
+ all_article_dirs = [
+ article_dir
+ for sub_dir in sub_dirs
+ for article_dir in sorted(sub_dir.iterdir())
+ if article_dir.is_dir() and article_dir.name.endswith(".png.tiles")
+ ]
+
+ for article_dir in tqdm(
+ all_article_dirs, desc="Chunking", disable=not all_article_dirs
+ ):
+ total_articles += 1
+
+ result = chunk_article(str(article_dir), dry_run=dry_run, force=force)
+ if result is None:
+ skipped_articles += 1
+ continue
- chunked_articles += 1
- total_tiles += result["num_tiles"]
- total_chunks += result["num_chunks"]
- total_files += result["files_written"]
+ chunked_articles += 1
+ total_tiles += result["num_tiles"]
+ total_chunks += result["num_chunks"]
+ total_files += result["files_written"]
# Delete tiles after chunking the whole shard
tiles_deleted = 0
diff --git a/embed/src/pixelrag_embed/embed_cpu.py b/embed/src/pixelrag_embed/embed_cpu.py
index 21e7050..dfed21a 100644
--- a/embed/src/pixelrag_embed/embed_cpu.py
+++ b/embed/src/pixelrag_embed/embed_cpu.py
@@ -203,9 +203,6 @@ def embed_items(
pooled = pooled / pooled.norm()
embeddings[i] = pooled.cpu().numpy().astype(np.float16)
- if (i + 1) % 10 == 0:
- logger.info("Embedded %d/%d", i + 1, len(items))
-
return embeddings