Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix: handle non-numeric filename-stem IDs in local source pipeline (#66)

Nadi Adatepe9cc1feb7986e
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.py
index ab0fea7..f943ec8 100644
--- a/index/src/pixelrag_index/pipelines.py
+++ b/index/src/pixelrag_index/pipelines.py
@@ -98,17 +98,22 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:
if pdf_docs:
logger.info(" Rendered %d PDFs", len(pdf_docs))
- # Save articles.json for serve API — title + URL per article
+ # Save articles.json for serve API — title + URL per article.
+ # Use the pipeline's sequential *position index* (0, 1, 2, …) rather than
+ # int(a["id"]), because local sources use filename stems (e.g. "art_alice")
+ # as doc IDs, which are not numeric. int() on a filename stem raises ValueError
+ # and crashes the entire index build step.
articles_path = output / "articles.json"
- max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0
- article_entries = [{"title": "", "url": ""}] * max_idx
- for a in articles:
- idx = int(a["id"])
+ article_entries = []
+ for enum_idx, a in enumerate(articles):
title = a.get("metadata", {}).get("title", "")
if not title and a.get("url"):
title = a["url"].split("/")[-1].replace("_", " ").replace("%20", " ")
- url = a.get("url", "")
- article_entries[idx] = {"title": title or str(idx), "url": url}
+ if not title:
+ # Fall back to original doc id (e.g. filename stem) as display title
+ title = a.get("id", str(enum_idx))
+ url = a.get("url", "") or a.get("path", "")
+ article_entries.append({"title": title, "url": url})
with open(articles_path, "w") as f:
json.dump(article_entries, f)
logger.info(
diff --git a/tests/repro_bug1.py b/tests/repro_bug1.py
new file mode 100644
index 0000000..4758466
--- /dev/null
+++ b/tests/repro_bug1.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""
+repro_bug1.py — Minimal reproduction of the int() crash on non-numeric local IDs.
+
+Reproduce WITHOUT needing a full PixelRAG installation:
+ python repro_bug1.py
+
+Expected on upstream v0.2.1 (before fix):
+ ValueError: invalid literal for int() with base 10: 'my_photo'
+
+Expected after fix:
+ PASS — article_entries has 2 entries with display titles.
+"""
+
+# Simulate the upstream article list from the pipeline (strings from filename stems)
+articles = [
+ {"id": "my_photo", "url": "", "path": "/tmp/my_photo.jpg", "metadata": {}},
+ {"id": "chart_btcusdt", "url": "", "path": "/tmp/chart.png", "metadata": {}},
+]
+
+print("Testing upstream behavior (BEFORE fix):")
+print("=" * 50)
+try:
+ max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0
+ print(f" max_idx = {max_idx}")
+ article_entries = [{"title": "", "url": ""}] * max_idx
+ for a in articles:
+ idx = int(a["id"]) # <-- crashes here on non-numeric IDs
+ article_entries[idx] = {"title": str(idx), "url": a.get("url", "")}
+ print(" PASS (unexpected — upstream has numeric IDs in this path)")
+except ValueError as e:
+ print(f" FAIL (expected on upstream): {e}")
+
+print()
+print("Testing FIXED behavior (enumerate-based):")
+print("=" * 50)
+article_entries = []
+for enum_idx, a in enumerate(articles):
+ title = a.get("metadata", {}).get("title", "")
+ if not title and a.get("url"):
+ title = a["url"].split("/")[-1]
+ if not title:
+ title = a.get("id", str(enum_idx)) # filename stem as fallback title
+ url = a.get("url", "") or a.get("path", "")
+ article_entries.append({"title": title, "url": url})
+
+for i, entry in enumerate(article_entries):
+ print(f" [{i}] title={entry['title']!r:20s} url={entry['url']!r}")
+
+assert len(article_entries) == 2
+assert article_entries[0]["title"] == "my_photo"
+assert article_entries[1]["title"] == "chart_btcusdt"
+print()
+print("PASS — all assertions OK")