https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
fix: handle non-numeric filename-stem IDs in local source pipeline (#66)
Nadi Adatepe9cc1feb7986e
diff --git a/index/src/pixelrag_index/pipelines.py b/index/src/pixelrag_index/pipelines.pyindex ab0fea7..f943ec8 100644--- a/index/src/pixelrag_index/pipelines.py+++ b/index/src/pixelrag_index/pipelines.py@@ -98,17 +98,22 @@ def build(config: dict, limit: int | None = None, force: bool = False) -> Path:if pdf_docs:logger.info(" Rendered %d PDFs", len(pdf_docs))- # Save articles.json for serve API — title + URL per article+ # Save articles.json for serve API — title + URL per article.+ # Use the pipeline's sequential *position index* (0, 1, 2, …) rather than+ # int(a["id"]), because local sources use filename stems (e.g. "art_alice")+ # as doc IDs, which are not numeric. int() on a filename stem raises ValueError+ # and crashes the entire index build step.articles_path = output / "articles.json"- max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0- article_entries = [{"title": "", "url": ""}] * max_idx- for a in articles:- idx = int(a["id"])+ article_entries = []+ for enum_idx, a in enumerate(articles):title = a.get("metadata", {}).get("title", "")if not title and a.get("url"):title = a["url"].split("/")[-1].replace("_", " ").replace("%20", " ")- url = a.get("url", "")- article_entries[idx] = {"title": title or str(idx), "url": url}+ if not title:+ # Fall back to original doc id (e.g. filename stem) as display title+ title = a.get("id", str(enum_idx))+ url = a.get("url", "") or a.get("path", "")+ article_entries.append({"title": title, "url": url})with open(articles_path, "w") as f:json.dump(article_entries, f)logger.info(diff --git a/tests/repro_bug1.py b/tests/repro_bug1.pynew file mode 100644index 0000000..4758466--- /dev/null+++ b/tests/repro_bug1.py@@ -0,0 +1,54 @@+#!/usr/bin/env python3+"""+repro_bug1.py — Minimal reproduction of the int() crash on non-numeric local IDs.++Reproduce WITHOUT needing a full PixelRAG installation:+ python repro_bug1.py++Expected on upstream v0.2.1 (before fix):+ ValueError: invalid literal for int() with base 10: 'my_photo'++Expected after fix:+ PASS — article_entries has 2 entries with display titles.+"""++# Simulate the upstream article list from the pipeline (strings from filename stems)+articles = [+ {"id": "my_photo", "url": "", "path": "/tmp/my_photo.jpg", "metadata": {}},+ {"id": "chart_btcusdt", "url": "", "path": "/tmp/chart.png", "metadata": {}},+]++print("Testing upstream behavior (BEFORE fix):")+print("=" * 50)+try:+ max_idx = max(int(a["id"]) for a in articles) + 1 if articles else 0+ print(f" max_idx = {max_idx}")+ article_entries = [{"title": "", "url": ""}] * max_idx+ for a in articles:+ idx = int(a["id"]) # <-- crashes here on non-numeric IDs+ article_entries[idx] = {"title": str(idx), "url": a.get("url", "")}+ print(" PASS (unexpected — upstream has numeric IDs in this path)")+except ValueError as e:+ print(f" FAIL (expected on upstream): {e}")++print()+print("Testing FIXED behavior (enumerate-based):")+print("=" * 50)+article_entries = []+for enum_idx, a in enumerate(articles):+ title = a.get("metadata", {}).get("title", "")+ if not title and a.get("url"):+ title = a["url"].split("/")[-1]+ if not title:+ title = a.get("id", str(enum_idx)) # filename stem as fallback title+ url = a.get("url", "") or a.get("path", "")+ article_entries.append({"title": title, "url": url})++for i, entry in enumerate(article_entries):+ print(f" [{i}] title={entry['title']!r:20s} url={entry['url']!r}")++assert len(article_entries) == 2+assert article_entries[0]["title"] == "my_photo"+assert article_entries[1]["title"] == "chart_btcusdt"+print()+print("PASS — all assertions OK")