https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat: tell the agent each article's page map upfront (article_pages) (#25)
Zhifei Licb3736b6e6c4
diff --git a/serve/src/pixelrag_serve/api.py b/serve/src/pixelrag_serve/api.pyindex 8be402f..96a5ef9 100644--- a/serve/src/pixelrag_serve/api.py+++ b/serve/src/pixelrag_serve/api.py@@ -33,6 +33,7 @@ Usage:import argparseimport base64+import functoolsimport ioimport jsonimport logging@@ -115,6 +116,10 @@ class Hit(BaseModel):tile_height: intpath: strurl: str+ # Which (tile, chunk) coordinates actually exist on disk for this article,+ # e.g. "0:0-8,1:0-4" — lets agents page through an article without+ # guessing coordinates past its end.+ article_pages: str | None = Noneimage_base64: str | None = None@@ -291,6 +296,30 @@ def _resolve_path(article_id: int, tile_index: int, chunk_index: int) -> str:)[email protected]_cache(maxsize=8192)+def _article_pages(article_id: int) -> str | None:+ """Map of chunk files that exist on disk for an article: "0:0-8,1:0-4".++ Lets clients (notably the chat agent) page through an article without+ probing nonexistent coordinates. Returns None when tiles are rendered+ on demand (no files on disk) or the article dir can't be found.+ """+ if _state.get("ondemand") is not None:+ return None+ probe = _resolve_path(article_id, 0, 0)+ d = os.path.dirname(probe)+ if not os.path.isdir(d):+ return None+ tiles: dict[int, list[int]] = {}+ for name in os.listdir(d):+ m = re.match(r"chunk_(\d{4})_(\d{2})\.(?:png|jpg|jpeg)$", name)+ if m:+ tiles.setdefault(int(m.group(1)), []).append(int(m.group(2)))+ if not tiles:+ return None+ return ",".join(f"{t}:{min(cs)}-{max(cs)}" for t, cs in sorted(tiles.items()))++def _resolve_url(article_id: int) -> str:"""Resolve URL or title from article_id."""articles = _state["articles"]@@ -394,6 +423,7 @@ async def search(req: SearchRequest):tile_height=th,path=rel_path,url=url,+ article_pages=_article_pages(aid),image_base64=img_b64,))diff --git a/web/agent-server.mjs b/web/agent-server.mjsindex 066fffb..0fcbc46 100644--- a/web/agent-server.mjs+++ b/web/agent-server.mjs@@ -73,7 +73,7 @@ function log(...args) {function createTools(onEvent, uploadedImage) {const searchTool = tool("pixelrag_search",- "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",+ "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs, tile positions, and `pages` — the article's valid tile:chunk ranges (e.g. '0:0-7,1:0-4' = tile 0 has chunks 0-7, tile 1 has chunks 0-4). Use this first, then pixelrag_tile to view tiles.",{query: z.string().optional().describe("Natural language search query. Omit only when searching purely by an uploaded image."),use_uploaded_image: z.boolean().optional().describe("Set true to search the index BY the image the user uploaded (visual similarity). Requires an image in this conversation."),@@ -112,6 +112,7 @@ function createTools(onEvent, uploadedImage) {article_id: h.article_id,tile_index: h.tile_index,chunk_index: h.chunk_index,+ pages: h.article_pages,}})onEvent("search_results", { query: label, hits })@@ -123,7 +124,7 @@ function createTools(onEvent, uploadedImage) {const tileTool = tool("pixelrag_tile",- "View a Wikipedia screenshot tile by its coordinates. Returns the tile as an image so you can read the visual content. Use after pixelrag_search to read the actual article content.",+ "View a Wikipedia screenshot tile by its coordinates. Returns the tile as an image so you can read the visual content. Only request coordinates within the article's `pages` ranges from search results (e.g. pages '0:0-7,1:0-4' means tile 1 ends at chunk 4) — coordinates beyond them do not exist.",{article_id: z.number().int().describe("Article ID from search results"),tile_index: z.number().int().describe("Tile index from search results"),diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.tsindex 08731f8..728dd54 100644--- a/web/app/api/chat/route.ts+++ b/web/app/api/chat/route.ts@@ -16,6 +16,7 @@ interface SearchHit {chunk_index: numberurl: stringtile_height: number+ article_pages?: string | null}const SYSTEM_PROMPT = `You are PixelRAG's research assistant. You answer using a visual Wikipedia search engine — you read Wikipedia content as rendered screenshot tiles. Don't answer factual questions from memory; find and read the tiles.@@ -43,7 +44,7 @@ function createTools() {const searchTool = tool("pixelrag_search",- "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",+ "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs, tile positions, and `pages` — the article's valid tile:chunk ranges (e.g. '0:0-7,1:0-4' = tile 0 has chunks 0-7, tile 1 has chunks 0-4). Use this first, then pixelrag_tile to view tiles.",{query: z.string()@@ -127,6 +128,7 @@ function createTools(article_id: h.article_id,tile_index: h.tile_index,chunk_index: h.chunk_index,+ pages: h.article_pages,}})@@ -149,7 +151,7 @@ function createTools(const tileTool = tool("pixelrag_tile",- "View a Wikipedia screenshot tile by its coordinates. Returns the tile as an image so you can read the visual content. Use after pixelrag_search to read the actual article content.",+ "View a Wikipedia screenshot tile by its coordinates. Returns the tile as an image so you can read the visual content. Only request coordinates within the article's `pages` ranges from search results (e.g. pages '0:0-7,1:0-4' means tile 1 ends at chunk 4) — coordinates beyond them do not exist.",{article_id: z.number().int().describe("Article ID from search results"),tile_index: z.number().int().describe("Tile index from search results"),