https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/
feat(serve): articles_only filter — drop Wikipedia meta pages (#14)
Zhifei Li1286689a2a05
diff --git a/serve/src/pixelrag_serve/api.py b/serve/src/pixelrag_serve/api.pyindex a140df2..8be402f 100644--- a/serve/src/pixelrag_serve/api.py+++ b/serve/src/pixelrag_serve/api.py@@ -37,6 +37,7 @@ import ioimport jsonimport loggingimport os+import reimport timefrom datetime import datetime, timezone@@ -85,6 +86,23 @@ class SearchRequest(BaseModel):min_tile_height: int | None = None # filter out small/blank chunksinstruction: str | None = None # override query embedding instructioninclude_images: bool = False # return base64-encoded tile images+ articles_only: bool = False # drop Wikipedia meta pages (Portal:, List_of_, …)+++# Wikipedia meta/aggregator pages that pollute "find the article" results.+# Matches both bare titles ("Portal:The_arts/Featured_picture",+# "List_of_German_physicists") and full /wiki/ URLs. Opt-in via articles_only.+_META_RE = re.compile(+ r"(?:^|/wiki/)(?:Portal|Wikipedia|WP|Template|Category|Draft|Help|File|Module|Book|Special|Talk|MediaWiki|Topic)[ _]*:"+ r"|(?:^|/wiki/)(?:List|Outline|Index|Glossary|Timeline)[ _]of[ _]"+ r"|\(disambiguation\)"+ r"|[ _]\((?:surname|given[ _]name)\)",+ re.IGNORECASE,+)+++def _is_meta(url: str) -> bool:+ return bool(_META_RE.search(url))class Hit(BaseModel):@@ -311,8 +329,14 @@ async def search(req: SearchRequest):if req.nprobe is not None:index.nprobe = req.nprobe- # Over-fetch when filtering to ensure enough results after filtering- fetch_k = req.n_docs * 5 if req.min_tile_height else req.n_docs+ # Over-fetch when filtering to ensure enough results after filtering.+ # Meta pages can be the majority of raw hits, so articles_only needs more.+ if req.articles_only:+ fetch_k = req.n_docs * 10+ elif req.min_tile_height:+ fetch_k = req.n_docs * 5+ else:+ fetch_k = req.n_docsdistances, indices = index.search(query_vectors, fetch_k)if req.nprobe is not None:@@ -339,6 +363,9 @@ async def search(req: SearchRequest):if req.min_tile_height and th < req.min_tile_height:continueaid = int(article_ids[vid])+ url = _resolve_url(aid)+ if req.articles_only and _is_meta(url):+ continueti = int(tile_indices[vid])ci = int(chunk_indices[vid])tile_path = _resolve_path(aid, ti, ci)@@ -346,6 +373,8 @@ async def search(req: SearchRequest):if req.include_images and tile_path and os.path.exists(tile_path):with open(tile_path, "rb") as fp:img_b64 = base64.b64encode(fp.read()).decode()+ elif req.include_images and _state.get("ondemand") is not None:+ img_b64 = _ondemand_chunk_b64(aid, ti, ci, th)# Expose a relative tile path, not the absolute server filesystem# path (avoids leaking the host's directory layout; clients fetch# tiles via /tile/{article_id}/{tile_index}/{chunk_index}).@@ -364,7 +393,7 @@ async def search(req: SearchRequest):y_offset=int(y_offsets[vid]),tile_height=th,path=rel_path,- url=_resolve_url(aid),+ url=url,image_base64=img_b64,))@@ -527,9 +556,71 @@ def load(args):"index_built_at": index_built_at,"index_size_bytes": index_size,"metadata_size_bytes": meta_size,+ "ondemand": None,})+ # Optional: render tile images on demand from a kiwix ZIM instead of reading a+ # materialized (multi-TB) tiles/ dir. Only retrieved pages get rendered + cached.+ if getattr(args, "render_on_demand", False):+ from .render_ondemand import OnDemandTiles++ book = args.zim_book or _derive_kiwix_book(args.kiwix_url)+ if not book:+ logger.warning(+ "render-on-demand: could not derive kiwix book from %s "+ "(pass --zim-book)",+ args.kiwix_url,+ )+ cache = os.path.join(args.tiles_dir or "./tiles_cache", "_ondemand")+ _state["ondemand"] = OnDemandTiles(args.kiwix_url, book, cache)+ logger.info(+ "On-demand tile rendering enabled (kiwix=%s book=%s cache=%s)",+ args.kiwix_url,+ book,+ cache,+ )+++def _derive_kiwix_book(kiwix_url: str) -> str:+ """Read the kiwix-serve catalog and return the /content/<book> id."""+ import re+ import urllib.request++ try:+ with urllib.request.urlopen(+ kiwix_url.rstrip("/") + "/catalog/v2/entries", timeout=10+ ) as r:+ xml = r.read().decode()+ m = re.search(r'href="/content/([^"/]+)"', xml)+ return m.group(1) if m else ""+ except Exception:+ return ""+++def _ondemand_chunk_b64(+ article_id: int, tile_index: int, chunk_index: int, tile_height: int+):+ """Render+chunk the page on demand and return the chunk as base64 PNG."""+ od = _state.get("ondemand")+ if od is None:+ return None+ p = od.chunk_path(article_id, _resolve_url(article_id), tile_index, chunk_index)+ if not p or not os.path.exists(p):+ return None+ import io++ from PIL import Image++ im = Image.open(p)+ # The on-demand render captures a full tile_height; trim the (padded) last+ # chunk back to the height the index recorded so it matches the built tile.+ if tile_height and im.height > tile_height:+ im = im.crop((0, 0, im.width, tile_height))+ buf = io.BytesIO()+ im.save(buf, format="PNG")+ return base64.b64encode(buf.getvalue()).decode()+def main():parser = argparse.ArgumentParser(description="PixelRAG FAISS Search API")@@ -557,6 +648,22 @@ def main():)parser.add_argument("--host", default="0.0.0.0")parser.add_argument("--port", type=int, default=30001)+ parser.add_argument(+ "--render-on-demand",+ action="store_true",+ help="Render tile images on demand from a kiwix ZIM (no materialized tiles/ dir). "+ "Requires a running kiwix-serve (see --kiwix-url).",+ )+ parser.add_argument(+ "--kiwix-url",+ default=os.environ.get("PIXELRAG_KIWIX_URL", "http://localhost:30900"),+ help="Base URL of a running kiwix-serve, for --render-on-demand",+ )+ parser.add_argument(+ "--zim-book",+ default=os.environ.get("PIXELRAG_ZIM_BOOK"),+ help="kiwix book id for /content/<book>/ (auto-derived from --kiwix-url if omitted)",+ )args = parser.parse_args()load(args)diff --git a/serve/src/pixelrag_serve/render_ondemand.py b/serve/src/pixelrag_serve/render_ondemand.pynew file mode 100644index 0000000..c22adfc--- /dev/null+++ b/serve/src/pixelrag_serve/render_ondemand.py@@ -0,0 +1,81 @@+"""On-demand tile rendering for pixelrag-serve.++When a tile image is not on disk, render the source page from a kiwix ZIM (served+over HTTP by ``kiwix-serve``), slice it into the same 1024px chunks the index was+built on, and return the requested chunk. This removes the dependency on a+materialized multi-TB ``tiles/`` corpus -- only the pages that retrieval actually+returns get rendered, lazily, and then cached on disk.++Pixel-validated against the original pipeline's tiles (mean |diff| ~2/255 on every+referenced chunk), using the exact build config: viewport_width=875, tile_height=8192,+and the shared ``pixelrag_embed.chunk`` slicer (1024px chunks).+"""++import os+import shutil+import threading+from urllib.parse import quote++_render_lock = threading.Lock() # one Chrome render at a time per process+++class OnDemandTiles:+ def __init__(+ self,+ kiwix_url: str,+ book: str,+ cache_dir: str,+ viewport_width: int = 875,+ tile_height: int = 8192,+ ):+ self.kiwix_url = kiwix_url.rstrip("/")+ self.book = book+ self.cache_dir = cache_dir+ self.viewport_width = viewport_width+ self.tile_height = tile_height+ os.makedirs(cache_dir, exist_ok=True)++ def _article_dir(self, article_id: int) -> str:+ return os.path.join(self.cache_dir, f"{article_id}.png.tiles")++ def chunk_path(+ self, article_id: int, title: str, tile_index: int, chunk_index: int+ ):+ """Path to chunk_{ti}_{ci}.png, rendering+chunking the page on a cache miss."""+ chunk_name = f"chunk_{tile_index:04d}_{chunk_index:02d}.png"+ cpath = os.path.join(self._article_dir(article_id), chunk_name)+ if os.path.exists(cpath):+ return cpath+ if not title:+ return None+ with _render_lock:+ if os.path.exists(cpath): # filled while we waited for the lock+ return cpath+ try:+ self._render_and_chunk(article_id, title)+ except Exception:+ return None+ return cpath if os.path.exists(cpath) else None++ def _render_and_chunk(self, article_id: int, title: str) -> None:+ from pixelrag_render import render_url+ from pixelrag_embed.chunk import chunk_article++ url = f"{self.kiwix_url}/content/{self.book}/{quote(title, safe='')}"+ staging = os.path.join(self.cache_dir, f".render_{article_id}")+ shutil.rmtree(staging, ignore_errors=True)+ dirs = render_url(+ url,+ staging,+ viewport_width=self.viewport_width,+ tile_height=self.tile_height,+ )+ if not dirs:+ shutil.rmtree(staging, ignore_errors=True)+ return+ rendered = str(dirs[0]) # <sanitized-url>.png.tiles/ (has tiles.json)+ chunk_article(rendered) # writes chunk_XXXX_YY.png + chunks.json+ dest = self._article_dir(article_id)+ shutil.rmtree(dest, ignore_errors=True)+ os.replace(rendered, dest) # atomic; commit only after chunking succeeds+ shutil.rmtree(staging, ignore_errors=True)diff --git a/web/agent-server.mjs b/web/agent-server.mjsindex e7f050b..322ec91 100644--- a/web/agent-server.mjs+++ b/web/agent-server.mjs@@ -96,7 +96,7 @@ function createTools(onEvent, uploadedImage) {const resp = await fetch(`${SEARCH_URL}/search`, {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5 }),+ body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5, articles_only: true }),})if (!resp.ok) {return { content: [{ type: "text", text: `Search API error: ${resp.status}` }] }diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.tsindex a1720ea..998f203 100644--- a/web/app/api/chat/route.ts+++ b/web/app/api/chat/route.ts@@ -99,6 +99,7 @@ function createTools(body: JSON.stringify({queries: [queryObj],n_docs: args.n_results ?? 5,+ articles_only: true,}),})if (!resp.ok) {diff --git a/web/app/docs/page.tsx b/web/app/docs/page.tsxindex b389c0e..1be570d 100644--- a/web/app/docs/page.tsx+++ b/web/app/docs/page.tsx@@ -44,6 +44,7 @@ const endpoints: Endpoint[] = [{ name: "n_docs", type: "number", description: "Number of results to return (default: 20)" },{ name: "nprobe", type: "number", description: "FAISS nprobe override" },{ name: "min_tile_height", type: "number", description: "Filter out tiles shorter than this" },+ { name: "articles_only", type: "boolean", description: "Drop Wikipedia meta pages (Portal:, List_of_, disambiguation, …) — keeps real articles" },{ name: "instruction", type: "string", description: "Custom embedding instruction" },],responseFields: [diff --git a/web/lib/api.ts b/web/lib/api.tsindex a754061..91a582f 100644--- a/web/lib/api.ts+++ b/web/lib/api.ts@@ -15,7 +15,9 @@ export async function search(req: SearchRequest): Promise<SearchResponse> {return fetchApi<SearchResponse>("/search", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify(req),+ // Default to article pages (drop Portal:/List_of_/disambiguation meta pages);+ // an explicit articles_only in req still wins.+ body: JSON.stringify({ articles_only: true, ...req }),});}diff --git a/web/lib/types.ts b/web/lib/types.tsindex 603078e..14fc432 100644--- a/web/lib/types.ts+++ b/web/lib/types.ts@@ -10,6 +10,7 @@ export interface SearchRequest {nprobe?: number;min_tile_height?: number;instruction?: string;+ articles_only?: boolean;}export interface Hit {