Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

feat(serve): articles_only filter — drop Wikipedia meta pages (#14)

Zhifei Li1286689a2a05
diff --git a/serve/src/pixelrag_serve/api.py b/serve/src/pixelrag_serve/api.py
index a140df2..8be402f 100644
--- a/serve/src/pixelrag_serve/api.py
+++ b/serve/src/pixelrag_serve/api.py
@@ -37,6 +37,7 @@ import io
import json
import logging
import os
+import re
import time
from datetime import datetime, timezone
@@ -85,6 +86,23 @@ class SearchRequest(BaseModel):
min_tile_height: int | None = None # filter out small/blank chunks
instruction: str | None = None # override query embedding instruction
include_images: bool = False # return base64-encoded tile images
+ articles_only: bool = False # drop Wikipedia meta pages (Portal:, List_of_, …)
+
+
+# Wikipedia meta/aggregator pages that pollute "find the article" results.
+# Matches both bare titles ("Portal:The_arts/Featured_picture",
+# "List_of_German_physicists") and full /wiki/ URLs. Opt-in via articles_only.
+_META_RE = re.compile(
+ r"(?:^|/wiki/)(?:Portal|Wikipedia|WP|Template|Category|Draft|Help|File|Module|Book|Special|Talk|MediaWiki|Topic)[ _]*:"
+ r"|(?:^|/wiki/)(?:List|Outline|Index|Glossary|Timeline)[ _]of[ _]"
+ r"|\(disambiguation\)"
+ r"|[ _]\((?:surname|given[ _]name)\)",
+ re.IGNORECASE,
+)
+
+
+def _is_meta(url: str) -> bool:
+ return bool(_META_RE.search(url))
class Hit(BaseModel):
@@ -311,8 +329,14 @@ async def search(req: SearchRequest):
if req.nprobe is not None:
index.nprobe = req.nprobe
- # Over-fetch when filtering to ensure enough results after filtering
- fetch_k = req.n_docs * 5 if req.min_tile_height else req.n_docs
+ # Over-fetch when filtering to ensure enough results after filtering.
+ # Meta pages can be the majority of raw hits, so articles_only needs more.
+ if req.articles_only:
+ fetch_k = req.n_docs * 10
+ elif req.min_tile_height:
+ fetch_k = req.n_docs * 5
+ else:
+ fetch_k = req.n_docs
distances, indices = index.search(query_vectors, fetch_k)
if req.nprobe is not None:
@@ -339,6 +363,9 @@ async def search(req: SearchRequest):
if req.min_tile_height and th < req.min_tile_height:
continue
aid = int(article_ids[vid])
+ url = _resolve_url(aid)
+ if req.articles_only and _is_meta(url):
+ continue
ti = int(tile_indices[vid])
ci = int(chunk_indices[vid])
tile_path = _resolve_path(aid, ti, ci)
@@ -346,6 +373,8 @@ async def search(req: SearchRequest):
if req.include_images and tile_path and os.path.exists(tile_path):
with open(tile_path, "rb") as fp:
img_b64 = base64.b64encode(fp.read()).decode()
+ elif req.include_images and _state.get("ondemand") is not None:
+ img_b64 = _ondemand_chunk_b64(aid, ti, ci, th)
# Expose a relative tile path, not the absolute server filesystem
# path (avoids leaking the host's directory layout; clients fetch
# tiles via /tile/{article_id}/{tile_index}/{chunk_index}).
@@ -364,7 +393,7 @@ async def search(req: SearchRequest):
y_offset=int(y_offsets[vid]),
tile_height=th,
path=rel_path,
- url=_resolve_url(aid),
+ url=url,
image_base64=img_b64,
)
)
@@ -527,9 +556,71 @@ def load(args):
"index_built_at": index_built_at,
"index_size_bytes": index_size,
"metadata_size_bytes": meta_size,
+ "ondemand": None,
}
)
+ # Optional: render tile images on demand from a kiwix ZIM instead of reading a
+ # materialized (multi-TB) tiles/ dir. Only retrieved pages get rendered + cached.
+ if getattr(args, "render_on_demand", False):
+ from .render_ondemand import OnDemandTiles
+
+ book = args.zim_book or _derive_kiwix_book(args.kiwix_url)
+ if not book:
+ logger.warning(
+ "render-on-demand: could not derive kiwix book from %s "
+ "(pass --zim-book)",
+ args.kiwix_url,
+ )
+ cache = os.path.join(args.tiles_dir or "./tiles_cache", "_ondemand")
+ _state["ondemand"] = OnDemandTiles(args.kiwix_url, book, cache)
+ logger.info(
+ "On-demand tile rendering enabled (kiwix=%s book=%s cache=%s)",
+ args.kiwix_url,
+ book,
+ cache,
+ )
+
+
+def _derive_kiwix_book(kiwix_url: str) -> str:
+ """Read the kiwix-serve catalog and return the /content/<book> id."""
+ import re
+ import urllib.request
+
+ try:
+ with urllib.request.urlopen(
+ kiwix_url.rstrip("/") + "/catalog/v2/entries", timeout=10
+ ) as r:
+ xml = r.read().decode()
+ m = re.search(r'href="/content/([^"/]+)"', xml)
+ return m.group(1) if m else ""
+ except Exception:
+ return ""
+
+
+def _ondemand_chunk_b64(
+ article_id: int, tile_index: int, chunk_index: int, tile_height: int
+):
+ """Render+chunk the page on demand and return the chunk as base64 PNG."""
+ od = _state.get("ondemand")
+ if od is None:
+ return None
+ p = od.chunk_path(article_id, _resolve_url(article_id), tile_index, chunk_index)
+ if not p or not os.path.exists(p):
+ return None
+ import io
+
+ from PIL import Image
+
+ im = Image.open(p)
+ # The on-demand render captures a full tile_height; trim the (padded) last
+ # chunk back to the height the index recorded so it matches the built tile.
+ if tile_height and im.height > tile_height:
+ im = im.crop((0, 0, im.width, tile_height))
+ buf = io.BytesIO()
+ im.save(buf, format="PNG")
+ return base64.b64encode(buf.getvalue()).decode()
+
def main():
parser = argparse.ArgumentParser(description="PixelRAG FAISS Search API")
@@ -557,6 +648,22 @@ def main():
)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=30001)
+ parser.add_argument(
+ "--render-on-demand",
+ action="store_true",
+ help="Render tile images on demand from a kiwix ZIM (no materialized tiles/ dir). "
+ "Requires a running kiwix-serve (see --kiwix-url).",
+ )
+ parser.add_argument(
+ "--kiwix-url",
+ default=os.environ.get("PIXELRAG_KIWIX_URL", "http://localhost:30900"),
+ help="Base URL of a running kiwix-serve, for --render-on-demand",
+ )
+ parser.add_argument(
+ "--zim-book",
+ default=os.environ.get("PIXELRAG_ZIM_BOOK"),
+ help="kiwix book id for /content/<book>/ (auto-derived from --kiwix-url if omitted)",
+ )
args = parser.parse_args()
load(args)
diff --git a/serve/src/pixelrag_serve/render_ondemand.py b/serve/src/pixelrag_serve/render_ondemand.py
new file mode 100644
index 0000000..c22adfc
--- /dev/null
+++ b/serve/src/pixelrag_serve/render_ondemand.py
@@ -0,0 +1,81 @@
+"""On-demand tile rendering for pixelrag-serve.
+
+When a tile image is not on disk, render the source page from a kiwix ZIM (served
+over HTTP by ``kiwix-serve``), slice it into the same 1024px chunks the index was
+built on, and return the requested chunk. This removes the dependency on a
+materialized multi-TB ``tiles/`` corpus -- only the pages that retrieval actually
+returns get rendered, lazily, and then cached on disk.
+
+Pixel-validated against the original pipeline's tiles (mean |diff| ~2/255 on every
+referenced chunk), using the exact build config: viewport_width=875, tile_height=8192,
+and the shared ``pixelrag_embed.chunk`` slicer (1024px chunks).
+"""
+
+import os
+import shutil
+import threading
+from urllib.parse import quote
+
+_render_lock = threading.Lock() # one Chrome render at a time per process
+
+
+class OnDemandTiles:
+ def __init__(
+ self,
+ kiwix_url: str,
+ book: str,
+ cache_dir: str,
+ viewport_width: int = 875,
+ tile_height: int = 8192,
+ ):
+ self.kiwix_url = kiwix_url.rstrip("/")
+ self.book = book
+ self.cache_dir = cache_dir
+ self.viewport_width = viewport_width
+ self.tile_height = tile_height
+ os.makedirs(cache_dir, exist_ok=True)
+
+ def _article_dir(self, article_id: int) -> str:
+ return os.path.join(self.cache_dir, f"{article_id}.png.tiles")
+
+ def chunk_path(
+ self, article_id: int, title: str, tile_index: int, chunk_index: int
+ ):
+ """Path to chunk_{ti}_{ci}.png, rendering+chunking the page on a cache miss."""
+ chunk_name = f"chunk_{tile_index:04d}_{chunk_index:02d}.png"
+ cpath = os.path.join(self._article_dir(article_id), chunk_name)
+ if os.path.exists(cpath):
+ return cpath
+ if not title:
+ return None
+ with _render_lock:
+ if os.path.exists(cpath): # filled while we waited for the lock
+ return cpath
+ try:
+ self._render_and_chunk(article_id, title)
+ except Exception:
+ return None
+ return cpath if os.path.exists(cpath) else None
+
+ def _render_and_chunk(self, article_id: int, title: str) -> None:
+ from pixelrag_render import render_url
+ from pixelrag_embed.chunk import chunk_article
+
+ url = f"{self.kiwix_url}/content/{self.book}/{quote(title, safe='')}"
+ staging = os.path.join(self.cache_dir, f".render_{article_id}")
+ shutil.rmtree(staging, ignore_errors=True)
+ dirs = render_url(
+ url,
+ staging,
+ viewport_width=self.viewport_width,
+ tile_height=self.tile_height,
+ )
+ if not dirs:
+ shutil.rmtree(staging, ignore_errors=True)
+ return
+ rendered = str(dirs[0]) # <sanitized-url>.png.tiles/ (has tiles.json)
+ chunk_article(rendered) # writes chunk_XXXX_YY.png + chunks.json
+ dest = self._article_dir(article_id)
+ shutil.rmtree(dest, ignore_errors=True)
+ os.replace(rendered, dest) # atomic; commit only after chunking succeeds
+ shutil.rmtree(staging, ignore_errors=True)
diff --git a/web/agent-server.mjs b/web/agent-server.mjs
index e7f050b..322ec91 100644
--- a/web/agent-server.mjs
+++ b/web/agent-server.mjs
@@ -96,7 +96,7 @@ function createTools(onEvent, uploadedImage) {
const resp = await fetch(`${SEARCH_URL}/search`, {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5 }),
+ body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5, articles_only: true }),
})
if (!resp.ok) {
return { content: [{ type: "text", text: `Search API error: ${resp.status}` }] }
diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.ts
index a1720ea..998f203 100644
--- a/web/app/api/chat/route.ts
+++ b/web/app/api/chat/route.ts
@@ -99,6 +99,7 @@ function createTools(
body: JSON.stringify({
queries: [queryObj],
n_docs: args.n_results ?? 5,
+ articles_only: true,
}),
})
if (!resp.ok) {
diff --git a/web/app/docs/page.tsx b/web/app/docs/page.tsx
index b389c0e..1be570d 100644
--- a/web/app/docs/page.tsx
+++ b/web/app/docs/page.tsx
@@ -44,6 +44,7 @@ const endpoints: Endpoint[] = [
{ name: "n_docs", type: "number", description: "Number of results to return (default: 20)" },
{ name: "nprobe", type: "number", description: "FAISS nprobe override" },
{ name: "min_tile_height", type: "number", description: "Filter out tiles shorter than this" },
+ { name: "articles_only", type: "boolean", description: "Drop Wikipedia meta pages (Portal:, List_of_, disambiguation, …) — keeps real articles" },
{ name: "instruction", type: "string", description: "Custom embedding instruction" },
],
responseFields: [
diff --git a/web/lib/api.ts b/web/lib/api.ts
index a754061..91a582f 100644
--- a/web/lib/api.ts
+++ b/web/lib/api.ts
@@ -15,7 +15,9 @@ export async function search(req: SearchRequest): Promise<SearchResponse> {
return fetchApi<SearchResponse>("/search", {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify(req),
+ // Default to article pages (drop Portal:/List_of_/disambiguation meta pages);
+ // an explicit articles_only in req still wins.
+ body: JSON.stringify({ articles_only: true, ...req }),
});
}
diff --git a/web/lib/types.ts b/web/lib/types.ts
index 603078e..14fc432 100644
--- a/web/lib/types.ts
+++ b/web/lib/types.ts
@@ -10,6 +10,7 @@ export interface SearchRequest {
nprobe?: number;
min_tile_height?: number;
instruction?: string;
+ articles_only?: boolean;
}
export interface Hit {