Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(agent): enable image-to-image and joint image+text search (#4)

Zhifei Li8029339a0f48
diff --git a/web/agent-server.mjs b/web/agent-server.mjs
index f88c20c..e7f050b 100644
--- a/web/agent-server.mjs
+++ b/web/agent-server.mjs
@@ -54,7 +54,9 @@ function rateLimit(ip, now) {
const SYSTEM_PROMPT = `You are PixelRAG's research assistant. You answer using a visual Wikipedia search engine — you read Wikipedia content as rendered screenshot tiles. Don't answer factual questions from memory; find and read the tiles.
For every user question, without exception:
-1. Call pixelrag_search to find relevant Wikipedia articles. This applies to visual and comparison questions too — for "what/who is this?" or "which X does this most resemble?" about an uploaded image, search for the likely subjects or candidates (e.g. well-known computer scientists) so you can compare them against their Wikipedia tiles.
+1. Call pixelrag_search to find relevant Wikipedia articles.
+ - If the user uploaded an image, set use_uploaded_image: true to include it in the search (visual similarity) — the best way to answer "what/who is this?" or "which X does this most resemble?". You can add a text query in the SAME call to combine image + text into one joint search, and you may also run separate text searches for likely candidates to compare against. Always do the image search first when an image is present.
+ - Otherwise pass a natural-language query.
2. Call pixelrag_tile to VIEW the screenshot tiles of the top results — this is how you read and compare. View at least 2-3 tiles.
3. Answer from what the tiles show, and cite the Wikipedia URLs. If the tiles don't contain the answer, say so honestly.
@@ -68,20 +70,33 @@ function log(...args) {
console.log(new Date().toISOString(), ...args)
}
-function createTools(onEvent) {
+function createTools(onEvent, uploadedImage) {
const searchTool = tool(
"pixelrag_search",
- "Search the visual Wikipedia index. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",
+ "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",
{
- query: z.string().describe("Natural language search query"),
+ query: z.string().optional().describe("Natural language search query. Omit only when searching purely by an uploaded image."),
+ use_uploaded_image: z.boolean().optional().describe("Set true to search the index BY the image the user uploaded (visual similarity). Requires an image in this conversation."),
n_results: z.number().int().min(1).max(20).optional().describe("Number of results (default 5)"),
},
async (args) => {
- onEvent("searching", { query: args.query })
+ if (args.use_uploaded_image && !uploadedImage) {
+ return { content: [{ type: "text", text: "No image was uploaded in this conversation — use a text query instead." }] }
+ }
+ const searchByImage = Boolean(args.use_uploaded_image && uploadedImage)
+ if (!searchByImage && !args.query) {
+ return { content: [{ type: "text", text: "Provide a text query, or set use_uploaded_image:true when the user uploaded an image." }] }
+ }
+ // Text and image can be combined in one query for joint image+text retrieval.
+ const queryObj = {}
+ if (searchByImage) queryObj.image = uploadedImage
+ if (args.query) queryObj.text = args.query
+ const label = searchByImage && args.query ? `${args.query} + uploaded image` : args.query || "uploaded image"
+ onEvent("searching", { query: label })
const resp = await fetch(`${SEARCH_URL}/search`, {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ queries: [{ text: args.query }], n_docs: args.n_results ?? 5 }),
+ body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5 }),
})
if (!resp.ok) {
return { content: [{ type: "text", text: `Search API error: ${resp.status}` }] }
@@ -99,9 +114,9 @@ function createTools(onEvent) {
chunk_index: h.chunk_index,
}
})
- onEvent("search_results", { query: args.query, hits })
+ onEvent("search_results", { query: label, hits })
return {
- content: [{ type: "text", text: JSON.stringify({ query: args.query, results, count: results.length }, null, 2) }],
+ content: [{ type: "text", text: JSON.stringify({ query: label, results, count: results.length }, null, 2) }],
}
}
)
@@ -222,7 +237,8 @@ const server = http.createServer(async (req, res) => {
})
const send = (event, data) => res.write(sse(event, data))
- const tools = createTools(send)
+ const uploadedImage = last?.image && typeof last.image === "string" ? last.image : null
+ const tools = createTools(send, uploadedImage)
const mcpServer = createSdkMcpServer({ name: "pixelrag", version: "1.0.0", tools })
inFlight++
diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.ts
index 45fc6f1..a1720ea 100644
--- a/web/app/api/chat/route.ts
+++ b/web/app/api/chat/route.ts
@@ -1,4 +1,9 @@
-import { query, tool, createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk"
+import {
+ query,
+ tool,
+ createSdkMcpServer,
+ type SDKUserMessage,
+} from "@anthropic-ai/claude-agent-sdk"
import { z } from "zod"
const SEARCH_URL =
@@ -16,7 +21,9 @@ interface SearchHit {
const SYSTEM_PROMPT = `You are PixelRAG's research assistant. You answer using a visual Wikipedia search engine — you read Wikipedia content as rendered screenshot tiles. Don't answer factual questions from memory; find and read the tiles.
For every user question, without exception:
-1. Call pixelrag_search to find relevant Wikipedia articles. This applies to visual and comparison questions too — for "what/who is this?" or "which X does this most resemble?" about an uploaded image, search for the likely subjects or candidates (e.g. well-known computer scientists) so you can compare them against their Wikipedia tiles.
+1. Call pixelrag_search to find relevant Wikipedia articles.
+ - If the user uploaded an image, set use_uploaded_image: true to include it in the search (visual similarity) — the best way to answer "what/who is this?" or "which X does this most resemble?". You can add a text query in the SAME call to combine image + text into one joint search, and you may also run separate text searches for likely candidates to compare against. Always do the image search first when an image is present.
+ - Otherwise pass a natural-language query.
2. Call pixelrag_tile to VIEW the screenshot tiles of the top results — this is how you read and compare. View at least 2-3 tiles.
3. Answer from what the tiles show, and cite the Wikipedia URLs. If the tiles don't contain the answer, say so honestly.
@@ -30,12 +37,22 @@ function sseEvent(event: string, data: unknown): string {
return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`
}
-function createTools(onEvent: (event: string, data: unknown) => void) {
+function createTools(
+ onEvent: (event: string, data: unknown) => void,
+ uploadedImage: string | null
+) {
const searchTool = tool(
"pixelrag_search",
- "Search the visual Wikipedia index. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",
+ "Search the visual Wikipedia index by text OR by the image the user uploaded. Returns ranked results with article URLs and tile positions. Use this first to find relevant articles, then use pixelrag_tile to view specific tiles.",
{
- query: z.string().describe("Natural language search query"),
+ query: z
+ .string()
+ .optional()
+ .describe("Natural language search query. Omit only when searching purely by an uploaded image."),
+ use_uploaded_image: z
+ .boolean()
+ .optional()
+ .describe("Set true to search the index BY the image the user uploaded (visual similarity). Requires an image in this conversation."),
n_results: z
.number()
.int()
@@ -45,13 +62,42 @@ function createTools(onEvent: (event: string, data: unknown) => void) {
.describe("Number of results (default 5)"),
},
async (args) => {
- onEvent("searching", { query: args.query })
+ if (args.use_uploaded_image && !uploadedImage) {
+ return {
+ content: [
+ {
+ type: "text" as const,
+ text: "No image was uploaded in this conversation — use a text query instead.",
+ },
+ ],
+ }
+ }
+ const searchByImage = Boolean(args.use_uploaded_image && uploadedImage)
+ if (!searchByImage && !args.query) {
+ return {
+ content: [
+ {
+ type: "text" as const,
+ text: "Provide a text query, or set use_uploaded_image:true when the user uploaded an image.",
+ },
+ ],
+ }
+ }
+ // Text and image can be combined in one query for joint image+text retrieval.
+ const queryObj: { image?: string; text?: string } = {}
+ if (searchByImage && uploadedImage) queryObj.image = uploadedImage
+ if (args.query) queryObj.text = args.query
+ const label =
+ searchByImage && args.query
+ ? `${args.query} + uploaded image`
+ : args.query || "uploaded image"
+ onEvent("searching", { query: label })
const resp = await fetch(`${SEARCH_URL}/search`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
- queries: [{ text: args.query }],
+ queries: [queryObj],
n_docs: args.n_results ?? 5,
}),
})
@@ -83,14 +129,14 @@ function createTools(onEvent: (event: string, data: unknown) => void) {
}
})
- onEvent("search_results", { query: args.query, hits })
+ onEvent("search_results", { query: label, hits })
return {
content: [
{
type: "text" as const,
text: JSON.stringify(
- { query: args.query, results, count: results.length },
+ { query: label, results, count: results.length },
null,
2
),
@@ -204,11 +250,37 @@ export async function POST(req: Request) {
.map((m: { role: string; content: string }) => `${m.role}: ${m.content}`)
.join("\n\n")
- const prompt =
+ const lastMessage = clientMessages[clientMessages.length - 1]
+ const textPrompt =
clientMessages.length === 1
- ? clientMessages[0].content
+ ? lastMessage.content || ""
: `Previous conversation:\n${conversationHistory}\n\nRespond to the last user message.`
+ // If the last user message carries an image, send a streaming prompt with an
+ // image content block so Claude can see it (and search the index by it).
+ const uploadedImage: string | null =
+ typeof lastMessage?.image === "string" ? lastMessage.image : null
+
+ let prompt: string | AsyncGenerator<SDKUserMessage>
+ if (uploadedImage) {
+ const m = uploadedImage.match(/^data:(image\/[a-z.+-]+);base64,(.+)$/i)
+ const mediaType = m ? m[1] : "image/png"
+ const data = m ? m[2] : uploadedImage
+ const content = [
+ ...(textPrompt ? [{ type: "text", text: textPrompt }] : []),
+ { type: "image", source: { type: "base64", media_type: mediaType, data } },
+ ]
+ prompt = (async function* () {
+ yield {
+ type: "user",
+ message: { role: "user", content },
+ parent_tool_use_id: null,
+ } as unknown as SDKUserMessage
+ })()
+ } else {
+ prompt = textPrompt
+ }
+
const stream = new ReadableStream({
async start(controller) {
const encoder = new TextEncoder()
@@ -216,7 +288,7 @@ export async function POST(req: Request) {
controller.enqueue(encoder.encode(sseEvent(event, data)))
}
- const tools = createTools(send)
+ const tools = createTools(send, uploadedImage)
const mcpServer = createSdkMcpServer({
name: "pixelrag",
version: "1.0.0",