Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

support cursor web search and images

Raine Virta0206ccb998ca
diff --git a/src/providers/cursor/client.test.ts b/src/providers/cursor/client.test.ts
index 006d816..ff575e3 100644
--- a/src/providers/cursor/client.test.ts
+++ b/src/providers/cursor/client.test.ts
@@ -54,13 +54,112 @@ describe("Cursor protocol client", () => {
const clientMessages = sentFrames.map(decodeFrameJson) as Array<Record<string, any>>;
expect(clientMessages[0]?.runRequest.conversationId).toBe("conversation");
+ expect(clientMessages[0]?.runRequest.clientSupportsInlineImages).toBe(true);
expect(clientMessages[1]).toEqual({ execClientControlMessage: { heartbeat: {} } });
expect(clientMessages[2]?.execClientMessage.requestContextResult.success.requestContext).toBeDefined();
+ expect(clientMessages[2]?.execClientMessage.requestContextResult.success.requestContext.webSearchEnabled).toBe(true);
+ expect(clientMessages[2]?.execClientMessage.requestContextResult.success.requestContext.webFetchEnabled).toBe(true);
expect(clientMessages[3]).toEqual({ execClientControlMessage: { streamClose: {} } });
expect(clientMessages[4]).toEqual({ kvClientMessage: { setBlobResult: {} } });
expect(clientMessages[5]).toEqual({ kvClientMessage: { getBlobResult: {}, id: 2 } });
});
+ it("sends selected images in the Cursor run request", async () => {
+ const sentFrames: Uint8Array[] = [];
+
+ const upstream = await runCursorAgent({
+ prompt: "describe image",
+ mode: "AGENT_MODE_AGENT",
+ conversationId: "conversation",
+ model: { modelId: "composer-2.5" },
+ selectedImages: [
+ {
+ data: "aGVsbG8=",
+ uuid: "image-id",
+ path: "claude-image-1.png",
+ mimeType: "image/png",
+ },
+ ],
+ auth: { accessToken: "token", source: "test" },
+ ctx: fakeCtx(),
+ proto: fakeProto,
+ openRunStream: async () => ({
+ readable: streamFromChunks([encodeConnectFrame(jsonBytes({}), 2)]),
+ status: Promise.resolve({ status: 200 }),
+ async write(frame) {
+ sentFrames.push(frame);
+ },
+ close() {},
+ }),
+ });
+ await drain(upstream);
+
+ const clientMessages = sentFrames.map(decodeFrameJson) as Array<Record<string, any>>;
+ expect(clientMessages[0]?.runRequest.action.userMessageAction.userMessage.selectedContext).toEqual({
+ selectedImages: [
+ {
+ data: "aGVsbG8=",
+ uuid: "image-id",
+ path: "claude-image-1.png",
+ mimeType: "image/png",
+ },
+ ],
+ });
+ expect(clientMessages[0]?.runRequest.clientSupportsInlineImages).toBe(true);
+ });
+
+ it("approves Cursor web search and web fetch interaction queries", async () => {
+ const sentFrames: Uint8Array[] = [];
+
+ const upstream = await runCursorAgent({
+ prompt: "search the web",
+ mode: "AGENT_MODE_AGENT",
+ conversationId: "conversation",
+ model: { modelId: "composer-2.5" },
+ auth: { accessToken: "token", source: "test" },
+ ctx: fakeCtx(),
+ proto: fakeProto,
+ openRunStream: async () => ({
+ readable: streamFromChunks([
+ frame({
+ message: {
+ case: "interactionQuery",
+ value: { id: 11, query: { case: "webSearchRequestQuery", value: { args: { searchTerm: "cursor" } } } },
+ },
+ }),
+ frame({
+ message: {
+ case: "interactionQuery",
+ value: { id: 12, query: { case: "webFetchRequestQuery", value: { args: { url: "https://example.com" } } } },
+ },
+ }),
+ frame({
+ message: {
+ case: "interactionQuery",
+ value: { id: 13, query: { case: "generateImageRequestQuery", value: {} } },
+ },
+ }),
+ encodeConnectFrame(jsonBytes({}), 2),
+ ]),
+ status: Promise.resolve({ status: 200 }),
+ async write(frame) {
+ sentFrames.push(frame);
+ },
+ close() {},
+ }),
+ });
+ await drain(upstream);
+
+ const clientMessages = sentFrames.map(decodeFrameJson) as Array<Record<string, any>>;
+ expect(clientMessages).toContainEqual({
+ interactionResponse: { id: 11, webSearchRequestResponse: { approved: {} } },
+ });
+ expect(clientMessages).toContainEqual({
+ interactionResponse: { id: 12, webFetchRequestResponse: { approved: {} } },
+ });
+ expect(clientMessages.some((message) => message.interactionResponse?.id === 13)).toBe(false);
+ });
+
it("answers Cursor readArgs with file content and closes the exec stream", async () => {
const sentFrames: Uint8Array[] = [];
const dir = await mkdtemp(join(tmpdir(), "cursor-read-"));
diff --git a/src/providers/cursor/client.ts b/src/providers/cursor/client.ts
index 66066c6..a645785 100644
--- a/src/providers/cursor/client.ts
+++ b/src/providers/cursor/client.ts
@@ -8,12 +8,14 @@ import type { Logger } from "../../log.ts";
import type { CursorProto } from "./proto-loader.ts";
import { loadCursorProto } from "./proto-loader.ts";
import type { CursorAuth } from "./auth/token-store.ts";
+import type { CursorSelectedImage } from "./translate/request.ts";
export interface CursorRunOptions {
prompt: string;
mode: CursorAgentMode;
conversationId: string;
model: CursorModelRequest;
+ selectedImages?: CursorSelectedImage[];
auth: CursorAuth;
ctx: RequestContext;
proto?: CursorProto;
@@ -165,7 +167,7 @@ export async function runCursorAgent(opts: CursorRunOptions): Promise<ReadableSt
userMessage: {
text: opts.prompt,
messageId: crypto.randomUUID(),
- selectedContext: {},
+ selectedContext: selectedContext(opts.selectedImages),
mode: opts.mode,
},
},
@@ -176,6 +178,7 @@ export async function runCursorAgent(opts: CursorRunOptions): Promise<ReadableSt
excludeWorkspaceContext: false,
selectedSubagentModels: selectedSubagentModels(opts.model),
conversationGroupId: opts.conversationId,
+ clientSupportsInlineImages: true,
},
});
@@ -465,6 +468,9 @@ async function processServerControlFrames(
if (typeof kv.id === "number" && kv.id !== 0) msg.id = kv.id;
await append({ kvClientMessage: msg });
}
+ } else if (oneof?.case === "interactionQuery") {
+ const response = buildInteractionResponse(oneof.value);
+ if (response) await append({ interactionResponse: response });
}
}
}
@@ -891,6 +897,8 @@ function buildRequestContextResult(exec: {
projectFolder: process.cwd(),
processWorkingDirectory: process.cwd(),
},
+ webSearchEnabled: true,
+ webFetchEnabled: true,
repositoryInfoComplete: true,
rulesInfoComplete: true,
envInfoComplete: true,
@@ -905,6 +913,21 @@ function buildRequestContextResult(exec: {
};
}
+function buildInteractionResponse(query: {
+ id?: number;
+ query?: { case?: string; value?: unknown };
+} | undefined): Record<string, unknown> | undefined {
+ if (!query || typeof query.id !== "number") return undefined;
+ switch (query.query?.case) {
+ case "webSearchRequestQuery":
+ return { id: query.id, webSearchRequestResponse: { approved: {} } };
+ case "webFetchRequestQuery":
+ return { id: query.id, webFetchRequestResponse: { approved: {} } };
+ default:
+ return undefined;
+ }
+}
+
const controlFrameState = new WeakMap<
(messageJson: unknown) => Promise<void>,
{ buffer: Buffer; execHeartbeatSent: boolean; requestContextAcked: boolean }
@@ -917,6 +940,7 @@ interface CursorOneofMessage {
id?: number;
execId?: string;
message?: { case?: string; value?: unknown };
+ query?: { case?: string; value?: unknown };
};
};
}
@@ -927,6 +951,7 @@ function summarizeCursorOneofMessage(message: CursorOneofMessage): Record<string
return {
case: oneof?.case ?? "unknown",
innerCase: value?.message?.case,
+ queryCase: value?.query?.case,
id: value?.id,
execId: value?.execId,
};
@@ -1112,6 +1137,18 @@ export function encodeConnectFrame(payload: Uint8Array, flags = 0): Uint8Array {
return out;
}
+function selectedContext(images: CursorSelectedImage[] | undefined): Record<string, unknown> {
+ if (!images?.length) return {};
+ return {
+ selectedImages: images.map((image) => ({
+ data: image.data,
+ uuid: image.uuid,
+ path: image.path,
+ mimeType: image.mimeType,
+ })),
+ };
+}
+
function selectedSubagentModels(model: CursorModelRequest): CursorModelRequest[] {
return [
{ modelId: "default" },
diff --git a/src/providers/cursor/index.ts b/src/providers/cursor/index.ts
index d58d67d..66d529d 100644
--- a/src/providers/cursor/index.ts
+++ b/src/providers/cursor/index.ts
@@ -15,7 +15,7 @@ import {
loadCursorAuth,
missingAuthMessage,
} from "./auth/token-store.ts";
-import { renderCursorPrompt } from "./translate/request.ts";
+import { cursorSelectedImages, renderCursorPrompt } from "./translate/request.ts";
import { CURSOR_SUPPORTED_MODELS, resolveCursorModel } from "./translate/model.ts";
import {
accumulateCursorResponse,
@@ -71,6 +71,7 @@ async function handleMessages(
const selection = resolveCursorModel(body);
const prompt = renderCursorPrompt(body);
+ const selectedImages = cursorSelectedImages(body);
const wantStream = wantsDownstreamStream(body);
const conversationId = cursorConversationForRequest(body, ctx.sessionId);
@@ -82,6 +83,7 @@ async function handleMessages(
stream: wantStream,
messageCount: body.messages.length,
promptChars: prompt.length,
+ selectedImageCount: selectedImages.length,
});
if (logVerbose()) log.debug("cursor prompt", { prompt });
@@ -118,6 +120,7 @@ async function handleMessages(
mode: selection.mode,
conversationId,
model: selection.requestedModel,
+ selectedImages,
auth,
ctx,
readHandler: bridgeRead
diff --git a/src/providers/cursor/translate/request.test.ts b/src/providers/cursor/translate/request.test.ts
index 895c632..8615299 100644
--- a/src/providers/cursor/translate/request.test.ts
+++ b/src/providers/cursor/translate/request.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "bun:test";
-import { renderCursorPrompt } from "./request.ts";
+import { cursorSelectedImages, renderCursorPrompt } from "./request.ts";
import type { AnthropicRequest } from "../../../anthropic/schema.ts";
describe("Cursor prompt rendering", () => {
@@ -38,4 +38,28 @@ describe("Cursor prompt rendering", () => {
expect(prompt).toContain('<tool_result tool_use_id="toolu_1">');
expect(prompt).toContain('"name":"Read"');
});
+
+ it("extracts base64 image blocks for Cursor selected context", () => {
+ const req: AnthropicRequest = {
+ model: "cursor",
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Describe this" },
+ { type: "image", source: { type: "base64", media_type: "image/png", data: "aGVsbG8=" } },
+ { type: "image", source: { type: "url", url: "https://example.invalid/image.png" } },
+ ],
+ },
+ ],
+ };
+
+ const images = cursorSelectedImages(req);
+
+ expect(images).toHaveLength(1);
+ expect(images[0]?.data).toBe("aGVsbG8=");
+ expect(images[0]?.mimeType).toBe("image/png");
+ expect(images[0]?.path).toBe("claude-image-1.png");
+ expect(typeof images[0]?.uuid).toBe("string");
+ });
});
diff --git a/src/providers/cursor/translate/request.ts b/src/providers/cursor/translate/request.ts
index 4cd8055..474ac4a 100644
--- a/src/providers/cursor/translate/request.ts
+++ b/src/providers/cursor/translate/request.ts
@@ -1,11 +1,19 @@
import type {
AnthropicContentBlock,
+ AnthropicImageBlock,
AnthropicMessage,
AnthropicRequest,
AnthropicTextBlock,
AnthropicToolResultContentBlock,
} from "../../../anthropic/schema.ts";
+export interface CursorSelectedImage {
+ data: string;
+ uuid: string;
+ path: string;
+ mimeType: string;
+}
+
export function renderCursorPrompt(req: AnthropicRequest): string {
const sections: string[] = [];
const system = renderSystem(req.system);
@@ -33,6 +41,27 @@ export function renderCursorPrompt(req: AnthropicRequest): string {
return sections.join("\n\n");
}
+export function cursorSelectedImages(req: AnthropicRequest): CursorSelectedImage[] {
+ const images: CursorSelectedImage[] = [];
+ let index = 0;
+ for (const message of req.messages) {
+ for (const block of messageBlocks(message)) {
+ collectImageBlocks(block, (image) => {
+ if (image.source.type !== "base64") return;
+ const uuid = crypto.randomUUID();
+ const extension = imageExtension(image.source.media_type);
+ images.push({
+ data: image.source.data,
+ uuid,
+ path: `claude-image-${++index}.${extension}`,
+ mimeType: image.source.media_type,
+ });
+ });
+ }
+ }
+ return images;
+}
+
function renderSystem(system: AnthropicRequest["system"]): string | undefined {
if (!system) return undefined;
const blocks: AnthropicTextBlock[] =
@@ -46,9 +75,7 @@ function renderSystem(system: AnthropicRequest["system"]): string | undefined {
}
function renderContent(message: AnthropicMessage): string {
- const blocks: AnthropicContentBlock[] =
- typeof message.content === "string" ? [{ type: "text", text: message.content }] : message.content;
- return blocks.map(renderBlock).filter(Boolean).join("\n\n");
+ return messageBlocks(message).map(renderBlock).filter(Boolean).join("\n\n");
}
function renderBlock(block: AnthropicContentBlock): string {
@@ -79,3 +106,35 @@ function renderToolResultBlock(block: AnthropicToolResultContentBlock): string {
const type = typeof block.type === "string" ? block.type : "unknown";
return `[unsupported tool result block: ${type}]`;
}
+
+function messageBlocks(message: AnthropicMessage): AnthropicContentBlock[] {
+ return typeof message.content === "string" ? [{ type: "text", text: message.content }] : message.content;
+}
+
+function collectImageBlocks(block: AnthropicContentBlock, visit: (block: AnthropicImageBlock) => void): void {
+ if (block.type === "image") {
+ visit(block);
+ return;
+ }
+ if (block.type !== "tool_result" || typeof block.content === "string") return;
+ for (const child of block.content) {
+ if (child.type === "text" || child.type === "image" || child.type === "tool_use" || child.type === "tool_result" || child.type === "thinking") {
+ collectImageBlocks(child as AnthropicContentBlock, visit);
+ }
+ }
+}
+
+function imageExtension(mediaType: string): string {
+ switch (mediaType) {
+ case "image/jpeg":
+ return "jpg";
+ case "image/png":
+ return "png";
+ case "image/gif":
+ return "gif";
+ case "image/webp":
+ return "webp";
+ default:
+ return "img";
+ }
+}