Open source CapCut alternative.
TypeScript
refactor(site): gemini adapter sheds desktop-agent tooling [rebuild]
David Le17ffc805da98
diff --git a/docs/guides/backend-apis.md b/docs/guides/backend-apis.mdindex 049dc269..75191d6d 100644--- a/docs/guides/backend-apis.md+++ b/docs/guides/backend-apis.md@@ -97,32 +97,7 @@ route reads an object only under the caller's own prefix. And storage is how alarge request succeeds, never why a small one fails: a client that can't reachstorage sends the bytes inline, as it always could.-### Computer use--Computer use is a built-in tool of Gemini's main flash model, so one model-drives both a browser and a guarded macOS desktop. The Mac app sends a request-tool; the adapter maps it to an environment and returns the model's native-action calls for the app to execute.--| Request tool | Environment | Used for |-|---|---|---|-| `donkey_gemini_browser_interaction` | browser | web interaction |-| `donkey_gemini_mac_desktop_interaction` | desktop | guarded macOS desktop interaction |-| `donkey_debug_ui_inspection` | — | developer-only, read-only UI inspection |--The first two return action calls the app executes. UI inspection is read-only:-the adapter routes it through a vision-capable model but must never forward or-run a UI action call.--### Screenshot parsing--Screenshot parsing is its own route, `POST /api/inference/screenshots/parse/`.-It takes a scoped app, window, or system-navigation screenshot — never a-whole-desktop capture — and returns read-only UI evidence in the Mac app's local-UI-understanding shape. The default provider is Gemini Flash, configured with-hosted Google credentials or `GEMINI_API_KEY`.--### Gemini and OpenAI adapters+### Gemini adapterThe Gemini adapter uses the official `@google/genai` SDK. It runs on Vertex AI'sglobal endpoint only when `GOOGLE_APPLICATION_CREDENTIALS_JSON` carries a@@ -133,17 +108,11 @@ Model choice is code (see Handler Rules on `process.env`):| Call | Model ||---|---|-| Fast structured decisions: task-intent, follow-ups | `gemini-3.1-flash-lite` |-| General chat, non-decision structured calls, computer use | `gemini-3.5-flash` |--Structured-decision requests normalize their JSON schema for Gemini and retry-without a provider-enforced schema when Vertex rejects the schema parameters.-Whether the returned JSON is actually executable is decided by Mac-side runtime-validation, not the backend.+| Gate-judged simple chat turns, fast structured decisions | `gemini-3.1-flash-lite` |+| General chat, non-decision structured calls | `gemini-3.5-flash` |-The OpenAI Responses adapter exists only for developer read-only UI inspection-and uses `OPENAI_API_KEY`. Keep that key in the hosted deploy; the Mac app never-carries it.+Structured requests normalize their JSON schema for Gemini and retry without a+provider-enforced schema when Vertex rejects the schema parameters.## Hosted Model Creditsdiff --git a/site/src/app/api/inference/responses/route.ts b/site/src/app/api/inference/responses/route.tsindex a22222e8..c79629c3 100644--- a/site/src/app/api/inference/responses/route.ts+++ b/site/src/app/api/inference/responses/route.ts@@ -10,7 +10,7 @@ import {} from "@/lib/credits/inference";import { createProviderRegistry } from "@/lib/inference/router";import { resolveInferenceBlobs } from "@/lib/inference/blobs";-import { isJsonObject, toJsonObject, toJsonValue } from "@/lib/inference/json";+import { toJsonObject } from "@/lib/inference/json";import {inferenceErrorCode,inferenceProviderErrorResponse,@@ -18,7 +18,6 @@ import {validationErrorResponse,} from "@/lib/inference/responses";import { InferenceProviderError } from "@/lib/inference/providers";-import type { JsonObject, JsonValue } from "@/lib/inference/providers";import { responseCreateRequestSchema } from "@/lib/inference/schemas";import {shouldBypassDonkeyInferenceCredits,@@ -55,19 +54,6 @@ export const POST = withDonkeyAuth(async (request) => {}}- const debugInspection = isDebugUIInspectionRequest(parsed.data.body);- const debugInspectionStartedAt = performance.now();- if (debugInspection) {- console.info(- [- "[debug-ui-inspection] start",- `at=${new Date().toISOString()}`,- `provider=${parsed.data.donkeyProvider ?? "default"}`,- `requestedModel=${requestedModel}`,- ].join(" "),- );- }-let failedUsageProvider = parsed.data.donkeyProvider ?? "default";try {@@ -182,7 +168,6 @@ export const POST = withDonkeyAuth(async (request) => {});}- const providerStartedAt = performance.now();const result = await provider.createResponse?.(data);if (!result) {if (!bypassCredits) {@@ -224,21 +209,6 @@ export const POST = withDonkeyAuth(async (request) => {usageHeaders = creditUsageHeaders(recordedUsage);}- if (debugInspection) {- const providerEndedAt = performance.now();- console.info(- [- "[debug-ui-inspection] end",- `at=${new Date().toISOString()}`,- `provider=${result.provider}`,- `model=${result.model}`,- `providerMs=${Math.round(providerEndedAt - providerStartedAt)}`,- `totalMs=${Math.round(providerEndedAt - debugInspectionStartedAt)}`,- debugInspectionResultSummary(result.body),- ].join(" "),- );- }-return NextResponse.json(result.body, {headers: {"X-Donkey-Inference-Provider": result.provider,@@ -270,46 +240,3 @@ export const POST = withDonkeyAuth(async (request) => {throw error;}});--function isDebugUIInspectionRequest(body: JsonObject) {- if (!Array.isArray(body.tools)) {- return false;- }-- return body.tools.some((tool) => {- return isJsonObject(tool) && tool.type === "donkey_debug_ui_inspection";- });-}--function debugInspectionResultSummary(body: JsonValue) {- const object: JsonObject = isJsonObject(body) ? body : {};- const outputText = typeof object.output_text === "string" ? object.output_text : "";- const elementCount =- elementCountFromFrame(object) ??- elementCountFromOutputText(outputText);-- return [- `outputTextChars=${outputText.length}`,- `elements=${elementCount ?? "unknown"}`,- ].join(" ");-}--function elementCountFromOutputText(outputText: string) {- if (!outputText.trim()) {- return null;- }-- try {- return elementCountFromFrame(toJsonValue(JSON.parse(outputText)));- } catch {- return null;- }-}--function elementCountFromFrame(value: JsonValue) {- if (!isJsonObject(value) || !Array.isArray(value.elements)) {- return null;- }-- return value.elements.length;-}diff --git a/site/src/app/api/inference/screenshots/parse/route.ts b/site/src/app/api/inference/screenshots/parse/route.tsdeleted file mode 100644index b310779a..00000000--- a/site/src/app/api/inference/screenshots/parse/route.ts+++ /dev/null@@ -1,235 +0,0 @@-import { NextResponse } from "next/server";--import {- creditErrorResponse,- creditUsageHeaders,- inferenceUsageRoutes,- recordFailedInferenceUsage,- recordInferenceUsage,- requireInferenceCredits,-} from "@/lib/credits/inference";-import { withDonkeyAuth } from "@/lib/donkey-api-auth";-import { InferenceProviderError } from "@/lib/inference/providers";-import {- checkInMemoryRateLimit,- rateLimitResponse,-} from "@/lib/inference/rate-limit";-import { readLimitedJsonBody } from "@/lib/inference/request-body";-import {- inferenceErrorCode,- inferenceProviderErrorResponse,- requireInferenceClientId,- validationErrorResponse,-} from "@/lib/inference/responses";-import {- createScreenshotParserProvider,- parseScreenshot,- parseScreenshotStream,-} from "@/lib/inference/screenshot-parsing";-import { screenshotParseRequestSchema } from "@/lib/inference/screenshot-parsing/schema";-import type { ScreenshotParseRequest } from "@/lib/inference/screenshot-parsing/schema";--export const dynamic = "force-dynamic";--const screenshotParseMaxBodyBytes = 6_250_000;-const screenshotParseRateLimit = {- limit: 4,- windowMs: 3_000,-};--export const POST = withDonkeyAuth(async (request) => {- const client = requireInferenceClientId(request.donkey.clientId);- if (!client.ok) {- return client.response;- }-- const rateLimit = checkInMemoryRateLimit({- key: `${request.donkey.userId}:${client.clientId}`,- ...screenshotParseRateLimit,- });- if (!rateLimit.ok) {- return rateLimitResponse(rateLimit.retryAfterSeconds);- }-- const body = await readLimitedJsonBody(request, screenshotParseMaxBodyBytes);- if (!body.ok) {- return body.response;- }-- const parsed = screenshotParseRequestSchema.safeParse(body.json);- if (!parsed.success) {- return validationErrorResponse(parsed.error);- }- if (parsed.data.metadata["screenshot.scope"] === "desktop") {- return NextResponse.json(- {- error: "Unsupported screenshot scope",- message: "Screenshot parsing accepts app/window or system-navigation screenshots, not full-desktop captures.",- },- { status: 400 },- );- }-- const parserProvider = createScreenshotParserProvider();- const model = parserProvider.modelForRequest(parsed.data);- const credits = await requireInferenceCredits({- model,- provider: parserProvider.inferenceProvider,- route: inferenceUsageRoutes.screenshotParse,- userId: request.donkey.userId,- });- if (!credits.ok) {- return credits.response;- }-- if (parsed.data.stream) {- return streamScreenshotParseResponse({- clientId: client.clientId,- conversationId: request.donkey.conversationId,- model,- parserProvider,- request: parsed.data,- userId: request.donkey.userId,- });- }-- try {- const result = await parseScreenshot(parsed.data, parserProvider);- const recordedUsage = await recordInferenceUsage({- clientId: client.clientId,- conversationId: request.donkey.conversationId,- metadata: {- parserProvider: parserProvider.id,- },- model: result.model,- provider: result.provider,- requestKind: "screenshot_parse",- route: inferenceUsageRoutes.screenshotParse,- status: "succeeded",- usage: result.usage,- userId: request.donkey.userId,- });-- return NextResponse.json(result.result, {- headers: {- "X-Donkey-Inference-Provider": result.provider,- "X-Donkey-Inference-Model": result.model,- ...creditUsageHeaders(recordedUsage),- },- });- } catch (error) {- await recordFailedInferenceUsage({- clientId: client.clientId,- conversationId: request.donkey.conversationId,- errorCode: inferenceErrorCode(error),- metadata: {- parserProvider: parserProvider.id,- },- model,- provider: parserProvider.inferenceProvider,- requestKind: "screenshot_parse",- route: inferenceUsageRoutes.screenshotParse,- userId: request.donkey.userId,- });- const creditResponse = creditErrorResponse(error);- if (creditResponse) {- return creditResponse;- }- if (error instanceof InferenceProviderError) {- return inferenceProviderErrorResponse(error);- }-- throw error;- }-});--function streamScreenshotParseResponse(input: {- clientId: string;- conversationId: string | null;- model: string;- parserProvider: ReturnType<typeof createScreenshotParserProvider>;- request: ScreenshotParseRequest;- userId: string;-}) {- const encoder = new TextEncoder();- const stream = new ReadableStream<Uint8Array>({- async start(controller) {- try {- for await (const event of parseScreenshotStream(input.request, input.parserProvider)) {- if (event.type === "partial") {- controller.enqueue(encoder.encode(serverSentEvent("partial", event.result)));- continue;- }-- const recordedUsage = await recordInferenceUsage({- clientId: input.clientId,- conversationId: input.conversationId,- metadata: {- parserProvider: input.parserProvider.id,- streaming: true,- },- model: event.model,- provider: event.provider,- requestKind: "screenshot_parse",- route: inferenceUsageRoutes.screenshotParse,- status: "succeeded",- usage: event.usage,- userId: input.userId,- });- controller.enqueue(- encoder.encode(- serverSentEvent("final", {- ...event.result,- metadata: {- ...event.result.metadata,- ...creditUsageHeaders(recordedUsage),- },- }),- ),- );- }- } catch (error) {- await recordFailedInferenceUsage({- clientId: input.clientId,- conversationId: input.conversationId,- errorCode: inferenceErrorCode(error),- metadata: {- parserProvider: input.parserProvider.id,- streaming: true,- },- model: input.model,- provider: input.parserProvider.inferenceProvider,- requestKind: "screenshot_parse",- route: inferenceUsageRoutes.screenshotParse,- userId: input.userId,- });- controller.enqueue(- encoder.encode(- serverSentEvent("error", {- error: inferenceErrorCode(error),- message: error instanceof Error ? error.message : "Screenshot parsing failed.",- details: error instanceof InferenceProviderError ? error.details : null,- }),- ),- );- } finally {- controller.close();- }- },- });-- return new Response(stream, {- headers: {- "Cache-Control": "no-cache, no-transform",- "Connection": "keep-alive",- "Content-Type": "text/event-stream; charset=utf-8",- "X-Accel-Buffering": "no",- "X-Donkey-Inference-Model": input.model,- "X-Donkey-Inference-Provider": input.parserProvider.inferenceProvider,- },- });-}--function serverSentEvent(event: string, data: unknown) {- return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;-}diff --git a/site/src/lib/credits/inference.ts b/site/src/lib/credits/inference.tsindex d80c2f9c..5a87bbd9 100644--- a/site/src/lib/credits/inference.ts+++ b/site/src/lib/credits/inference.ts@@ -24,7 +24,6 @@ export const inferenceUsageRoutes = {browserRun: "/api/browser/run/",chatCompletions: "/api/inference/chat/completions/",responses: "/api/inference/responses/",- screenshotParse: "/api/inference/screenshots/parse/",} as const;type CreditsDatabase = PrismaClient | Prisma.TransactionClient;diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.tsindex 16259200..96763a12 100644--- a/site/src/lib/credits/provider-pricing.ts+++ b/site/src/lib/credits/provider-pricing.ts@@ -16,7 +16,6 @@ import {type GeminiOmniModel,type GeminiTtsModel,} from "@/lib/inference/gemini-models";-import { openaiModels, type OpenAIRunModel } from "@/lib/inference/openai-models";import { browserUsePerStepUsd } from "@/lib/browser/pricing";export type ProviderCreditPricing = {@@ -73,19 +72,6 @@ export function providerCreditPricing(return undefined;}-// Every OpenAI model the gateway selects must appear here: the Record is keyed by the-// OpenAIRunModel union, so adding a run model without a price fails the type-check (and the-// build). Matched exactly (not by prefix) so it never shadows a more specific table entry.-const openaiRunModelPricing: Record<OpenAIRunModel, ProviderCreditPricing> = {- [openaiModels.debugInspection]: textTokenPricing({- model: "gpt-5.4",- input: "2.5",- cachedInput: "0.25",- output: "15",- longContext: { input: "5", cachedInput: "0.5", output: "22.5" },- }),-};-function browserUseCreditPricing(): ProviderCreditPricing {// Browser Use Cloud bills ~$0.01/task init + a per-step LLM fee, and the API// exposes stepCount (not a USD cost), so we price per step. The per-step rate@@ -102,11 +88,6 @@ function openAICreditPricing(model: string): ProviderCreditPricing | undefined {return audioPricing;}- const runModelPricing = openaiRunModelPricing[model as OpenAIRunModel];- if (runModelPricing) {- return runModelPricing;- }-const matched = openAITextCreditRates.find((rate) => modelMatches(model, rate.model));if (!matched) {return undefined;diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.tsindex fd1e7b21..db079f48 100644--- a/site/src/lib/inference/adapters/gemini-context-cache.ts+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts@@ -97,18 +97,13 @@ async function resolveCachedSystemInstruction(args: {/*** Swap a large, repeated inline system instruction for a cached reference, in place on `requestParameters`.- * Applies only to the text/decision path (no Gemini tools registered) where the same instruction recurs- * every step; tool/computer-use calls and short instructions are left untouched. A no-op on any failure, so- * the request always stays valid.+ * Applies where the same instruction recurs every step; short instructions are left untouched.+ * A no-op on any failure, so the request always stays valid.*/export async function applyContextCacheToRequest(requestParameters: GenerateContentParameters,- registeredTools: string[],client: GeminiClient,): Promise<void> {- if (registeredTools.length > 0) {- return;- }const config = requestParameters.config;const systemInstruction = config?.systemInstruction;if (!config || typeof systemInstruction !== "string" || systemInstruction.length < MIN_CACHEABLE_CHARS) {diff --git a/site/src/lib/inference/adapters/gemini-computer-use.ts b/site/src/lib/inference/adapters/gemini-responses.tssimilarity index 80%rename from site/src/lib/inference/adapters/gemini-computer-use.tsrename to site/src/lib/inference/adapters/gemini-responses.tsindex fc599de8..3e1e231f 100644--- a/site/src/lib/inference/adapters/gemini-computer-use.ts+++ b/site/src/lib/inference/adapters/gemini-responses.ts@@ -1,6 +1,5 @@import {ApiError,- Environment,GoogleGenAI,ThinkingLevel,} from "@google/genai";@@ -47,45 +46,11 @@ import {type TextStreamResult,} from "@/lib/inference/providers";-const providerID = "gemini-computer-use";+const providerID = "gemini-responses";const geminiProviderID = "gemini";-const defaultDecisionResponsesModel = geminiModelRoles.fastDecision;const defaultVertexResponsesModel = geminiModelRoles.chat;-const defaultComputerUseModel = geminiModelRoles.computerUse;--export const geminiBrowserInteractionToolType = "donkey_gemini_browser_interaction";-export const geminiMacDesktopInteractionToolType = "donkey_gemini_mac_desktop_interaction";-export const debugUIInspectionToolType = "donkey_debug_ui_inspection";--// Note: `generic_harness_planning` is deliberately NOT fast-decision. Planning is the most-// reasoning-heavy step (routing, plan steps, applying loaded skill guidance, and the act-vs-confirm-// call), and the flash-lite model follows that nuanced guidance poorly — it tends to ask for-// clarification on requests a skill can resolve directly. Route planning to the stronger model.-const fastDecisionSchemaNames = new Set([- "task_intent_v1",- "task_followup_resolution_v1",-]);--const fastDecisionPromptVersions = new Set([- "task-followup-resolution-v1",-]);-const browserOnlyFunctionExclusions = [- "drag_and_drop",-];--// On the macOS desktop the agent never drives a web browser chrome, so the-// browser-navigation predefined functions are excluded — leaving the GUI actions-// (click, type, scroll, drag_and_drop, key_combination, …) the desktop needs.-const desktopExcludedFunctions = [- "open_web_browser",- "navigate",- "go_back",- "go_forward",- "search",-];--export function createGeminiComputerUseProvider(+export function createGeminiResponsesProvider(environment: AdapterEnvironment = process.env,clientFactory: GeminiClientFactory = (options) => new GoogleGenAI(options),): InferenceProvider {@@ -98,12 +63,11 @@ export function createGeminiComputerUseProvider(return [];}- // gemini-3.5-flash serves both general Responses and computer use, so list it once with the- // computer-use capability — listing it twice would let dedupeModels drop the computer-use entry.- // The fast-decision model (flash-lite) is the only other entry.+ // The full flash model serves general Responses; the fast-decision model+ // (flash-lite) is the only other entry.return [- staticModel(defaultDecisionResponsesModel, false),- staticModel(defaultComputerUseModel, true),+ staticModel(geminiModelRoles.fastDecision),+ staticModel(defaultVertexResponsesModel),];}@@ -113,31 +77,22 @@ export function createGeminiComputerUseProvider(// Caller-defined function tools are honored only when the caller explicitly// asked for Gemini; the default Responses route keeps sending them elsewhere.const allowFunctionTools = request.donkeyProvider === geminiProviderID;- const registeredTools = registeredToolTypes(request.body.tools);if (hasExplicitUnsupportedTools(request.body.tools, allowFunctionTools)) {throw new InferenceProviderError("Gemini Responses received unsupported tool declarations.", {statusCode: 400,code: "gemini_tool_unsupported",details: {- supportedTools: [- geminiBrowserInteractionToolType,- geminiMacDesktopInteractionToolType,- debugUIInspectionToolType,- ],+ supportedTools: allowFunctionTools ? ["function"] : [],},});}- const model = requestedModel(- request.body,- defaultResponseModel(request.body, registeredTools),- );+ const model = requestedModel(request.body, defaultVertexResponsesModel);const requestParameters = geminiGenerateContentParameters(request.body,- registeredTools,model,);- return { registeredTools, model, requestParameters };+ return { model, requestParameters };}async function createResponse(@@ -145,14 +100,14 @@ export function createGeminiComputerUseProvider(): Promise<ResponseCreateResult> {ensureConfigured(configured);- const { registeredTools, model, requestParameters } = responseRequestSetup(request);+ const { model, requestParameters } = responseRequestSetup(request);const client = clientFactory(clientConfig.options);// Cache the planner's large, byte-identical system instruction once and reference it on later steps, so// its ~11K tokens bill at the cached rate instead of full price on every step. Best-effort: a no-op for- // tool/computer-use calls and short instructions, and a clean fallback to the inline instruction on any+ // short instructions, and a clean fallback to the inline instruction on any// caching error — so the request is never made invalid by this.- await applyContextCacheToRequest(requestParameters, registeredTools, client);+ await applyContextCacheToRequest(requestParameters, client);let rawResponse: unknown;let retriedWithoutSchema = false;@@ -174,7 +129,7 @@ export function createGeminiComputerUseProvider(}const rawBody = toJsonValue(rawResponse);- const body = normalizedGeminiResponse(toJsonValue(rawBody), registeredTools);+ const body = normalizedGeminiResponse(toJsonValue(rawBody));return {provider: geminiProviderID,model,@@ -184,7 +139,6 @@ export function createGeminiComputerUseProvider(provider: geminiProviderID,api: "google-genai-sdk",service: clientConfig.service,- registeredTools,structuredSchemaRetry: String(retriedWithoutSchema),},};@@ -200,9 +154,9 @@ export function createGeminiComputerUseProvider(): Promise<ResponseStreamResult> {ensureConfigured(configured);- const { registeredTools, model, requestParameters } = responseRequestSetup(request);+ const { model, requestParameters } = responseRequestSetup(request);const client = clientFactory(clientConfig.options);- await applyContextCacheToRequest(requestParameters, registeredTools, client);+ await applyContextCacheToRequest(requestParameters, client);let iterator: AsyncGenerator<GenerateContentResponse>;try {@@ -214,12 +168,11 @@ export function createGeminiComputerUseProvider(return {provider: geminiProviderID,model,- events: responseStreamEvents(iterator, registeredTools),+ events: responseStreamEvents(iterator),metadata: {provider: geminiProviderID,api: "google-genai-sdk",service: clientConfig.service,- registeredTools,},};}@@ -249,7 +202,7 @@ export function createGeminiComputerUseProvider(}const rawBody = toJsonValue(rawResponse);- const normalized = normalizedGeminiResponse(rawBody, []);+ const normalized = normalizedGeminiResponse(rawBody);const outputText = stringValue(normalized.output_text) ?? "";return {provider: geminiProviderID,@@ -353,7 +306,6 @@ export function createGeminiComputerUseProvider(// signatures. Non-thought text also goes out live as delta events.async function* responseStreamEvents(iterator: AsyncGenerator<GenerateContentResponse>,- registeredTools: string[],): AsyncGenerator<ResponseStreamEvent> {const parts: JsonObject[] = [];let responseId: string | undefined;@@ -395,7 +347,6 @@ async function* responseStreamEvents(...(usageMetadata !== undefined ? { usageMetadata } : {}),candidates: [{ content: { parts } }],},- registeredTools,);yield { type: "completed", body, usage: body.usage };}@@ -409,10 +360,9 @@ function streamChunkText(chunk: GenerateContentResponse): string {function geminiGenerateContentParameters(body: JsonObject,- registeredTools: string[],model: string,): GenerateContentParameters {- const tools = geminiTools(registeredTools, body.tools);+ const tools = geminiTools(body.tools);const generationConfig = generationConfigFromBody(body);const systemInstruction = systemInstructionFromBody(body);const config: GenerateContentConfig = {@@ -432,33 +382,9 @@ function geminiGenerateContentParameters(};}-function geminiTools(registeredTools: string[], rawTools: JsonValue | undefined): Tool[] {+function geminiTools(rawTools: JsonValue | undefined): Tool[] {const tools: Tool[] = [];- if (registeredTools.includes(geminiBrowserInteractionToolType)) {- tools.push({- computerUse: {- environment: Environment.ENVIRONMENT_BROWSER,- excludedPredefinedFunctions: excludedPredefinedFunctions(- rawTools,- browserOnlyFunctionExclusions,- ),- },- });- }-- if (registeredTools.includes(geminiMacDesktopInteractionToolType)) {- tools.push({- computerUse: {- environment: Environment.ENVIRONMENT_DESKTOP,- excludedPredefinedFunctions: excludedPredefinedFunctions(- rawTools,- desktopExcludedFunctions,- ),- },- });- }-const declarations = functionDeclarationsFromTools(rawTools);if (declarations.length > 0) {tools.push({ functionDeclarations: declarations });@@ -762,7 +688,7 @@ function generationConfigFromBody(body: JsonObject): Partial<GenerateContentConf}// Bound reasoning so thinking tokens (which count against maxOutputTokens) can't starve the// structured output. Callers driving tight per-turn loops pass a small budget; 0 disables thinking.- // Gemini 3.x models (e.g. gemini-3.5-flash) take thinking_level (minimal|low|medium|high), NOT the+ // Gemini 3.x models (e.g. gemini-3.7-flash) take thinking_level (minimal|low|medium|high), NOT the// integer thinking_budget — passing the budget to them is silently ignored. Prefer the level when the// caller sets it; the two are mutually exclusive. Older 2.x models still use thinking_budget. In both// cases request the thought summary so callers can persist the reasoning (the normalized response@@ -926,44 +852,6 @@ function responseFormatFromBody(body: JsonObject): { json: boolean; schema?: Jsoreturn null;}-function defaultResponseModel(body: JsonObject, registeredTools: string[]) {- if (- registeredTools.includes(geminiBrowserInteractionToolType) ||- registeredTools.includes(geminiMacDesktopInteractionToolType)- ) {- return defaultComputerUseModel;- }-- if (registeredTools.includes(debugUIInspectionToolType)) {- return defaultDecisionResponsesModel;- }-- if (isFastDecisionRequest(body)) {- return defaultDecisionResponsesModel;- }-- return defaultVertexResponsesModel;-}--function isFastDecisionRequest(body: JsonObject) {- const metadata = isJsonObject(body.metadata) ? body.metadata : {};- const promptVersion =- stringValue(metadata.prompt_version) ??- stringValue(metadata.promptVersion);- if (promptVersion && fastDecisionPromptVersions.has(promptVersion)) {- return true;- }-- const format =- isJsonObject(body.text) && isJsonObject(body.text.format)- ? body.text.format- : isJsonObject(body.response_format)- ? body.response_format- : null;- const schemaName = format ? stringValue(format.name) : undefined;- return Boolean(schemaName && fastDecisionSchemaNames.has(schemaName));-}-function systemInstructionFromBody(body: JsonObject): string | undefined {const instruction = [stringValue(body.instructions),@@ -976,10 +864,7 @@ function systemInstructionFromBody(body: JsonObject): string | undefined {return instruction;}-function normalizedGeminiResponse(- raw: JsonValue,- registeredTools: string[],-): JsonObject {+function normalizedGeminiResponse(raw: JsonValue): JsonObject {const partObjects = geminiCandidateParts(geminiCandidates(raw)[0]);// Thought summaries (parts flagged `thought: true` when includeThoughts is on) must NOT land in// output_text — that field carries the structured JSON the caller parses. Keep them separate so the@@ -1015,10 +900,6 @@ function normalizedGeminiResponse(...call,})),],- computer_use: {- registered_tools: registeredTools,- calls,- },provider_output: raw,usage: isJsonObject(raw) ? raw.usageMetadata ?? null : null,};@@ -1051,49 +932,6 @@ function functionCallFromPart(part: JsonObject): JsonObject | null {return call;}-function registeredToolTypes(tools: JsonValue | undefined): string[] {- if (!Array.isArray(tools)) {- return [];- }-- const registered = new Set<string>();- for (const tool of tools) {- if (!isJsonObject(tool)) {- continue;- }- if (tool.type === geminiBrowserInteractionToolType) {- registered.add(geminiBrowserInteractionToolType);- } else if (tool.type === geminiMacDesktopInteractionToolType) {- registered.add(geminiMacDesktopInteractionToolType);- } else if (tool.type === debugUIInspectionToolType) {- registered.add(debugUIInspectionToolType);- }- }- return [...registered];-}--function excludedPredefinedFunctions(rawTools: JsonValue | undefined, defaults: string[]) {- const excluded = new Set(defaults);- if (Array.isArray(rawTools)) {- for (const tool of rawTools) {- if (!isJsonObject(tool)) {- continue;- }- const values = Array.isArray(tool.excludedPredefinedFunctions)- ? tool.excludedPredefinedFunctions- : Array.isArray(tool.excluded_predefined_functions)- ? tool.excluded_predefined_functions- : [];- for (const value of values) {- if (typeof value === "string" && value.trim()) {- excluded.add(value.trim());- }- }- }- }- return [...excluded];-}-function geminiRole(value: string | undefined) {return value === "assistant" || value === "model" ? "model" : "user";}@@ -1147,16 +985,11 @@ function hasExplicitUnsupportedTools(if (!isJsonObject(tool)) {return true;}- if (allowFunctionTools && tool.type === "function") {- return false;- }- return tool.type !== geminiBrowserInteractionToolType &&- tool.type !== geminiMacDesktopInteractionToolType &&- tool.type !== debugUIInspectionToolType;+ return !(allowFunctionTools && tool.type === "function");});}-function staticModel(model: string, computerUse: boolean): InferenceModel {+function staticModel(model: string): InferenceModel {return {id: model,name: model,@@ -1168,16 +1001,7 @@ function staticModel(model: string, computerUse: boolean): InferenceModel {metadata: {provider: geminiProviderID,api: "generateContent",- ...(computerUse- ? {- computerUse,- registeredTools: [- geminiBrowserInteractionToolType,- geminiMacDesktopInteractionToolType,- debugUIInspectionToolType,- ],- }- : { structuredOutputs: true }),+ structuredOutputs: true,},};}diff --git a/site/src/lib/inference/adapters/hosted-responses.ts b/site/src/lib/inference/adapters/hosted-responses.tsdeleted file mode 100644index 4ce6dc75..00000000--- a/site/src/lib/inference/adapters/hosted-responses.ts+++ /dev/null@@ -1,279 +0,0 @@-import OpenAI, { APIError } from "openai";-import type { ResponseCreateParamsNonStreaming } from "openai/resources/responses/responses";--import {- ensureConfigured,- type FetchLike,-} from "@/lib/inference/http";-import { toJsonValue } from "@/lib/inference/json";-import { openaiModels } from "@/lib/inference/openai-models";-import {- InferenceProviderError,- type InferenceModality,- type InferenceModel,- type InferenceProvider,- type JsonObject,- type JsonValue,- type ResponseCreateRequest,- type ResponseCreateResult,-} from "@/lib/inference/providers";--type AdapterEnvironment = Record<string, string | undefined>;--const providerID = "hosted-responses";-const openAIProviderID = "openai";-const openAIBaseURL = "https://api.openai.com/v1";-const debugUIInspectionToolType = "donkey_debug_ui_inspection";-const defaultOpenAIDebugInspectionModel = openaiModels.debugInspection;-const unsupportedOpenAIComputerUseParameters = ["temperature", "top_p", "topP"];--export function createHostedResponsesProvider(- environment: AdapterEnvironment = process.env,- fetcher: FetchLike = fetch,-): InferenceProvider {- const openAIKey = environment.OPENAI_API_KEY?.trim() ?? "";- const configured = openAIKey.length > 0;-- async function listModels(modalities: InferenceModality[]) {- const requested = modalities.length > 0 ? modalities : ["text"];- if (!requested.includes("text") && !requested.includes("image")) {- return [];- }-- if (!openAIKey) {- return [];- }-- return [- staticModel(defaultOpenAIDebugInspectionModel),- ];- }-- async function createResponse(- request: ResponseCreateRequest,- ): Promise<ResponseCreateResult> {- ensureConfigured(configured);- if (!isOpenAIResponsesToolRequest(request.body)) {- throw new InferenceProviderError(- "OpenAI Responses is only configured for read-only debug UI inspection.",- {- statusCode: 400,- code: "openai_computer_tool_required",- details: {- supportedTools: [- debugUIInspectionToolType,- ],- },- },- );- }-- const client = new OpenAI({- apiKey: openAIKey,- baseURL: openAIBaseURL,- fetch: fetcher,- });-- try {- const model = defaultOpenAIModel(request.body);- const body = requestBody(request.body, model);- const response = await client.responses.create(- body as unknown as ResponseCreateParamsNonStreaming,- );- const value = toJsonValue(response);- return {- provider: openAIProviderID,- model,- body: value,- usage: usageFromResponse(value),- metadata: {- provider: openAIProviderID,- baseURL: openAIBaseURL,- store: String(body.store ?? false),- },- };- } catch (error) {- throw providerError("OpenAI Responses computer-use request failed.", error);- }- }-- return {- id: providerID,- configured,- capabilities: ["text", "image"],- responseProviderIDs: [openAIProviderID],- canCreateResponse: (request) => isOpenAIResponsesToolRequest(request.body),- listModels,- createResponse,- };-}--function defaultOpenAIModel(_body: JsonObject) {- return defaultOpenAIDebugInspectionModel;-}--function requestBody(body: JsonObject, model: string): JsonObject {- const normalizedBody = normalizeOpenAIComputerUseBody(body);- return {- ...normalizedBody,- model,- stream: false,- store: boolValue(normalizedBody.store, false),- };-}--function normalizeOpenAIComputerUseBody(body: JsonObject): JsonObject {- const hasDebugInspectionTool = hasDebugUIInspectionTool(body);- if (!hasDebugInspectionTool) {- return body;- }-- const tools = body.tools;- const toolChoice = openAIComputerToolChoice(body.tool_choice);- const normalized: JsonObject = {- ...body,- instructions: openAIInstructions(body.instructions),- ...(toolChoice === undefined ? {} : { tool_choice: toolChoice }),- };- for (const parameter of unsupportedOpenAIComputerUseParameters) {- delete normalized[parameter];- }- if (Array.isArray(tools)) {- const normalizedTools = tools- .map(openAIComputerToolReference)- .filter((tool): tool is JsonObject => isJsonObject(tool));- if (normalizedTools.length > 0) {- normalized.tools = normalizedTools;- } else {- delete normalized.tools;- }- }- return normalized;-}--function openAIComputerToolChoice(value: JsonValue | undefined): JsonValue | undefined {- if (!isJsonObject(value)) {- return value;- }-- const reference = openAIComputerToolReference(value);- if (reference !== value) {- return reference;- }-- if (value.type !== "allowed_tools" || !Array.isArray(value.tools)) {- return value;- }-- return {- ...value,- tools: value.tools.map(openAIComputerToolReference),- };-}--// The debug-inspection tool is read-only and has no OpenAI-native equivalent, so it is stripped from-// the request (the work is driven entirely by the inspection instructions). Any other tool passes-// through untouched.-function openAIComputerToolReference(value: JsonValue): JsonValue | null {- if (!isJsonObject(value)) {- return value;- }-- if (value.type === debugUIInspectionToolType) {- return null;- }-- return value;-}--function openAIInstructions(value: JsonValue | undefined) {- return [- stringValue(value),- [- "Perform read-only macOS UI inspection from the provided screenshot.",- "Return strict JSON only and do not return computer_call, function_call, click, type, scroll, drag, or navigation actions.",- ].join(" "),- ].filter(Boolean).join("\n\n");-}--function staticModel(model: string): InferenceModel {- return {- id: model,- name: model,- provider: openAIProviderID,- inputModalities: ["text", "image"],- outputModalities: ["text"],- contextLength: null,- pricing: null,- metadata: {- provider: openAIProviderID,- baseURL: openAIBaseURL,- api: "responses",- registeredTools: [- debugUIInspectionToolType,- ],- },- };-}--function isOpenAIResponsesToolRequest(body: JsonObject) {- const tools = body.tools;- if (!Array.isArray(tools)) {- return false;- }-- return tools.some((tool) => {- return isJsonObject(tool) && tool.type === debugUIInspectionToolType;- });-}--function hasDebugUIInspectionTool(body: JsonObject) {- const tools = body.tools;- if (!Array.isArray(tools)) {- return false;- }-- return tools.some((tool) => {- return isJsonObject(tool) && tool.type === debugUIInspectionToolType;- });-}--function usageFromResponse(value: JsonValue): JsonValue | undefined {- if (!isJsonObject(value)) {- return undefined;- }- return value.usage;-}--function boolValue(value: JsonValue | undefined, fallback: boolean) {- return typeof value === "boolean" ? value : fallback;-}--function stringValue(value: JsonValue | undefined): string | undefined {- return typeof value === "string" && value.trim() ? value : undefined;-}--function isJsonObject(value: unknown): value is JsonObject {- return typeof value === "object" && value !== null && !Array.isArray(value);-}--function providerError(message: string, error: unknown) {- if (error instanceof APIError) {- return new InferenceProviderError(message, {- statusCode: error.status ?? 502,- code: error.code ?? "provider_error",- details: {- body: toJsonValue(error.error ?? {}),- requestID: error.requestID ?? null,- status: error.status ?? null,- type: error.type ?? null,- },- });- }-- return new InferenceProviderError(message, {- details: {- message: error instanceof Error ? error.message : "Unknown error",- },- });-}diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.tsindex 25128895..321a9c14 100644--- a/site/src/lib/inference/gemini-models.ts+++ b/site/src/lib/inference/gemini-models.ts@@ -67,11 +67,6 @@ export const geminiModelRoles = {chatSimple: geminiModels.flashLite,// Fast structured task-intent and follow-up decisions.fastDecision: geminiModels.flashLite,- // Computer Use tool calls (browser and macOS desktop environments). Computer- // use is a built-in tool of the main flash model, so both share one model.- computerUse: geminiModels.flash,- // Screenshot parsing into read-only UI evidence.- screenshotParse: geminiModels.flash,// Generative image editing and generation.imageGeneration: geminiModels.proImage,// Production review: the director judging rendered takes and minted framesdiff --git a/site/src/lib/inference/openai-models.ts b/site/src/lib/inference/openai-models.tsdeleted file mode 100644index 508d7906..00000000--- a/site/src/lib/inference/openai-models.ts+++ /dev/null@@ -1,9 +0,0 @@-// Central registry of OpenAI model IDs the gateway runs (mirrors gemini-models.ts).-// Anything the code selects lives here so it is priced exhaustively in provider-pricing.ts;-// bumping a model is a one-line change here.-export const openaiModels = {- // Developer UI inspection Responses calls.- debugInspection: "gpt-5.4",-} as const;--export type OpenAIRunModel = (typeof openaiModels)[keyof typeof openaiModels];diff --git a/site/src/lib/inference/router.ts b/site/src/lib/inference/router.tsindex 6138821b..aad4ecdf 100644--- a/site/src/lib/inference/router.ts+++ b/site/src/lib/inference/router.ts@@ -1,9 +1,8 @@-import { createGeminiComputerUseProvider } from "@/lib/inference/adapters/gemini-computer-use";+import { createGeminiResponsesProvider } from "@/lib/inference/adapters/gemini-responses";import { createGeminiImageAssetProvider } from "@/lib/inference/adapters/gemini-image";import { createGeminiMusicAssetProvider } from "@/lib/inference/adapters/gemini-music";import { createGeminiSpeechAssetProvider } from "@/lib/inference/adapters/gemini-speech";import { createGeminiOmniVideoAssetProvider } from "@/lib/inference/adapters/gemini-omni-video";-import { createHostedResponsesProvider } from "@/lib/inference/adapters/hosted-responses";import {InferenceProviderError,type AssetGenerationRequest,@@ -216,7 +215,7 @@ export class ProviderRegistry {export function createProviderRegistry() {return new ProviderRegistry([// Asset selection is by capability + generateAsset, not list order: the image asset- // provider serves kind="image" (gemini-computer-use lists "image" as an input modality+ // provider serves kind="image" (gemini-responses lists "image" as an input modality// but has no generateAsset, so it is never chosen for asset generation).createGeminiImageAssetProvider(),// Omni (Gemini Omni Flash) serves kind="video"; its distinct provider id@@ -231,8 +230,7 @@ export function createProviderRegistry() {// adapter picks the clip or the longer pro model from the requested length so// a bed can still span a longer video (see gemini-music.ts).createGeminiMusicAssetProvider(),- createGeminiComputerUseProvider(),- createHostedResponsesProvider(),+ createGeminiResponsesProvider(),]);}diff --git a/site/src/lib/inference/screenshot-parsing/gemini-flash.ts b/site/src/lib/inference/screenshot-parsing/gemini-flash.tsdeleted file mode 100644index 8b143dc1..00000000--- a/site/src/lib/inference/screenshot-parsing/gemini-flash.ts+++ /dev/null@@ -1,681 +0,0 @@-import {- ApiError,- GoogleGenAI,- MediaResolution,- Type,-} from "@google/genai";-import { JWT, type JWTInput } from "google-auth-library";-import type {- GenerateContentParameters,- GoogleGenAIOptions,- Schema,-} from "@google/genai";--import { geminiModelRoles } from "@/lib/inference/gemini-models";-import { ensureConfigured } from "@/lib/inference/http";-import {- isJsonObject,- toJsonValue,-} from "@/lib/inference/json";-import {- InferenceProviderError,- type JsonValue,-} from "@/lib/inference/providers";-import { normalizedScreenshotResult } from "@/lib/inference/screenshot-parsing/normalize";-import {- geminiScreenshotParseOutputSchema,- geminiScreenshotControlSchema,- type GeminiScreenshotParseOutput,- type ScreenshotParseRequest,-} from "@/lib/inference/screenshot-parsing/schema";-import type { ScreenshotParserProvider } from "@/lib/inference/screenshot-parsing/types";--type AdapterEnvironment = Record<string, string | undefined>;-type GeminiClient = Pick<GoogleGenAI, "models">;-type GeminiClientFactory = (options: GoogleGenAIOptions) => GeminiClient;--const geminiProviderID = "gemini";-const screenshotProviderID = "gemini-flash";-const defaultScreenshotParseModel = geminiModelRoles.screenshotParse;-const defaultDebugOverlayScreenshotParseModel = geminiModelRoles.screenshotParse;-const defaultVertexLocation = "global";-const vertexAIScope = "https://www.googleapis.com/auth/cloud-platform";-// Client-side request timeout so a stalled parse aborts instead of hanging the-// per-step agent loop; screenshot parsing is normally a few seconds, so this only-// trips on a true hang.-const requestTimeoutMs = 120_000;--export function createGeminiFlashScreenshotParser(- environment: AdapterEnvironment = process.env,- clientFactory: GeminiClientFactory = (options) => new GoogleGenAI(options),-): ScreenshotParserProvider {- const config = geminiClientConfig(environment);-- return {- id: screenshotProviderID,- inferenceProvider: geminiProviderID,- configured: config.configured,- modelForRequest: (request) => screenshotParseModelForRequest(request),- async parse(request) {- ensureConfigured(config.configured);- const client = clientFactory(config.options);- const model = screenshotParseModelForRequest(request);- const startedAt = performance.now();-- let rawResponse: unknown;- try {- rawResponse = await client.models.generateContent(- geminiRequestParameters(request, model),- );- } catch (error) {- throw geminiProviderError(error);- }-- const rawBody = toJsonValue(rawResponse);- const output = parseGeminiOutput(rawBody);- const result = normalizedScreenshotResult(request, output, {- provider: geminiProviderID,- parserProvider: screenshotProviderID,- model,- service: config.service,- location: config.location,- latencyMs: String(Math.round(performance.now() - startedAt)),- });-- return {- provider: geminiProviderID,- model,- result,- usage: usageFromGeminiResponse(rawBody),- metadata: result.metadata,- };- },- async *stream(request) {- ensureConfigured(config.configured);- const client = clientFactory(config.options);- const model = screenshotParseModelForRequest(request);- const startedAt = performance.now();- const generator = await streamGeminiContent(client, request, model);- let accumulatedText = "";- let emittedControlCount = 0;- let latestUsage: JsonValue | undefined;-- for await (const chunk of generator) {- accumulatedText += geminiResponseText(chunk);- latestUsage = usageFromGeminiResponse(toJsonValue(chunk)) ?? latestUsage;-- const partialOutput = partialGeminiOutput(accumulatedText);- if (partialOutput.controls.length <= emittedControlCount) {- continue;- }-- emittedControlCount = partialOutput.controls.length;- const result = normalizedScreenshotResult(request, partialOutput, {- provider: geminiProviderID,- parserProvider: screenshotProviderID,- model,- service: config.service,- location: config.location,- latencyMs: String(Math.round(performance.now() - startedAt)),- "screenshotParser.stream": "partial",- });- yield {- type: "partial",- provider: geminiProviderID,- model,- result,- metadata: result.metadata,- };- }-- const output = parseGeminiOutputText(accumulatedText);- const result = normalizedScreenshotResult(request, output, {- provider: geminiProviderID,- parserProvider: screenshotProviderID,- model,- service: config.service,- location: config.location,- latencyMs: String(Math.round(performance.now() - startedAt)),- "screenshotParser.stream": "final",- });- yield {- type: "final",- provider: geminiProviderID,- model,- result,- usage: latestUsage,- metadata: result.metadata,- };- },- };-}--export function screenshotParseModelForRequest(request: ScreenshotParseRequest) {- if (isDebugOverlayRequest(request)) {- return defaultDebugOverlayScreenshotParseModel;- }-- return defaultScreenshotParseModel;-}--function geminiRequestParameters(- request: ScreenshotParseRequest,- model: string,-): GenerateContentParameters {- const profile = screenshotParseProfile(request);- return {- model,- contents: [- {- role: "user",- parts: [- {- text: screenshotParsingPrompt(request),- },- {- inlineData: {- data: request.imageBase64,- mimeType: request.contentType,- },- },- ],- },- ],- config: {- mediaResolution: profile.mediaResolution,- responseMimeType: "application/json",- responseSchema: geminiOutputSchema,- temperature: 0,- topP: 0.1,- candidateCount: 1,- maxOutputTokens: profile.maxOutputTokens,- thinkingConfig: {- thinkingBudget: 0,- },- },- };-}--function screenshotParsingPrompt(request: ScreenshotParseRequest) {- if (isDebugOverlayRequest(request)) {- return [- "You are a read-only UI screenshot parser for a macOS developer inspection overlay.",- "The screenshot is a single app/window capture, never the entire desktop.",- "Return compact JSON matching the provided schema.",- "For this overlay profile, visibleText must be [] and formFields must be []; put readable names only in control labels.",- "Return bounding boxes for the full visible clickable/control region, not just the text glyphs or icon pixels.",- "Treat compound controls as one control when a symbol/icon and nearby text visually act together; the box should cover the icon, text, and full hit target.",- "Treat standalone symbols/icons as controls when they use standard UI shapes or placement, even if there is no visible text label.",- "For nav/sidebar/list/file rows, box the entire row target including icon, label, and selection background when visible.",- "Always include macOS titlebar controls and toolbar controls when visible: red/yellow/green traffic lights, sidebar toggle, back/forward arrows, title actions, and top-right toolbar icons.",- "Always include bottom composer/input controls when visible: the text input surface, add/attachment button, mode selector, model selector, mic/voice button, and send/stop button.",- "Deliberately scan the top edge of the image from left to right for small window chrome controls, navigation icons, segmented controls, menus, status buttons, and titlebar actions.",- "Deliberately scan the bottom edge of the image from left to right for composer/input areas, accessory icons, dropdown selectors, microphone controls, submit/stop controls, and other docked toolbars.",- "Small circular or icon-only controls near the top or bottom edges should be included even when they have no readable text; infer a concise generic label such as close, minimize, zoom, back, forward, add, voice, submit, or stop when visually supported.",- "Prioritize visible controls and selectable regions that are useful to draw debug boxes: buttons, icon buttons, nav/sidebar rows, tabs, menus, file rows, list rows, links, inputs, titlebar controls, and bottom composer controls.",- "Do not include decorative backgrounds, separators, or long static prose blocks unless they are clickable/selectable regions.",- "Prefer 25-55 high-confidence controls over an exhaustive map.",- "Use box_2d as [ymin, xmin, ymax, xmax] normalized to 0-1000 over the full image.",- "Use concise labels copied from visible text when possible.",- `Image pixel size: ${request.pixelSize.width}x${request.pixelSize.height}.`,- `Target ID: ${request.targetID ?? "unknown"}.`,- ].join("\n");- }-- return [- "You are a read-only UI screenshot parser for a macOS app automation system.",- "The screenshot is scoped to an app/window or a system-navigation surface, never the entire desktop.",- "Extract only visible, grounded UI evidence from that scoped screenshot.",- "Do not infer user intent, do not suggest actions, and do not return executable commands.",- "Return JSON matching the provided schema.",- "Be dense: return all visible navigation rows, sidebar items, tabs, buttons, icon buttons, menus, file/change rows, input fields, checkboxes, links, cards, list items, and important text regions.",- "Treat controls as visual affordances, not only text. Many controls are a symbol/icon plus a label, a symbol inside a button shape, or an icon-only target; box the complete hit target rather than only the glyphs.",- "Do not stop after the primary content. Include left navigation, right panels, bottom composers/toolbars, title bars, and repeated rows when visible.",- "Use kind=listItem for selectable rows or navigation entries, kind=group for non-clickable labeled regions/cards, and kind=unknown only when no better kind fits.",- "Prefer returning more grounded UI elements over a sparse summary. Aim for 40-120 controls on a complex desktop app screenshot when that many distinct regions are visible.",- "visibleText should contain the most important readable text snippets, including text from controls.",- "Use box_2d as [ymin, xmin, ymax, xmax] normalized to 0-1000 over the full image.",- "Use concise labels copied from visible text when possible.",- `Image pixel size: ${request.pixelSize.width}x${request.pixelSize.height}.`,- `Screenshot scope: ${request.metadata["screenshot.scope"] ?? "targetWindow"}.`,- `Target ID: ${request.targetID ?? "unknown"}.`,- ].join("\n");-}--function screenshotParseProfile(request: ScreenshotParseRequest) {- if (isDebugOverlayRequest(request)) {- return {- mediaResolution: MediaResolution.MEDIA_RESOLUTION_LOW,- maxOutputTokens: 4_096,- };- }-- return {- mediaResolution: MediaResolution.MEDIA_RESOLUTION_HIGH,- maxOutputTokens: 16_384,- };-}--function isDebugOverlayRequest(request: ScreenshotParseRequest) {- return request.metadata.source === "debug-ui-inspection-overlay";-}--function parseGeminiOutput(rawBody: JsonValue) {- return parseGeminiOutputText(geminiText(rawBody));-}--function parseGeminiOutputText(text: string) {- const candidates = jsonObjectCandidates(text).reverse();- let parsed: unknown;- for (const candidate of candidates) {- try {- parsed = JSON.parse(candidate);- break;- } catch {- continue;- }- }-- if (parsed === undefined) {- throw new InferenceProviderError("Gemini returned invalid screenshot parser JSON.", {- statusCode: 502,- code: "invalid_provider_output",- });- }-- const output = geminiScreenshotParseOutputSchema.safeParse(parsed);- if (!output.success) {- throw new InferenceProviderError("Gemini screenshot parser output did not match schema.", {- statusCode: 502,- code: "invalid_provider_output",- details: {- issues: output.error.issues.map((issue) => ({- path: issue.path.join("."),- message: issue.message,- })),- },- });- }-- return output.data;-}--function jsonObjectCandidates(text: string) {- const trimmed = text.trim();- const candidates = trimmed ? [trimmed] : [];- let objectStart = -1;- let depth = 0;- let inString = false;- let escaped = false;-- for (let index = 0; index < text.length; index += 1) {- const character = text[index];-- if (inString) {- if (escaped) {- escaped = false;- } else if (character === "\\") {- escaped = true;- } else if (character === "\"") {- inString = false;- }- continue;- }-- if (character === "\"") {- inString = true;- } else if (character === "{") {- if (depth === 0) {- objectStart = index;- }- depth += 1;- } else if (character === "}" && depth > 0) {- depth -= 1;- if (depth === 0 && objectStart >= 0) {- candidates.push(text.slice(objectStart, index + 1));- objectStart = -1;- }- }- }-- return Array.from(new Set(candidates));-}--async function streamGeminiContent(- client: GeminiClient,- request: ScreenshotParseRequest,- model: string,-) {- try {- return await client.models.generateContentStream(- geminiRequestParameters(request, model),- );- } catch (error) {- throw geminiProviderError(error);- }-}--function partialGeminiOutput(text: string): GeminiScreenshotParseOutput {- const controls = extractArrayObjectTexts(text, "controls")- .map((controlText) => {- try {- return JSON.parse(controlText) as unknown;- } catch {- return null;- }- })- .map((control) => geminiScreenshotControlSchema.safeParse(control))- .filter((control) => control.success)- .map((control) => control.data);-- return {- visibleText: [],- controls,- formFields: [],- confidence: controls.reduce(- (confidence, control) => Math.max(confidence, control.confidence),- 0,- ),- };-}--function extractArrayObjectTexts(jsonText: string, propertyName: string) {- const propertyIndex = jsonText.indexOf(`"${propertyName}"`);- if (propertyIndex < 0) {- return [];- }-- const arrayStart = jsonText.indexOf("[", propertyIndex);- if (arrayStart < 0) {- return [];- }-- const objects: string[] = [];- let objectStart = -1;- let objectDepth = 0;- let arrayDepth = 0;- let inString = false;- let escaped = false;-- for (let index = arrayStart; index < jsonText.length; index += 1) {- const character = jsonText[index];-- if (inString) {- if (escaped) {- escaped = false;- } else if (character === "\\") {- escaped = true;- } else if (character === "\"") {- inString = false;- }- continue;- }-- if (character === "\"") {- inString = true;- continue;- }-- if (character === "[") {- arrayDepth += 1;- continue;- }-- if (character === "]") {- arrayDepth -= 1;- if (arrayDepth === 0) {- break;- }- continue;- }-- if (arrayDepth !== 1) {- continue;- }-- if (character === "{") {- if (objectDepth === 0) {- objectStart = index;- }- objectDepth += 1;- continue;- }-- if (character === "}") {- objectDepth -= 1;- if (objectDepth === 0 && objectStart >= 0) {- objects.push(jsonText.slice(objectStart, index + 1));- objectStart = -1;- }- }- }-- return objects;-}--function geminiText(rawBody: JsonValue) {- if (isJsonObject(rawBody) && typeof rawBody.text === "string") {- return rawBody.text;- }- const candidates = isJsonObject(rawBody) && Array.isArray(rawBody.candidates)- ? rawBody.candidates- : [];- const first = candidates.find(isJsonObject);- const content = first ? first.content : undefined;- const parts = content !== undefined && isJsonObject(content) && Array.isArray(content.parts)- ? content.parts- : [];- return parts- .filter(isJsonObject)- .map((part) => typeof part.text === "string" ? part.text : "")- .join("")- .trim();-}--function geminiResponseText(response: unknown) {- const directText = response !== null- && typeof response === "object"- && "text" in response- && typeof response.text === "string"- ? response.text- : undefined;- return directText ?? geminiText(toJsonValue(response));-}--function usageFromGeminiResponse(rawBody: JsonValue): JsonValue | undefined {- if (!isJsonObject(rawBody)) {- return undefined;- }- const usage = rawBody.usageMetadata ?? rawBody.usage;- return usage === undefined ? undefined : toJsonValue(usage);-}---const geminiOutputSchema: Schema = {- type: Type.OBJECT,- required: ["visibleText", "controls", "formFields", "confidence"],- properties: {- visibleText: {- type: Type.ARRAY,- items: {- type: Type.OBJECT,- required: ["text", "confidence"],- properties: {- id: { type: Type.STRING },- text: { type: Type.STRING },- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },- },- },- },- controls: {- type: Type.ARRAY,- items: {- type: Type.OBJECT,- required: ["label", "kind", "confidence", "box_2d"],- properties: {- id: { type: Type.STRING },- label: { type: Type.STRING },- kind: { type: Type.STRING },- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },- box_2d: {- type: Type.ARRAY,- items: { type: Type.NUMBER, minimum: 0, maximum: 1000 },- },- },- },- },- formFields: {- type: Type.ARRAY,- items: {- type: Type.OBJECT,- required: ["label", "isRequired", "confidence"],- properties: {- id: { type: Type.STRING },- label: { type: Type.STRING },- isRequired: { type: Type.BOOLEAN },- currentValue: { type: Type.STRING },- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },- box_2d: {- type: Type.ARRAY,- items: { type: Type.NUMBER, minimum: 0, maximum: 1000 },- },- },- },- },- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },- },-};--export function geminiClientConfig(environment: AdapterEnvironment = process.env): {- configured: boolean;- options: GoogleGenAIOptions;- service: "vertex-ai" | "gemini-api";- location: string;-} {- const googleCredentials = googleCredentialsFromEnvironment(environment);- const project = googleCredentials?.project_id;- const apiKey =- environment.GEMINI_API_KEY?.trim()- || environment.GOOGLE_API_KEY?.trim()- || "";-- if (project) {- const options: GoogleGenAIOptions = {- vertexai: true,- location: defaultVertexLocation,- project,- httpOptions: { timeout: requestTimeoutMs },- };- if (googleCredentials) {- options.googleAuthOptions = {- authClient: googleAuthClient(googleCredentials),- };- }- return {- configured: true,- options,- service: "vertex-ai",- location: defaultVertexLocation,- };- }-- const options: GoogleGenAIOptions = {- httpOptions: { timeout: requestTimeoutMs },- };- if (apiKey) {- options.apiKey = apiKey;- }-- return {- configured: Boolean(apiKey),- options,- service: "gemini-api",- location: "global",- };-}--function googleCredentialsFromEnvironment(- environment: AdapterEnvironment,-): JWTInput | undefined {- const rawCredentials = environment.GOOGLE_APPLICATION_CREDENTIALS_JSON?.trim();- if (!rawCredentials) {- return undefined;- }-- return serviceAccountCredentials(rawCredentials);-}--function googleAuthClient(credentials: JWTInput) {- return new JWT({- email: credentials.client_email,- key: credentials.private_key,- keyId: credentials.private_key_id,- scopes: [vertexAIScope],- });-}--function serviceAccountCredentials(rawCredentials: string): JWTInput {- let parsed: unknown;- try {- parsed = JSON.parse(rawCredentials);- } catch {- throw new InferenceProviderError("Google service account JSON is invalid.", {- statusCode: 500,- code: "invalid_google_service_account_json",- });- }-- const credentials = toJsonValue(parsed);- if (!isJsonObject(credentials)) {- throw new InferenceProviderError("Google service account JSON must be an object.", {- statusCode: 500,- code: "invalid_google_service_account_json",- });- }-- const clientEmail = stringValue(credentials.client_email);- const privateKey = stringValue(credentials.private_key);- if (!clientEmail || !privateKey) {- throw new InferenceProviderError(- "Google service account JSON must include client_email and private_key.",- {- statusCode: 500,- code: "invalid_google_service_account_json",- },- );- }-- return {- type: stringValue(credentials.type),- project_id: stringValue(credentials.project_id),- private_key_id: stringValue(credentials.private_key_id),- private_key: privateKey,- client_email: clientEmail,- client_id: stringValue(credentials.client_id),- universe_domain: stringValue(credentials.universe_domain),- };-}--function geminiProviderError(error: unknown) {- if (error instanceof ApiError) {- return new InferenceProviderError("Gemini screenshot parsing failed.", {- statusCode: error.status,- code: "provider_error",- details: {- status: error.status,- message: error.message,- },- });- }-- return new InferenceProviderError("Gemini screenshot parsing failed.", {- details: {- message: error instanceof Error ? error.message : "Unknown error",- },- });-}--function stringValue(value: JsonValue | undefined) {- return typeof value === "string" && value.trim() ? value.trim() : undefined;-}-diff --git a/site/src/lib/inference/screenshot-parsing/index.ts b/site/src/lib/inference/screenshot-parsing/index.tsdeleted file mode 100644index 4fef8e2a..00000000--- a/site/src/lib/inference/screenshot-parsing/index.ts+++ /dev/null@@ -1,59 +0,0 @@-import { InferenceProviderError } from "@/lib/inference/providers";-import { createGeminiFlashScreenshotParser } from "@/lib/inference/screenshot-parsing/gemini-flash";-import type {- ScreenshotParserProvider,- ScreenshotParserProviderResult,- ScreenshotParserProviderStreamEvent,-} from "@/lib/inference/screenshot-parsing/types";-import type { ScreenshotParseRequest } from "@/lib/inference/screenshot-parsing/schema";--export function createScreenshotParserProvider(): ScreenshotParserProvider {- return createGeminiFlashScreenshotParser();-}--export async function parseScreenshot(- request: ScreenshotParseRequest,- provider: ScreenshotParserProvider = createScreenshotParserProvider(),-): Promise<ScreenshotParserProviderResult> {- if (!provider.configured) {- throw new InferenceProviderError("Screenshot parser provider is not configured.", {- statusCode: 503,- code: "missing_provider_credentials",- details: {- provider: provider.id,- },- });- }-- return provider.parse(request);-}--export async function* parseScreenshotStream(- request: ScreenshotParseRequest,- provider: ScreenshotParserProvider = createScreenshotParserProvider(),-): AsyncGenerator<ScreenshotParserProviderStreamEvent> {- if (!provider.configured) {- throw new InferenceProviderError("Screenshot parser provider is not configured.", {- statusCode: 503,- code: "missing_provider_credentials",- details: {- provider: provider.id,- },- });- }-- if (!provider.stream) {- const result = await provider.parse(request);- yield {- type: "final",- provider: result.provider,- model: result.model,- result: result.result,- usage: result.usage,- metadata: result.metadata,- };- return;- }-- yield* provider.stream(request);-}diff --git a/site/src/lib/inference/screenshot-parsing/normalize.ts b/site/src/lib/inference/screenshot-parsing/normalize.tsdeleted file mode 100644index 59070da0..00000000--- a/site/src/lib/inference/screenshot-parsing/normalize.ts+++ /dev/null@@ -1,131 +0,0 @@-import { toJsonObject } from "@/lib/inference/json";-import type {- GeminiScreenshotParseOutput,- HotLoopRectJSON,- ScreenshotParseRequest,-} from "@/lib/inference/screenshot-parsing/schema";-import type { ScreenshotParserResult } from "@/lib/inference/screenshot-parsing/types";--// Provider-agnostic normalization shared by the screenshot-parsing providers-// (gemini-flash, vision). Turns the common parse output — visibleText,-// controls with box_2d in [ymin, xmin, ymax, xmax]/1000 space, and form fields-// — into the ScreenshotParserResult contract, stamping each result with its-// originating `source`.-export function normalizedScreenshotResult(- request: ScreenshotParseRequest,- output: GeminiScreenshotParseOutput,- metadata: Record<string, string>,- source = "gemini-screenshot-parser",-): ScreenshotParserResult {- const visibleText = output.visibleText.reduce<Record<string, string>>((result, item, index) => {- const key = item.id?.trim() || (index === 0 ? "visibleText" : `visibleText.${index + 1}`);- result[key] = item.text;- return result;- }, {});-- const usedControlIDs = new Set<string>();- const controls = output.controls.map((control) => {- // Opaque, unique-per-result control id. searchField keeps the well-known- // "search" id that the local-app action plan defaults to.- const controlID = control.kind === "searchField"- ? "search"- : uniqueShortID(usedControlIDs);- return {- id: controlID,- label: control.label,- kind: control.kind,- frame: rectFromGeminiBox(control.box_2d, request.pixelSize.width, request.pixelSize.height),- confidence: clamp01(control.confidence),- metadata: {- controlID,- segmentID: controlID,- source,- boxFormat: "ymin,xmin,ymax,xmax/1000",- "localUIElement.actionEligibility": "guardedAction",- directInputActionsAllowed: "true",- },- };- });-- const formFields = output.formFields.map((field, index) => {- const id = field.id?.trim() || `form-field-${index + 1}`;- return {- id,- label: field.label,- isRequired: field.isRequired,- currentValue: field.currentValue ?? null,- metadata: {- source,- confidence: String(clamp01(field.confidence)),- },- };- });-- return {- visibleText,- controls,- formFields,- confidence: clamp01(output.confidence || controls.map((control) => control.confidence).reduce(- (max, value) => Math.max(max, value),- 0,- )),- metadata: toJsonObject({- ...metadata,- "runtime.backend": source,- "directInputActionsAllowed": "true",- "screenshotParser.controlCount": String(controls.length),- "screenshotParser.formFieldCount": String(formFields.length),- }),- };-}--export function rectFromGeminiBox(- box: [number, number, number, number],- imageWidth: number,- imageHeight: number,-): HotLoopRectJSON | null {- const [ymin, xmin, ymax, xmax] = box;- const x1 = clamp(xmin, 0, 1000) / 1000 * imageWidth;- const y1 = clamp(ymin, 0, 1000) / 1000 * imageHeight;- const x2 = clamp(xmax, 0, 1000) / 1000 * imageWidth;- const y2 = clamp(ymax, 0, 1000) / 1000 * imageHeight;- const width = Math.max(0, x2 - x1);- const height = Math.max(0, y2 - y1);- if (width <= 0 || height <= 0) {- return null;- }-- return {- origin: {- x: x1,- y: y1,- space: "window",- },- size: {- width,- height,- space: "window",- },- };-}--function uniqueShortID(used: Set<string>): string {- let id = shortID();- while (used.has(id)) {- id = shortID();- }- used.add(id);- return id;-}--function shortID(): string {- return Math.random().toString(36).slice(2, 8).padEnd(6, "0");-}--function clamp01(value: number) {- return clamp(value, 0, 1);-}--function clamp(value: number, lower: number, upper: number) {- return Math.min(Math.max(value, lower), upper);-}diff --git a/site/src/lib/inference/screenshot-parsing/schema.ts b/site/src/lib/inference/screenshot-parsing/schema.tsdeleted file mode 100644index 2f0bf68c..00000000--- a/site/src/lib/inference/screenshot-parsing/schema.ts+++ /dev/null@@ -1,145 +0,0 @@-import { z } from "zod";--import { toJsonObject } from "@/lib/inference/json";--const metadataSchema = z.record(z.string().min(1).max(64), z.string().max(512));-const coordinateSpaceSchema = z.enum(["screen", "window", "crop", "normalizedTarget"]);-type ControlKind =- | "button"- | "textField"- | "searchField"- | "checkbox"- | "link"- | "menuItem"- | "listItem"- | "group"- | "unknown";--export const hotLoopSizeSchema = z.object({- width: z.number().positive().max(20_000),- height: z.number().positive().max(20_000),- space: coordinateSpaceSchema.default("window"),-});--export const hotLoopRectSchema = z.object({- origin: z.object({- x: z.number().finite(),- y: z.number().finite(),- space: coordinateSpaceSchema,- }),- size: hotLoopSizeSchema,-});--export const screenshotParseRequestSchema = z.object({- imageBase64: z.string().min(1).max(6_000_000),- contentType: z.enum(["image/png", "image/jpeg", "image/webp"]).default("image/png"),- pixelSize: hotLoopSizeSchema,- traceID: z.string().min(1).max(128).optional(),- targetID: z.string().min(1).max(256).optional(),- cropBounds: hotLoopRectSchema.optional(),- metadata: metadataSchema.optional().default({}),- stream: z.boolean().optional().default(false),-}).transform((value) => ({- ...value,- metadata: toJsonObject(value.metadata),-}));--export const geminiScreenshotTextSchema = z.object({- id: clippedString(128).optional(),- text: clippedString(2_000),- confidence: z.number().min(0).max(1).optional().default(0.5),-});--export const geminiScreenshotControlSchema = z.object({- id: clippedString(128).optional(),- label: clippedString(256),- kind: z.string().optional().default("unknown").transform(normalizedControlKind),- confidence: z.number().min(0).max(1),- box_2d: z.tuple([- z.number().min(0).max(1000),- z.number().min(0).max(1000),- z.number().min(0).max(1000),- z.number().min(0).max(1000),- ]),-});--export const geminiScreenshotFormFieldSchema = z.object({- id: clippedString(128).optional(),- label: clippedString(256),- isRequired: z.boolean().optional().default(false),- currentValue: clippedString(1_000).nullable().optional(),- confidence: z.number().min(0).max(1).optional().default(0.5),- box_2d: z.tuple([- z.number().min(0).max(1000),- z.number().min(0).max(1000),- z.number().min(0).max(1000),- z.number().min(0).max(1000),- ]).optional(),-});--export const geminiScreenshotParseOutputSchema = z.object({- visibleText: z.array(geminiScreenshotTextSchema).max(200).default([]),- controls: z.array(geminiScreenshotControlSchema).max(200).default([]),- formFields: z.array(geminiScreenshotFormFieldSchema).max(80).default([]),- confidence: z.number().min(0).max(1).default(0),-});--export type ScreenshotParseRequest = z.infer<typeof screenshotParseRequestSchema>;-export type GeminiScreenshotParseOutput = z.infer<typeof geminiScreenshotParseOutputSchema>;-export type HotLoopRectJSON = z.infer<typeof hotLoopRectSchema>;--function clippedString(maxLength: number) {- return z.string()- .transform((value) => value.trim().slice(0, maxLength))- .pipe(z.string().min(1));-}--function normalizedControlKind(value: string): ControlKind {- const normalized = value.trim().toLowerCase().replace(/[^a-z]/g, "");- switch (normalized) {- case "button":- case "iconbutton":- case "toolbarbutton":- case "reviewbutton":- case "sendbutton":- return "button";- case "textfield":- case "textinput":- case "input":- case "composer":- case "textarea":- return "textField";- case "searchfield":- case "search":- case "searchinput":- return "searchField";- case "checkbox":- case "toggle":- return "checkbox";- case "link":- return "link";- case "menuitem":- case "menu":- case "dropdown":- return "menuItem";- case "listitem":- case "navitem":- case "navigationitem":- case "row":- case "filerow":- case "changerow":- case "threadrow":- return "listItem";- case "group":- case "card":- case "panel":- case "region":- case "label":- case "text":- case "textregion":- case "statictext":- return "group";- default:- return "unknown";- }-}diff --git a/site/src/lib/inference/screenshot-parsing/types.ts b/site/src/lib/inference/screenshot-parsing/types.tsdeleted file mode 100644index 5af0960d..00000000--- a/site/src/lib/inference/screenshot-parsing/types.ts+++ /dev/null@@ -1,64 +0,0 @@-import type { JsonObject, JsonValue } from "@/lib/inference/providers";-import type {- HotLoopRectJSON,- ScreenshotParseRequest,-} from "@/lib/inference/screenshot-parsing/schema";--export type ScreenshotParserResult = {- visibleText: Record<string, string>;- controls: ScreenshotParserControl[];- formFields: ScreenshotParserFormField[];- confidence: number;- metadata: JsonObject;-};--export type ScreenshotParserControl = {- id: string;- label: string;- kind: string;- frame: HotLoopRectJSON | null;- confidence: number;- metadata: Record<string, string>;-};--export type ScreenshotParserFormField = {- id: string;- label: string;- isRequired: boolean;- currentValue?: string | null;- metadata: Record<string, string>;-};--export type ScreenshotParserProviderResult = {- provider: string;- model: string;- result: ScreenshotParserResult;- usage?: JsonValue;- metadata: JsonObject;-};--export type ScreenshotParserProviderStreamEvent =- | {- type: "partial";- provider: string;- model: string;- result: ScreenshotParserResult;- metadata: JsonObject;- }- | {- type: "final";- provider: string;- model: string;- result: ScreenshotParserResult;- usage?: JsonValue;- metadata: JsonObject;- };--export type ScreenshotParserProvider = {- id: string;- inferenceProvider: string;- configured: boolean;- modelForRequest: (request: ScreenshotParseRequest) => string;- parse: (request: ScreenshotParseRequest) => Promise<ScreenshotParserProviderResult>;- stream?: (request: ScreenshotParseRequest) => AsyncGenerator<ScreenshotParserProviderStreamEvent>;-};diff --git a/site/src/lib/inference/web-search.ts b/site/src/lib/inference/web-search.tsindex 1670bf61..1f946224 100644--- a/site/src/lib/inference/web-search.ts+++ b/site/src/lib/inference/web-search.ts@@ -1,7 +1,7 @@import { GoogleGenAI } from "@google/genai";+import { geminiClientConfig } from "@/lib/inference/adapters/gemini-client";import { geminiModelRoles } from "@/lib/inference/gemini-models";-import { geminiClientConfig } from "@/lib/inference/screenshot-parsing/gemini-flash";export type WebSearchSource = {title: string;@@ -17,12 +17,12 @@ export type WebSearchResult = {/*** Web search via Gemini's built-in Google Search grounding on Vertex AI. This uses the backend's- * service-account credential (the same one behind live-token and screenshot parsing) — no API key- * and no separate Programmable Search Engine. The model searches the web for the query and returns a+ * service-account credential (the same one behind live-token) — no API key and no separate+ * Programmable Search Engine. The model searches the web for the query and returns a* grounded summary plus the source pages it used, which the caller turns into ranked results.*/export async function searchWeb(query: string): Promise<WebSearchResult | null> {- const config = geminiClientConfig();+ const config = geminiClientConfig(process.env);if (!config.configured) {return null;}