Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): gemini adapter sheds desktop-agent tooling [rebuild]

David Le17ffc805da98
diff --git a/docs/guides/backend-apis.md b/docs/guides/backend-apis.md
index 049dc269..75191d6d 100644
--- a/docs/guides/backend-apis.md
+++ b/docs/guides/backend-apis.md
@@ -97,32 +97,7 @@ route reads an object only under the caller's own prefix. And storage is how a
large request succeeds, never why a small one fails: a client that can't reach
storage sends the bytes inline, as it always could.
-### Computer use
-
-Computer use is a built-in tool of Gemini's main flash model, so one model
-drives both a browser and a guarded macOS desktop. The Mac app sends a request
-tool; the adapter maps it to an environment and returns the model's native
-action calls for the app to execute.
-
-| Request tool | Environment | Used for |
-|---|---|---|
-| `donkey_gemini_browser_interaction` | browser | web interaction |
-| `donkey_gemini_mac_desktop_interaction` | desktop | guarded macOS desktop interaction |
-| `donkey_debug_ui_inspection` | — | developer-only, read-only UI inspection |
-
-The first two return action calls the app executes. UI inspection is read-only:
-the adapter routes it through a vision-capable model but must never forward or
-run a UI action call.
-
-### Screenshot parsing
-
-Screenshot parsing is its own route, `POST /api/inference/screenshots/parse/`.
-It takes a scoped app, window, or system-navigation screenshot — never a
-whole-desktop capture — and returns read-only UI evidence in the Mac app's local
-UI-understanding shape. The default provider is Gemini Flash, configured with
-hosted Google credentials or `GEMINI_API_KEY`.
-
-### Gemini and OpenAI adapters
+### Gemini adapter
The Gemini adapter uses the official `@google/genai` SDK. It runs on Vertex AI's
global endpoint only when `GOOGLE_APPLICATION_CREDENTIALS_JSON` carries a
@@ -133,17 +108,11 @@ Model choice is code (see Handler Rules on `process.env`):
| Call | Model |
|---|---|
-| Fast structured decisions: task-intent, follow-ups | `gemini-3.1-flash-lite` |
-| General chat, non-decision structured calls, computer use | `gemini-3.5-flash` |
-
-Structured-decision requests normalize their JSON schema for Gemini and retry
-without a provider-enforced schema when Vertex rejects the schema parameters.
-Whether the returned JSON is actually executable is decided by Mac-side runtime
-validation, not the backend.
+| Gate-judged simple chat turns, fast structured decisions | `gemini-3.1-flash-lite` |
+| General chat, non-decision structured calls | `gemini-3.5-flash` |
-The OpenAI Responses adapter exists only for developer read-only UI inspection
-and uses `OPENAI_API_KEY`. Keep that key in the hosted deploy; the Mac app never
-carries it.
+Structured requests normalize their JSON schema for Gemini and retry without a
+provider-enforced schema when Vertex rejects the schema parameters.
## Hosted Model Credits
diff --git a/site/src/app/api/inference/responses/route.ts b/site/src/app/api/inference/responses/route.ts
index a22222e8..c79629c3 100644
--- a/site/src/app/api/inference/responses/route.ts
+++ b/site/src/app/api/inference/responses/route.ts
@@ -10,7 +10,7 @@ import {
} from "@/lib/credits/inference";
import { createProviderRegistry } from "@/lib/inference/router";
import { resolveInferenceBlobs } from "@/lib/inference/blobs";
-import { isJsonObject, toJsonObject, toJsonValue } from "@/lib/inference/json";
+import { toJsonObject } from "@/lib/inference/json";
import {
inferenceErrorCode,
inferenceProviderErrorResponse,
@@ -18,7 +18,6 @@ import {
validationErrorResponse,
} from "@/lib/inference/responses";
import { InferenceProviderError } from "@/lib/inference/providers";
-import type { JsonObject, JsonValue } from "@/lib/inference/providers";
import { responseCreateRequestSchema } from "@/lib/inference/schemas";
import {
shouldBypassDonkeyInferenceCredits,
@@ -55,19 +54,6 @@ export const POST = withDonkeyAuth(async (request) => {
}
}
- const debugInspection = isDebugUIInspectionRequest(parsed.data.body);
- const debugInspectionStartedAt = performance.now();
- if (debugInspection) {
- console.info(
- [
- "[debug-ui-inspection] start",
- `at=${new Date().toISOString()}`,
- `provider=${parsed.data.donkeyProvider ?? "default"}`,
- `requestedModel=${requestedModel}`,
- ].join(" "),
- );
- }
-
let failedUsageProvider = parsed.data.donkeyProvider ?? "default";
try {
@@ -182,7 +168,6 @@ export const POST = withDonkeyAuth(async (request) => {
});
}
- const providerStartedAt = performance.now();
const result = await provider.createResponse?.(data);
if (!result) {
if (!bypassCredits) {
@@ -224,21 +209,6 @@ export const POST = withDonkeyAuth(async (request) => {
usageHeaders = creditUsageHeaders(recordedUsage);
}
- if (debugInspection) {
- const providerEndedAt = performance.now();
- console.info(
- [
- "[debug-ui-inspection] end",
- `at=${new Date().toISOString()}`,
- `provider=${result.provider}`,
- `model=${result.model}`,
- `providerMs=${Math.round(providerEndedAt - providerStartedAt)}`,
- `totalMs=${Math.round(providerEndedAt - debugInspectionStartedAt)}`,
- debugInspectionResultSummary(result.body),
- ].join(" "),
- );
- }
-
return NextResponse.json(result.body, {
headers: {
"X-Donkey-Inference-Provider": result.provider,
@@ -270,46 +240,3 @@ export const POST = withDonkeyAuth(async (request) => {
throw error;
}
});
-
-function isDebugUIInspectionRequest(body: JsonObject) {
- if (!Array.isArray(body.tools)) {
- return false;
- }
-
- return body.tools.some((tool) => {
- return isJsonObject(tool) && tool.type === "donkey_debug_ui_inspection";
- });
-}
-
-function debugInspectionResultSummary(body: JsonValue) {
- const object: JsonObject = isJsonObject(body) ? body : {};
- const outputText = typeof object.output_text === "string" ? object.output_text : "";
- const elementCount =
- elementCountFromFrame(object) ??
- elementCountFromOutputText(outputText);
-
- return [
- `outputTextChars=${outputText.length}`,
- `elements=${elementCount ?? "unknown"}`,
- ].join(" ");
-}
-
-function elementCountFromOutputText(outputText: string) {
- if (!outputText.trim()) {
- return null;
- }
-
- try {
- return elementCountFromFrame(toJsonValue(JSON.parse(outputText)));
- } catch {
- return null;
- }
-}
-
-function elementCountFromFrame(value: JsonValue) {
- if (!isJsonObject(value) || !Array.isArray(value.elements)) {
- return null;
- }
-
- return value.elements.length;
-}
diff --git a/site/src/app/api/inference/screenshots/parse/route.ts b/site/src/app/api/inference/screenshots/parse/route.ts
deleted file mode 100644
index b310779a..00000000
--- a/site/src/app/api/inference/screenshots/parse/route.ts
+++ /dev/null
@@ -1,235 +0,0 @@
-import { NextResponse } from "next/server";
-
-import {
- creditErrorResponse,
- creditUsageHeaders,
- inferenceUsageRoutes,
- recordFailedInferenceUsage,
- recordInferenceUsage,
- requireInferenceCredits,
-} from "@/lib/credits/inference";
-import { withDonkeyAuth } from "@/lib/donkey-api-auth";
-import { InferenceProviderError } from "@/lib/inference/providers";
-import {
- checkInMemoryRateLimit,
- rateLimitResponse,
-} from "@/lib/inference/rate-limit";
-import { readLimitedJsonBody } from "@/lib/inference/request-body";
-import {
- inferenceErrorCode,
- inferenceProviderErrorResponse,
- requireInferenceClientId,
- validationErrorResponse,
-} from "@/lib/inference/responses";
-import {
- createScreenshotParserProvider,
- parseScreenshot,
- parseScreenshotStream,
-} from "@/lib/inference/screenshot-parsing";
-import { screenshotParseRequestSchema } from "@/lib/inference/screenshot-parsing/schema";
-import type { ScreenshotParseRequest } from "@/lib/inference/screenshot-parsing/schema";
-
-export const dynamic = "force-dynamic";
-
-const screenshotParseMaxBodyBytes = 6_250_000;
-const screenshotParseRateLimit = {
- limit: 4,
- windowMs: 3_000,
-};
-
-export const POST = withDonkeyAuth(async (request) => {
- const client = requireInferenceClientId(request.donkey.clientId);
- if (!client.ok) {
- return client.response;
- }
-
- const rateLimit = checkInMemoryRateLimit({
- key: `${request.donkey.userId}:${client.clientId}`,
- ...screenshotParseRateLimit,
- });
- if (!rateLimit.ok) {
- return rateLimitResponse(rateLimit.retryAfterSeconds);
- }
-
- const body = await readLimitedJsonBody(request, screenshotParseMaxBodyBytes);
- if (!body.ok) {
- return body.response;
- }
-
- const parsed = screenshotParseRequestSchema.safeParse(body.json);
- if (!parsed.success) {
- return validationErrorResponse(parsed.error);
- }
- if (parsed.data.metadata["screenshot.scope"] === "desktop") {
- return NextResponse.json(
- {
- error: "Unsupported screenshot scope",
- message: "Screenshot parsing accepts app/window or system-navigation screenshots, not full-desktop captures.",
- },
- { status: 400 },
- );
- }
-
- const parserProvider = createScreenshotParserProvider();
- const model = parserProvider.modelForRequest(parsed.data);
- const credits = await requireInferenceCredits({
- model,
- provider: parserProvider.inferenceProvider,
- route: inferenceUsageRoutes.screenshotParse,
- userId: request.donkey.userId,
- });
- if (!credits.ok) {
- return credits.response;
- }
-
- if (parsed.data.stream) {
- return streamScreenshotParseResponse({
- clientId: client.clientId,
- conversationId: request.donkey.conversationId,
- model,
- parserProvider,
- request: parsed.data,
- userId: request.donkey.userId,
- });
- }
-
- try {
- const result = await parseScreenshot(parsed.data, parserProvider);
- const recordedUsage = await recordInferenceUsage({
- clientId: client.clientId,
- conversationId: request.donkey.conversationId,
- metadata: {
- parserProvider: parserProvider.id,
- },
- model: result.model,
- provider: result.provider,
- requestKind: "screenshot_parse",
- route: inferenceUsageRoutes.screenshotParse,
- status: "succeeded",
- usage: result.usage,
- userId: request.donkey.userId,
- });
-
- return NextResponse.json(result.result, {
- headers: {
- "X-Donkey-Inference-Provider": result.provider,
- "X-Donkey-Inference-Model": result.model,
- ...creditUsageHeaders(recordedUsage),
- },
- });
- } catch (error) {
- await recordFailedInferenceUsage({
- clientId: client.clientId,
- conversationId: request.donkey.conversationId,
- errorCode: inferenceErrorCode(error),
- metadata: {
- parserProvider: parserProvider.id,
- },
- model,
- provider: parserProvider.inferenceProvider,
- requestKind: "screenshot_parse",
- route: inferenceUsageRoutes.screenshotParse,
- userId: request.donkey.userId,
- });
- const creditResponse = creditErrorResponse(error);
- if (creditResponse) {
- return creditResponse;
- }
- if (error instanceof InferenceProviderError) {
- return inferenceProviderErrorResponse(error);
- }
-
- throw error;
- }
-});
-
-function streamScreenshotParseResponse(input: {
- clientId: string;
- conversationId: string | null;
- model: string;
- parserProvider: ReturnType<typeof createScreenshotParserProvider>;
- request: ScreenshotParseRequest;
- userId: string;
-}) {
- const encoder = new TextEncoder();
- const stream = new ReadableStream<Uint8Array>({
- async start(controller) {
- try {
- for await (const event of parseScreenshotStream(input.request, input.parserProvider)) {
- if (event.type === "partial") {
- controller.enqueue(encoder.encode(serverSentEvent("partial", event.result)));
- continue;
- }
-
- const recordedUsage = await recordInferenceUsage({
- clientId: input.clientId,
- conversationId: input.conversationId,
- metadata: {
- parserProvider: input.parserProvider.id,
- streaming: true,
- },
- model: event.model,
- provider: event.provider,
- requestKind: "screenshot_parse",
- route: inferenceUsageRoutes.screenshotParse,
- status: "succeeded",
- usage: event.usage,
- userId: input.userId,
- });
- controller.enqueue(
- encoder.encode(
- serverSentEvent("final", {
- ...event.result,
- metadata: {
- ...event.result.metadata,
- ...creditUsageHeaders(recordedUsage),
- },
- }),
- ),
- );
- }
- } catch (error) {
- await recordFailedInferenceUsage({
- clientId: input.clientId,
- conversationId: input.conversationId,
- errorCode: inferenceErrorCode(error),
- metadata: {
- parserProvider: input.parserProvider.id,
- streaming: true,
- },
- model: input.model,
- provider: input.parserProvider.inferenceProvider,
- requestKind: "screenshot_parse",
- route: inferenceUsageRoutes.screenshotParse,
- userId: input.userId,
- });
- controller.enqueue(
- encoder.encode(
- serverSentEvent("error", {
- error: inferenceErrorCode(error),
- message: error instanceof Error ? error.message : "Screenshot parsing failed.",
- details: error instanceof InferenceProviderError ? error.details : null,
- }),
- ),
- );
- } finally {
- controller.close();
- }
- },
- });
-
- return new Response(stream, {
- headers: {
- "Cache-Control": "no-cache, no-transform",
- "Connection": "keep-alive",
- "Content-Type": "text/event-stream; charset=utf-8",
- "X-Accel-Buffering": "no",
- "X-Donkey-Inference-Model": input.model,
- "X-Donkey-Inference-Provider": input.parserProvider.inferenceProvider,
- },
- });
-}
-
-function serverSentEvent(event: string, data: unknown) {
- return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
-}
diff --git a/site/src/lib/credits/inference.ts b/site/src/lib/credits/inference.ts
index d80c2f9c..5a87bbd9 100644
--- a/site/src/lib/credits/inference.ts
+++ b/site/src/lib/credits/inference.ts
@@ -24,7 +24,6 @@ export const inferenceUsageRoutes = {
browserRun: "/api/browser/run/",
chatCompletions: "/api/inference/chat/completions/",
responses: "/api/inference/responses/",
- screenshotParse: "/api/inference/screenshots/parse/",
} as const;
type CreditsDatabase = PrismaClient | Prisma.TransactionClient;
diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.ts
index 16259200..96763a12 100644
--- a/site/src/lib/credits/provider-pricing.ts
+++ b/site/src/lib/credits/provider-pricing.ts
@@ -16,7 +16,6 @@ import {
type GeminiOmniModel,
type GeminiTtsModel,
} from "@/lib/inference/gemini-models";
-import { openaiModels, type OpenAIRunModel } from "@/lib/inference/openai-models";
import { browserUsePerStepUsd } from "@/lib/browser/pricing";
export type ProviderCreditPricing = {
@@ -73,19 +72,6 @@ export function providerCreditPricing(
return undefined;
}
-// Every OpenAI model the gateway selects must appear here: the Record is keyed by the
-// OpenAIRunModel union, so adding a run model without a price fails the type-check (and the
-// build). Matched exactly (not by prefix) so it never shadows a more specific table entry.
-const openaiRunModelPricing: Record<OpenAIRunModel, ProviderCreditPricing> = {
- [openaiModels.debugInspection]: textTokenPricing({
- model: "gpt-5.4",
- input: "2.5",
- cachedInput: "0.25",
- output: "15",
- longContext: { input: "5", cachedInput: "0.5", output: "22.5" },
- }),
-};
-
function browserUseCreditPricing(): ProviderCreditPricing {
// Browser Use Cloud bills ~$0.01/task init + a per-step LLM fee, and the API
// exposes stepCount (not a USD cost), so we price per step. The per-step rate
@@ -102,11 +88,6 @@ function openAICreditPricing(model: string): ProviderCreditPricing | undefined {
return audioPricing;
}
- const runModelPricing = openaiRunModelPricing[model as OpenAIRunModel];
- if (runModelPricing) {
- return runModelPricing;
- }
-
const matched = openAITextCreditRates.find((rate) => modelMatches(model, rate.model));
if (!matched) {
return undefined;
diff --git a/site/src/lib/inference/adapters/gemini-context-cache.ts b/site/src/lib/inference/adapters/gemini-context-cache.ts
index fd1e7b21..db079f48 100644
--- a/site/src/lib/inference/adapters/gemini-context-cache.ts
+++ b/site/src/lib/inference/adapters/gemini-context-cache.ts
@@ -97,18 +97,13 @@ async function resolveCachedSystemInstruction(args: {
/**
* Swap a large, repeated inline system instruction for a cached reference, in place on `requestParameters`.
- * Applies only to the text/decision path (no Gemini tools registered) where the same instruction recurs
- * every step; tool/computer-use calls and short instructions are left untouched. A no-op on any failure, so
- * the request always stays valid.
+ * Applies where the same instruction recurs every step; short instructions are left untouched.
+ * A no-op on any failure, so the request always stays valid.
*/
export async function applyContextCacheToRequest(
requestParameters: GenerateContentParameters,
- registeredTools: string[],
client: GeminiClient,
): Promise<void> {
- if (registeredTools.length > 0) {
- return;
- }
const config = requestParameters.config;
const systemInstruction = config?.systemInstruction;
if (!config || typeof systemInstruction !== "string" || systemInstruction.length < MIN_CACHEABLE_CHARS) {
diff --git a/site/src/lib/inference/adapters/gemini-computer-use.ts b/site/src/lib/inference/adapters/gemini-responses.ts
similarity index 80%
rename from site/src/lib/inference/adapters/gemini-computer-use.ts
rename to site/src/lib/inference/adapters/gemini-responses.ts
index fc599de8..3e1e231f 100644
--- a/site/src/lib/inference/adapters/gemini-computer-use.ts
+++ b/site/src/lib/inference/adapters/gemini-responses.ts
@@ -1,6 +1,5 @@
import {
ApiError,
- Environment,
GoogleGenAI,
ThinkingLevel,
} from "@google/genai";
@@ -47,45 +46,11 @@ import {
type TextStreamResult,
} from "@/lib/inference/providers";
-const providerID = "gemini-computer-use";
+const providerID = "gemini-responses";
const geminiProviderID = "gemini";
-const defaultDecisionResponsesModel = geminiModelRoles.fastDecision;
const defaultVertexResponsesModel = geminiModelRoles.chat;
-const defaultComputerUseModel = geminiModelRoles.computerUse;
-
-export const geminiBrowserInteractionToolType = "donkey_gemini_browser_interaction";
-export const geminiMacDesktopInteractionToolType = "donkey_gemini_mac_desktop_interaction";
-export const debugUIInspectionToolType = "donkey_debug_ui_inspection";
-
-// Note: `generic_harness_planning` is deliberately NOT fast-decision. Planning is the most
-// reasoning-heavy step (routing, plan steps, applying loaded skill guidance, and the act-vs-confirm
-// call), and the flash-lite model follows that nuanced guidance poorly — it tends to ask for
-// clarification on requests a skill can resolve directly. Route planning to the stronger model.
-const fastDecisionSchemaNames = new Set([
- "task_intent_v1",
- "task_followup_resolution_v1",
-]);
-
-const fastDecisionPromptVersions = new Set([
- "task-followup-resolution-v1",
-]);
-const browserOnlyFunctionExclusions = [
- "drag_and_drop",
-];
-
-// On the macOS desktop the agent never drives a web browser chrome, so the
-// browser-navigation predefined functions are excluded — leaving the GUI actions
-// (click, type, scroll, drag_and_drop, key_combination, …) the desktop needs.
-const desktopExcludedFunctions = [
- "open_web_browser",
- "navigate",
- "go_back",
- "go_forward",
- "search",
-];
-
-export function createGeminiComputerUseProvider(
+export function createGeminiResponsesProvider(
environment: AdapterEnvironment = process.env,
clientFactory: GeminiClientFactory = (options) => new GoogleGenAI(options),
): InferenceProvider {
@@ -98,12 +63,11 @@ export function createGeminiComputerUseProvider(
return [];
}
- // gemini-3.5-flash serves both general Responses and computer use, so list it once with the
- // computer-use capability — listing it twice would let dedupeModels drop the computer-use entry.
- // The fast-decision model (flash-lite) is the only other entry.
+ // The full flash model serves general Responses; the fast-decision model
+ // (flash-lite) is the only other entry.
return [
- staticModel(defaultDecisionResponsesModel, false),
- staticModel(defaultComputerUseModel, true),
+ staticModel(geminiModelRoles.fastDecision),
+ staticModel(defaultVertexResponsesModel),
];
}
@@ -113,31 +77,22 @@ export function createGeminiComputerUseProvider(
// Caller-defined function tools are honored only when the caller explicitly
// asked for Gemini; the default Responses route keeps sending them elsewhere.
const allowFunctionTools = request.donkeyProvider === geminiProviderID;
- const registeredTools = registeredToolTypes(request.body.tools);
if (hasExplicitUnsupportedTools(request.body.tools, allowFunctionTools)) {
throw new InferenceProviderError("Gemini Responses received unsupported tool declarations.", {
statusCode: 400,
code: "gemini_tool_unsupported",
details: {
- supportedTools: [
- geminiBrowserInteractionToolType,
- geminiMacDesktopInteractionToolType,
- debugUIInspectionToolType,
- ],
+ supportedTools: allowFunctionTools ? ["function"] : [],
},
});
}
- const model = requestedModel(
- request.body,
- defaultResponseModel(request.body, registeredTools),
- );
+ const model = requestedModel(request.body, defaultVertexResponsesModel);
const requestParameters = geminiGenerateContentParameters(
request.body,
- registeredTools,
model,
);
- return { registeredTools, model, requestParameters };
+ return { model, requestParameters };
}
async function createResponse(
@@ -145,14 +100,14 @@ export function createGeminiComputerUseProvider(
): Promise<ResponseCreateResult> {
ensureConfigured(configured);
- const { registeredTools, model, requestParameters } = responseRequestSetup(request);
+ const { model, requestParameters } = responseRequestSetup(request);
const client = clientFactory(clientConfig.options);
// Cache the planner's large, byte-identical system instruction once and reference it on later steps, so
// its ~11K tokens bill at the cached rate instead of full price on every step. Best-effort: a no-op for
- // tool/computer-use calls and short instructions, and a clean fallback to the inline instruction on any
+ // short instructions, and a clean fallback to the inline instruction on any
// caching error — so the request is never made invalid by this.
- await applyContextCacheToRequest(requestParameters, registeredTools, client);
+ await applyContextCacheToRequest(requestParameters, client);
let rawResponse: unknown;
let retriedWithoutSchema = false;
@@ -174,7 +129,7 @@ export function createGeminiComputerUseProvider(
}
const rawBody = toJsonValue(rawResponse);
- const body = normalizedGeminiResponse(toJsonValue(rawBody), registeredTools);
+ const body = normalizedGeminiResponse(toJsonValue(rawBody));
return {
provider: geminiProviderID,
model,
@@ -184,7 +139,6 @@ export function createGeminiComputerUseProvider(
provider: geminiProviderID,
api: "google-genai-sdk",
service: clientConfig.service,
- registeredTools,
structuredSchemaRetry: String(retriedWithoutSchema),
},
};
@@ -200,9 +154,9 @@ export function createGeminiComputerUseProvider(
): Promise<ResponseStreamResult> {
ensureConfigured(configured);
- const { registeredTools, model, requestParameters } = responseRequestSetup(request);
+ const { model, requestParameters } = responseRequestSetup(request);
const client = clientFactory(clientConfig.options);
- await applyContextCacheToRequest(requestParameters, registeredTools, client);
+ await applyContextCacheToRequest(requestParameters, client);
let iterator: AsyncGenerator<GenerateContentResponse>;
try {
@@ -214,12 +168,11 @@ export function createGeminiComputerUseProvider(
return {
provider: geminiProviderID,
model,
- events: responseStreamEvents(iterator, registeredTools),
+ events: responseStreamEvents(iterator),
metadata: {
provider: geminiProviderID,
api: "google-genai-sdk",
service: clientConfig.service,
- registeredTools,
},
};
}
@@ -249,7 +202,7 @@ export function createGeminiComputerUseProvider(
}
const rawBody = toJsonValue(rawResponse);
- const normalized = normalizedGeminiResponse(rawBody, []);
+ const normalized = normalizedGeminiResponse(rawBody);
const outputText = stringValue(normalized.output_text) ?? "";
return {
provider: geminiProviderID,
@@ -353,7 +306,6 @@ export function createGeminiComputerUseProvider(
// signatures. Non-thought text also goes out live as delta events.
async function* responseStreamEvents(
iterator: AsyncGenerator<GenerateContentResponse>,
- registeredTools: string[],
): AsyncGenerator<ResponseStreamEvent> {
const parts: JsonObject[] = [];
let responseId: string | undefined;
@@ -395,7 +347,6 @@ async function* responseStreamEvents(
...(usageMetadata !== undefined ? { usageMetadata } : {}),
candidates: [{ content: { parts } }],
},
- registeredTools,
);
yield { type: "completed", body, usage: body.usage };
}
@@ -409,10 +360,9 @@ function streamChunkText(chunk: GenerateContentResponse): string {
function geminiGenerateContentParameters(
body: JsonObject,
- registeredTools: string[],
model: string,
): GenerateContentParameters {
- const tools = geminiTools(registeredTools, body.tools);
+ const tools = geminiTools(body.tools);
const generationConfig = generationConfigFromBody(body);
const systemInstruction = systemInstructionFromBody(body);
const config: GenerateContentConfig = {
@@ -432,33 +382,9 @@ function geminiGenerateContentParameters(
};
}
-function geminiTools(registeredTools: string[], rawTools: JsonValue | undefined): Tool[] {
+function geminiTools(rawTools: JsonValue | undefined): Tool[] {
const tools: Tool[] = [];
- if (registeredTools.includes(geminiBrowserInteractionToolType)) {
- tools.push({
- computerUse: {
- environment: Environment.ENVIRONMENT_BROWSER,
- excludedPredefinedFunctions: excludedPredefinedFunctions(
- rawTools,
- browserOnlyFunctionExclusions,
- ),
- },
- });
- }
-
- if (registeredTools.includes(geminiMacDesktopInteractionToolType)) {
- tools.push({
- computerUse: {
- environment: Environment.ENVIRONMENT_DESKTOP,
- excludedPredefinedFunctions: excludedPredefinedFunctions(
- rawTools,
- desktopExcludedFunctions,
- ),
- },
- });
- }
-
const declarations = functionDeclarationsFromTools(rawTools);
if (declarations.length > 0) {
tools.push({ functionDeclarations: declarations });
@@ -762,7 +688,7 @@ function generationConfigFromBody(body: JsonObject): Partial<GenerateContentConf
}
// Bound reasoning so thinking tokens (which count against maxOutputTokens) can't starve the
// structured output. Callers driving tight per-turn loops pass a small budget; 0 disables thinking.
- // Gemini 3.x models (e.g. gemini-3.5-flash) take thinking_level (minimal|low|medium|high), NOT the
+ // Gemini 3.x models (e.g. gemini-3.7-flash) take thinking_level (minimal|low|medium|high), NOT the
// integer thinking_budget — passing the budget to them is silently ignored. Prefer the level when the
// caller sets it; the two are mutually exclusive. Older 2.x models still use thinking_budget. In both
// cases request the thought summary so callers can persist the reasoning (the normalized response
@@ -926,44 +852,6 @@ function responseFormatFromBody(body: JsonObject): { json: boolean; schema?: Jso
return null;
}
-function defaultResponseModel(body: JsonObject, registeredTools: string[]) {
- if (
- registeredTools.includes(geminiBrowserInteractionToolType) ||
- registeredTools.includes(geminiMacDesktopInteractionToolType)
- ) {
- return defaultComputerUseModel;
- }
-
- if (registeredTools.includes(debugUIInspectionToolType)) {
- return defaultDecisionResponsesModel;
- }
-
- if (isFastDecisionRequest(body)) {
- return defaultDecisionResponsesModel;
- }
-
- return defaultVertexResponsesModel;
-}
-
-function isFastDecisionRequest(body: JsonObject) {
- const metadata = isJsonObject(body.metadata) ? body.metadata : {};
- const promptVersion =
- stringValue(metadata.prompt_version) ??
- stringValue(metadata.promptVersion);
- if (promptVersion && fastDecisionPromptVersions.has(promptVersion)) {
- return true;
- }
-
- const format =
- isJsonObject(body.text) && isJsonObject(body.text.format)
- ? body.text.format
- : isJsonObject(body.response_format)
- ? body.response_format
- : null;
- const schemaName = format ? stringValue(format.name) : undefined;
- return Boolean(schemaName && fastDecisionSchemaNames.has(schemaName));
-}
-
function systemInstructionFromBody(body: JsonObject): string | undefined {
const instruction = [
stringValue(body.instructions),
@@ -976,10 +864,7 @@ function systemInstructionFromBody(body: JsonObject): string | undefined {
return instruction;
}
-function normalizedGeminiResponse(
- raw: JsonValue,
- registeredTools: string[],
-): JsonObject {
+function normalizedGeminiResponse(raw: JsonValue): JsonObject {
const partObjects = geminiCandidateParts(geminiCandidates(raw)[0]);
// Thought summaries (parts flagged `thought: true` when includeThoughts is on) must NOT land in
// output_text — that field carries the structured JSON the caller parses. Keep them separate so the
@@ -1015,10 +900,6 @@ function normalizedGeminiResponse(
...call,
})),
],
- computer_use: {
- registered_tools: registeredTools,
- calls,
- },
provider_output: raw,
usage: isJsonObject(raw) ? raw.usageMetadata ?? null : null,
};
@@ -1051,49 +932,6 @@ function functionCallFromPart(part: JsonObject): JsonObject | null {
return call;
}
-function registeredToolTypes(tools: JsonValue | undefined): string[] {
- if (!Array.isArray(tools)) {
- return [];
- }
-
- const registered = new Set<string>();
- for (const tool of tools) {
- if (!isJsonObject(tool)) {
- continue;
- }
- if (tool.type === geminiBrowserInteractionToolType) {
- registered.add(geminiBrowserInteractionToolType);
- } else if (tool.type === geminiMacDesktopInteractionToolType) {
- registered.add(geminiMacDesktopInteractionToolType);
- } else if (tool.type === debugUIInspectionToolType) {
- registered.add(debugUIInspectionToolType);
- }
- }
- return [...registered];
-}
-
-function excludedPredefinedFunctions(rawTools: JsonValue | undefined, defaults: string[]) {
- const excluded = new Set(defaults);
- if (Array.isArray(rawTools)) {
- for (const tool of rawTools) {
- if (!isJsonObject(tool)) {
- continue;
- }
- const values = Array.isArray(tool.excludedPredefinedFunctions)
- ? tool.excludedPredefinedFunctions
- : Array.isArray(tool.excluded_predefined_functions)
- ? tool.excluded_predefined_functions
- : [];
- for (const value of values) {
- if (typeof value === "string" && value.trim()) {
- excluded.add(value.trim());
- }
- }
- }
- }
- return [...excluded];
-}
-
function geminiRole(value: string | undefined) {
return value === "assistant" || value === "model" ? "model" : "user";
}
@@ -1147,16 +985,11 @@ function hasExplicitUnsupportedTools(
if (!isJsonObject(tool)) {
return true;
}
- if (allowFunctionTools && tool.type === "function") {
- return false;
- }
- return tool.type !== geminiBrowserInteractionToolType &&
- tool.type !== geminiMacDesktopInteractionToolType &&
- tool.type !== debugUIInspectionToolType;
+ return !(allowFunctionTools && tool.type === "function");
});
}
-function staticModel(model: string, computerUse: boolean): InferenceModel {
+function staticModel(model: string): InferenceModel {
return {
id: model,
name: model,
@@ -1168,16 +1001,7 @@ function staticModel(model: string, computerUse: boolean): InferenceModel {
metadata: {
provider: geminiProviderID,
api: "generateContent",
- ...(computerUse
- ? {
- computerUse,
- registeredTools: [
- geminiBrowserInteractionToolType,
- geminiMacDesktopInteractionToolType,
- debugUIInspectionToolType,
- ],
- }
- : { structuredOutputs: true }),
+ structuredOutputs: true,
},
};
}
diff --git a/site/src/lib/inference/adapters/hosted-responses.ts b/site/src/lib/inference/adapters/hosted-responses.ts
deleted file mode 100644
index 4ce6dc75..00000000
--- a/site/src/lib/inference/adapters/hosted-responses.ts
+++ /dev/null
@@ -1,279 +0,0 @@
-import OpenAI, { APIError } from "openai";
-import type { ResponseCreateParamsNonStreaming } from "openai/resources/responses/responses";
-
-import {
- ensureConfigured,
- type FetchLike,
-} from "@/lib/inference/http";
-import { toJsonValue } from "@/lib/inference/json";
-import { openaiModels } from "@/lib/inference/openai-models";
-import {
- InferenceProviderError,
- type InferenceModality,
- type InferenceModel,
- type InferenceProvider,
- type JsonObject,
- type JsonValue,
- type ResponseCreateRequest,
- type ResponseCreateResult,
-} from "@/lib/inference/providers";
-
-type AdapterEnvironment = Record<string, string | undefined>;
-
-const providerID = "hosted-responses";
-const openAIProviderID = "openai";
-const openAIBaseURL = "https://api.openai.com/v1";
-const debugUIInspectionToolType = "donkey_debug_ui_inspection";
-const defaultOpenAIDebugInspectionModel = openaiModels.debugInspection;
-const unsupportedOpenAIComputerUseParameters = ["temperature", "top_p", "topP"];
-
-export function createHostedResponsesProvider(
- environment: AdapterEnvironment = process.env,
- fetcher: FetchLike = fetch,
-): InferenceProvider {
- const openAIKey = environment.OPENAI_API_KEY?.trim() ?? "";
- const configured = openAIKey.length > 0;
-
- async function listModels(modalities: InferenceModality[]) {
- const requested = modalities.length > 0 ? modalities : ["text"];
- if (!requested.includes("text") && !requested.includes("image")) {
- return [];
- }
-
- if (!openAIKey) {
- return [];
- }
-
- return [
- staticModel(defaultOpenAIDebugInspectionModel),
- ];
- }
-
- async function createResponse(
- request: ResponseCreateRequest,
- ): Promise<ResponseCreateResult> {
- ensureConfigured(configured);
- if (!isOpenAIResponsesToolRequest(request.body)) {
- throw new InferenceProviderError(
- "OpenAI Responses is only configured for read-only debug UI inspection.",
- {
- statusCode: 400,
- code: "openai_computer_tool_required",
- details: {
- supportedTools: [
- debugUIInspectionToolType,
- ],
- },
- },
- );
- }
-
- const client = new OpenAI({
- apiKey: openAIKey,
- baseURL: openAIBaseURL,
- fetch: fetcher,
- });
-
- try {
- const model = defaultOpenAIModel(request.body);
- const body = requestBody(request.body, model);
- const response = await client.responses.create(
- body as unknown as ResponseCreateParamsNonStreaming,
- );
- const value = toJsonValue(response);
- return {
- provider: openAIProviderID,
- model,
- body: value,
- usage: usageFromResponse(value),
- metadata: {
- provider: openAIProviderID,
- baseURL: openAIBaseURL,
- store: String(body.store ?? false),
- },
- };
- } catch (error) {
- throw providerError("OpenAI Responses computer-use request failed.", error);
- }
- }
-
- return {
- id: providerID,
- configured,
- capabilities: ["text", "image"],
- responseProviderIDs: [openAIProviderID],
- canCreateResponse: (request) => isOpenAIResponsesToolRequest(request.body),
- listModels,
- createResponse,
- };
-}
-
-function defaultOpenAIModel(_body: JsonObject) {
- return defaultOpenAIDebugInspectionModel;
-}
-
-function requestBody(body: JsonObject, model: string): JsonObject {
- const normalizedBody = normalizeOpenAIComputerUseBody(body);
- return {
- ...normalizedBody,
- model,
- stream: false,
- store: boolValue(normalizedBody.store, false),
- };
-}
-
-function normalizeOpenAIComputerUseBody(body: JsonObject): JsonObject {
- const hasDebugInspectionTool = hasDebugUIInspectionTool(body);
- if (!hasDebugInspectionTool) {
- return body;
- }
-
- const tools = body.tools;
- const toolChoice = openAIComputerToolChoice(body.tool_choice);
- const normalized: JsonObject = {
- ...body,
- instructions: openAIInstructions(body.instructions),
- ...(toolChoice === undefined ? {} : { tool_choice: toolChoice }),
- };
- for (const parameter of unsupportedOpenAIComputerUseParameters) {
- delete normalized[parameter];
- }
- if (Array.isArray(tools)) {
- const normalizedTools = tools
- .map(openAIComputerToolReference)
- .filter((tool): tool is JsonObject => isJsonObject(tool));
- if (normalizedTools.length > 0) {
- normalized.tools = normalizedTools;
- } else {
- delete normalized.tools;
- }
- }
- return normalized;
-}
-
-function openAIComputerToolChoice(value: JsonValue | undefined): JsonValue | undefined {
- if (!isJsonObject(value)) {
- return value;
- }
-
- const reference = openAIComputerToolReference(value);
- if (reference !== value) {
- return reference;
- }
-
- if (value.type !== "allowed_tools" || !Array.isArray(value.tools)) {
- return value;
- }
-
- return {
- ...value,
- tools: value.tools.map(openAIComputerToolReference),
- };
-}
-
-// The debug-inspection tool is read-only and has no OpenAI-native equivalent, so it is stripped from
-// the request (the work is driven entirely by the inspection instructions). Any other tool passes
-// through untouched.
-function openAIComputerToolReference(value: JsonValue): JsonValue | null {
- if (!isJsonObject(value)) {
- return value;
- }
-
- if (value.type === debugUIInspectionToolType) {
- return null;
- }
-
- return value;
-}
-
-function openAIInstructions(value: JsonValue | undefined) {
- return [
- stringValue(value),
- [
- "Perform read-only macOS UI inspection from the provided screenshot.",
- "Return strict JSON only and do not return computer_call, function_call, click, type, scroll, drag, or navigation actions.",
- ].join(" "),
- ].filter(Boolean).join("\n\n");
-}
-
-function staticModel(model: string): InferenceModel {
- return {
- id: model,
- name: model,
- provider: openAIProviderID,
- inputModalities: ["text", "image"],
- outputModalities: ["text"],
- contextLength: null,
- pricing: null,
- metadata: {
- provider: openAIProviderID,
- baseURL: openAIBaseURL,
- api: "responses",
- registeredTools: [
- debugUIInspectionToolType,
- ],
- },
- };
-}
-
-function isOpenAIResponsesToolRequest(body: JsonObject) {
- const tools = body.tools;
- if (!Array.isArray(tools)) {
- return false;
- }
-
- return tools.some((tool) => {
- return isJsonObject(tool) && tool.type === debugUIInspectionToolType;
- });
-}
-
-function hasDebugUIInspectionTool(body: JsonObject) {
- const tools = body.tools;
- if (!Array.isArray(tools)) {
- return false;
- }
-
- return tools.some((tool) => {
- return isJsonObject(tool) && tool.type === debugUIInspectionToolType;
- });
-}
-
-function usageFromResponse(value: JsonValue): JsonValue | undefined {
- if (!isJsonObject(value)) {
- return undefined;
- }
- return value.usage;
-}
-
-function boolValue(value: JsonValue | undefined, fallback: boolean) {
- return typeof value === "boolean" ? value : fallback;
-}
-
-function stringValue(value: JsonValue | undefined): string | undefined {
- return typeof value === "string" && value.trim() ? value : undefined;
-}
-
-function isJsonObject(value: unknown): value is JsonObject {
- return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function providerError(message: string, error: unknown) {
- if (error instanceof APIError) {
- return new InferenceProviderError(message, {
- statusCode: error.status ?? 502,
- code: error.code ?? "provider_error",
- details: {
- body: toJsonValue(error.error ?? {}),
- requestID: error.requestID ?? null,
- status: error.status ?? null,
- type: error.type ?? null,
- },
- });
- }
-
- return new InferenceProviderError(message, {
- details: {
- message: error instanceof Error ? error.message : "Unknown error",
- },
- });
-}
diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.ts
index 25128895..321a9c14 100644
--- a/site/src/lib/inference/gemini-models.ts
+++ b/site/src/lib/inference/gemini-models.ts
@@ -67,11 +67,6 @@ export const geminiModelRoles = {
chatSimple: geminiModels.flashLite,
// Fast structured task-intent and follow-up decisions.
fastDecision: geminiModels.flashLite,
- // Computer Use tool calls (browser and macOS desktop environments). Computer
- // use is a built-in tool of the main flash model, so both share one model.
- computerUse: geminiModels.flash,
- // Screenshot parsing into read-only UI evidence.
- screenshotParse: geminiModels.flash,
// Generative image editing and generation.
imageGeneration: geminiModels.proImage,
// Production review: the director judging rendered takes and minted frames
diff --git a/site/src/lib/inference/openai-models.ts b/site/src/lib/inference/openai-models.ts
deleted file mode 100644
index 508d7906..00000000
--- a/site/src/lib/inference/openai-models.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-// Central registry of OpenAI model IDs the gateway runs (mirrors gemini-models.ts).
-// Anything the code selects lives here so it is priced exhaustively in provider-pricing.ts;
-// bumping a model is a one-line change here.
-export const openaiModels = {
- // Developer UI inspection Responses calls.
- debugInspection: "gpt-5.4",
-} as const;
-
-export type OpenAIRunModel = (typeof openaiModels)[keyof typeof openaiModels];
diff --git a/site/src/lib/inference/router.ts b/site/src/lib/inference/router.ts
index 6138821b..aad4ecdf 100644
--- a/site/src/lib/inference/router.ts
+++ b/site/src/lib/inference/router.ts
@@ -1,9 +1,8 @@
-import { createGeminiComputerUseProvider } from "@/lib/inference/adapters/gemini-computer-use";
+import { createGeminiResponsesProvider } from "@/lib/inference/adapters/gemini-responses";
import { createGeminiImageAssetProvider } from "@/lib/inference/adapters/gemini-image";
import { createGeminiMusicAssetProvider } from "@/lib/inference/adapters/gemini-music";
import { createGeminiSpeechAssetProvider } from "@/lib/inference/adapters/gemini-speech";
import { createGeminiOmniVideoAssetProvider } from "@/lib/inference/adapters/gemini-omni-video";
-import { createHostedResponsesProvider } from "@/lib/inference/adapters/hosted-responses";
import {
InferenceProviderError,
type AssetGenerationRequest,
@@ -216,7 +215,7 @@ export class ProviderRegistry {
export function createProviderRegistry() {
return new ProviderRegistry([
// Asset selection is by capability + generateAsset, not list order: the image asset
- // provider serves kind="image" (gemini-computer-use lists "image" as an input modality
+ // provider serves kind="image" (gemini-responses lists "image" as an input modality
// but has no generateAsset, so it is never chosen for asset generation).
createGeminiImageAssetProvider(),
// Omni (Gemini Omni Flash) serves kind="video"; its distinct provider id
@@ -231,8 +230,7 @@ export function createProviderRegistry() {
// adapter picks the clip or the longer pro model from the requested length so
// a bed can still span a longer video (see gemini-music.ts).
createGeminiMusicAssetProvider(),
- createGeminiComputerUseProvider(),
- createHostedResponsesProvider(),
+ createGeminiResponsesProvider(),
]);
}
diff --git a/site/src/lib/inference/screenshot-parsing/gemini-flash.ts b/site/src/lib/inference/screenshot-parsing/gemini-flash.ts
deleted file mode 100644
index 8b143dc1..00000000
--- a/site/src/lib/inference/screenshot-parsing/gemini-flash.ts
+++ /dev/null
@@ -1,681 +0,0 @@
-import {
- ApiError,
- GoogleGenAI,
- MediaResolution,
- Type,
-} from "@google/genai";
-import { JWT, type JWTInput } from "google-auth-library";
-import type {
- GenerateContentParameters,
- GoogleGenAIOptions,
- Schema,
-} from "@google/genai";
-
-import { geminiModelRoles } from "@/lib/inference/gemini-models";
-import { ensureConfigured } from "@/lib/inference/http";
-import {
- isJsonObject,
- toJsonValue,
-} from "@/lib/inference/json";
-import {
- InferenceProviderError,
- type JsonValue,
-} from "@/lib/inference/providers";
-import { normalizedScreenshotResult } from "@/lib/inference/screenshot-parsing/normalize";
-import {
- geminiScreenshotParseOutputSchema,
- geminiScreenshotControlSchema,
- type GeminiScreenshotParseOutput,
- type ScreenshotParseRequest,
-} from "@/lib/inference/screenshot-parsing/schema";
-import type { ScreenshotParserProvider } from "@/lib/inference/screenshot-parsing/types";
-
-type AdapterEnvironment = Record<string, string | undefined>;
-type GeminiClient = Pick<GoogleGenAI, "models">;
-type GeminiClientFactory = (options: GoogleGenAIOptions) => GeminiClient;
-
-const geminiProviderID = "gemini";
-const screenshotProviderID = "gemini-flash";
-const defaultScreenshotParseModel = geminiModelRoles.screenshotParse;
-const defaultDebugOverlayScreenshotParseModel = geminiModelRoles.screenshotParse;
-const defaultVertexLocation = "global";
-const vertexAIScope = "https://www.googleapis.com/auth/cloud-platform";
-// Client-side request timeout so a stalled parse aborts instead of hanging the
-// per-step agent loop; screenshot parsing is normally a few seconds, so this only
-// trips on a true hang.
-const requestTimeoutMs = 120_000;
-
-export function createGeminiFlashScreenshotParser(
- environment: AdapterEnvironment = process.env,
- clientFactory: GeminiClientFactory = (options) => new GoogleGenAI(options),
-): ScreenshotParserProvider {
- const config = geminiClientConfig(environment);
-
- return {
- id: screenshotProviderID,
- inferenceProvider: geminiProviderID,
- configured: config.configured,
- modelForRequest: (request) => screenshotParseModelForRequest(request),
- async parse(request) {
- ensureConfigured(config.configured);
- const client = clientFactory(config.options);
- const model = screenshotParseModelForRequest(request);
- const startedAt = performance.now();
-
- let rawResponse: unknown;
- try {
- rawResponse = await client.models.generateContent(
- geminiRequestParameters(request, model),
- );
- } catch (error) {
- throw geminiProviderError(error);
- }
-
- const rawBody = toJsonValue(rawResponse);
- const output = parseGeminiOutput(rawBody);
- const result = normalizedScreenshotResult(request, output, {
- provider: geminiProviderID,
- parserProvider: screenshotProviderID,
- model,
- service: config.service,
- location: config.location,
- latencyMs: String(Math.round(performance.now() - startedAt)),
- });
-
- return {
- provider: geminiProviderID,
- model,
- result,
- usage: usageFromGeminiResponse(rawBody),
- metadata: result.metadata,
- };
- },
- async *stream(request) {
- ensureConfigured(config.configured);
- const client = clientFactory(config.options);
- const model = screenshotParseModelForRequest(request);
- const startedAt = performance.now();
- const generator = await streamGeminiContent(client, request, model);
- let accumulatedText = "";
- let emittedControlCount = 0;
- let latestUsage: JsonValue | undefined;
-
- for await (const chunk of generator) {
- accumulatedText += geminiResponseText(chunk);
- latestUsage = usageFromGeminiResponse(toJsonValue(chunk)) ?? latestUsage;
-
- const partialOutput = partialGeminiOutput(accumulatedText);
- if (partialOutput.controls.length <= emittedControlCount) {
- continue;
- }
-
- emittedControlCount = partialOutput.controls.length;
- const result = normalizedScreenshotResult(request, partialOutput, {
- provider: geminiProviderID,
- parserProvider: screenshotProviderID,
- model,
- service: config.service,
- location: config.location,
- latencyMs: String(Math.round(performance.now() - startedAt)),
- "screenshotParser.stream": "partial",
- });
- yield {
- type: "partial",
- provider: geminiProviderID,
- model,
- result,
- metadata: result.metadata,
- };
- }
-
- const output = parseGeminiOutputText(accumulatedText);
- const result = normalizedScreenshotResult(request, output, {
- provider: geminiProviderID,
- parserProvider: screenshotProviderID,
- model,
- service: config.service,
- location: config.location,
- latencyMs: String(Math.round(performance.now() - startedAt)),
- "screenshotParser.stream": "final",
- });
- yield {
- type: "final",
- provider: geminiProviderID,
- model,
- result,
- usage: latestUsage,
- metadata: result.metadata,
- };
- },
- };
-}
-
-export function screenshotParseModelForRequest(request: ScreenshotParseRequest) {
- if (isDebugOverlayRequest(request)) {
- return defaultDebugOverlayScreenshotParseModel;
- }
-
- return defaultScreenshotParseModel;
-}
-
-function geminiRequestParameters(
- request: ScreenshotParseRequest,
- model: string,
-): GenerateContentParameters {
- const profile = screenshotParseProfile(request);
- return {
- model,
- contents: [
- {
- role: "user",
- parts: [
- {
- text: screenshotParsingPrompt(request),
- },
- {
- inlineData: {
- data: request.imageBase64,
- mimeType: request.contentType,
- },
- },
- ],
- },
- ],
- config: {
- mediaResolution: profile.mediaResolution,
- responseMimeType: "application/json",
- responseSchema: geminiOutputSchema,
- temperature: 0,
- topP: 0.1,
- candidateCount: 1,
- maxOutputTokens: profile.maxOutputTokens,
- thinkingConfig: {
- thinkingBudget: 0,
- },
- },
- };
-}
-
-function screenshotParsingPrompt(request: ScreenshotParseRequest) {
- if (isDebugOverlayRequest(request)) {
- return [
- "You are a read-only UI screenshot parser for a macOS developer inspection overlay.",
- "The screenshot is a single app/window capture, never the entire desktop.",
- "Return compact JSON matching the provided schema.",
- "For this overlay profile, visibleText must be [] and formFields must be []; put readable names only in control labels.",
- "Return bounding boxes for the full visible clickable/control region, not just the text glyphs or icon pixels.",
- "Treat compound controls as one control when a symbol/icon and nearby text visually act together; the box should cover the icon, text, and full hit target.",
- "Treat standalone symbols/icons as controls when they use standard UI shapes or placement, even if there is no visible text label.",
- "For nav/sidebar/list/file rows, box the entire row target including icon, label, and selection background when visible.",
- "Always include macOS titlebar controls and toolbar controls when visible: red/yellow/green traffic lights, sidebar toggle, back/forward arrows, title actions, and top-right toolbar icons.",
- "Always include bottom composer/input controls when visible: the text input surface, add/attachment button, mode selector, model selector, mic/voice button, and send/stop button.",
- "Deliberately scan the top edge of the image from left to right for small window chrome controls, navigation icons, segmented controls, menus, status buttons, and titlebar actions.",
- "Deliberately scan the bottom edge of the image from left to right for composer/input areas, accessory icons, dropdown selectors, microphone controls, submit/stop controls, and other docked toolbars.",
- "Small circular or icon-only controls near the top or bottom edges should be included even when they have no readable text; infer a concise generic label such as close, minimize, zoom, back, forward, add, voice, submit, or stop when visually supported.",
- "Prioritize visible controls and selectable regions that are useful to draw debug boxes: buttons, icon buttons, nav/sidebar rows, tabs, menus, file rows, list rows, links, inputs, titlebar controls, and bottom composer controls.",
- "Do not include decorative backgrounds, separators, or long static prose blocks unless they are clickable/selectable regions.",
- "Prefer 25-55 high-confidence controls over an exhaustive map.",
- "Use box_2d as [ymin, xmin, ymax, xmax] normalized to 0-1000 over the full image.",
- "Use concise labels copied from visible text when possible.",
- `Image pixel size: ${request.pixelSize.width}x${request.pixelSize.height}.`,
- `Target ID: ${request.targetID ?? "unknown"}.`,
- ].join("\n");
- }
-
- return [
- "You are a read-only UI screenshot parser for a macOS app automation system.",
- "The screenshot is scoped to an app/window or a system-navigation surface, never the entire desktop.",
- "Extract only visible, grounded UI evidence from that scoped screenshot.",
- "Do not infer user intent, do not suggest actions, and do not return executable commands.",
- "Return JSON matching the provided schema.",
- "Be dense: return all visible navigation rows, sidebar items, tabs, buttons, icon buttons, menus, file/change rows, input fields, checkboxes, links, cards, list items, and important text regions.",
- "Treat controls as visual affordances, not only text. Many controls are a symbol/icon plus a label, a symbol inside a button shape, or an icon-only target; box the complete hit target rather than only the glyphs.",
- "Do not stop after the primary content. Include left navigation, right panels, bottom composers/toolbars, title bars, and repeated rows when visible.",
- "Use kind=listItem for selectable rows or navigation entries, kind=group for non-clickable labeled regions/cards, and kind=unknown only when no better kind fits.",
- "Prefer returning more grounded UI elements over a sparse summary. Aim for 40-120 controls on a complex desktop app screenshot when that many distinct regions are visible.",
- "visibleText should contain the most important readable text snippets, including text from controls.",
- "Use box_2d as [ymin, xmin, ymax, xmax] normalized to 0-1000 over the full image.",
- "Use concise labels copied from visible text when possible.",
- `Image pixel size: ${request.pixelSize.width}x${request.pixelSize.height}.`,
- `Screenshot scope: ${request.metadata["screenshot.scope"] ?? "targetWindow"}.`,
- `Target ID: ${request.targetID ?? "unknown"}.`,
- ].join("\n");
-}
-
-function screenshotParseProfile(request: ScreenshotParseRequest) {
- if (isDebugOverlayRequest(request)) {
- return {
- mediaResolution: MediaResolution.MEDIA_RESOLUTION_LOW,
- maxOutputTokens: 4_096,
- };
- }
-
- return {
- mediaResolution: MediaResolution.MEDIA_RESOLUTION_HIGH,
- maxOutputTokens: 16_384,
- };
-}
-
-function isDebugOverlayRequest(request: ScreenshotParseRequest) {
- return request.metadata.source === "debug-ui-inspection-overlay";
-}
-
-function parseGeminiOutput(rawBody: JsonValue) {
- return parseGeminiOutputText(geminiText(rawBody));
-}
-
-function parseGeminiOutputText(text: string) {
- const candidates = jsonObjectCandidates(text).reverse();
- let parsed: unknown;
- for (const candidate of candidates) {
- try {
- parsed = JSON.parse(candidate);
- break;
- } catch {
- continue;
- }
- }
-
- if (parsed === undefined) {
- throw new InferenceProviderError("Gemini returned invalid screenshot parser JSON.", {
- statusCode: 502,
- code: "invalid_provider_output",
- });
- }
-
- const output = geminiScreenshotParseOutputSchema.safeParse(parsed);
- if (!output.success) {
- throw new InferenceProviderError("Gemini screenshot parser output did not match schema.", {
- statusCode: 502,
- code: "invalid_provider_output",
- details: {
- issues: output.error.issues.map((issue) => ({
- path: issue.path.join("."),
- message: issue.message,
- })),
- },
- });
- }
-
- return output.data;
-}
-
-function jsonObjectCandidates(text: string) {
- const trimmed = text.trim();
- const candidates = trimmed ? [trimmed] : [];
- let objectStart = -1;
- let depth = 0;
- let inString = false;
- let escaped = false;
-
- for (let index = 0; index < text.length; index += 1) {
- const character = text[index];
-
- if (inString) {
- if (escaped) {
- escaped = false;
- } else if (character === "\\") {
- escaped = true;
- } else if (character === "\"") {
- inString = false;
- }
- continue;
- }
-
- if (character === "\"") {
- inString = true;
- } else if (character === "{") {
- if (depth === 0) {
- objectStart = index;
- }
- depth += 1;
- } else if (character === "}" && depth > 0) {
- depth -= 1;
- if (depth === 0 && objectStart >= 0) {
- candidates.push(text.slice(objectStart, index + 1));
- objectStart = -1;
- }
- }
- }
-
- return Array.from(new Set(candidates));
-}
-
-async function streamGeminiContent(
- client: GeminiClient,
- request: ScreenshotParseRequest,
- model: string,
-) {
- try {
- return await client.models.generateContentStream(
- geminiRequestParameters(request, model),
- );
- } catch (error) {
- throw geminiProviderError(error);
- }
-}
-
-function partialGeminiOutput(text: string): GeminiScreenshotParseOutput {
- const controls = extractArrayObjectTexts(text, "controls")
- .map((controlText) => {
- try {
- return JSON.parse(controlText) as unknown;
- } catch {
- return null;
- }
- })
- .map((control) => geminiScreenshotControlSchema.safeParse(control))
- .filter((control) => control.success)
- .map((control) => control.data);
-
- return {
- visibleText: [],
- controls,
- formFields: [],
- confidence: controls.reduce(
- (confidence, control) => Math.max(confidence, control.confidence),
- 0,
- ),
- };
-}
-
-function extractArrayObjectTexts(jsonText: string, propertyName: string) {
- const propertyIndex = jsonText.indexOf(`"${propertyName}"`);
- if (propertyIndex < 0) {
- return [];
- }
-
- const arrayStart = jsonText.indexOf("[", propertyIndex);
- if (arrayStart < 0) {
- return [];
- }
-
- const objects: string[] = [];
- let objectStart = -1;
- let objectDepth = 0;
- let arrayDepth = 0;
- let inString = false;
- let escaped = false;
-
- for (let index = arrayStart; index < jsonText.length; index += 1) {
- const character = jsonText[index];
-
- if (inString) {
- if (escaped) {
- escaped = false;
- } else if (character === "\\") {
- escaped = true;
- } else if (character === "\"") {
- inString = false;
- }
- continue;
- }
-
- if (character === "\"") {
- inString = true;
- continue;
- }
-
- if (character === "[") {
- arrayDepth += 1;
- continue;
- }
-
- if (character === "]") {
- arrayDepth -= 1;
- if (arrayDepth === 0) {
- break;
- }
- continue;
- }
-
- if (arrayDepth !== 1) {
- continue;
- }
-
- if (character === "{") {
- if (objectDepth === 0) {
- objectStart = index;
- }
- objectDepth += 1;
- continue;
- }
-
- if (character === "}") {
- objectDepth -= 1;
- if (objectDepth === 0 && objectStart >= 0) {
- objects.push(jsonText.slice(objectStart, index + 1));
- objectStart = -1;
- }
- }
- }
-
- return objects;
-}
-
-function geminiText(rawBody: JsonValue) {
- if (isJsonObject(rawBody) && typeof rawBody.text === "string") {
- return rawBody.text;
- }
- const candidates = isJsonObject(rawBody) && Array.isArray(rawBody.candidates)
- ? rawBody.candidates
- : [];
- const first = candidates.find(isJsonObject);
- const content = first ? first.content : undefined;
- const parts = content !== undefined && isJsonObject(content) && Array.isArray(content.parts)
- ? content.parts
- : [];
- return parts
- .filter(isJsonObject)
- .map((part) => typeof part.text === "string" ? part.text : "")
- .join("")
- .trim();
-}
-
-function geminiResponseText(response: unknown) {
- const directText = response !== null
- && typeof response === "object"
- && "text" in response
- && typeof response.text === "string"
- ? response.text
- : undefined;
- return directText ?? geminiText(toJsonValue(response));
-}
-
-function usageFromGeminiResponse(rawBody: JsonValue): JsonValue | undefined {
- if (!isJsonObject(rawBody)) {
- return undefined;
- }
- const usage = rawBody.usageMetadata ?? rawBody.usage;
- return usage === undefined ? undefined : toJsonValue(usage);
-}
-
-
-const geminiOutputSchema: Schema = {
- type: Type.OBJECT,
- required: ["visibleText", "controls", "formFields", "confidence"],
- properties: {
- visibleText: {
- type: Type.ARRAY,
- items: {
- type: Type.OBJECT,
- required: ["text", "confidence"],
- properties: {
- id: { type: Type.STRING },
- text: { type: Type.STRING },
- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },
- },
- },
- },
- controls: {
- type: Type.ARRAY,
- items: {
- type: Type.OBJECT,
- required: ["label", "kind", "confidence", "box_2d"],
- properties: {
- id: { type: Type.STRING },
- label: { type: Type.STRING },
- kind: { type: Type.STRING },
- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },
- box_2d: {
- type: Type.ARRAY,
- items: { type: Type.NUMBER, minimum: 0, maximum: 1000 },
- },
- },
- },
- },
- formFields: {
- type: Type.ARRAY,
- items: {
- type: Type.OBJECT,
- required: ["label", "isRequired", "confidence"],
- properties: {
- id: { type: Type.STRING },
- label: { type: Type.STRING },
- isRequired: { type: Type.BOOLEAN },
- currentValue: { type: Type.STRING },
- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },
- box_2d: {
- type: Type.ARRAY,
- items: { type: Type.NUMBER, minimum: 0, maximum: 1000 },
- },
- },
- },
- },
- confidence: { type: Type.NUMBER, minimum: 0, maximum: 1 },
- },
-};
-
-export function geminiClientConfig(environment: AdapterEnvironment = process.env): {
- configured: boolean;
- options: GoogleGenAIOptions;
- service: "vertex-ai" | "gemini-api";
- location: string;
-} {
- const googleCredentials = googleCredentialsFromEnvironment(environment);
- const project = googleCredentials?.project_id;
- const apiKey =
- environment.GEMINI_API_KEY?.trim()
- || environment.GOOGLE_API_KEY?.trim()
- || "";
-
- if (project) {
- const options: GoogleGenAIOptions = {
- vertexai: true,
- location: defaultVertexLocation,
- project,
- httpOptions: { timeout: requestTimeoutMs },
- };
- if (googleCredentials) {
- options.googleAuthOptions = {
- authClient: googleAuthClient(googleCredentials),
- };
- }
- return {
- configured: true,
- options,
- service: "vertex-ai",
- location: defaultVertexLocation,
- };
- }
-
- const options: GoogleGenAIOptions = {
- httpOptions: { timeout: requestTimeoutMs },
- };
- if (apiKey) {
- options.apiKey = apiKey;
- }
-
- return {
- configured: Boolean(apiKey),
- options,
- service: "gemini-api",
- location: "global",
- };
-}
-
-function googleCredentialsFromEnvironment(
- environment: AdapterEnvironment,
-): JWTInput | undefined {
- const rawCredentials = environment.GOOGLE_APPLICATION_CREDENTIALS_JSON?.trim();
- if (!rawCredentials) {
- return undefined;
- }
-
- return serviceAccountCredentials(rawCredentials);
-}
-
-function googleAuthClient(credentials: JWTInput) {
- return new JWT({
- email: credentials.client_email,
- key: credentials.private_key,
- keyId: credentials.private_key_id,
- scopes: [vertexAIScope],
- });
-}
-
-function serviceAccountCredentials(rawCredentials: string): JWTInput {
- let parsed: unknown;
- try {
- parsed = JSON.parse(rawCredentials);
- } catch {
- throw new InferenceProviderError("Google service account JSON is invalid.", {
- statusCode: 500,
- code: "invalid_google_service_account_json",
- });
- }
-
- const credentials = toJsonValue(parsed);
- if (!isJsonObject(credentials)) {
- throw new InferenceProviderError("Google service account JSON must be an object.", {
- statusCode: 500,
- code: "invalid_google_service_account_json",
- });
- }
-
- const clientEmail = stringValue(credentials.client_email);
- const privateKey = stringValue(credentials.private_key);
- if (!clientEmail || !privateKey) {
- throw new InferenceProviderError(
- "Google service account JSON must include client_email and private_key.",
- {
- statusCode: 500,
- code: "invalid_google_service_account_json",
- },
- );
- }
-
- return {
- type: stringValue(credentials.type),
- project_id: stringValue(credentials.project_id),
- private_key_id: stringValue(credentials.private_key_id),
- private_key: privateKey,
- client_email: clientEmail,
- client_id: stringValue(credentials.client_id),
- universe_domain: stringValue(credentials.universe_domain),
- };
-}
-
-function geminiProviderError(error: unknown) {
- if (error instanceof ApiError) {
- return new InferenceProviderError("Gemini screenshot parsing failed.", {
- statusCode: error.status,
- code: "provider_error",
- details: {
- status: error.status,
- message: error.message,
- },
- });
- }
-
- return new InferenceProviderError("Gemini screenshot parsing failed.", {
- details: {
- message: error instanceof Error ? error.message : "Unknown error",
- },
- });
-}
-
-function stringValue(value: JsonValue | undefined) {
- return typeof value === "string" && value.trim() ? value.trim() : undefined;
-}
-
diff --git a/site/src/lib/inference/screenshot-parsing/index.ts b/site/src/lib/inference/screenshot-parsing/index.ts
deleted file mode 100644
index 4fef8e2a..00000000
--- a/site/src/lib/inference/screenshot-parsing/index.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-import { InferenceProviderError } from "@/lib/inference/providers";
-import { createGeminiFlashScreenshotParser } from "@/lib/inference/screenshot-parsing/gemini-flash";
-import type {
- ScreenshotParserProvider,
- ScreenshotParserProviderResult,
- ScreenshotParserProviderStreamEvent,
-} from "@/lib/inference/screenshot-parsing/types";
-import type { ScreenshotParseRequest } from "@/lib/inference/screenshot-parsing/schema";
-
-export function createScreenshotParserProvider(): ScreenshotParserProvider {
- return createGeminiFlashScreenshotParser();
-}
-
-export async function parseScreenshot(
- request: ScreenshotParseRequest,
- provider: ScreenshotParserProvider = createScreenshotParserProvider(),
-): Promise<ScreenshotParserProviderResult> {
- if (!provider.configured) {
- throw new InferenceProviderError("Screenshot parser provider is not configured.", {
- statusCode: 503,
- code: "missing_provider_credentials",
- details: {
- provider: provider.id,
- },
- });
- }
-
- return provider.parse(request);
-}
-
-export async function* parseScreenshotStream(
- request: ScreenshotParseRequest,
- provider: ScreenshotParserProvider = createScreenshotParserProvider(),
-): AsyncGenerator<ScreenshotParserProviderStreamEvent> {
- if (!provider.configured) {
- throw new InferenceProviderError("Screenshot parser provider is not configured.", {
- statusCode: 503,
- code: "missing_provider_credentials",
- details: {
- provider: provider.id,
- },
- });
- }
-
- if (!provider.stream) {
- const result = await provider.parse(request);
- yield {
- type: "final",
- provider: result.provider,
- model: result.model,
- result: result.result,
- usage: result.usage,
- metadata: result.metadata,
- };
- return;
- }
-
- yield* provider.stream(request);
-}
diff --git a/site/src/lib/inference/screenshot-parsing/normalize.ts b/site/src/lib/inference/screenshot-parsing/normalize.ts
deleted file mode 100644
index 59070da0..00000000
--- a/site/src/lib/inference/screenshot-parsing/normalize.ts
+++ /dev/null
@@ -1,131 +0,0 @@
-import { toJsonObject } from "@/lib/inference/json";
-import type {
- GeminiScreenshotParseOutput,
- HotLoopRectJSON,
- ScreenshotParseRequest,
-} from "@/lib/inference/screenshot-parsing/schema";
-import type { ScreenshotParserResult } from "@/lib/inference/screenshot-parsing/types";
-
-// Provider-agnostic normalization shared by the screenshot-parsing providers
-// (gemini-flash, vision). Turns the common parse output — visibleText,
-// controls with box_2d in [ymin, xmin, ymax, xmax]/1000 space, and form fields
-// — into the ScreenshotParserResult contract, stamping each result with its
-// originating `source`.
-export function normalizedScreenshotResult(
- request: ScreenshotParseRequest,
- output: GeminiScreenshotParseOutput,
- metadata: Record<string, string>,
- source = "gemini-screenshot-parser",
-): ScreenshotParserResult {
- const visibleText = output.visibleText.reduce<Record<string, string>>((result, item, index) => {
- const key = item.id?.trim() || (index === 0 ? "visibleText" : `visibleText.${index + 1}`);
- result[key] = item.text;
- return result;
- }, {});
-
- const usedControlIDs = new Set<string>();
- const controls = output.controls.map((control) => {
- // Opaque, unique-per-result control id. searchField keeps the well-known
- // "search" id that the local-app action plan defaults to.
- const controlID = control.kind === "searchField"
- ? "search"
- : uniqueShortID(usedControlIDs);
- return {
- id: controlID,
- label: control.label,
- kind: control.kind,
- frame: rectFromGeminiBox(control.box_2d, request.pixelSize.width, request.pixelSize.height),
- confidence: clamp01(control.confidence),
- metadata: {
- controlID,
- segmentID: controlID,
- source,
- boxFormat: "ymin,xmin,ymax,xmax/1000",
- "localUIElement.actionEligibility": "guardedAction",
- directInputActionsAllowed: "true",
- },
- };
- });
-
- const formFields = output.formFields.map((field, index) => {
- const id = field.id?.trim() || `form-field-${index + 1}`;
- return {
- id,
- label: field.label,
- isRequired: field.isRequired,
- currentValue: field.currentValue ?? null,
- metadata: {
- source,
- confidence: String(clamp01(field.confidence)),
- },
- };
- });
-
- return {
- visibleText,
- controls,
- formFields,
- confidence: clamp01(output.confidence || controls.map((control) => control.confidence).reduce(
- (max, value) => Math.max(max, value),
- 0,
- )),
- metadata: toJsonObject({
- ...metadata,
- "runtime.backend": source,
- "directInputActionsAllowed": "true",
- "screenshotParser.controlCount": String(controls.length),
- "screenshotParser.formFieldCount": String(formFields.length),
- }),
- };
-}
-
-export function rectFromGeminiBox(
- box: [number, number, number, number],
- imageWidth: number,
- imageHeight: number,
-): HotLoopRectJSON | null {
- const [ymin, xmin, ymax, xmax] = box;
- const x1 = clamp(xmin, 0, 1000) / 1000 * imageWidth;
- const y1 = clamp(ymin, 0, 1000) / 1000 * imageHeight;
- const x2 = clamp(xmax, 0, 1000) / 1000 * imageWidth;
- const y2 = clamp(ymax, 0, 1000) / 1000 * imageHeight;
- const width = Math.max(0, x2 - x1);
- const height = Math.max(0, y2 - y1);
- if (width <= 0 || height <= 0) {
- return null;
- }
-
- return {
- origin: {
- x: x1,
- y: y1,
- space: "window",
- },
- size: {
- width,
- height,
- space: "window",
- },
- };
-}
-
-function uniqueShortID(used: Set<string>): string {
- let id = shortID();
- while (used.has(id)) {
- id = shortID();
- }
- used.add(id);
- return id;
-}
-
-function shortID(): string {
- return Math.random().toString(36).slice(2, 8).padEnd(6, "0");
-}
-
-function clamp01(value: number) {
- return clamp(value, 0, 1);
-}
-
-function clamp(value: number, lower: number, upper: number) {
- return Math.min(Math.max(value, lower), upper);
-}
diff --git a/site/src/lib/inference/screenshot-parsing/schema.ts b/site/src/lib/inference/screenshot-parsing/schema.ts
deleted file mode 100644
index 2f0bf68c..00000000
--- a/site/src/lib/inference/screenshot-parsing/schema.ts
+++ /dev/null
@@ -1,145 +0,0 @@
-import { z } from "zod";
-
-import { toJsonObject } from "@/lib/inference/json";
-
-const metadataSchema = z.record(z.string().min(1).max(64), z.string().max(512));
-const coordinateSpaceSchema = z.enum(["screen", "window", "crop", "normalizedTarget"]);
-type ControlKind =
- | "button"
- | "textField"
- | "searchField"
- | "checkbox"
- | "link"
- | "menuItem"
- | "listItem"
- | "group"
- | "unknown";
-
-export const hotLoopSizeSchema = z.object({
- width: z.number().positive().max(20_000),
- height: z.number().positive().max(20_000),
- space: coordinateSpaceSchema.default("window"),
-});
-
-export const hotLoopRectSchema = z.object({
- origin: z.object({
- x: z.number().finite(),
- y: z.number().finite(),
- space: coordinateSpaceSchema,
- }),
- size: hotLoopSizeSchema,
-});
-
-export const screenshotParseRequestSchema = z.object({
- imageBase64: z.string().min(1).max(6_000_000),
- contentType: z.enum(["image/png", "image/jpeg", "image/webp"]).default("image/png"),
- pixelSize: hotLoopSizeSchema,
- traceID: z.string().min(1).max(128).optional(),
- targetID: z.string().min(1).max(256).optional(),
- cropBounds: hotLoopRectSchema.optional(),
- metadata: metadataSchema.optional().default({}),
- stream: z.boolean().optional().default(false),
-}).transform((value) => ({
- ...value,
- metadata: toJsonObject(value.metadata),
-}));
-
-export const geminiScreenshotTextSchema = z.object({
- id: clippedString(128).optional(),
- text: clippedString(2_000),
- confidence: z.number().min(0).max(1).optional().default(0.5),
-});
-
-export const geminiScreenshotControlSchema = z.object({
- id: clippedString(128).optional(),
- label: clippedString(256),
- kind: z.string().optional().default("unknown").transform(normalizedControlKind),
- confidence: z.number().min(0).max(1),
- box_2d: z.tuple([
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- ]),
-});
-
-export const geminiScreenshotFormFieldSchema = z.object({
- id: clippedString(128).optional(),
- label: clippedString(256),
- isRequired: z.boolean().optional().default(false),
- currentValue: clippedString(1_000).nullable().optional(),
- confidence: z.number().min(0).max(1).optional().default(0.5),
- box_2d: z.tuple([
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- z.number().min(0).max(1000),
- ]).optional(),
-});
-
-export const geminiScreenshotParseOutputSchema = z.object({
- visibleText: z.array(geminiScreenshotTextSchema).max(200).default([]),
- controls: z.array(geminiScreenshotControlSchema).max(200).default([]),
- formFields: z.array(geminiScreenshotFormFieldSchema).max(80).default([]),
- confidence: z.number().min(0).max(1).default(0),
-});
-
-export type ScreenshotParseRequest = z.infer<typeof screenshotParseRequestSchema>;
-export type GeminiScreenshotParseOutput = z.infer<typeof geminiScreenshotParseOutputSchema>;
-export type HotLoopRectJSON = z.infer<typeof hotLoopRectSchema>;
-
-function clippedString(maxLength: number) {
- return z.string()
- .transform((value) => value.trim().slice(0, maxLength))
- .pipe(z.string().min(1));
-}
-
-function normalizedControlKind(value: string): ControlKind {
- const normalized = value.trim().toLowerCase().replace(/[^a-z]/g, "");
- switch (normalized) {
- case "button":
- case "iconbutton":
- case "toolbarbutton":
- case "reviewbutton":
- case "sendbutton":
- return "button";
- case "textfield":
- case "textinput":
- case "input":
- case "composer":
- case "textarea":
- return "textField";
- case "searchfield":
- case "search":
- case "searchinput":
- return "searchField";
- case "checkbox":
- case "toggle":
- return "checkbox";
- case "link":
- return "link";
- case "menuitem":
- case "menu":
- case "dropdown":
- return "menuItem";
- case "listitem":
- case "navitem":
- case "navigationitem":
- case "row":
- case "filerow":
- case "changerow":
- case "threadrow":
- return "listItem";
- case "group":
- case "card":
- case "panel":
- case "region":
- case "label":
- case "text":
- case "textregion":
- case "statictext":
- return "group";
- default:
- return "unknown";
- }
-}
diff --git a/site/src/lib/inference/screenshot-parsing/types.ts b/site/src/lib/inference/screenshot-parsing/types.ts
deleted file mode 100644
index 5af0960d..00000000
--- a/site/src/lib/inference/screenshot-parsing/types.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import type { JsonObject, JsonValue } from "@/lib/inference/providers";
-import type {
- HotLoopRectJSON,
- ScreenshotParseRequest,
-} from "@/lib/inference/screenshot-parsing/schema";
-
-export type ScreenshotParserResult = {
- visibleText: Record<string, string>;
- controls: ScreenshotParserControl[];
- formFields: ScreenshotParserFormField[];
- confidence: number;
- metadata: JsonObject;
-};
-
-export type ScreenshotParserControl = {
- id: string;
- label: string;
- kind: string;
- frame: HotLoopRectJSON | null;
- confidence: number;
- metadata: Record<string, string>;
-};
-
-export type ScreenshotParserFormField = {
- id: string;
- label: string;
- isRequired: boolean;
- currentValue?: string | null;
- metadata: Record<string, string>;
-};
-
-export type ScreenshotParserProviderResult = {
- provider: string;
- model: string;
- result: ScreenshotParserResult;
- usage?: JsonValue;
- metadata: JsonObject;
-};
-
-export type ScreenshotParserProviderStreamEvent =
- | {
- type: "partial";
- provider: string;
- model: string;
- result: ScreenshotParserResult;
- metadata: JsonObject;
- }
- | {
- type: "final";
- provider: string;
- model: string;
- result: ScreenshotParserResult;
- usage?: JsonValue;
- metadata: JsonObject;
- };
-
-export type ScreenshotParserProvider = {
- id: string;
- inferenceProvider: string;
- configured: boolean;
- modelForRequest: (request: ScreenshotParseRequest) => string;
- parse: (request: ScreenshotParseRequest) => Promise<ScreenshotParserProviderResult>;
- stream?: (request: ScreenshotParseRequest) => AsyncGenerator<ScreenshotParserProviderStreamEvent>;
-};
diff --git a/site/src/lib/inference/web-search.ts b/site/src/lib/inference/web-search.ts
index 1670bf61..1f946224 100644
--- a/site/src/lib/inference/web-search.ts
+++ b/site/src/lib/inference/web-search.ts
@@ -1,7 +1,7 @@
import { GoogleGenAI } from "@google/genai";
+import { geminiClientConfig } from "@/lib/inference/adapters/gemini-client";
import { geminiModelRoles } from "@/lib/inference/gemini-models";
-import { geminiClientConfig } from "@/lib/inference/screenshot-parsing/gemini-flash";
export type WebSearchSource = {
title: string;
@@ -17,12 +17,12 @@ export type WebSearchResult = {
/**
* Web search via Gemini's built-in Google Search grounding on Vertex AI. This uses the backend's
- * service-account credential (the same one behind live-token and screenshot parsing) — no API key
- * and no separate Programmable Search Engine. The model searches the web for the query and returns a
+ * service-account credential (the same one behind live-token) — no API key and no separate
+ * Programmable Search Engine. The model searches the web for the query and returns a
* grounded summary plus the source pages it used, which the caller turns into ranked results.
*/
export async function searchWeb(query: string): Promise<WebSearchResult | null> {
- const config = geminiClientConfig();
+ const config = geminiClientConfig(process.env);
if (!config.configured) {
return null;
}