Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

5.8 KB
Raw Blame
/**
 * Vision Proxy Service
 *
 * Describes images via the Anthropic API so non-vision models can receive
 * a rich text description in place of image_url blocks.
 *
 * Each image is described in a separate API call for simplicity and reliability.
 * All errors are caught and logged; callers receive null on failure (fall back to stripping).
 */

import { log } from "../logger.js";
import { findVisionAlias } from "../providers/catalog-query.js";

/** Lazy on every call — `findVisionAlias` is memoized at the catalog-query layer. */
function getVisionModel(): string {
  return findVisionAlias("sonnet")?.modelId ?? "claude-sonnet-4-6";
}
const MAX_TOKENS_PER_IMAGE = 1024;
const VISION_ENDPOINT = "https://api.anthropic.com/v1/messages";
const TIMEOUT_MS = 30_000;

const DESCRIPTION_PROMPT = `Describe this image in detail for a model that cannot see images. Provide:
- All visible text content (exact quotes where possible)
- Layout and structure (how elements are arranged spatially)
- Colors, visual style, and key visual elements
- If code: include the complete code text
- If a diagram or chart: describe relationships, nodes, flow, and data
- If a screenshot or UI: describe each UI element, its state, and labels
- If a photograph: describe subjects, setting, and any relevant context

Be comprehensive - this description will be the only information the model has about the image.`;

/**
 * Auth headers extracted from the original Claude Code request.
 * Passed through unchanged to the Anthropic vision API call.
 */
export interface VisionProxyAuthHeaders {
  "x-api-key"?: string;
}

/**
 * An image block in OpenAI format, as produced by convertMessagesToOpenAI().
 * The url field is always a data URL: "data:<media_type>;base64,<data>"
 */
export interface OpenAIImageBlock {
  type: "image_url";
  image_url: { url: string };
}

/**
 * Parse a data URL into media type and base64 data.
 * Input: "data:image/png;base64,<data>"
 * Output: { mediaType: "image/png", data: "<data>" }
 * Returns null for malformed URLs.
 */
function parseDataUrl(dataUrl: string): { mediaType: string; data: string } | null {
  if (!dataUrl.startsWith("data:")) return null;

  const withoutPrefix = dataUrl.slice("data:".length);
  const semicolonIdx = withoutPrefix.indexOf(";");
  if (semicolonIdx === -1) return null;

  const mediaType = withoutPrefix.slice(0, semicolonIdx);
  const rest = withoutPrefix.slice(semicolonIdx + 1);

  if (!rest.startsWith("base64,")) return null;

  const data = rest.slice("base64,".length);
  if (!mediaType || !data) return null;

  return { mediaType, data };
}

/**
 * Describe a single image via the Anthropic API.
 * Returns a description string on success, or null on failure.
 */
async function describeImage(
  image: OpenAIImageBlock,
  auth: VisionProxyAuthHeaders
): Promise<string | null> {
  const parsed = parseDataUrl(image.image_url.url);
  if (!parsed) {
    log("[VisionProxy] Skipping image: malformed or non-base64 data URL");
    return null;
  }

  const { mediaType, data } = parsed;

  const requestBody = {
    model: getVisionModel(),
    max_tokens: MAX_TOKENS_PER_IMAGE,
    stream: false,
    messages: [
      {
        role: "user",
        content: [
          {
            type: "image",
            source: {
              type: "base64",
              media_type: mediaType,
              data,
            },
          },
          {
            type: "text",
            text: DESCRIPTION_PROMPT,
          },
        ],
      },
    ],
  };

  const headers: Record<string, string> = {
    "content-type": "application/json",
    "anthropic-version": "2023-06-01",
  };
  if (auth["x-api-key"]) headers["x-api-key"] = auth["x-api-key"];

  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), TIMEOUT_MS);

  try {
    const response = await fetch(VISION_ENDPOINT, {
      method: "POST",
      headers,
      body: JSON.stringify(requestBody),
      signal: controller.signal,
    });

    clearTimeout(timeoutId);

    if (!response.ok) {
      const errorText = await response.text();
      log(`[VisionProxy] API error ${response.status}: ${errorText}`);
      return null;
    }

    const json = (await response.json()) as { content?: Array<{ type: string; text?: string }> };
    const textBlock = json.content?.find((block) => block.type === "text");
    if (!textBlock || !textBlock.text) {
      log("[VisionProxy] No text content in response");
      return null;
    }

    return textBlock.text;
  } catch (err: any) {
    clearTimeout(timeoutId);
    if (err.name === "AbortError") {
      log(`[VisionProxy] Request timed out after ${TIMEOUT_MS}ms`);
    } else {
      log(`[VisionProxy] Fetch error: ${err.message}`);
    }
    return null;
  }
}

/**
 * Describes all provided images via the Anthropic API, one call per image.
 *
 * @param images  - Array of OpenAI-format image blocks (in order)
 * @param auth    - Auth headers from the original request (passed through)
 * @returns       - Array of text descriptions, one per image, in order.
 *                  Returns null if any API call fails critically (caller strips images instead).
 *                  Individual images that fail get empty string descriptions.
 */
export async function describeImages(
  images: OpenAIImageBlock[],
  auth: VisionProxyAuthHeaders
): Promise<string[] | null> {
  if (images.length === 0) return [];

  try {
    const results = await Promise.all(images.map((img) => describeImage(img, auth)));
    // If any result is null, return null to trigger fallback
    if (results.some((r) => r === null)) {
      log("[VisionProxy] One or more image descriptions failed, falling back");
      return null;
    }

    log(`[VisionProxy] Successfully described ${results.length} image(s)`);
    return results as string[];
  } catch (err: any) {
    log(`[VisionProxy] Unexpected error: ${err.message}`);
    return null;
  }
}