Skip to content

Soul-driven AI agent with permission-hardened tools, token budgets, and multi-channel access. Runs 24/7 from CLI, Telegram or More.

4.2 KB
Raw Blame
/**
 * How the current model takes screenshots (ADR-043): whether it sees them,
 * how they reach it, and how big they should be.
 *
 * A model is billed for an image by its pixels, by rules that differ per
 * family, so one fixed size (Optimus used 1280×800 for every model) costs
 * far more than needed on some and is never seen at all by others:
 *
 *   Claude   ≈ width × height / 750 tokens        1024×640 ≈ 874 (1280×800: 1,365)
 *   OpenAI   85 + 170 per 512-px tile (high)      1024×640 = 765 (1280×800: 1,105)
 *   Gemini   258 per 768-px tile                   768×480 = 258  (1280×800: 1,032)
 *   Qwen-VL and other local vision models: 28-px patches, slow on big images.
 *
 * Route:
 *   tool-result   the provider takes images inside tool results (Anthropic,
 *                 OpenAI Responses): the screenshot stays with its tool call
 *   user-message  the model sees images but its API only takes them from the
 *                 user (OpenAI-compatible chat: OpenRouter, LiteLLM, Mercury
 *                 Cloud, Ollama, LM Studio…): the screenshot follows the tool
 *                 result as an image message (payload-policy.ts)
 *   none          the model can't see images: no screenshot is taken at all;
 *                 the tool still says which window is in front
 *
 * The size is passed to tools that declare `screenshots: true` in their
 * plugin manifest; they shrink the capture themselves, so the coordinates
 * the model clicks stay in the picture it saw.
 */

export type ImageRoute = 'tool-result' | 'user-message' | 'none';

export interface ScreenshotBox {
  width: number;
  height: number;
}

export interface VisionProfile {
  route: ImageRoute;
  box: ScreenshotBox;
  /** Which pricing rule applied (for logs and estimates). */
  family: 'claude' | 'openai' | 'gemini' | 'patch' | 'default';
}

const BOXES: Record<VisionProfile['family'], ScreenshotBox> = {
  claude: { width: 1024, height: 640 },
  openai: { width: 1024, height: 640 },
  gemini: { width: 768, height: 480 },
  patch: { width: 896, height: 560 },
  default: { width: 1024, height: 640 },
};

/** The family whose image pricing applies to this model id. */
export function visionFamily(model: string): VisionProfile['family'] {
  const m = model.toLowerCase();
  if (/claude/.test(m)) return 'claude';
  if (/gemini|gemma/.test(m)) return 'gemini';
  if (/gpt|chatgpt|(^|[/:])o[1-9]\b|grok/.test(m)) return 'openai';
  if (/qwen|llava|minicpm|internvl|moondream|molmo|pixtral|mistral|llama|nemotron|kimi|glm|(^|[-_.:/])vl($|[-_.:/])/.test(m)) return 'patch';
  return 'default';
}

/** "1280x800" from MERCURY_SCREENSHOT_BOX: the owner's own size for every model. */
function boxOverride(env: string | undefined): ScreenshotBox | undefined {
  const m = /^\s*(\d{3,4})\s*[x×]\s*(\d{3,4})\s*$/i.exec(env ?? '');
  if (!m) return undefined;
  const width = Math.min(Math.max(Number(m[1]), 320), 3840);
  const height = Math.min(Math.max(Number(m[2]), 240), 2160);
  return { width, height };
}

export function visionProfile(input: { model: string; seesImages: boolean; toolResultImages: boolean }, env: NodeJS.ProcessEnv = process.env): VisionProfile {
  const family = visionFamily(input.model);
  const route: ImageRoute = !input.seesImages ? 'none' : input.toolResultImages ? 'tool-result' : 'user-message';
  return { route, family, box: boxOverride(env.MERCURY_SCREENSHOT_BOX) ?? BOXES[family] };
}

/** Roughly what an image of this size costs this family, in input tokens. */
export function estimateImageTokens(family: VisionProfile['family'], w: number, h: number): number {
  switch (family) {
    case 'claude':
      return Math.ceil((w * h) / 750);
    case 'openai': {
      // Fit in 2048², then shortest side to 768, then 512-px tiles.
      let s = Math.min(1, 2048 / Math.max(w, h));
      let [x, y] = [w * s, h * s];
      s = Math.min(1, 768 / Math.min(x, y));
      [x, y] = [x * s, y * s];
      return 85 + 170 * Math.ceil(x / 512) * Math.ceil(y / 512);
    }
    case 'gemini':
      return w <= 384 && h <= 384 ? 258 : 258 * Math.ceil(w / 768) * Math.ceil(h / 768);
    case 'patch':
      return Math.ceil(w / 28) * Math.ceil(h / 28);
    default:
      return Math.ceil((w * h) / 750);
  }
}