Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
12.0 KB
Raw Blame
import { timed } from "@omnirush/timeline";
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { currentTestEvidence } from "./ambient.ts";
import type { ScreenshotArtifact } from "./screenshot.ts";

const REPO_ROOT = fileURLToPath(new URL("../../../..", import.meta.url));
const CACHE_DIR = join(REPO_ROOT, "evals", "results", ".vision-cache");
// models.dev's checked-in catalog marks both current defaults as accepting
// image input. The provider-specific IDs keep the no-SDK transport portable.
const OPENAI_DEFAULT_MODEL = "gpt-5.6";
const ANTHROPIC_DEFAULT_MODEL = "claude-sonnet-4-6";

export interface VisualExpectationResult {
  expectation: string;
  passed: boolean;
  evidence: string;
}

export interface VisualEvidenceResult {
  ok: boolean;
  description: string;
  results: VisualExpectationResult[];
  why: string;
  model: string;
  cached: boolean;
  deferred?: boolean;
  pendingExpectations?: string[];
}

export interface VisionRequest {
  prompt: string;
  png: Buffer;
  model: string;
}

export interface ValidateOptions {
  ask?: (req: VisionRequest) => Promise<string>;
  bypassCache?: boolean;
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function parseJsonResponse(raw: string, label: string): unknown {
  const trimmed = raw.trim();
  const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
  const candidate = fenced?.[1] ?? trimmed;
  try {
    return JSON.parse(candidate);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    throw new Error(`${label} was not valid JSON: ${message}. Response: ${trimmed.slice(0, 300)}`);
  }
}

function parseDescription(raw: string): string {
  const value = parseJsonResponse(raw, "Vision model description response");
  if (!isRecord(value) || typeof value.description !== "string" || !value.description.trim()) {
    throw new Error(`Vision model description response must be {"description":"..."}. Response: ${raw.trim().slice(0, 300)}`);
  }
  return value.description.trim();
}

function parseResults(value: unknown, expectations: string[], label: string): VisualExpectationResult[] {
  if (!isRecord(value) || !Array.isArray(value.results) || value.results.length !== expectations.length) {
    throw new Error(`${label} must contain exactly ${expectations.length} result entries.`);
  }
  const parsed: VisualExpectationResult[] = [];
  for (let index = 0; index < expectations.length; index += 1) {
    const expectation = expectations[index];
    const entry = value.results[index];
    if (
      !expectation
      || !isRecord(entry)
      || entry.expectation !== expectation
      || typeof entry.passed !== "boolean"
      || typeof entry.evidence !== "string"
      || !entry.evidence.trim()
    ) {
      throw new Error(`${label} result ${index + 1} must preserve the expectation and include boolean passed plus non-empty evidence.`);
    }
    parsed.push({ expectation, passed: entry.passed, evidence: entry.evidence.trim() });
  }
  return parsed;
}

function parseVerdict(raw: string, expectations: string[]): VisualExpectationResult[] {
  return parseResults(parseJsonResponse(raw, "Vision model verdict response"), expectations, "Vision model verdict response");
}

function parseCachedVisualEvidence(value: unknown, expectations: string[], model: string): VisualEvidenceResult {
  if (
    !isRecord(value)
    || typeof value.description !== "string"
    || typeof value.why !== "string"
    || typeof value.ok !== "boolean"
    || value.model !== model
  ) {
    throw new Error("Vision cache entry has an invalid VisualEvidenceResult shape.");
  }
  const results = parseResults(value, expectations, "Vision cache entry");
  const ok = results.every((result) => result.passed);
  if (value.ok !== ok || (ok && value.why !== "")) {
    throw new Error("Vision cache entry has an inconsistent verdict.");
  }
  return {
    ok,
    description: value.description,
    results,
    why: value.why,
    model,
    cached: true,
  };
}

function providerJson(raw: string, provider: string): unknown {
  try {
    return JSON.parse(raw);
  } catch {
    throw new Error(`${provider} returned a non-JSON response: ${raw.slice(0, 300)}`);
  }
}

function isTimeoutError(error: unknown): boolean {
  if (error instanceof Error && error.name === "TimeoutError") return true;
  return error instanceof DOMException && (error.name === "TimeoutError" || error.name === "AbortError");
}

async function askOpenAi(req: VisionRequest, key: string): Promise<string> {
  const response = await fetch("https://api.openai.com/v1/chat/completions", {
    method: "POST",
    // Unbounded vision calls stalled runs for undici's full 300s header
    // timeout; 120s comfortably covers slow multimodal responses.
    signal: AbortSignal.timeout(120_000),
    headers: {
      authorization: `Bearer ${key}`,
      "content-type": "application/json",
    },
    body: JSON.stringify({
      model: req.model,
      messages: [{
        role: "user",
        content: [
          { type: "text", text: req.prompt },
          { type: "image_url", image_url: { url: `data:image/png;base64,${req.png.toString("base64")}` } },
        ],
      }],
      response_format: { type: "json_object" },
    }),
  });
  const raw = await response.text();
  if (!response.ok) throw new Error(`OpenAI vision request failed with HTTP ${response.status}: ${raw.slice(0, 500)}`);
  const body = providerJson(raw, "OpenAI");
  const choices = isRecord(body) ? body.choices : null;
  const first = Array.isArray(choices) ? choices[0] : null;
  const message = isRecord(first) ? first.message : null;
  const content = isRecord(message) ? message.content : null;
  if (typeof content !== "string" || !content.trim()) {
    throw new Error(`OpenAI vision response did not contain message content: ${raw.slice(0, 500)}`);
  }
  return content;
}

async function askAnthropic(req: VisionRequest, key: string): Promise<string> {
  const response = await fetch("https://api.anthropic.com/v1/messages", {
    method: "POST",
    signal: AbortSignal.timeout(120_000),
    headers: {
      "anthropic-version": "2023-06-01",
      "content-type": "application/json",
      "x-api-key": key,
    },
    body: JSON.stringify({
      model: req.model,
      max_tokens: 2_048,
      messages: [{
        role: "user",
        content: [
          { type: "image", source: { type: "base64", media_type: "image/png", data: req.png.toString("base64") } },
          { type: "text", text: req.prompt },
        ],
      }],
    }),
  });
  const raw = await response.text();
  if (!response.ok) throw new Error(`Anthropic vision request failed with HTTP ${response.status}: ${raw.slice(0, 500)}`);
  const body = providerJson(raw, "Anthropic");
  const content = isRecord(body) ? body.content : null;
  const text = Array.isArray(content)
    ? content.flatMap((part) => isRecord(part) && part.type === "text" && typeof part.text === "string" ? [part.text] : []).join("\n")
    : "";
  if (!text.trim()) throw new Error(`Anthropic vision response did not contain text content: ${raw.slice(0, 500)}`);
  return text;
}

function failureSummary(results: VisualExpectationResult[]): string {
  const failures = results.filter((result) => !result.passed);
  if (failures.length === 0) return "";
  return `Failed ${failures.length}/${results.length} visual expectations: ${failures
    .map((result) => `${result.expectation} — ${result.evidence}`)
    .join("; ")}`;
}

function visionModel(): string {
  const openAiKey = process.env.OPENAI_API_KEY?.trim() ?? "";
  const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim() ?? "";
  return process.env.OMNIRUSH_EVAL_VISION_MODEL?.trim()
    || (anthropicKey && !openAiKey ? ANTHROPIC_DEFAULT_MODEL : OPENAI_DEFAULT_MODEL);
}

export async function judgeVision(
  screenshotArtifact: ScreenshotArtifact,
  expectations: string[],
  opts: ValidateOptions = {},
): Promise<VisualEvidenceResult> {
  // Vision latency is the main per-screenshot cost; record it so results show it.
  const openAiKey = process.env.OPENAI_API_KEY?.trim() ?? "";
  const anthropicKey = process.env.ANTHROPIC_API_KEY?.trim() ?? "";
  const model = visionModel();
  let ask = opts.ask;
  if (!ask) {
    if (openAiKey) ask = (req) => askOpenAi(req, openAiKey);
    else if (anthropicKey) ask = (req) => askAnthropic(req, anthropicKey);
    else {
      throw new Error(
        "Vision validation requires OPENAI_API_KEY or ANTHROPIC_API_KEY, unless ValidateOptions.ask is provided.",
      );
    }
  }
  const askOnce = ask;
  ask = async (req) => {
    try {
      return await askOnce(req);
    } catch (error) {
      if (!isTimeoutError(error)) throw error;
      await delay(10_000);
      try {
        return await askOnce(req);
      } catch (retryError) {
        if (!isTimeoutError(retryError)) throw retryError;
        throw new Error("Vision request timed out on both attempts.", { cause: retryError });
      }
    }
  };

  const key = createHash("sha256")
    .update(screenshotArtifact.hash + JSON.stringify(expectations) + model)
    .digest("hex");
  const cachePath = join(CACHE_DIR, `${key}.json`);
  let visualEvidence: VisualEvidenceResult | null = null;
  if (!opts.bypassCache) {
    try {
      const cached = JSON.parse(await readFile(cachePath, "utf8"));
      visualEvidence = parseCachedVisualEvidence(cached, expectations, model);
    } catch (error) {
      if (isRecord(error) && error.code === "ENOENT") {
        // Cache miss: perform the two independent model requests below.
      } else {
        throw error;
      }
    }
  }
  if (!visualEvidence) {
    const description = parseDescription(await ask({
      prompt: [
        "Objectively describe only what is visibly present in this screenshot.",
        "Do not infer hidden state, correctness, intent, or expected behavior.",
        "Return only JSON matching: {\"description\":\"concise visible description\"}",
      ].join("\n"),
      png: screenshotArtifact.png,
      model,
    }));
    const results = parseVerdict(await ask({
      prompt: [
        "Judge each expectation against the screenshot and the expectation-free description below.",
        `Independent description: ${JSON.stringify(description)}`,
        `Expectations: ${JSON.stringify(expectations)}`,
        "Return only JSON matching: {\"results\":[{\"expectation\":\"exact expectation text\",\"passed\":true,\"evidence\":\"short visible quote or observation\"}]}",
        "Return exactly one result per expectation, in the same order. Treat negative expectations literally and do not assume success.",
      ].join("\n"),
      png: screenshotArtifact.png,
      model,
    }), expectations);
    visualEvidence = {
      ok: results.every((result) => result.passed),
      description,
      results,
      why: failureSummary(results),
      model,
      cached: false,
    };
    await mkdir(CACHE_DIR, { recursive: true });
    await writeFile(cachePath, `${JSON.stringify(visualEvidence, null, 2)}\n`, "utf8");
  }
  return visualEvidence;
}

export async function validate(
  screenshotArtifact: ScreenshotArtifact,
  expectations: string[],
  opts: ValidateOptions = {},
): Promise<VisualEvidenceResult> {
  // A caller-provided ask is a deterministic witness, not an LLM call, so defer never applies to it.
  if (!opts.ask && process.env.OMNIRUSH_EVAL_VISION?.trim() === "defer") {
    const visualEvidence: VisualEvidenceResult = {
      ok: true,
      description: "",
      results: [],
      why: "vision judgment deferred",
      model: visionModel(),
      cached: false,
      deferred: true,
      pendingExpectations: expectations,
    };
    currentTestEvidence()?.recordVisualValidation(screenshotArtifact.hash, visualEvidence);
    return visualEvidence;
  }

  const visualEvidence = await judgeVision(screenshotArtifact, expectations, opts);
  currentTestEvidence()?.recordVisualValidation(screenshotArtifact.hash, visualEvidence);
  return visualEvidence;
}