Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

3.9 KB
Raw Blame
/**
 * Vision probe classification: the three-valued verdict that decides whether the models
 * dialog turns "supports vision" on, off, or leaves it alone.
 *
 * The distinction under test is the whole point of the feature: a model that ANSWERS "I
 * cannot take images" has told us something definitive (switch it off), while a probe that
 * failed on auth or the network has told us nothing (leave the user's setting untouched).
 * Getting that backwards would silently disable vision on a working model.
 */
import { describe, expect, it } from "vitest";
import type { LLMOutcome } from "@adelie/core";
import {
  VISION_PROBE_IMAGE,
  VISION_PROBE_MAX_TOKENS,
  classifyVisionProbe,
  classifyVisionProbeError,
  isImageRejection,
} from "../src/services/vision-detect.js";

const failed = (errorMessage: string): LLMOutcome => ({ status: "fatal", errorMessage });

describe("VISION_PROBE_IMAGE", () => {
  it("is a tiny inline PNG data URL, so the probe sends the least it can", () => {
    expect(VISION_PROBE_IMAGE.startsWith("data:image/png;base64,")).toBe(true);
    const b64 = VISION_PROBE_IMAGE.slice("data:image/png;base64,".length);
    const bytes = Buffer.from(b64, "base64");
    // A real PNG signature, and well under 200 bytes.
    expect([...bytes.subarray(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]);
    expect(bytes.length).toBeLessThan(200);
    // Round-trips, i.e. it is valid base64 rather than a truncated paste.
    expect(bytes.toString("base64")).toBe(b64);
  });

  it("caps the probe's output so a success cannot run up a bill", () => {
    expect(VISION_PROBE_MAX_TOKENS).toBeLessThanOrEqual(16);
  });
});

describe("classifyVisionProbe", () => {
  it("treats a completed answer as vision support", () => {
    expect(classifyVisionProbe({ status: "completed" }, false)).toBe("supported");
  });

  it("treats streamed content as support even if the run then trips its tiny cap", () => {
    // Same allowance the connectivity probe makes: the image was plainly accepted, which
    // is the only question being asked.
    expect(classifyVisionProbe(failed("max tokens reached"), true)).toBe("supported");
  });

  it("reads an image-specific rejection as a definitive no", () => {
    for (const msg of [
      "Invalid content type 'image_url' for this model",
      "This model does not support image input",
      "400 Bad Request: unsupported image content",
      "The selected model has no vision capability",
      "modality not supported: image",
    ]) {
      expect(classifyVisionProbe(failed(msg), false)).toBe("unsupported");
    }
  });

  it("does NOT read an unrelated failure as a no — the setting must survive it", () => {
    for (const msg of [
      "401 Unauthorized: invalid api key",
      "fetch failed: ECONNREFUSED",
      "request timed out",
      "429 rate limit exceeded",
      "Invalid request: missing parameter 'model'",
      "500 Internal Server Error",
    ]) {
      expect(classifyVisionProbe(failed(msg), false)).toBe("failed");
    }
  });
});

describe("classifyVisionProbeError (the SDK can throw before streaming)", () => {
  it("keeps the same split for thrown errors", () => {
    expect(classifyVisionProbeError(new Error("image_url is not supported"))).toBe("unsupported");
    expect(classifyVisionProbeError(new Error("OPENAI_API_KEY is not set"))).toBe("failed");
    expect(classifyVisionProbeError("some string failure")).toBe("failed");
  });
});

describe("isImageRejection", () => {
  it("matches case-insensitively", () => {
    expect(isImageRejection("UNSUPPORTED IMAGE")).toBe(true);
    expect(isImageRejection("Vision")).toBe(true);
  });

  it("never fires on a bare validation error with no image concept in it", () => {
    // The asymmetry that matters: a false positive switches vision off on a working model.
    expect(isImageRejection("invalid request")).toBe(false);
    expect(isImageRejection("bad request")).toBe(false);
    expect(isImageRejection("unauthorized")).toBe(false);
  });
});