Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
/**
* Smoke test probe implementations.
*
* Three probes: tool calling, reasoning, vision.
* Each returns a ProbeResult and uses AbortSignal for timeout.
*/
import type {
AnthropicResponse,
OllamaResponse,
OpenAIResponse,
ProbeFn,
ProbeResult,
SmokeProviderConfig,
} from "./types.js";
// 32x32 solid red PNG, base64-encoded (no filesystem dependency)
// 1x1 is rejected by many providers as too small
const TEST_IMAGE_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NsQ0AAAzCMP5/un0CNkuZ41wybXsHAAAAAAAAAAAAxR4yw/wuPL6QkAAAAABJRU5ErkJggg==";
const TEST_IMAGE_MEDIA_TYPE = "image/png";
// Error phrases that indicate vision is not supported
const VISION_ERROR_PHRASES = [
"not support",
"cannot process",
"unable to analyze",
"does not support image",
"image type not supported",
"cannot view image",
"cannot see image",
];
/**
* Determine if a model ID indicates a reasoning/thinking model.
*/
function isReasoningModel(modelId: string): boolean {
return /\br1\b|qwq|thinking|o1(?:[-/]|\b)|reasoning/i.test(modelId);
}
/**
* Build auth headers based on the provider's auth scheme.
*/
function buildHeaders(config: SmokeProviderConfig): Record<string, string> {
const headers: Record<string, string> = {
"Content-Type": "application/json",
...config.extraHeaders,
};
switch (config.authScheme) {
case "x-api-key":
headers["x-api-key"] = config.apiKey;
headers["anthropic-version"] = "2023-06-01";
break;
case "bearer":
headers.Authorization = `Bearer ${config.apiKey}`;
headers["anthropic-version"] = "2023-06-01";
break;
case "openai":
headers.Authorization = `Bearer ${config.apiKey}`;
break;
}
return headers;
}
/**
* Make an HTTP POST to the provider and return the parsed JSON response.
* Throws on non-2xx status codes.
*/
export async function callProvider(
config: SmokeProviderConfig,
body: Record<string, unknown>,
signal: AbortSignal
): Promise<unknown> {
const url = config.baseUrl + config.apiPath;
const headers = buildHeaders(config);
const response = await fetch(url, {
method: "POST",
headers,
body: JSON.stringify(body),
signal,
});
if (!response.ok) {
const text = await response.text();
throw new Error(`HTTP ${response.status}: ${text.slice(0, 200)}`);
}
return response.json();
}
/**
* Wrap a probe function with timeout and error handling.
*/
export async function runProbe(
capability: ProbeResult["capability"],
fn: ProbeFn,
config: SmokeProviderConfig,
timeoutMs = 30_000
): Promise<ProbeResult> {
const controller = new AbortController();
const t0 = Date.now();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const result = await fn(config, controller.signal);
return result;
} catch (err: unknown) {
const elapsed = Date.now() - t0;
const error = err as { name?: string; message?: string };
if (error.name === "AbortError") {
return {
capability,
status: "fail",
durationMs: timeoutMs,
reason: `timeout after ${timeoutMs}ms`,
};
}
return {
capability,
status: "fail",
durationMs: elapsed,
reason: error.message ?? String(err),
};
} finally {
clearTimeout(timer);
}
}
// ─────────────────────────────────────────────────────────────
// Probe 1: Tool Calling
// ─────────────────────────────────────────────────────────────
export const runToolCallingProbe: ProbeFn = async (
config: SmokeProviderConfig,
signal: AbortSignal
): Promise<ProbeResult> => {
const t0 = Date.now();
if (!config.capabilities.supportsTools) {
return {
capability: "tool_calling",
status: "skip",
durationMs: 0,
reason: "provider does not support tools",
};
}
let body: Record<string, unknown>;
if (config.wireFormat === "anthropic-compat") {
body = {
model: config.representativeModel,
max_tokens: 256,
stream: false,
system: "You are a helpful assistant. When asked about weather, use the get_weather tool.",
messages: [{ role: "user", content: "What's the weather in Tokyo?" }],
tools: [
{
name: "get_weather",
description: "Get current weather for a city",
input_schema: {
type: "object",
properties: {
city: { type: "string", description: "City name" },
},
required: ["city"],
},
},
],
};
} else if (config.wireFormat === "ollama") {
body = {
model: config.representativeModel,
stream: false,
messages: [
{
role: "system",
content:
"You are a helpful assistant. When asked about weather, use the get_weather tool.",
},
{ role: "user", content: "What's the weather in Tokyo?" },
],
tools: [
{
type: "function",
function: {
name: "get_weather",
description: "Get current weather for a city",
parameters: {
type: "object",
properties: {
city: { type: "string", description: "City name" },
},
required: ["city"],
},
},
},
],
};
} else {
body = {
model: config.representativeModel,
max_tokens: 256,
stream: false,
messages: [
{
role: "system",
content:
"You are a helpful assistant. When asked about weather, use the get_weather tool.",
},
{ role: "user", content: "What's the weather in Tokyo?" },
],
tools: [
{
type: "function",
function: {
name: "get_weather",
description: "Get current weather for a city",
parameters: {
type: "object",
properties: {
city: { type: "string", description: "City name" },
},
required: ["city"],
},
},
},
],
tool_choice: "auto",
};
}
const raw = await callProvider(config, body, signal);
const elapsed = Date.now() - t0;
if (config.wireFormat === "anthropic-compat") {
const resp = raw as AnthropicResponse;
const toolBlock = resp.content?.find((b) => b.type === "tool_use") as
| { type: "tool_use"; name: string; input: Record<string, unknown> }
| undefined;
if (
resp.stop_reason === "tool_use" &&
toolBlock &&
toolBlock.name === "get_weather" &&
toolBlock.input &&
Object.keys(toolBlock.input).length > 0
) {
return {
capability: "tool_calling",
status: "pass",
durationMs: elapsed,
reason: "tool_use detected",
excerpt: `tool: ${toolBlock.name}, input: ${JSON.stringify(toolBlock.input).slice(0, 100)}`,
};
}
return {
capability: "tool_calling",
status: "fail",
durationMs: elapsed,
reason: `no tool_use block (stop_reason was: ${resp.stop_reason})`,
excerpt: JSON.stringify(resp.content).slice(0, 200),
};
}
if (config.wireFormat === "ollama") {
const resp = raw as OllamaResponse;
const toolCalls = resp.message?.tool_calls;
if (
toolCalls &&
toolCalls.length > 0 &&
toolCalls[0].function.name === "get_weather" &&
Object.keys(toolCalls[0].function.arguments).length > 0
) {
return {
capability: "tool_calling",
status: "pass",
durationMs: elapsed,
reason: "tool_calls detected",
excerpt: `tool: ${toolCalls[0].function.name}, args: ${JSON.stringify(toolCalls[0].function.arguments).slice(0, 100)}`,
};
}
return {
capability: "tool_calling",
status: "fail",
durationMs: elapsed,
reason: `no tool_calls (done_reason was: ${resp.done_reason ?? "unknown"})`,
excerpt: JSON.stringify(resp.message).slice(0, 200),
};
}
const resp = raw as OpenAIResponse;
const choice = resp.choices?.[0];
const toolCalls = choice?.message?.tool_calls;
// Some providers (e.g. opencode-zen) return finish_reason: null even when
// tool_calls is present. Check tool_calls presence first; finish_reason is
// informational only.
if (
toolCalls &&
toolCalls.length > 0 &&
toolCalls[0].function.name === "get_weather" &&
toolCalls[0].function.arguments.length > 0
) {
return {
capability: "tool_calling",
status: "pass",
durationMs: elapsed,
reason: "tool_calls detected",
excerpt: `tool: ${toolCalls[0].function.name}, args: ${toolCalls[0].function.arguments.slice(0, 100)}`,
};
}
return {
capability: "tool_calling",
status: "fail",
durationMs: elapsed,
reason: `no tool_calls (finish_reason was: ${choice?.finish_reason ?? "unknown"})`,
excerpt: JSON.stringify(choice?.message).slice(0, 200),
};
};
// ─────────────────────────────────────────────────────────────
// Probe 2: Reasoning
// ─────────────────────────────────────────────────────────────
export const runReasoningProbe: ProbeFn = async (
config: SmokeProviderConfig,
signal: AbortSignal
): Promise<ProbeResult> => {
const t0 = Date.now();
let body: Record<string, unknown>;
if (config.wireFormat === "anthropic-compat") {
body = {
model: config.representativeModel,
max_tokens: 512,
stream: false,
system: "You are a helpful math assistant.",
messages: [{ role: "user", content: "What is 17 × 23? Show your reasoning step by step." }],
};
} else if (config.wireFormat === "ollama") {
body = {
model: config.representativeModel,
stream: false,
messages: [
{ role: "system", content: "You are a helpful math assistant." },
{ role: "user", content: "What is 17 × 23? Show your reasoning step by step." },
],
};
} else {
body = {
model: config.representativeModel,
max_tokens: 512,
stream: false,
messages: [
{ role: "system", content: "You are a helpful math assistant." },
{ role: "user", content: "What is 17 × 23? Show your reasoning step by step." },
],
};
}
const raw = await callProvider(config, body, signal);
const elapsed = Date.now() - t0;
const isReasoning = isReasoningModel(config.representativeModel);
if (config.wireFormat === "ollama") {
const resp = raw as OllamaResponse;
const content = resp.message?.content ?? "";
if (content.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
excerpt: content.slice(0, 200),
};
}
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "empty response",
};
}
if (config.wireFormat === "anthropic-compat") {
const resp = raw as AnthropicResponse;
const thinkingBlock = resp.content?.find((b) => b.type === "thinking") as
| { type: "thinking"; thinking: string }
| undefined;
const textBlock = resp.content?.find((b) => b.type === "text") as
| { type: "text"; text: string }
| undefined;
if (isReasoning) {
if (thinkingBlock && thinkingBlock.thinking.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
reason: "thinking tokens detected",
excerpt: thinkingBlock.thinking.slice(0, 200),
};
}
if (textBlock && textBlock.text.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
reason: "text response (reasoning not surfaced as tokens)",
excerpt: textBlock.text.slice(0, 200),
};
}
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "no thinking block and no text response",
};
}
// Non-reasoning model: any non-empty text response is a pass
if (textBlock && textBlock.text.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
excerpt: textBlock.text.slice(0, 200),
};
}
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "empty response",
};
}
const resp = raw as OpenAIResponse;
const msg = resp.choices?.[0]?.message;
if (!msg) {
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "no choices in response",
};
}
if (isReasoning) {
if (msg.reasoning_content && msg.reasoning_content.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
reason: "reasoning_content tokens detected",
excerpt: msg.reasoning_content.slice(0, 200),
};
}
if (msg.content && msg.content.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
reason: "text response (reasoning not surfaced as tokens)",
excerpt: msg.content.slice(0, 200),
};
}
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "empty response for reasoning model",
};
}
// Non-reasoning model: any non-empty content or reasoning_content is a pass
// Some providers (e.g. opencode-zen-go) put all output in reasoning_content
// even for models not classified as "reasoning".
const textOut = msg.content || msg.reasoning_content || "";
if (textOut.length > 0) {
return {
capability: "reasoning",
status: "pass",
durationMs: elapsed,
excerpt: textOut.slice(0, 200),
};
}
return {
capability: "reasoning",
status: "fail",
durationMs: elapsed,
reason: "empty response",
};
};
// ─────────────────────────────────────────────────────────────
// Probe 3: Vision
// ─────────────────────────────────────────────────────────────
export const runVisionProbe: ProbeFn = async (
config: SmokeProviderConfig,
signal: AbortSignal
): Promise<ProbeResult> => {
const t0 = Date.now();
if (!config.capabilities.supportsVision) {
return {
capability: "vision",
status: "skip",
durationMs: 0,
reason: "provider does not support vision",
};
}
let body: Record<string, unknown>;
if (config.wireFormat === "anthropic-compat") {
body = {
model: config.representativeModel,
max_tokens: 128,
stream: false,
messages: [
{
role: "user",
content: [
{
type: "image",
source: {
type: "base64",
media_type: TEST_IMAGE_MEDIA_TYPE,
data: TEST_IMAGE_BASE64,
},
},
{
type: "text",
text: "Describe what you see in this image in one sentence.",
},
],
},
],
};
} else if (config.wireFormat === "ollama") {
body = {
model: config.representativeModel,
stream: false,
messages: [
{
role: "user",
content: "Describe what you see in this image in one sentence.",
images: [TEST_IMAGE_BASE64],
},
],
};
} else {
body = {
model: config.representativeModel,
max_tokens: 128,
stream: false,
messages: [
{
role: "user",
content: [
{
type: "image_url",
image_url: {
url: `data:${TEST_IMAGE_MEDIA_TYPE};base64,${TEST_IMAGE_BASE64}`,
},
},
{
type: "text",
text: "Describe what you see in this image in one sentence.",
},
],
},
],
};
}
const raw = await callProvider(config, body, signal);
const elapsed = Date.now() - t0;
// Extract text content from the response
let textContent = "";
if (config.wireFormat === "anthropic-compat") {
const resp = raw as AnthropicResponse;
const textBlock = resp.content?.find((b) => b.type === "text") as
| { type: "text"; text: string }
| undefined;
textContent = textBlock?.text ?? "";
} else if (config.wireFormat === "ollama") {
const resp = raw as OllamaResponse;
textContent = resp.message?.content ?? "";
} else {
const resp = raw as OpenAIResponse;
textContent = resp.choices?.[0]?.message?.content ?? "";
}
if (!textContent) {
return {
capability: "vision",
status: "fail",
durationMs: elapsed,
reason: "empty response",
};
}
// Check for error phrases indicating vision is not supported
const lowerText = textContent.toLowerCase();
for (const phrase of VISION_ERROR_PHRASES) {
if (lowerText.includes(phrase)) {
return {
capability: "vision",
status: "fail",
durationMs: elapsed,
reason: `vision error phrase detected: "${phrase}"`,
excerpt: textContent.slice(0, 200),
};
}
}
return {
capability: "vision",
status: "pass",
durationMs: elapsed,
excerpt: textContent.slice(0, 200),
};
};