🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it, vi } from "vitest";
import type { DecisionModelRequest } from "../../evals/decisionModelEvaluatorExecution";
import { createOpenAIDecisionModelClient } from "./openAIDecisionModelClient";
function jsonResponse(body: unknown, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { "Content-Type": "application/json" },
});
}
const request: DecisionModelRequest = {
state: { input: "I was charged twice.", output: "Refund issued." },
questions: {
department: {
type: "choice",
instructions: "Which team should handle `input`?",
choices: [
{ value: "billing", description: "Payments and refunds" },
{ value: "other" },
],
},
severity: {
type: "score",
instructions: "How severe is the issue?",
levels: [{ description: "Appearance only" }, { description: "Blocked" }],
},
refund: {
type: "predicate",
instructions: "Does `input` request a refund?",
criteria: { true: "Asks for money back", false: { note: "money" } },
},
},
};
describe("createOpenAIDecisionModelClient", () => {
it("posts text input and maps predicate, choice, and score answers", async () => {
const fetchImpl = vi.fn().mockResolvedValue(
jsonResponse({
model: "gpt-6-luna",
answers: [
{
type: "choice",
name: "department",
choice: "billing",
confidence: 0.8,
probabilities: [
{ value: "billing", probability: 0.9 },
{ value: "other", probability: 0.1 },
],
},
{
type: "score",
name: "severity",
score: 0.4,
confidence: 0.7,
probabilities: [
{ value: 0, probability: 0.6 },
{ value: 1, probability: 0.4 },
],
},
{ type: "predicate", name: "refund", probability: 0.95 },
],
usage: { input_tokens: 120, output_tokens: 3 },
}),
);
const client = createOpenAIDecisionModelClient({
apiKey: "sk-test",
model: "gpt-6-luna",
fetchImpl,
});
const evaluation = await client.evaluate(request);
const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit];
expect(url).toBe("https://api.openai.com/v1/decisions");
expect(new Headers(init.headers).get("authorization")).toBe(
"Bearer sk-test",
);
expect(JSON.parse(init.body as string)).toEqual({
model: "gpt-6-luna",
input: [
{
role: "user",
content: [
{ type: "input_text", text: JSON.stringify(request.state) },
],
},
],
questions: [
{
type: "choice",
name: "department",
instructions: "Which team should handle `input`?",
choices: [
{ value: "billing", description: "Payments and refunds" },
{ value: "other" },
],
},
{
type: "score",
name: "severity",
instructions: "How severe is the issue?",
levels: [
{ label: "0", description: "Appearance only" },
{ label: "1", description: "Blocked" },
],
},
{
type: "predicate",
name: "refund",
instructions:
'Does `input` request a refund?\n\nCriteria for true:\nAsks for money back\n\nCriteria for false:\n{"note":"money"}',
},
],
});
expect(evaluation).toEqual({
model: "gpt-6-luna",
answers: {
department: {
type: "choice",
choice: "billing",
probabilities: { billing: 0.9, other: 0.1 },
confidence: 0.8,
},
severity: {
type: "score",
score: 0.4,
probabilities: { "0": 0.6, "1": 0.4 },
confidence: 0.7,
},
refund: { type: "boolean", probability: 0.95 },
},
usage: { inputTokens: 120, outputTokens: 3 },
});
});
it.each([
[401, false],
[500, true],
])(
"surfaces HTTP %s as an AI SDK call error (retryable: %s)",
async (status, isRetryable) => {
const fetchImpl = vi
.fn()
.mockImplementation(() =>
jsonResponse(
{ error: { message: "nope", type: "invalid_request_error" } },
status,
),
);
const client = createOpenAIDecisionModelClient({
apiKey: "sk-bad",
model: "gpt-6-luna",
fetchImpl,
});
await expect(client.evaluate(request)).rejects.toMatchObject({
name: "AI_APICallError",
statusCode: status,
isRetryable,
});
},
);
it("fails the call when any question is refused", async () => {
const fetchImpl = vi.fn().mockImplementation(() =>
jsonResponse({
model: "gpt-6-luna",
answers: [
{
type: "choice",
name: "department",
choice: "billing",
probabilities: [
{ value: "billing", probability: 1 },
{ value: "other", probability: 0 },
],
},
{
type: "score",
name: "severity",
score: 0,
probabilities: [
{ value: 0, probability: 1 },
{ value: 1, probability: 0 },
],
},
{ type: "refusal", name: "refund" },
],
}),
);
const client = createOpenAIDecisionModelClient({
apiKey: "sk-test",
model: "gpt-6-luna",
fetchImpl,
});
await expect(client.evaluate(request)).rejects.toMatchObject({
name: "DecisionModelEvaluatorError",
message: "OpenAI declined to answer: refund.",
});
});
it("appends /decisions to a custom base URL", async () => {
const fetchImpl = vi.fn().mockResolvedValue(
jsonResponse({
model: "gpt-6-luna",
answers: [{ type: "predicate", name: "refund", probability: 0.5 }],
}),
);
const client = createOpenAIDecisionModelClient({
apiKey: "sk-test",
model: "gpt-6-luna",
baseURL: "https://llm-proxy.example.com/v1/",
fetchImpl,
});
await client.evaluate({
state: request.state,
questions: { refund: request.questions.refund },
});
const [url] = fetchImpl.mock.calls[0] as [string, RequestInit];
expect(url).toBe("https://llm-proxy.example.com/v1/decisions");
});
});