🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it } from "vitest";
import {
compilePersistedEvalOutputDefinition,
buildEvalOutputResultSchema,
createBooleanEvalOutputDefinition,
createCategoricalEvalOutputDefinition,
createNumericEvalOutputDefinition,
EvalTargetObject,
PersistedEvalOutputDefinitionSchema,
ScoreDataTypeEnum,
validateEvalOutputResult,
} from "@langfuse/shared";
import {
buildEvalExecutionData,
type EvaluatorLlmErrorClassification,
} from "@langfuse/shared/src/server";
import { parseDispatchResult } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";
import { createDeterministicEvalScoreId } from "../../../../packages/shared/src/server/evals/evalScoreIds";
import { buildEvalScoreWritePayloads } from "./evalScoreEvent";
import {
buildEvalExecutionSpanAttributes,
buildEvaluatorLlmErrorSpanAttributes,
} from "./evalSpanAttributes";
describe("evaluation helpers", () => {
describe("buildEvalOutputResultSchema", () => {
it("should build numeric response schema with descriptions", () => {
const schema = buildEvalOutputResultSchema(
createNumericEvalOutputDefinition({
scoreDescription: "A number between 0 and 1 indicating accuracy",
reasoningDescription: "Explanation of the score",
}),
);
expect(schema.shape.score).toBeDefined();
expect(schema.shape.reasoning).toBeDefined();
});
it("should validate correct response", () => {
const schema = buildEvalOutputResultSchema(
createNumericEvalOutputDefinition({
scoreDescription: "Score between 0 and 1",
reasoningDescription: "The reasoning",
}),
);
const result = schema.safeParse({
score: 0.75,
reasoning: "Good accuracy overall",
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.score).toBe(0.75);
expect(result.data.reasoning).toBe("Good accuracy overall");
}
});
it("should reject numeric responses outside the configured range", () => {
const schema = buildEvalOutputResultSchema({
version: 2,
dataType: ScoreDataTypeEnum.NUMERIC,
score: {
description: "Score between -1 and 1",
minValue: -1,
maxValue: 1,
},
reasoning: { description: "The reasoning" },
});
expect(
schema.safeParse({ score: -1, reasoning: "At the minimum" }).success,
).toBe(true);
expect(
schema.safeParse({ score: 1, reasoning: "At the maximum" }).success,
).toBe(true);
expect(
schema.safeParse({ score: 1.1, reasoning: "Above the maximum" })
.success,
).toBe(false);
expect(
schema.safeParse({ score: -1.1, reasoning: "Below the minimum" })
.success,
).toBe(false);
});
it("should reject invalid response - missing score", () => {
const schema = buildEvalOutputResultSchema(
createNumericEvalOutputDefinition({
scoreDescription: "Score between 0 and 1",
reasoningDescription: "The reasoning",
}),
);
const result = schema.safeParse({
reasoning: "Missing score",
});
expect(result.success).toBe(false);
});
it("should reject invalid response - string score", () => {
const schema = buildEvalOutputResultSchema(
createNumericEvalOutputDefinition({
scoreDescription: "Score between 0 and 1",
reasoningDescription: "The reasoning",
}),
);
const result = schema.safeParse({
score: "0.75",
reasoning: "Score is string",
});
expect(result.success).toBe(false);
});
it("should validate boolean responses", () => {
const schema = buildEvalOutputResultSchema(
createBooleanEvalOutputDefinition({
scoreDescription:
"Return true if the answer is correct, otherwise false",
reasoningDescription: "Explain the verdict",
}),
);
expect(
schema.safeParse({
score: true,
reasoning: "The answer satisfies the criteria.",
}).success,
).toBe(true);
});
it("should reject string values for boolean responses", () => {
const schema = buildEvalOutputResultSchema(
createBooleanEvalOutputDefinition({
scoreDescription:
"Return true if the answer is correct, otherwise false",
reasoningDescription: "Explain the verdict",
}),
);
expect(
schema.safeParse({
score: "true",
reasoning: "String booleans should be rejected.",
}).success,
).toBe(false);
});
it("should validate categorical responses against allowed values", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose the best matching category",
reasoningDescription: "Explain the selected category",
categories: ["correct", "partial"],
}),
);
expect(
schema.safeParse({
score: "correct",
reasoning: "The answer is fully supported.",
}).success,
).toBe(true);
expect(
schema.safeParse({
score: "incorrect",
reasoning: "The answer is unsupported.",
}).success,
).toBe(false);
});
it("should reject categorical arrays when only a single match is allowed", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose the best matching category",
reasoningDescription: "Explain the selected category",
categories: ["correct", "partial"],
}),
);
expect(
schema.safeParse({
score: ["correct", "partial"],
reasoning: "This should fail because only one category is allowed.",
}).success,
).toBe(false);
});
it("should validate categorical multi-match responses against allowed values", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Explain the selected categories",
categories: ["correct", "partial"],
shouldAllowMultipleMatches: true,
}),
);
expect(
schema.safeParse({
score: ["correct", "partial"],
reasoning: "The answer is partly right and partly complete.",
}).success,
).toBe(true);
expect(
schema.safeParse({
score: ["correct", "correct"],
reasoning: "Duplicate categories should be rejected.",
}).success,
).toBe(false);
});
it("should reject duplicate categorical multi-match values", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Explain the selected categories",
categories: ["correct", "partial"],
shouldAllowMultipleMatches: true,
}),
);
expect(
schema.safeParse({
score: ["correct", "correct"],
reasoning: "This should be invalid.",
}).success,
).toBe(false);
});
it("should reject categorical scalars when multiple matches are required to be returned as an array", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Explain the selected categories",
categories: ["correct", "partial"],
shouldAllowMultipleMatches: true,
}),
);
expect(
schema.safeParse({
score: "correct",
reasoning: "This should fail because multi-match uses an array.",
}).success,
).toBe(false);
});
it("should reject empty categorical multi-match arrays", () => {
const schema = buildEvalOutputResultSchema(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Explain the selected categories",
categories: ["correct", "partial"],
shouldAllowMultipleMatches: true,
}),
);
expect(
schema.safeParse({
score: [],
reasoning: "At least one category is required.",
}).success,
).toBe(false);
});
});
describe("buildEvalExecutionData", () => {
it("should include all provided fields", () => {
const params = {
type: "JOB" as const,
jobExecutionId: "exec-123",
jobConfigurationId: "config-456",
targetTraceId: "trace-789",
targetObservationId: "obs-abc",
targetDatasetItemId: "dataset-def",
};
const result = buildEvalExecutionData(params);
expect(result).toEqual({
executionMetadata: {
job_execution_id: "exec-123",
job_configuration_id: "config-456",
target_trace_id: "trace-789",
target_observation_id: "obs-abc",
target_dataset_item_id: "dataset-def",
},
evaluationContext: {
evaluationRuleId: "config-456",
evaluatorExecutionIsTest: false,
},
});
});
it("should exclude null/undefined fields", () => {
const params = {
type: "JOB" as const,
jobExecutionId: "exec-123",
jobConfigurationId: "config-456",
targetTraceId: null,
targetObservationId: undefined,
targetDatasetItemId: null,
};
const result = buildEvalExecutionData(params);
expect(result).toEqual({
executionMetadata: {
job_execution_id: "exec-123",
job_configuration_id: "config-456",
},
evaluationContext: {
evaluationRuleId: "config-456",
evaluatorExecutionIsTest: false,
},
});
expect(Object.keys(result.executionMetadata)).not.toContain(
"target_trace_id",
);
expect(Object.keys(result.executionMetadata)).not.toContain(
"target_observation_id",
);
expect(Object.keys(result.executionMetadata)).not.toContain(
"target_dataset_item_id",
);
});
});
describe("buildEvalExecutionSpanAttributes", () => {
it("should include target object, filter dimensions, and trace variable source fields", () => {
const attributes = buildEvalExecutionSpanAttributes({
config: {
id: "config-123",
targetObject: EvalTargetObject.TRACE,
filter: [
{
type: "string",
column: "name",
operator: "=",
value: "checkout",
},
{
type: "stringObject",
column: "metadata",
key: "tenant",
operator: "=",
value: "langfuse",
},
{
type: "numberObject",
column: "scores_avg",
key: "quality",
operator: ">",
value: 0.8,
},
{
type: "stringObject",
column: "metadata",
key: "tenant",
operator: "contains",
value: "lang",
},
],
variableMapping: [
{
templateVariable: "traceInput",
langfuseObject: "trace",
selectedColumnId: "input",
jsonSelector: "messages.0.content",
},
{
templateVariable: "answer",
langfuseObject: "generation",
objectName: "answer-generator",
selectedColumnId: "output",
jsonSelector: null,
},
],
},
});
expect(attributes).toMatchObject({
"eval.job_configuration.id": "config-123",
"eval.job_configuration.target_object": EvalTargetObject.TRACE,
"eval.job_configuration.filter.dimensions": [
"name",
"metadata",
"scores_avg",
],
"eval.job_configuration.filter.dimension_count": 3,
"eval.variable.source_fields": ["trace.input", "generation.output"],
"eval.variable.source_field_count": 2,
});
});
it("should use observation variable mappings for event and experiment targets", () => {
const attributes = buildEvalExecutionSpanAttributes({
config: {
id: "config-456",
targetObject: EvalTargetObject.EVENT,
filter: [
{
type: "positionInTrace",
column: "position",
operator: "=",
key: "root",
},
],
variableMapping: [
{
templateVariable: "input",
selectedColumnId: "input",
jsonSelector: "question",
},
{
templateVariable: "output",
selectedColumnId: "output",
jsonSelector: null,
},
],
},
});
expect(attributes).toMatchObject({
"eval.job_configuration.id": "config-456",
"eval.job_configuration.target_object": EvalTargetObject.EVENT,
"eval.job_configuration.filter.dimensions": ["position"],
"eval.job_configuration.filter.dimension_count": 1,
"eval.variable.source_fields": ["input", "output"],
"eval.variable.source_field_count": 2,
});
});
});
describe("buildEvaluatorLlmErrorSpanAttributes", () => {
it("exposes native AI SDK retry metadata without leaking the error message", () => {
const classification = {
kind: "provider",
message: "sensitive provider response",
statusCode: 429,
isRetryable: true,
error: new Error("sensitive provider response"),
retryError: {
reason: "maxRetriesExceeded",
errors: [new Error("attempt 1"), new Error("attempt 2")],
},
blockReason: null,
} as EvaluatorLlmErrorClassification;
const attributes = buildEvaluatorLlmErrorSpanAttributes(classification);
expect(attributes).toEqual({
"eval.llm.error.kind": "provider",
"eval.llm.error.retryable": true,
"eval.llm.error.status_code": 429,
"eval.llm.retry.reason": "maxRetriesExceeded",
"eval.llm.retry.attempt_count": 2,
"eval.llm.blocked": false,
});
expect(JSON.stringify(attributes)).not.toContain(
"sensitive provider response",
);
expect(attributes).not.toHaveProperty("http.response.status_code");
});
it("exposes the terminal evaluator block decision", () => {
const classification = {
kind: "validation",
message: "endpoint unavailable",
statusCode: 400,
isRetryable: false,
error: new Error("endpoint unavailable"),
blockReason: "LLM_CONNECTION_ENDPOINT_UNREACHABLE",
} as EvaluatorLlmErrorClassification;
expect(buildEvaluatorLlmErrorSpanAttributes(classification)).toEqual({
"eval.llm.error.kind": "validation",
"eval.llm.error.retryable": false,
"eval.llm.error.status_code": 400,
"eval.llm.blocked": true,
"eval.llm.block.reason": "LLM_CONNECTION_ENDPOINT_UNREACHABLE",
"eval.llm.block.source": "llm_completion_error",
});
});
it("uses a low-cardinality fallback for unknown errors", () => {
expect(buildEvaluatorLlmErrorSpanAttributes(null)).toEqual({
"eval.llm.error.kind": "unknown",
"eval.llm.blocked": false,
});
});
});
describe("parseDispatchResult", () => {
it("should preserve optional per-score metadata from code eval runners", () => {
const result = parseDispatchResult({
scores: [
{
name: "quality",
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
metadata: { rubric: "strict" },
},
],
});
expect(result.scores[0]).toMatchObject({
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
metadata: { rubric: "strict" },
});
});
it("should reject non-object score metadata from code eval runners", () => {
expect(() =>
parseDispatchResult({
scores: [
{
name: "quality",
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
metadata: "not-a-dict",
},
],
}),
).toThrow("Invalid code eval result");
});
});
describe("buildEvalScoreWritePayloads", () => {
const evaluationContext = {
evaluationRuleId: "rule-1",
evaluatorExecutionIsTest: false,
};
it("should build stable code eval score IDs when different score names reorder", () => {
const originalPayloads = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
name: "accuracy",
},
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.7,
name: "fluency",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
const reorderedPayloads = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.7,
name: "fluency",
},
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
name: "accuracy",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
expect(originalPayloads[0].scoreId).toBe(reorderedPayloads[1].scoreId);
expect(originalPayloads[1].scoreId).toBe(reorderedPayloads[0].scoreId);
});
it("should build distinct deterministic code eval score IDs for duplicate score names", () => {
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "correct",
name: "accuracy",
},
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.7,
name: "fluency",
},
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "partial",
name: "accuracy",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
const scoreIds = result.map((payload) => payload.scoreId);
expect(new Set(scoreIds).size).toBe(3);
expect(scoreIds).toEqual([
createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 0,
}),
createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "fluency",
occurrenceIndex: 0,
}),
createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 1,
}),
]);
});
it("should build deterministic score IDs as part of payload creation", () => {
const expectedScoreIds = [
createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 0,
}),
createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 1,
}),
];
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "correct",
name: "accuracy",
},
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "partial",
name: "accuracy",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
expect(result.map((payload) => payload.scoreId)).toEqual(
expectedScoreIds,
);
expect(result.map((payload) => payload.event.body.id)).toEqual(
expectedScoreIds,
);
});
it("should build a single numeric score payload", () => {
const scoreId = createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 0,
});
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.85,
name: "accuracy",
comment: "High accuracy observed",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: null,
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
eventId: expect.any(String),
scoreId,
event: expect.objectContaining({
body: expect.objectContaining({
value: 0.85,
dataType: ScoreDataTypeEnum.NUMERIC,
}),
}),
});
});
it("should build a single boolean score payload", () => {
const scoreId = createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "correctness",
occurrenceIndex: 0,
});
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.BOOLEAN,
value: 1,
name: "correctness",
comment: "The answer satisfies the criteria",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: null,
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
eventId: expect.any(String),
scoreId,
event: expect.objectContaining({
body: expect.objectContaining({
value: 1,
dataType: ScoreDataTypeEnum.BOOLEAN,
}),
}),
});
});
it("should build one categorical payload per match while preserving shared metadata", () => {
const firstScoreId = createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 0,
});
const secondScoreId = createDeterministicEvalScoreId({
jobExecutionId: "job-1",
scoreName: "accuracy",
occurrenceIndex: 1,
});
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "correct",
name: "accuracy",
comment: "Both categories apply",
},
{
dataType: ScoreDataTypeEnum.CATEGORICAL,
value: "partial",
name: "accuracy",
comment: "Both categories apply",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
evaluationContext,
});
expect(result).toHaveLength(2);
expect(result[0].scoreId).toBe(firstScoreId);
expect(result[1].scoreId).toBe(secondScoreId);
expect(result[0].event.body.value).toBe("correct");
expect(result[1].event.body.value).toBe("partial");
expect(result[0].event.body.comment).toBe("Both categories apply");
expect(result[1].event.body.comment).toBe("Both categories apply");
expect(result[0].event.body.metadata).toEqual({
job_execution_id: "job-1",
});
expect(result[1].event.body.metadata).toEqual({
job_execution_id: "job-1",
});
});
it("should merge returned score metadata with execution metadata", () => {
const result = buildEvalScoreWritePayloads({
scores: [
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.9,
name: "accuracy",
metadata: {
rubric: "strict",
tags: ["math", "strict"],
job_execution_id: "user-supplied-job",
},
},
{
dataType: ScoreDataTypeEnum.NUMERIC,
value: 0.7,
name: "fluency",
},
],
jobExecutionId: "job-1",
traceId: "trace-456",
observationId: "obs-789",
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: {
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
},
evaluationContext: {
evaluatorId: "evaluator-1",
evaluationRuleId: "legacy-rule-1",
evaluatorExecutionIsTest: false,
},
});
expect(result[0].event.body.metadata).toEqual({
rubric: "strict",
tags: ["math", "strict"],
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
});
expect(result[0].event.body).toMatchObject({
evaluatorId: "evaluator-1",
evaluationRuleId: "legacy-rule-1",
});
expect(result[1].event.body.metadata).toEqual({
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
});
expect(result[1].event.body).toMatchObject({
evaluatorId: "evaluator-1",
evaluationRuleId: "legacy-rule-1",
});
});
});
describe("validateEvalOutputResult", () => {
it("should validate correct response", () => {
const outputDefinition = createNumericEvalOutputDefinition({
scoreDescription: "Score 0-1",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: { score: 0.8, reasoning: "Good" },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data).toEqual({
dataType: ScoreDataTypeEnum.NUMERIC,
score: 0.8,
reasoning: "Good",
});
}
});
it("should return error for invalid response", () => {
const outputDefinition = createNumericEvalOutputDefinition({
scoreDescription: "Score 0-1",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: { score: "invalid", reasoning: "Test" },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error).toBeDefined();
}
});
it("should return error for null response", () => {
const outputDefinition = createNumericEvalOutputDefinition({
scoreDescription: "Score 0-1",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: null,
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
});
it("should return error for missing fields", () => {
const outputDefinition = createNumericEvalOutputDefinition({
scoreDescription: "Score 0-1",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: { score: 0.5 },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
});
it("should normalize boolean responses", () => {
const outputDefinition = createBooleanEvalOutputDefinition({
scoreDescription:
"Return true if the answer is correct, otherwise false",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: { score: false, reasoning: "The answer is incorrect." },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data).toEqual({
dataType: ScoreDataTypeEnum.BOOLEAN,
score: false,
reasoning: "The answer is incorrect.",
});
}
});
it("should reject invalid boolean responses", () => {
const outputDefinition = createBooleanEvalOutputDefinition({
scoreDescription:
"Return true if the answer is correct, otherwise false",
reasoningDescription: "Why",
});
const result = validateEvalOutputResult({
response: {
score: "false",
reasoning: "String booleans are invalid.",
},
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
});
it("should normalize categorical responses to a matches array", () => {
const outputDefinition = createCategoricalEvalOutputDefinition({
scoreDescription: "Choose the best matching category",
reasoningDescription: "Why",
categories: ["correct", "incorrect"],
});
const result = validateEvalOutputResult({
response: { score: "correct", reasoning: "Supported by the context" },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data).toEqual({
dataType: ScoreDataTypeEnum.CATEGORICAL,
matches: ["correct"],
reasoning: "Supported by the context",
});
}
});
it("should validate categorical multi-match responses", () => {
const outputDefinition = createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Why",
categories: ["correct", "incorrect"],
shouldAllowMultipleMatches: true,
});
const result = validateEvalOutputResult({
response: {
score: ["correct", "incorrect"],
reasoning: "Both labels apply in this synthetic test.",
},
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data).toEqual({
dataType: ScoreDataTypeEnum.CATEGORICAL,
matches: ["correct", "incorrect"],
reasoning: "Both labels apply in this synthetic test.",
});
}
});
it("should reject categorical responses with categories outside the allowed set", () => {
const outputDefinition = createCategoricalEvalOutputDefinition({
scoreDescription: "Choose the best matching category",
reasoningDescription: "Why",
categories: ["correct", "incorrect"],
});
const result = validateEvalOutputResult({
response: { score: "partial", reasoning: "Not an allowed category" },
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
});
it("should reject categorical multi-match responses with duplicate categories", () => {
const outputDefinition = createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Why",
categories: ["correct", "incorrect"],
shouldAllowMultipleMatches: true,
});
const result = validateEvalOutputResult({
response: {
score: ["correct", "correct"],
reasoning: "Duplicates should be rejected.",
},
compiledOutputDefinition:
compilePersistedEvalOutputDefinition(outputDefinition),
});
expect(result.success).toBe(false);
});
});
describe("PersistedEvalOutputDefinitionSchema", () => {
it("should validate correct output definition", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
score: "A number between 0 and 1",
reasoning: "Explanation of the score",
});
expect(result.success).toBe(true);
});
it("should default missing legacy score to an empty string", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
reasoning: "Only reasoning",
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.score).toBe("");
}
});
it("should default missing legacy reasoning to an empty string", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
score: "Only score",
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.reasoning).toBe("");
}
});
it("should reject non-string score", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
score: 0.5,
reasoning: "Explanation",
});
expect(result.success).toBe(false);
});
it("should accept versioned categorical schemas", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose the best matching category",
reasoningDescription: "Explain the selected category",
categories: ["correct", "partial"],
}),
);
expect(result.success).toBe(true);
});
it("should accept versioned boolean schemas", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse(
createBooleanEvalOutputDefinition({
scoreDescription:
"Return true if the answer is correct, otherwise false",
reasoningDescription: "Explain the verdict",
}),
);
expect(result.success).toBe(true);
});
it("should reject numeric schemas with an inverted range", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
version: 2,
dataType: ScoreDataTypeEnum.NUMERIC,
reasoning: { description: "Explain the score" },
score: {
description: "Score within the configured range",
minValue: 1,
maxValue: 0,
},
});
expect(result.success).toBe(false);
});
it("should accept versioned categorical multi-match schemas", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse(
createCategoricalEvalOutputDefinition({
scoreDescription: "Choose all matching categories",
reasoningDescription: "Explain the selected categories",
categories: ["correct", "partial"],
shouldAllowMultipleMatches: true,
}),
);
expect(result.success).toBe(true);
});
it("should default missing categorical multi-match flag to false", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
version: 2,
dataType: ScoreDataTypeEnum.CATEGORICAL,
reasoning: {
description: "Explain the selected category",
},
score: {
description: "Choose the best matching category",
categories: ["correct", "partial"],
},
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.score.shouldAllowMultipleMatches).toBe(false);
}
});
it("should reject categorical schemas with duplicate categories after trimming whitespace", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
version: 2,
dataType: ScoreDataTypeEnum.CATEGORICAL,
reasoning: {
description: "Explain the selected category",
},
score: {
description: "Choose the best matching category",
categories: ["correct", " correct "],
},
});
expect(result.success).toBe(false);
});
it("should reject categorical schemas with blank categories", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
version: 2,
dataType: ScoreDataTypeEnum.CATEGORICAL,
reasoning: {
description: "Explain the selected category",
},
score: {
description: "Choose the best matching category",
categories: ["correct", " "],
},
});
expect(result.success).toBe(false);
});
it("should reject categorical schemas with fewer than two categories", () => {
const result = PersistedEvalOutputDefinitionSchema.safeParse({
version: 2,
dataType: ScoreDataTypeEnum.CATEGORICAL,
reasoning: {
description: "Explain the selected category",
},
score: {
description: "Choose the best matching category",
categories: ["correct"],
},
});
expect(result.success).toBe(false);
});
});
});