🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable no-nested-ternary */
import {
EvalTemplateType,
extractVariables,
InternalServerError,
InvalidRequestError,
observationVariableMappingList,
PersistedEvalOutputDefinitionSchema,
resolvePersistedEvalOutputDefinition,
variableMappingList,
type FilterState,
type ObservationVariableMapping,
} from "@langfuse/shared";
import { logger } from "@langfuse/shared/src/server";
import { EvaluatorDefinitionInputSchema } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
import { isLegacyEvalTarget } from "@/src/features/evals/utils/typeHelpers";
import type { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";
import type { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
import {
EvaluationRule,
type CreateEvaluationRuleBodyType,
} from "@/src/features/public-api/types/evaluation/evaluationRules";
import {
Evaluator,
EvaluatorVersion,
type EvaluatorDefinitionType,
} from "@/src/features/public-api/types/evaluation/evaluators";
import {
LegacyPromptVariableMapping,
PublicEvaluationRuleReadFilter,
PUBLIC_EVALUATOR_TYPE_CODE,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
type LegacyPromptVariableMappingType,
type PromptVariableMappingInputType,
type PromptVariableMappingReadType,
type PublicEvaluatorOutputDefinitionType,
} from "@/src/features/public-api/types/evaluation/publicEvalsContract";
const PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN: Record<
PromptVariableMappingInputType["source"],
ObservationVariableMapping["selectedColumnId"]
> = {
input: "input",
output: "output",
metadata: "metadata",
tool_calls: "toolCalls",
expected_output: "experimentItemExpectedOutput",
experiment_item_metadata: "experimentItemMetadata",
};
const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<
string,
PromptVariableMappingInputType["source"]
> = {
input: "input",
output: "output",
metadata: "metadata",
toolCalls: "tool_calls",
tool_calls: "tool_calls",
expected_output: "expected_output",
expectedOutput: "expected_output",
experiment_item_expected_output: "expected_output",
experimentItemExpectedOutput: "expected_output",
experimentItemMetadata: "experiment_item_metadata",
experiment_item_metadata: "experiment_item_metadata",
};
export const PUBLIC_API_EVALUATOR_TYPES: EvalTemplateType[] = [
EvalTemplateType.LLM_AS_JUDGE,
EvalTemplateType.CODE,
];
export function isPublicApiEvaluatorType(type: EvalTemplateType) {
return PUBLIC_API_EVALUATOR_TYPES.includes(type);
}
export function toPublicEvaluatorType(type: EvalTemplateType) {
switch (type) {
case EvalTemplateType.CODE:
return PUBLIC_EVALUATOR_TYPE_CODE;
case EvalTemplateType.LLM_AS_JUDGE:
return PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE;
case EvalTemplateType.DECISION_MODEL:
throw new InvalidRequestError(
"Decision-model evaluators are not available through the public API",
);
}
}
export function toStoredMappingList(
mappings: PromptVariableMappingInputType[],
) {
return observationVariableMappingList.parse(
mappings.map((mapping) => ({
templateVariable: mapping.variable,
selectedColumnId:
PUBLIC_MAPPING_SOURCE_TO_INTERNAL_COLUMN[mapping.source],
jsonSelector: mapping.jsonPath ?? null,
})),
);
}
export function toApiReadMappings(
mappings: unknown,
): PromptVariableMappingReadType[] {
const parsed = observationVariableMappingList.safeParse(mappings);
if (!parsed.success) {
logger.error("Failed to parse public evaluation rule mappings", {
issues: parsed.error.issues,
});
throw new InternalServerError("Evaluation rule mapping is corrupted");
}
return parsed.data.map((mapping) => {
const source =
INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE[mapping.selectedColumnId];
if (!source && mapping.selectedColumnId.trim()) {
throw new InternalServerError("Evaluation rule mapping is corrupted");
}
return {
variable: mapping.templateVariable,
source: source ?? null,
...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),
};
});
}
function toApiLegacyMappings(
mappings: unknown,
): LegacyPromptVariableMappingType[] {
const parsed = variableMappingList.safeParse(mappings);
if (!parsed.success) {
logger.error("Failed to parse public legacy evaluation rule mappings", {
issues: parsed.error.issues,
});
throw new InternalServerError("Evaluation rule mapping is corrupted");
}
return LegacyPromptVariableMapping.array().parse(
parsed.data.map((mapping) => ({
mappingType: "legacy",
variable: mapping.templateVariable,
langfuseObject: mapping.langfuseObject,
objectName: mapping.objectName ?? null,
source: mapping.selectedColumnId,
...(mapping.jsonSelector ? { jsonPath: mapping.jsonSelector } : {}),
})),
);
}
function toPublicOutputDefinition(outputDefinition: unknown) {
const parsed =
PersistedEvalOutputDefinitionSchema.safeParse(outputDefinition);
if (!parsed.success) {
logger.error("Failed to parse public evaluator output definition", {
issues: parsed.error.issues,
});
throw new InternalServerError("Evaluator output definition is corrupted");
}
const resolved = resolvePersistedEvalOutputDefinition(parsed.data);
const descriptions = {
...(resolved.reasoningDescription
? { scoreReasoningInstructions: resolved.reasoningDescription }
: {}),
...(resolved.scoreDescription
? { scoreValueInstructions: resolved.scoreDescription }
: {}),
};
if (resolved.dataType === "NUMERIC") {
return {
dataType: resolved.dataType,
...descriptions,
...(resolved.minValue === undefined
? {}
: { minValue: resolved.minValue }),
...(resolved.maxValue === undefined
? {}
: { maxValue: resolved.maxValue }),
};
}
if (resolved.dataType === "BOOLEAN") {
return { dataType: resolved.dataType, ...descriptions };
}
return {
dataType: resolved.dataType,
...descriptions,
categories: resolved.categories,
shouldAllowMultipleMatches: resolved.shouldAllowMultipleMatches,
};
}
function toCreator(
creator: {
id: string;
name: string | null;
} | null,
) {
return creator ? { id: creator.id, name: creator.name } : null;
}
function toInternalOutputDefinition(
outputDefinition: PublicEvaluatorOutputDefinitionType,
) {
const descriptions = {
reasoning: {
description: outputDefinition.scoreReasoningInstructions ?? "",
},
scoreDescription: outputDefinition.scoreValueInstructions ?? "",
};
if (outputDefinition.dataType === "NUMERIC") {
return {
dataType: outputDefinition.dataType,
reasoning: descriptions.reasoning,
score: {
description: descriptions.scoreDescription,
...(outputDefinition.minValue === undefined
? {}
: { minValue: outputDefinition.minValue }),
...(outputDefinition.maxValue === undefined
? {}
: { maxValue: outputDefinition.maxValue }),
},
};
}
if (outputDefinition.dataType === "BOOLEAN") {
return {
dataType: outputDefinition.dataType,
reasoning: descriptions.reasoning,
score: { description: descriptions.scoreDescription },
};
}
return {
dataType: outputDefinition.dataType,
reasoning: descriptions.reasoning,
score: {
description: descriptions.scoreDescription,
categories: outputDefinition.categories,
shouldAllowMultipleMatches: outputDefinition.shouldAllowMultipleMatches,
},
};
}
export function toEvaluatorServiceDefinition(
definition: EvaluatorDefinitionType,
) {
return EvaluatorDefinitionInputSchema.parse(
definition.type === PUBLIC_EVALUATOR_TYPE_CODE
? {
type: EvalTemplateType.CODE,
sourceCode: definition.sourceCode,
sourceCodeLanguage: definition.sourceCodeLanguage,
}
: {
type: EvalTemplateType.LLM_AS_JUDGE,
promptMessages: definition.prompt,
modelConfig: definition.modelConfig ?? null,
variableMapping:
definition.variableMapping == null
? null
: toStoredMappingList(definition.variableMapping),
outputDefinition: toInternalOutputDefinition(
definition.outputDefinition,
),
},
);
}
type ServiceEvaluator = Awaited<ReturnType<EvaluatorService["get"]>>;
type ServiceEvaluatorVersion =
| ServiceEvaluator["versions"][number]
| Awaited<ReturnType<EvaluatorService["listVersions"]>>["data"][number];
export function toPublicEvaluatorVersion(
evaluatorType: EvalTemplateType,
version: ServiceEvaluatorVersion,
) {
const common = {
id: version.id,
version: version.version,
createdAt: version.createdAt,
createdBy: toCreator(version.createdByUser),
};
if (evaluatorType === EvalTemplateType.DECISION_MODEL) {
throw new InvalidRequestError(
"Decision-model evaluators are not available through the public API",
);
}
if (evaluatorType === EvalTemplateType.CODE) {
if (!version.sourceCode || !version.sourceCodeLanguage) {
throw new InternalServerError("Code evaluator definition is corrupted");
}
return EvaluatorVersion.parse({
...common,
type: PUBLIC_EVALUATOR_TYPE_CODE,
sourceCode: version.sourceCode,
sourceCodeLanguage: version.sourceCodeLanguage,
});
}
if (!version.promptMessages) {
throw new InternalServerError("Evaluator prompt messages are corrupted");
}
const prompt = version.promptMessages;
return EvaluatorVersion.parse({
...common,
type: PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
prompt,
variables:
version.vars.length > 0
? version.vars
: extractVariables(prompt.map(({ content }) => content).join("\n")),
variableMapping:
version.variableMapping === null
? null
: toApiReadMappings(version.variableMapping),
modelConfig:
version.provider && version.model
? {
provider: version.provider,
model: version.model,
}
: null,
outputDefinition: toPublicOutputDefinition(version.outputDefinition),
});
}
export function toPublicEvaluator(evaluator: ServiceEvaluator) {
const latestVersion = evaluator.versions[0];
if (!latestVersion) {
throw new InternalServerError("Evaluator version is missing");
}
const common = {
id: evaluator.id,
name: evaluator.name,
description: evaluator.description,
createdBy: toCreator(evaluator.createdByUser),
status: evaluator.blockedAt ? ("paused" as const) : ("active" as const),
pausedAt: evaluator.blockedAt,
pausedReason: evaluator.blockReason,
pausedMessage: evaluator.blockMessage,
evaluationRuleAssignments: evaluator.assignments.map(
({ evaluationRule, variableMapping }) => ({
evaluationRuleId: evaluationRule.id,
...(variableMapping === null
? {}
: {
variableMappingOverride: isLegacyEvalTarget(
evaluationRule.targetObject,
)
? toApiLegacyMappings(variableMapping)
: toApiReadMappings(variableMapping),
}),
}),
),
createdAt: evaluator.createdAt,
updatedAt: evaluator.updatedAt,
};
const {
id: versionId,
createdAt: versionCreatedAt,
createdBy: versionCreatedBy,
...version
} = toPublicEvaluatorVersion(evaluator.type, latestVersion);
return Evaluator.parse({
...common,
...version,
versionId,
versionCreatedAt,
versionCreatedBy,
});
}
export function toInternalFilters(filters: FilterState) {
return filters.map((filter) =>
filter.column === "datasetId"
? { ...filter, column: "experimentDatasetId" }
: filter,
) as FilterState;
}
function toPublicFilters(filters: unknown) {
return PublicEvaluationRuleReadFilter.array()
.parse(filters)
.map((filter) =>
filter.column === "experimentDatasetId"
? { ...filter, column: "datasetId" }
: filter,
);
}
export function toInternalAssignments(
assignments: CreateEvaluationRuleBodyType["evaluatorAssignments"],
) {
return assignments.map((assignment) => ({
evaluatorId: assignment.evaluatorId,
variableMapping:
assignment.variableMapping === null
? null
: (toStoredMappingList(
assignment.variableMapping,
) as ObservationVariableMapping[]),
}));
}
type ServiceRule = Awaited<ReturnType<RuleService["get"]>>;
export function toPublicRule(rule: ServiceRule) {
return EvaluationRule.parse({
id: rule.id,
name: rule.name,
createdBy: toCreator(rule.createdByUser),
enabled: rule.enabled,
sampling: rule.sampling,
filter: toPublicFilters(rule.filter),
evaluatorAssignments: rule.assignments.map((assignment) => ({
evaluatorId: assignment.evaluator.id,
variableMapping:
assignment.variableMapping === null
? null
: isLegacyEvalTarget(rule.targetObject)
? toApiLegacyMappings(assignment.variableMapping)
: toApiReadMappings(assignment.variableMapping),
})),
createdAt: rule.createdAt,
updatedAt: rule.updatedAt,
});
}