🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { type AggregatedScoreData } from "@langfuse/shared";
import type Decimal from "decimal.js";
export type NumericDiff = {
type: "NUMERIC";
absoluteDifference: number;
direction: "+" | "-";
};
export type CategoricalDiff = {
type: "CATEGORICAL";
isDifferent: true;
/** The two values, when each side carries exactly one. */
from?: string;
to?: string;
};
export type BaselineDiff = NumericDiff | CategoricalDiff | null;
/**
* Calculate numeric diff between current and baseline values
* Handles number, Decimal, null, undefined
* Returns null if diff cannot be calculated or values are equal
*/
export function calculateNumericDiff(
current: number | Decimal | null | undefined,
baseline: number | Decimal | null | undefined,
): BaselineDiff | null {
if (current == null || baseline == null) return null;
// Convert to numbers
const currentNum = typeof current === "number" ? current : current.toNumber();
const baselineNum =
typeof baseline === "number" ? baseline : baseline.toNumber();
// Same value → no diff
const diff = currentNum - baselineNum;
if (diff === 0) return null;
return {
absoluteDifference: Math.abs(diff),
direction: diff > 0 ? "+" : "-",
type: "NUMERIC",
};
}
/**
* Calculate diff between current and baseline score aggregate
* Returns null if diff cannot be calculated
*/
export function calculateScoreDiff(
current: AggregatedScoreData | null,
baseline: AggregatedScoreData | null,
): BaselineDiff {
// Missing data → no diff
if (!current || !baseline) return null;
// Aggregate scores (no id) → skip
if (!current.id || !baseline.id) return null;
// Type mismatch → no diff
if (current.type !== baseline.type) return null;
if (current.type === "NUMERIC" && baseline.type === "NUMERIC") {
return calculateNumericDiff(current.average, baseline.average);
}
if (current.type === "CATEGORICAL" && baseline.type === "CATEGORICAL") {
const currentValue = current.values[0];
const baselineValue = baseline.values[0];
// Same value → no diff
if (currentValue === baselineValue) return null;
// A categorical score has no distance, so the diff is the move itself.
return {
type: "CATEGORICAL",
isDifferent: true,
...(current.values.length === 1 && baseline.values.length === 1
? { from: baselineValue, to: currentValue }
: {}),
};
}
return null;
}