🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it } from "vitest";
import { pickDefaultStripMetric } from "./pickDefaultStripMetric";
import { buildMetricOptions } from "@/src/features/experiments/utils/charts";
describe("pickDefaultStripMetric", () => {
it("opens on the first numeric score, not on cost", () => {
expect(
pickDefaultStripMetric(
buildMetricOptions({
obs_scores_avg: ["groundedness", "accuracy"],
obs_score_categories: { verdict: ["pass", "fail"] },
}),
),
).toBe("obs-score-numeric:accuracy");
});
it("falls back to a categorical score, then to cost", () => {
expect(
pickDefaultStripMetric(
buildMetricOptions({ obs_score_categories: { verdict: ["pass"] } }),
),
).toBe("obs-score-categorical:verdict");
expect(pickDefaultStripMetric(buildMetricOptions({}))).toBe("base:cost");
});
// The reason the rule changed: the alphabetically first score was a language
// check recorded on a handful of items, and it opened the page.
it("prefers coverage over the alphabetically first name", () => {
expect(
pickDefaultStripMetric(
buildMetricOptions({
obs_scores_avg: ["accuracy", "groundedness"],
}),
{
obs: new Map([
["accuracy", 3],
["groundedness", 120],
]),
},
),
).toBe("obs-score-numeric:groundedness");
});
it("ranks a boolean score below a numeric one whatever its coverage", () => {
expect(
pickDefaultStripMetric(
buildMetricOptions({
obs_scores_avg: ["answered_in_right_language", "groundedness"],
obs_score_booleans: ["answered_in_right_language"],
}),
{
obs: new Map([
["answered_in_right_language", 400],
["groundedness", 12],
]),
},
),
).toBe("obs-score-numeric:groundedness");
});
// Coverage is per level, and the aggregate keys it is counted under carry
// normalized names ("." and "-" become "_").
it("resolves coverage per level and through name normalization", () => {
const options = buildMetricOptions({
obs_scores_avg: ["judge.verdict"],
experiment_scores_avg: ["judge.verdict"],
});
expect(
pickDefaultStripMetric(options, {
obs: new Map([["judge_verdict", 2]]),
experiment: new Map([["judge_verdict", 90]]),
}),
).toBe("experiment-score-numeric:judge.verdict");
});
// Without coverage the order must still be total, so the strip does not
// switch metrics between renders while the row metrics are in flight.
it("is stable when no coverage is known", () => {
const options = buildMetricOptions({
obs_scores_avg: ["groundedness"],
experiment_scores_avg: ["groundedness"],
});
expect(pickDefaultStripMetric(options)).toBe(
"obs-score-numeric:groundedness",
);
expect(pickDefaultStripMetric(options)).toBe(
pickDefaultStripMetric(options),
);
});
});