🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// @vitest-environment node
import { describe, expect, it } from "vitest";
import {
MAX_SCORE_NAME_LENGTH,
MAX_SCORE_NAMES_PER_TYPE,
observedScoreNamesFromOptions,
withMetadataPathOptions,
type ObservedOptions,
} from "./observed-options";
describe("observedScoreNamesFromOptions", () => {
it("returns undefined while the observed map is still loading", () => {
expect(observedScoreNamesFromOptions(undefined)).toBeUndefined();
});
it("keeps a set undefined until its column has loaded, [] when loaded empty", () => {
// scores_avg loaded with names, score_categories loaded but empty, the two
// trace columns not loaded yet. Only loaded sets are defined — the server
// must not enforce a set whose column simply hasn't arrived.
const names = observedScoreNamesFromOptions({
scores_avg: [{ value: "helpfulness-rating" }, { value: "accuracy" }],
score_categories: [],
});
expect(names).toEqual({
numeric: ["helpfulness-rating", "accuracy"],
categorical: [],
traceNumeric: undefined,
traceCategorical: undefined,
});
});
it("sends an over-cap set as undefined instead of truncating it", () => {
// A truncated set would make the un-sent names look unknown and get their
// filters dropped; skipping enforcement is the safe degradation.
const names = observedScoreNamesFromOptions({
scores_avg: Array.from(
{ length: MAX_SCORE_NAMES_PER_TYPE + 1 },
(_, i) => ({
value: `score-${i}`,
}),
),
score_categories: [{ value: "sentiment" }],
});
expect(names?.numeric).toBeUndefined();
expect(names?.categorical).toEqual(["sentiment"]);
});
it("sends a set containing an overlong name as undefined", () => {
const names = observedScoreNamesFromOptions({
scores_avg: [{ value: "x".repeat(MAX_SCORE_NAME_LENGTH + 1) }],
});
expect(names?.numeric).toBeUndefined();
});
});
describe("withMetadataPathOptions", () => {
const observed: ObservedOptions = { level: [{ value: "ERROR" }] };
it("keeps undefined observed undefined (loading semantics untouched)", () => {
expect(
withMetadataPathOptions(undefined, { a: { type: "string" } }),
).toBeUndefined();
});
it("returns the observed map unchanged without observed paths", () => {
expect(withMetadataPathOptions(observed, undefined)).toBe(observed);
expect(withMetadataPathOptions(observed, {})).toBe(observed);
});
it("merges the metadata suggestions in, leaving other columns alone", () => {
const out = withMetadataPathOptions(observed, {
region: { type: "string", values: ["eu"] },
});
expect(out?.level).toBe(observed.level);
expect(out?.metadata).toEqual([{ value: "region", type: "string" }]);
expect(out?.["metadata.region"]).toEqual([{ value: "eu" }]);
});
});