🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
const {
mockAddScoreDelete,
mockAddBatchAction,
mockGetEventsExactFilterOptionsForColumns,
} = vi.hoisted(() => ({
mockAddScoreDelete: vi.fn(),
mockAddBatchAction: vi.fn(),
mockGetEventsExactFilterOptionsForColumns: vi.fn(async () => []),
}));
vi.mock("@langfuse/shared/src/server", async () => {
const originalModule = await vi.importActual("@langfuse/shared/src/server");
return {
...originalModule,
ScoreDeleteQueue: {
getInstance: vi.fn(() => ({
add: mockAddScoreDelete,
})),
},
BatchActionQueue: {
getInstance: vi.fn(() => ({
add: mockAddBatchAction,
})),
},
getEventsExactFilterOptionsForColumns:
mockGetEventsExactFilterOptionsForColumns,
};
});
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type { Session } from "next-auth";
import { prisma } from "@langfuse/shared/src/db";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { ScoreConfigDataType } from "@langfuse/shared";
import {
createEvent,
createEventsCh,
createObservation,
createObservationsCh,
createTrace,
createTraceScore,
createTracesCh,
createScoresCh,
ScoreDeleteQueue,
BatchActionQueue,
QueueJobs,
createOrgProjectAndApiKey,
getScoreById,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import { observationScopeFilter } from "@/src/features/filters/config/scores-config";
import { SCORES_FIELD_REGISTRY } from "@/src/features/scores/constants/scoresSearchRegistry";
import { planCommit } from "@/src/features/search-bar/lib/commit";
import { randomUUID } from "crypto";
const maybeEvents =
env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
? describe
: describe.skip;
describe("scores trpc", () => {
let projectId: string;
let orgId: string;
let caller: ReturnType<typeof appRouter.createCaller>;
beforeEach(async () => {
const setup = await createOrgProjectAndApiKey();
projectId = setup.projectId;
orgId = setup.orgId;
mockAddScoreDelete.mockClear();
mockAddBatchAction.mockClear();
mockGetEventsExactFilterOptionsForColumns.mockClear();
const session: Session = {
expires: "1",
user: {
id: "user-1",
canCreateOrganizations: true,
name: "Demo User",
organizations: [
{
id: orgId,
name: "Test Organization",
role: "OWNER",
plan: "cloud:hobby",
cloudConfig: undefined,
metadata: {},
aiFeaturesEnabled: false,
aiTelemetryEnabled: true,
projects: [
{
id: projectId,
role: "ADMIN",
retentionDays: 30,
deletedAt: null,
name: "Test Project",
hasTraces: false,
metadata: {},
createdAt: new Date().toISOString(),
},
],
},
],
featureFlags: testFeatureFlags(),
admin: true,
},
environment: {} as any,
};
const ctx = createInnerTRPCContext({ session, headers: {} });
caller = appRouter.createCaller({ ...ctx, prisma });
});
describe("scores.all", () => {
it("preserves evaluator-test filters for score rows and counts on both read paths", async () => {
const scores = [
createTraceScore({
project_id: projectId,
name: "missing-test-marker",
metadata: {},
}),
createTraceScore({
project_id: projectId,
name: "false-test-marker",
metadata: { evaluator_test: "false" },
}),
createTraceScore({
project_id: projectId,
name: "true-test-marker",
metadata: { evaluator_test: "true" },
}),
];
await createScoresCh(scores);
for (const { operator, value, expectedIds } of [
{
operator: "=" as const,
value: false,
expectedIds: [scores[0].id, scores[1].id],
},
{
operator: "=" as const,
value: true,
expectedIds: [scores[2].id],
},
{
operator: "<>" as const,
value: true,
expectedIds: [scores[0].id, scores[1].id],
},
{
operator: "<>" as const,
value: false,
expectedIds: [scores[2].id],
},
]) {
const payload = {
projectId,
filter: [
{
column: "isEvaluatorTest",
type: "boolean" as const,
operator,
value,
},
],
orderBy: { column: "timestamp", order: "DESC" as const },
page: 0,
limit: 50,
};
const [rows, eventRows, count, eventCount] = await Promise.all([
caller.scores.all(payload),
caller.scores.allFromEvents(payload),
caller.scores.countAll({ ...payload, orderBy: null }),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(rows.scores.map(({ id }) => id).sort()).toEqual(
[...expectedIds].sort(),
);
expect(eventRows.scores.map(({ id }) => id).sort()).toEqual(
[...expectedIds].sort(),
);
expect(count.totalCount).toBe(expectedIds.length);
expect(eventCount.totalCount).toBe(expectedIds.length);
}
});
it("pages through scores with identical timestamps without duplicates or gaps on both read paths", async () => {
const timestamp = new Date();
const scores = Array.from({ length: 8 }, () =>
createTraceScore({ project_id: projectId, name: "batch", timestamp }),
);
await createScoresCh(scores);
for (const procedure of [
caller.scores.all,
caller.scores.allFromEvents,
]) {
const pages = await Promise.all(
scores.map((_, page) =>
procedure({
projectId,
filter: [],
orderBy: { column: "timestamp", order: "DESC" },
page,
limit: 1,
}),
),
);
const pagedIds = pages.flatMap((p) => p.scores.map(({ id }) => id));
expect(pagedIds).toHaveLength(scores.length);
expect(new Set(pagedIds)).toEqual(new Set(scores.map(({ id }) => id)));
}
});
it("applies search-bar name matching and repeated numeric bounds to v4 rows and counts", async () => {
await createScoresCh(
[
{ name: "Rouge Score", value: 0.5 },
{ name: "Weighted Rouge Score", value: 0.7 },
{ name: "confidence", value: 0.6 },
{ name: "Rouge Score", value: 0.1 },
{ name: "confidence", value: 0.95 },
].map((score) =>
createTraceScore({
project_id: projectId,
data_type: "NUMERIC",
...score,
}),
),
);
for (const { query, names } of [
{
query: "Rouge Score value:>0.2 value:<0.8",
names: ["Rouge Score", "Weighted Rouge Score"],
},
{
query: 'name:("Rouge Score" OR confidence) value:>0.2 value:<0.8',
names: ["Rouge Score", "confidence"],
},
]) {
const committed = planCommit(query, undefined, SCORES_FIELD_REGISTRY);
expect(committed.status, query).toBe("committed");
if (committed.status !== "committed") {
throw new Error(`Invalid score search: ${query}`);
}
const payload = { projectId, filter: committed.filters };
const [rows, count] = await Promise.all([
caller.scores.allFromEvents({
...payload,
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
}),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(rows.scores.map((score) => score.name).sort(), query).toEqual(
names,
);
expect(count.totalCount, query).toBe(names.length);
}
});
it("counts v4 scores within a timestamp window, pruning other day-buckets and honoring the exact bound", async () => {
// Exercises the FINAL-free count path's date handling: the coarse
// toDate(timestamp) prune must drop other day-buckets pre-dedup, while the
// exact timestamp bound (re-applied post-dedup) must still exclude an
// in-bucket-but-out-of-window row. Regression guard for
// buildScoresDatePrune.
const from = new Date("2024-06-15T10:00:00.000Z");
const to = new Date("2024-06-15T14:00:00.000Z");
await createScoresCh([
createTraceScore({
project_id: projectId,
name: "in-window",
timestamp: new Date("2024-06-15T12:00:00.000Z"),
}),
// Same day (survives the coarse toDate prune) but after `to` — only the
// exact post-dedup bound excludes it.
createTraceScore({
project_id: projectId,
name: "same-day-after-window",
timestamp: new Date("2024-06-15T20:00:00.000Z"),
}),
createTraceScore({
project_id: projectId,
name: "earlier-day",
timestamp: new Date("2024-06-10T12:00:00.000Z"),
}),
]);
const payload = {
projectId,
filter: [
{
column: "timestamp",
type: "datetime" as const,
operator: ">=" as const,
value: from,
},
{
column: "timestamp",
type: "datetime" as const,
operator: "<" as const,
value: to,
},
],
};
const [rows, count] = await Promise.all([
caller.scores.allFromEvents({
...payload,
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
}),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(rows.scores.map((score) => score.name)).toEqual(["in-window"]);
expect(count.totalCount).toBe(rows.scores.length);
expect(count.totalCount).toBe(1);
});
it("reconstructs the latest version of a score id without FINAL", async () => {
// Two versions of one score id share the dedup key
// (project_id, toDate(timestamp), name, id) and differ only in event_ts.
// The FINAL-free rows path (ORDER BY event_ts DESC + LIMIT 1 BY) and count
// path (argMax) must both collapse them to the single latest version and
// return the whole latest row.
const scoreId = randomUUID();
const timestamp = new Date("2024-06-15T12:00:00.000Z");
await createScoresCh([
createTraceScore({
id: scoreId,
project_id: projectId,
name: "versioned-score",
timestamp,
event_ts: new Date("2024-06-15T12:00:00.000Z"),
value: 0.1,
comment: "stale",
}),
createTraceScore({
id: scoreId,
project_id: projectId,
name: "versioned-score",
timestamp,
event_ts: new Date("2024-06-15T12:05:00.000Z"),
value: 0.9,
comment: "latest",
}),
]);
const [rows, count] = await Promise.all([
caller.scores.allFromEvents({
projectId,
filter: [],
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
}),
caller.scores.countAllFromEvents({
projectId,
filter: [],
orderBy: null,
}),
]);
expect(rows.scores.map((score) => score.id)).toEqual([scoreId]);
expect(rows.scores[0].value).toBe(0.9);
expect(rows.scores[0].comment).toBe("latest");
expect(count.totalCount).toBe(1);
});
it("filters mutable columns after dedup, never against a stale version", async () => {
// value/comment/timestamp/trace_id are mutable across a score's versions,
// so a filter must run AFTER dedup. Each id below has a stale version that
// disagrees with its latest on `value > 0.5`; filtering the raw rows
// pre-dedup would wrongly keep `latest-low` (its stale 0.9) and could drop
// `latest-high`. The predicate must be decided on the latest version only.
const latestLowId = randomUUID(); // stale 0.9, latest 0.1 -> excluded
const latestHighId = randomUUID(); // stale 0.1, latest 0.9 -> included
const timestamp = new Date("2024-06-15T12:00:00.000Z");
await createScoresCh([
createTraceScore({
id: latestLowId,
project_id: projectId,
name: "latest-low",
timestamp,
event_ts: new Date("2024-06-15T12:00:00.000Z"),
value: 0.9,
}),
createTraceScore({
id: latestLowId,
project_id: projectId,
name: "latest-low",
timestamp,
event_ts: new Date("2024-06-15T12:05:00.000Z"),
value: 0.1,
}),
createTraceScore({
id: latestHighId,
project_id: projectId,
name: "latest-high",
timestamp,
event_ts: new Date("2024-06-15T12:00:00.000Z"),
value: 0.1,
}),
createTraceScore({
id: latestHighId,
project_id: projectId,
name: "latest-high",
timestamp,
event_ts: new Date("2024-06-15T12:05:00.000Z"),
value: 0.9,
}),
]);
const payload = {
projectId,
filter: [
{
column: "value",
type: "number" as const,
operator: ">" as const,
value: 0.5,
},
],
};
const [rows, count] = await Promise.all([
caller.scores.allFromEvents({
...payload,
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
}),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(rows.scores.map((score) => score.id)).toEqual([latestHighId]);
expect(count.totalCount).toBe(1);
});
it("returns the recorded evaluator and resolves legacy scores by rule assignment and score name", async () => {
const [recordedEvaluator, matchingEvaluator, otherEvaluator] =
await Promise.all(
["Recorded evaluator", "Matching evaluator", "Other evaluator"].map(
(name) =>
prisma.evaluator.create({
data: {
projectId,
name,
type: "LLM_AS_JUDGE",
},
}),
),
);
const rule = await prisma.evaluationRule.create({
data: {
projectId,
name: "Evaluation rule",
targetObject: "EVENT",
filter: [],
sampling: 1,
delay: 0,
assignments: {
create: [matchingEvaluator, otherEvaluator].map((evaluator) => ({
projectId,
evaluatorId: evaluator.id,
})),
},
},
});
const recordedScore = createTraceScore({
project_id: projectId,
name: recordedEvaluator.name,
metadata: { evaluator_id: recordedEvaluator.id },
});
const legacyScore = createTraceScore({
project_id: projectId,
name: matchingEvaluator.name,
metadata: {},
});
await Promise.all([
createScoresCh([recordedScore, legacyScore]),
prisma.jobExecution.createMany({
data: [recordedScore, legacyScore].map((score) => ({
projectId,
jobConfigurationId: rule.id,
jobOutputScoreId: score.id,
status: "COMPLETED",
})),
}),
]);
const result = await caller.scores.allFromEvents({
projectId,
filter: [],
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
});
const evaluatorIdByScoreId = new Map(
result.scores.map((score) => [score.id, score.evaluatorId]),
);
expect(evaluatorIdByScoreId.get(recordedScore.id)).toBe(
recordedEvaluator.id,
);
expect(evaluatorIdByScoreId.get(legacyScore.id)).toBe(
matchingEvaluator.id,
);
});
it("filters evaluator scores only by recorded evaluator metadata", async () => {
const [evaluator, otherEvaluator] = await Promise.all(
["Evaluator", "Other evaluator"].map((name) =>
prisma.evaluator.create({
data: { projectId, name, type: "CODE" },
}),
),
);
const rule = await prisma.evaluationRule.create({
data: {
projectId,
name: "Legacy rule",
targetObject: "EVENT",
filter: [],
sampling: 1,
delay: 0,
assignments: {
create: { projectId, evaluatorId: evaluator.id },
},
},
});
const directScore = createTraceScore({
project_id: projectId,
name: "direct-score",
metadata: { evaluator_id: evaluator.id },
});
const legacyScore = createTraceScore({
project_id: projectId,
name: "legacy-score-with-a-different-name",
metadata: { job_configuration_id: rule.id },
});
const otherScore = createTraceScore({
project_id: projectId,
name: "other-score",
metadata: { evaluator_id: otherEvaluator.id },
});
await createScoresCh([directScore, legacyScore, otherScore]);
const payload = {
projectId,
filter: [
{
column: "evaluatorId",
type: "stringOptions" as const,
operator: "any of" as const,
value: [evaluator.id, rule.id],
},
],
orderBy: { column: "timestamp", order: "DESC" as const },
page: 0,
limit: 50,
};
const [scores, eventScores, count, eventCount] = await Promise.all([
caller.scores.all(payload),
caller.scores.allFromEvents(payload),
caller.scores.countAll({ ...payload, orderBy: null }),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(new Set(scores.scores.map(({ id }) => id))).toEqual(
new Set([directScore.id]),
);
expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(
new Set([directScore.id]),
);
expect(count.totalCount).toBe(1);
expect(eventCount.totalCount).toBe(1);
});
it("does not match empty boolean representations when filtering boolean values", async () => {
const trueBooleanScore = createTraceScore({
project_id: projectId,
name: "boolean-score-true",
data_type: "BOOLEAN",
value: 1,
string_value: "True",
});
const falseBooleanScore = createTraceScore({
project_id: projectId,
name: "boolean-score-false",
data_type: "BOOLEAN",
value: 0,
string_value: "False",
});
const emptyBooleanScore = createTraceScore({
project_id: projectId,
name: "boolean-score-empty",
data_type: "BOOLEAN",
value: 1,
string_value: "",
});
const numericScore = createTraceScore({
project_id: projectId,
name: "numeric-score",
data_type: "NUMERIC",
value: 0.7,
string_value: null,
});
await createScoresCh([
trueBooleanScore,
falseBooleanScore,
emptyBooleanScore,
numericScore,
]);
const payload = {
projectId,
filter: [
{
column: "booleanValue",
type: "stringOptions" as const,
operator: "none of" as const,
value: ["false"],
},
],
orderBy: { column: "timestamp", order: "DESC" as const },
page: 0,
limit: 50,
};
const result = await caller.scores.all(payload);
const resultFromEvents = await caller.scores.allFromEvents(payload);
expect(result.scores.map((score) => score.id)).toEqual([
trueBooleanScore.id,
]);
expect(resultFromEvents.scores.map((score) => score.id)).toEqual([
trueBooleanScore.id,
]);
});
});
describe("observation scope filter", () => {
it("lists trace-level scores for a trace-level owner span and for no other span", async () => {
const traceId = randomUUID();
const rootObservationId = randomUUID();
const childObservationId = randomUUID();
const traceLevelScore = createTraceScore({
project_id: projectId,
trace_id: traceId,
name: "trace-level-score",
});
const rootObservationScore = createTraceScore({
project_id: projectId,
trace_id: traceId,
observation_id: rootObservationId,
name: "root-observation-score",
});
const childObservationScore = createTraceScore({
project_id: projectId,
trace_id: traceId,
observation_id: childObservationId,
name: "child-observation-score",
});
await createScoresCh([
traceLevelScore,
rootObservationScore,
childObservationScore,
]);
const scoreIdsFor = async (
observationId: string,
includeTraceLevelScores: boolean,
) => {
const payload = {
projectId,
filter: [
{
column: "traceId",
type: "string" as const,
operator: "=" as const,
value: traceId,
},
...observationScopeFilter(observationId, includeTraceLevelScores),
],
orderBy: { column: "timestamp", order: "DESC" as const },
page: 0,
limit: 50,
};
const [v3, v4] = await Promise.all([
caller.scores.all(payload),
caller.scores.allFromEvents(payload),
]);
return {
v3: v3.scores.map((score) => score.id).sort(),
v4: v4.scores.map((score) => score.id).sort(),
};
};
// Trace-level owner: its own score plus the trace-level one, each once.
const owner = await scoreIdsFor(rootObservationId, true);
const expectedOwnerScoreIds = [
traceLevelScore.id,
rootObservationScore.id,
].sort();
expect(owner.v3).toEqual(expectedOwnerScoreIds);
expect(owner.v4).toEqual(expectedOwnerScoreIds);
// Any other span stays observation-scoped.
const child = await scoreIdsFor(childObservationId, false);
expect(child.v3).toEqual([childObservationScore.id]);
expect(child.v4).toEqual([childObservationScore.id]);
});
});
maybeEvents("scores.allFromEvents trace-name filters", () => {
it("keeps scores for semantic roots without a stored trace name", async () => {
const traceId = randomUUID();
const score = createTraceScore({
project_id: projectId,
trace_id: traceId,
name: "semantic-root-score",
value: 0.9,
});
await createEventsCh([
createEvent({
trace_id: traceId,
project_id: projectId,
parent_span_id: "external-parent",
is_app_root: true,
name: "semantic-root-trace",
trace_name: "",
}),
]);
await createScoresCh([score]);
const result = await caller.scores.allFromEvents({
projectId,
filter: [
{
column: "traceName",
operator: "=",
value: "semantic-root-trace",
type: "string",
},
],
orderBy: { column: "timestamp", order: "DESC" },
page: 0,
limit: 50,
});
expect(result.scores.map(({ id }) => id)).toContain(score.id);
});
it("returns every score of a trace, not one per trace", async () => {
const traceName = `multi-score-trace-${randomUUID()}`;
const traceIds = [randomUUID(), randomUUID()];
const scores = traceIds.flatMap((traceId) =>
["a", "b", "c"].map((name) =>
createTraceScore({ project_id: projectId, trace_id: traceId, name }),
),
);
await createEventsCh(
traceIds.map((traceId) =>
createEvent({
trace_id: traceId,
project_id: projectId,
parent_span_id: "",
name: traceName,
trace_name: traceName,
}),
),
);
await createScoresCh(scores);
const payload = {
projectId,
filter: [
{
column: "traceName",
operator: "=" as const,
value: traceName,
type: "string" as const,
},
],
page: 0,
limit: 50,
};
const [result, count] = await Promise.all([
caller.scores.allFromEvents({
...payload,
orderBy: { column: "timestamp", order: "DESC" },
}),
caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
]);
expect(result.scores.map(({ id }) => id).sort()).toEqual(
scores.map(({ id }) => id).sort(),
);
expect(count.totalCount).toBe(scores.length);
});
});
describe("scores.createAnnotationScore", () => {
it("rejects empty stringValue for boolean annotation scores", async () => {
const configId = randomUUID();
const scoreName = `boolean-annotation-score-${configId.slice(0, 8)}`;
await expect(
caller.scores.createAnnotationScore({
projectId,
name: scoreName,
value: 1,
stringValue: "",
dataType: "BOOLEAN",
scoreTarget: { type: "trace", traceId: randomUUID() },
configId,
environment: "default",
} as any),
).rejects.toThrow();
});
it("accepts explicit boolean annotation stringValue labels", async () => {
const traceId = randomUUID();
const configId = randomUUID();
const scoreName = `boolean-annotation-score-${configId.slice(0, 8)}`;
await createTracesCh([
createTrace({
id: traceId,
project_id: projectId,
}),
]);
await prisma.scoreConfig.create({
data: {
id: configId,
projectId,
name: scoreName,
dataType: ScoreConfigDataType.BOOLEAN,
categories: [
{ label: "True", value: 1 },
{ label: "False", value: 0 },
],
},
});
const score = await caller.scores.createAnnotationScore({
projectId,
name: scoreName,
value: 1,
stringValue: "True",
dataType: "BOOLEAN",
scoreTarget: { type: "trace", traceId },
configId,
environment: "default",
});
expect(score.stringValue).toBe("True");
});
});
describe("scores.updateAnnotationScore", () => {
it("bumps updated_at and keeps created_at when editing an existing score", async () => {
const originalTime = new Date(Date.now() - 60 * 60 * 1000);
const configId = randomUUID();
const scoreName = `numeric-annotation-score-${configId.slice(0, 8)}`;
await prisma.scoreConfig.create({
data: {
id: configId,
projectId,
name: scoreName,
dataType: ScoreConfigDataType.NUMERIC,
},
});
const existing = createTraceScore({
project_id: projectId,
source: "ANNOTATION",
name: scoreName,
config_id: configId,
value: 1,
timestamp: originalTime.getTime(),
created_at: originalTime.getTime(),
updated_at: originalTime.getTime(),
});
await createScoresCh([existing]);
await caller.scores.updateAnnotationScore({
id: existing.id,
projectId,
name: scoreName,
value: 2,
stringValue: null,
dataType: "NUMERIC",
scoreTarget: { type: "trace", traceId: existing.trace_id! },
configId,
environment: "default",
});
const updated = await getScoreById({
projectId,
scoreId: existing.id,
});
expect(updated?.value).toBe(2);
expect(updated?.createdAt.getTime()).toBe(originalTime.getTime());
expect(updated?.updatedAt.getTime()).toBeGreaterThan(
originalTime.getTime(),
);
});
});
describe("scores.deleteAnnotationScore", () => {
it("deletes a correction ingested via the API", async () => {
const correction = createTraceScore({
project_id: projectId,
name: "output",
source: "API",
data_type: "CORRECTION",
value: 0,
long_string_value: "corrected response",
});
await createScoresCh([correction]);
await caller.scores.deleteAnnotationScore({
projectId,
id: correction.id,
});
expect(
await getScoreById({ projectId, scoreId: correction.id }),
).toBeUndefined();
});
});
describe("scores.deleteMany", () => {
it("should delete scores by ids", async () => {
// Setup
const createdScore = createTraceScore({
project_id: projectId,
});
await createScoresCh([createdScore]);
const scoreDeleteQueue = ScoreDeleteQueue.getInstance();
// When
await caller.scores.deleteMany({
projectId,
scoreIds: [createdScore.id],
});
expect(scoreDeleteQueue).not.toBeNull();
// Then
expect(scoreDeleteQueue!.add).toHaveBeenCalledWith(
QueueJobs.ScoreDelete,
expect.objectContaining({
payload: expect.objectContaining({
projectId,
scoreIds: [createdScore.id],
}),
}),
);
});
it("should delete scores via batch query", async () => {
// Setup
const scoreName = randomUUID();
const createdScore = createTraceScore({
project_id: projectId,
name: scoreName,
});
await createScoresCh([createdScore]);
const batchActionQueue = BatchActionQueue.getInstance();
// When
await caller.scores.deleteMany({
projectId,
scoreIds: null,
isBatchAction: true,
query: {
orderBy: { column: "timestamp", order: "ASC" },
filter: [
{
column: "name",
operator: "=",
value: scoreName,
type: "string",
},
],
},
});
expect(batchActionQueue).not.toBeNull();
// Then
expect(batchActionQueue!.add).toHaveBeenCalledWith(
QueueJobs.BatchActionProcessingJob,
expect.objectContaining({
payload: expect.objectContaining({
projectId,
actionId: "score-delete",
}),
}),
expect.objectContaining({}),
);
});
it("should throw an error if batchAction and scoreIds are missing", async () => {
// When
const promise = caller.scores.deleteMany({
projectId,
scoreIds: null,
isBatchAction: false,
});
// Then
await expect(promise).rejects.toMatchObject({
code: "BAD_REQUEST",
message:
"Either batchAction or scoreIds must be provided to delete scores.",
});
});
});
describe("scores.getScoreColumns", () => {
it("should distinguish trace-level from trace-scoped score discovery", async () => {
const traceId = randomUUID();
const observationId = randomUUID();
await createTracesCh([
createTrace({
id: traceId,
project_id: projectId,
}),
]);
await createObservationsCh([
createObservation({
id: observationId,
trace_id: traceId,
project_id: projectId,
}),
]);
await createScoresCh([
createTraceScore({
project_id: projectId,
trace_id: traceId,
observation_id: null,
name: "trace_level_score",
source: "API",
data_type: "NUMERIC",
value: 0.9,
}),
createTraceScore({
project_id: projectId,
trace_id: traceId,
observation_id: observationId,
name: "observation_level_score",
source: "API",
data_type: "NUMERIC",
value: 0.7,
}),
]);
const traceScopedColumns = await caller.scores.getScoreColumns({
projectId,
filter: [
{
column: "traceId",
operator: "is not null",
value: "",
type: "null",
},
],
});
const traceLevelColumns = await caller.scores.getScoreColumns({
projectId,
filter: [
{
column: "traceId",
operator: "is not null",
value: "",
type: "null",
},
{
column: "observationId",
operator: "is null",
value: "",
type: "null",
},
],
});
expect(
traceScopedColumns.scoreColumns.map((column) => column.name),
).toEqual(
expect.arrayContaining([
"trace_level_score",
"observation_level_score",
]),
);
expect(
traceLevelColumns.scoreColumns.map((column) => column.name),
).toEqual(["trace_level_score"]);
});
});
describe("scores.filterOptions", () => {
it("returns static boolean value options for both scores views", async () => {
await expect(
caller.scores.filterOptions({ projectId }),
).resolves.toMatchObject({
booleanValue: [{ value: "true" }, { value: "false" }],
});
await expect(
caller.scores.filterOptionsFromEvents({ projectId }),
).resolves.toMatchObject({
booleanValue: [{ value: "true" }, { value: "false" }],
});
});
});
describe("scoreConfigs.all", () => {
it("should paginate score configs deterministically when createdAt timestamps tie", async () => {
const sharedCreatedAt = new Date("2100-05-12T00:00:00.000Z");
const configIds: string[] = [randomUUID(), randomUUID(), randomUUID()];
await prisma.scoreConfig.createMany({
data: configIds.map((id, index) => ({
id,
projectId,
name: `trpc-tie-config-${index}-${id.slice(0, 8)}`,
description: `trpc tie config ${index}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: index,
maxValue: index + 1,
createdAt: sharedCreatedAt,
updatedAt: sharedCreatedAt,
})),
});
const firstPage = await caller.scoreConfigs.all({
projectId,
page: 0,
limit: 2,
});
const secondPage = await caller.scoreConfigs.all({
projectId,
page: 1,
limit: 2,
});
const tiedIds = [...firstPage.configs, ...secondPage.configs]
.filter((config) => configIds.includes(config.id))
.map((config) => config.id);
expect(tiedIds).toEqual(configIds.slice().sort());
expect(new Set(tiedIds).size).toBe(configIds.length);
});
});
describe("scoreConfigs.byId", () => {
it("returns NOT_FOUND when the config is missing in the project", async () => {
await expect(
caller.scoreConfigs.byId({
projectId,
id: randomUUID(),
}),
).rejects.toMatchObject({
code: "NOT_FOUND",
message: "No score config with this id in this project.",
});
});
it("returns the config when it exists in the project", async () => {
const config = await prisma.scoreConfig.create({
data: {
projectId,
name: `byid-${randomUUID().slice(0, 8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
});
await expect(
caller.scoreConfigs.byId({
projectId,
id: config.id,
}),
).resolves.toMatchObject({
id: config.id,
name: config.name,
dataType: ScoreConfigDataType.NUMERIC,
});
});
});
describe("scoreConfigs.appendCategory", () => {
it("keeps both categories when two appends race", async () => {
const config = await prisma.scoreConfig.create({
data: {
projectId,
name: `append-race-${randomUUID().slice(0, 8)}`,
dataType: ScoreConfigDataType.CATEGORICAL,
categories: [{ label: "internal_user", value: 0 }],
},
});
await Promise.all([
caller.scoreConfigs.appendCategory({
projectId,
id: config.id,
label: "pen_testing",
}),
caller.scoreConfigs.appendCategory({
projectId,
id: config.id,
label: "just_testing",
}),
]);
const latest = await caller.scoreConfigs.byId({
projectId,
id: config.id,
});
expect(
latest.categories?.map((category) => category.label).sort(),
).toEqual(["internal_user", "just_testing", "pen_testing"]);
});
it("rejects a duplicate label", async () => {
const config = await prisma.scoreConfig.create({
data: {
projectId,
name: `append-dup-${randomUUID().slice(0, 8)}`,
dataType: ScoreConfigDataType.CATEGORICAL,
categories: [{ label: "internal_user", value: 0 }],
},
});
await expect(
caller.scoreConfigs.appendCategory({
projectId,
id: config.id,
label: "internal_user",
}),
).rejects.toMatchObject({
code: "BAD_REQUEST",
message: "A category with this name already exists",
});
});
it("rejects a non-categorical config", async () => {
const config = await prisma.scoreConfig.create({
data: {
projectId,
name: `append-numeric-${randomUUID().slice(0, 8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
});
await expect(
caller.scoreConfigs.appendCategory({
projectId,
id: config.id,
label: "pen_testing",
}),
).rejects.toMatchObject({
code: "BAD_REQUEST",
message: "Only categorical score configs can append categories.",
});
});
});
});