Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(api): treat prompt version metrics dimensions as numbers (#16746)

Niklas Semmlerd91c904cb913
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index 0bd408632a..df551d3d5a 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -444,7 +444,7 @@ export const observationsView: ViewDeclarationType = {
promptVersion: {
sql: "observations.prompt_version",
alias: "promptVersion",
- type: "string",
+ type: "number",
description: "Version of the prompt used for the observation.",
},
userId: {
@@ -723,7 +723,7 @@ const scoreBaseDimensions: DimensionsDeclarationType = {
observationPromptVersion: {
sql: "observations.prompt_version",
alias: "observationPromptVersion",
- type: "string",
+ type: "number",
relationTable: "observations",
description: "Version of the prompt used for the observation.",
},
@@ -810,7 +810,7 @@ const scoresV2BaseDimensions: DimensionsDeclarationType = {
observationPromptVersion: {
sql: "events_observations.prompt_version",
alias: "observationPromptVersion",
- type: "string",
+ type: "number",
relationTable: "events_observations",
description: "Version of the prompt used for the observation.",
},
@@ -1308,7 +1308,7 @@ export const eventsObservationsView: ViewDeclarationType = {
promptVersion: {
sql: "events_observations.prompt_version",
alias: "promptVersion",
- type: "string",
+ type: "number",
description: "Version of the prompt used for the observation.",
},
startTimeMonth: {
diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
index 86f0ead80a..5085b671b2 100644
--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
@@ -46,6 +46,93 @@ describe("queryBuilder filter type validation", () => {
expect(view.dimensions.value).toBeUndefined();
});
+ it.each(["v1", "v2"] as const)(
+ "declares prompt version dimensions as numbers in the %s scores view",
+ (version) => {
+ const scoresView = getViewDeclaration("scores-numeric", version);
+ const observationsView = getViewDeclaration("observations", version);
+
+ expect(scoresView.dimensions.observationPromptVersion?.type).toBe(
+ "number",
+ );
+ expect(observationsView.dimensions.promptVersion?.type).toBe("number");
+ },
+ );
+
+ it.each(["v1", "v2"] as const)(
+ "lowers a numeric observationPromptVersion filter in the %s scores view",
+ async (version) => {
+ const { query } = await buildQueryWithFilter(
+ {
+ column: "observationPromptVersion",
+ operator: "=",
+ value: 2,
+ type: "number",
+ },
+ { view: "scores-numeric" },
+ version,
+ );
+
+ expect(query).toContain("prompt_version = {numberFilter");
+ expect(query).not.toContain("position(");
+ },
+ );
+
+ it.each(["v1", "v2"] as const)(
+ "rejects a string observationPromptVersion filter in the %s scores view",
+ async (version) => {
+ await expect(
+ buildQueryWithFilter(
+ {
+ column: "observationPromptVersion",
+ operator: "contains",
+ value: "2",
+ type: "string",
+ },
+ { view: "scores-numeric" },
+ version,
+ ),
+ ).rejects.toThrow(
+ "Filter type 'string' is not supported for dimension type 'number'",
+ );
+ },
+ );
+
+ it.each(["v1", "v2"] as const)(
+ "selects observationPromptVersion when grouping scores in %s",
+ async (version) => {
+ const { query } = await new QueryBuilder(undefined, version).build(
+ {
+ ...baseQuery,
+ view: "scores-numeric",
+ dimensions: [{ field: "observationPromptVersion" }],
+ } as QueryType,
+ "test-project",
+ );
+
+ expect(query).toContain("prompt_version");
+ expect(query).toContain("observationPromptVersion");
+ },
+ );
+
+ it.each(["v1", "v2"] as const)(
+ "lowers a numeric promptVersion filter in the %s observations view",
+ async (version) => {
+ const { query } = await buildQueryWithFilter(
+ {
+ column: "promptVersion",
+ operator: "=",
+ value: 3,
+ type: "number",
+ },
+ undefined,
+ version,
+ );
+
+ expect(query).toContain("prompt_version = {numberFilter");
+ },
+ );
+
it.each([
{
name: "arrayOptions on scalar string dimension",
diff --git a/packages/shared/src/features/query/server/queryBuilder.ts b/packages/shared/src/features/query/server/queryBuilder.ts
index 7ae8d7b394..fc4b902424 100644
--- a/packages/shared/src/features/query/server/queryBuilder.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.ts
@@ -564,7 +564,7 @@ export class QueryBuilder {
} else {
clickhouseSelect = dimension.sql;
}
- type = "string";
+ type = dimension.type ?? "string";
if (dimension.relationTable) {
clickhouseTableName = dimension.relationTable;
}
diff --git a/web/src/__tests__/server/metrics-v2-api.servertest.ts b/web/src/__tests__/server/metrics-v2-api.servertest.ts
index cd71c91d03..183256cf3f 100644
--- a/web/src/__tests__/server/metrics-v2-api.servertest.ts
+++ b/web/src/__tests__/server/metrics-v2-api.servertest.ts
@@ -1363,6 +1363,8 @@ describe("/api/public/v2/metrics API Endpoint", () => {
project_id: projectId,
name: "test-observation-for-score-v2",
provided_model_name: "gpt-4-turbo",
+ prompt_name: "ticket-intent-router",
+ prompt_version: 2,
start_time: Date.now() * 1000,
}),
]);
@@ -1406,6 +1408,16 @@ describe("/api/public/v2/metrics API Endpoint", () => {
"test-observation-for-score-v2",
(row: any) => row.observationName === "test-observation-for-score-v2",
],
+ [
+ "observationPromptName",
+ "ticket-intent-router",
+ (row: any) => row.observationPromptName === "ticket-intent-router",
+ ],
+ [
+ "observationPromptVersion",
+ 2,
+ (row: any) => Number(row.observationPromptVersion) === 2,
+ ],
])(
"should support %s dimension via events table",
async (dimensionField, expectedValue, findRowFn) => {
@@ -1439,5 +1451,91 @@ describe("/api/public/v2/metrics API Endpoint", () => {
expect(foundRow).toBeDefined();
},
);
+
+ it("should filter scores-numeric by observationPromptVersion as a number", async () => {
+ const query = {
+ view: "scores-numeric",
+ dimensions: [{ field: "name" }],
+ metrics: [{ measure: "count", aggregation: "count" }],
+ filters: [
+ {
+ column: "name",
+ operator: "=",
+ value: "score-with-observation-v2",
+ type: "string",
+ },
+ {
+ column: "observationPromptVersion",
+ operator: "=",
+ value: 2,
+ type: "number",
+ },
+ ],
+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),
+ toTimestamp: new Date().toISOString(),
+ };
+
+ const response = await makeZodVerifiedAPICall(
+ GetMetricsV1Response,
+ "GET",
+ `/api/public/v2/metrics?query=${encodeURIComponent(JSON.stringify(query))}`,
+ );
+
+ expect(response.status).toBe(200);
+ expect(response.body.data).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ name: "score-with-observation-v2",
+ }),
+ ]),
+ );
+
+ const mismatched = await makeZodVerifiedAPICall(
+ GetMetricsV1Response,
+ "GET",
+ `/api/public/v2/metrics?query=${encodeURIComponent(
+ JSON.stringify({
+ ...query,
+ filters: [
+ query.filters[0],
+ {
+ column: "observationPromptVersion",
+ operator: "=",
+ value: 99,
+ type: "number",
+ },
+ ],
+ }),
+ )}`,
+ );
+
+ expect(mismatched.status).toBe(200);
+ expect(mismatched.body.data).toHaveLength(0);
+ });
+
+ it("rejects a string filter on observationPromptVersion", async () => {
+ const query = {
+ view: "scores-numeric",
+ dimensions: [{ field: "name" }],
+ metrics: [{ measure: "count", aggregation: "count" }],
+ filters: [
+ {
+ column: "observationPromptVersion",
+ operator: "=",
+ value: "2",
+ type: "string",
+ },
+ ],
+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),
+ toTimestamp: new Date().toISOString(),
+ };
+
+ const response = await makeAPICall(
+ "GET",
+ `/api/public/v2/metrics?query=${encodeURIComponent(JSON.stringify(query))}`,
+ );
+
+ expect(response.status).toBe(400);
+ });
});
});