🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(api): treat prompt version metrics dimensions as numbers (#16746)
Niklas Semmlerd91c904cb913
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex 0bd408632a..df551d3d5a 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -444,7 +444,7 @@ export const observationsView: ViewDeclarationType = {promptVersion: {sql: "observations.prompt_version",alias: "promptVersion",- type: "string",+ type: "number",description: "Version of the prompt used for the observation.",},userId: {@@ -723,7 +723,7 @@ const scoreBaseDimensions: DimensionsDeclarationType = {observationPromptVersion: {sql: "observations.prompt_version",alias: "observationPromptVersion",- type: "string",+ type: "number",relationTable: "observations",description: "Version of the prompt used for the observation.",},@@ -810,7 +810,7 @@ const scoresV2BaseDimensions: DimensionsDeclarationType = {observationPromptVersion: {sql: "events_observations.prompt_version",alias: "observationPromptVersion",- type: "string",+ type: "number",relationTable: "events_observations",description: "Version of the prompt used for the observation.",},@@ -1308,7 +1308,7 @@ export const eventsObservationsView: ViewDeclarationType = {promptVersion: {sql: "events_observations.prompt_version",alias: "promptVersion",- type: "string",+ type: "number",description: "Version of the prompt used for the observation.",},startTimeMonth: {diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.tsindex 86f0ead80a..5085b671b2 100644--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts@@ -46,6 +46,93 @@ describe("queryBuilder filter type validation", () => {expect(view.dimensions.value).toBeUndefined();});+ it.each(["v1", "v2"] as const)(+ "declares prompt version dimensions as numbers in the %s scores view",+ (version) => {+ const scoresView = getViewDeclaration("scores-numeric", version);+ const observationsView = getViewDeclaration("observations", version);++ expect(scoresView.dimensions.observationPromptVersion?.type).toBe(+ "number",+ );+ expect(observationsView.dimensions.promptVersion?.type).toBe("number");+ },+ );++ it.each(["v1", "v2"] as const)(+ "lowers a numeric observationPromptVersion filter in the %s scores view",+ async (version) => {+ const { query } = await buildQueryWithFilter(+ {+ column: "observationPromptVersion",+ operator: "=",+ value: 2,+ type: "number",+ },+ { view: "scores-numeric" },+ version,+ );++ expect(query).toContain("prompt_version = {numberFilter");+ expect(query).not.toContain("position(");+ },+ );++ it.each(["v1", "v2"] as const)(+ "rejects a string observationPromptVersion filter in the %s scores view",+ async (version) => {+ await expect(+ buildQueryWithFilter(+ {+ column: "observationPromptVersion",+ operator: "contains",+ value: "2",+ type: "string",+ },+ { view: "scores-numeric" },+ version,+ ),+ ).rejects.toThrow(+ "Filter type 'string' is not supported for dimension type 'number'",+ );+ },+ );++ it.each(["v1", "v2"] as const)(+ "selects observationPromptVersion when grouping scores in %s",+ async (version) => {+ const { query } = await new QueryBuilder(undefined, version).build(+ {+ ...baseQuery,+ view: "scores-numeric",+ dimensions: [{ field: "observationPromptVersion" }],+ } as QueryType,+ "test-project",+ );++ expect(query).toContain("prompt_version");+ expect(query).toContain("observationPromptVersion");+ },+ );++ it.each(["v1", "v2"] as const)(+ "lowers a numeric promptVersion filter in the %s observations view",+ async (version) => {+ const { query } = await buildQueryWithFilter(+ {+ column: "promptVersion",+ operator: "=",+ value: 3,+ type: "number",+ },+ undefined,+ version,+ );++ expect(query).toContain("prompt_version = {numberFilter");+ },+ );+it.each([{name: "arrayOptions on scalar string dimension",diff --git a/packages/shared/src/features/query/server/queryBuilder.ts b/packages/shared/src/features/query/server/queryBuilder.tsindex 7ae8d7b394..fc4b902424 100644--- a/packages/shared/src/features/query/server/queryBuilder.ts+++ b/packages/shared/src/features/query/server/queryBuilder.ts@@ -564,7 +564,7 @@ export class QueryBuilder {} else {clickhouseSelect = dimension.sql;}- type = "string";+ type = dimension.type ?? "string";if (dimension.relationTable) {clickhouseTableName = dimension.relationTable;}diff --git a/web/src/__tests__/server/metrics-v2-api.servertest.ts b/web/src/__tests__/server/metrics-v2-api.servertest.tsindex cd71c91d03..183256cf3f 100644--- a/web/src/__tests__/server/metrics-v2-api.servertest.ts+++ b/web/src/__tests__/server/metrics-v2-api.servertest.ts@@ -1363,6 +1363,8 @@ describe("/api/public/v2/metrics API Endpoint", () => {project_id: projectId,name: "test-observation-for-score-v2",provided_model_name: "gpt-4-turbo",+ prompt_name: "ticket-intent-router",+ prompt_version: 2,start_time: Date.now() * 1000,}),]);@@ -1406,6 +1408,16 @@ describe("/api/public/v2/metrics API Endpoint", () => {"test-observation-for-score-v2",(row: any) => row.observationName === "test-observation-for-score-v2",],+ [+ "observationPromptName",+ "ticket-intent-router",+ (row: any) => row.observationPromptName === "ticket-intent-router",+ ],+ [+ "observationPromptVersion",+ 2,+ (row: any) => Number(row.observationPromptVersion) === 2,+ ],])("should support %s dimension via events table",async (dimensionField, expectedValue, findRowFn) => {@@ -1439,5 +1451,91 @@ describe("/api/public/v2/metrics API Endpoint", () => {expect(foundRow).toBeDefined();},);++ it("should filter scores-numeric by observationPromptVersion as a number", async () => {+ const query = {+ view: "scores-numeric",+ dimensions: [{ field: "name" }],+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [+ {+ column: "name",+ operator: "=",+ value: "score-with-observation-v2",+ type: "string",+ },+ {+ column: "observationPromptVersion",+ operator: "=",+ value: 2,+ type: "number",+ },+ ],+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),+ toTimestamp: new Date().toISOString(),+ };++ const response = await makeZodVerifiedAPICall(+ GetMetricsV1Response,+ "GET",+ `/api/public/v2/metrics?query=${encodeURIComponent(JSON.stringify(query))}`,+ );++ expect(response.status).toBe(200);+ expect(response.body.data).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ name: "score-with-observation-v2",+ }),+ ]),+ );++ const mismatched = await makeZodVerifiedAPICall(+ GetMetricsV1Response,+ "GET",+ `/api/public/v2/metrics?query=${encodeURIComponent(+ JSON.stringify({+ ...query,+ filters: [+ query.filters[0],+ {+ column: "observationPromptVersion",+ operator: "=",+ value: 99,+ type: "number",+ },+ ],+ }),+ )}`,+ );++ expect(mismatched.status).toBe(200);+ expect(mismatched.body.data).toHaveLength(0);+ });++ it("rejects a string filter on observationPromptVersion", async () => {+ const query = {+ view: "scores-numeric",+ dimensions: [{ field: "name" }],+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [+ {+ column: "observationPromptVersion",+ operator: "=",+ value: "2",+ type: "string",+ },+ ],+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),+ toTimestamp: new Date().toISOString(),+ };++ const response = await makeAPICall(+ "GET",+ `/api/public/v2/metrics?query=${encodeURIComponent(JSON.stringify(query))}`,+ );++ expect(response.status).toBe(400);+ });});});