Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(analytics-integrations): source generation usage units from usage_details (#17254)

Niklas Semmlera3985d3cd3e8
diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.ts
index 1f6e92b570..e565a2d25a 100644
--- a/packages/shared/src/server/repositories/observations.ts
+++ b/packages/shared/src/server/repositories/observations.ts
@@ -1888,9 +1888,9 @@ export const getGenerationsForAnalyticsIntegrations = async function* (
o.id as id,
o.total_cost as total_cost,
if(isNull(completion_start_time), NULL, date_diff('millisecond', start_time, completion_start_time)) as time_to_first_token,
- o.usage_details['total'] as input_tokens,
+ o.usage_details['input'] as input_tokens,
o.usage_details['output'] as output_tokens,
- o.cost_details['total'] as total_tokens,
+ o.usage_details['total'] as total_tokens,
o.project_id as project_id,
if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000) as latency,
o.provided_model_name as model,
diff --git a/worker/src/__tests__/analyticsIntegrationGenerationUsage.test.ts b/worker/src/__tests__/analyticsIntegrationGenerationUsage.test.ts
new file mode 100644
index 0000000000..1fab6ac379
--- /dev/null
+++ b/worker/src/__tests__/analyticsIntegrationGenerationUsage.test.ts
@@ -0,0 +1,60 @@
+import { randomUUID } from "crypto";
+import { describe, expect, it } from "vitest";
+import {
+ createObservation,
+ createObservationsCh,
+ createOrgProjectAndApiKey,
+ createTrace,
+ createTracesCh,
+ getGenerationsForAnalyticsIntegrations,
+} from "@langfuse/shared/src/server";
+
+// Regression for the legacy observations-based analytics export: generation
+// usage units must be sourced from usage_details (input/output/total), never
+// from usage_details['total'] for input or cost_details['total'] for total.
+describe("getGenerationsForAnalyticsIntegrations usage units", () => {
+ it("maps input/output/total units from usage_details, not cost_details", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+
+ const traceId = randomUUID();
+ const startTime = new Date();
+
+ await createTracesCh([
+ createTrace({
+ project_id: projectId,
+ id: traceId,
+ timestamp: startTime.getTime(),
+ }),
+ ]);
+
+ // Distinct values so a wrong source is unambiguous:
+ // usage input/output/total = 11/22/33, cost total = 3.
+ const observation = createObservation({
+ project_id: projectId,
+ trace_id: traceId,
+ type: "GENERATION",
+ start_time: startTime.getTime(),
+ usage_details: { input: 11, output: 22, total: 33 },
+ cost_details: { input: 1, output: 2, total: 3 },
+ total_cost: 3,
+ });
+ await createObservationsCh([observation]);
+
+ const records = [];
+ for await (const record of getGenerationsForAnalyticsIntegrations(
+ projectId,
+ "test-project",
+ new Date(startTime.getTime() - 60_000),
+ new Date(startTime.getTime() + 60_000),
+ )) {
+ records.push(record);
+ }
+
+ const record = records.find((r) => r.langfuse_id === observation.id);
+ expect(record).toBeDefined();
+ expect(Number(record!.langfuse_input_units)).toBe(11);
+ expect(Number(record!.langfuse_output_units)).toBe(22);
+ expect(Number(record!.langfuse_total_units)).toBe(33);
+ expect(Number(record!.langfuse_cost_usd)).toBe(3);
+ });
+});