🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): stop inferring usage for failed and cancelled AI gateway generations (#17959)
Hassieb Pakzadb7b4c7cc0cb1
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex b0f3f4942f..a61896e639 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -40,6 +40,8 @@ import { OtelIngestionQueue } from "../redis/otelIngestionQueue";import { isValidDateString, flattenJsonToPathArrays } from "./utils";import { convertDateToClickhouseDateTime } from "../clickhouse/client";+export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";+// Foreign level vocabularies observed from OTel senders (OTel severity// names, python logging, loguru, console) mapped onto the Langfuse enum.// The classic ingestion API keeps its strict enum; only the@@ -1891,10 +1893,9 @@ export class OtelIngestionProcessor {delete rawFilteredAttributes[key];});- // Gateway spans only use known Langfuse attributes for values that are- // represented by canonical trace and observation fields. Keep unknown- // attributes available for diagnostics.- if (instrumentationScopeName === "langfuse-ai-gateway") {+ // Gateway observation attributes are represented by canonical fields.+ // Keep unknown attributes available for diagnostics.+ if (instrumentationScopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME) {for (const key of Object.values(LangfuseOtelSpanAttributes)) {delete rawFilteredAttributes[key];}diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex a44ba13158..7fc90692df 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -60,6 +60,7 @@ import {sanitizeSdkMetricTagValue,type IngestionAttribution,type PricingTierMatchAttributes,+ AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,} from "@langfuse/shared/src/server";import { tokenCountAsync } from "../../features/tokenisation/async-usage";@@ -68,6 +69,7 @@ import {convertJsonSchemaToRecord,convertPostgresJsonToMetadataRecord,convertRecordValuesToString,+ hasAiGatewayScope,overwriteObject,} from "./utils";import { randomUUID } from "crypto";@@ -355,6 +357,8 @@ export class IngestionService {modelParameters,metadata,},+ isAiGatewayGeneration:+ eventData.scopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,observationRecord: {id: eventData.spanId,project_id: eventData.projectId,@@ -362,6 +366,7 @@ export class IngestionService {provided_model_name: eventData.modelName,provided_usage_details: eventData.providedUsageDetails ?? {},provided_cost_details: eventData.providedCostDetails ?? {},+ level: eventData.level,input,output,},@@ -1086,6 +1091,9 @@ export class IngestionService {const generationUsage = await this.getGenerationUsage({projectId,pricingMatchAttributeValues,+ isAiGatewayGeneration: timeSortedEvents.some((event) =>+ hasAiGatewayScope(event.body?.metadata),+ ),observationRecord: mergedObservationRecord,});const finalObservationRecord = {@@ -1321,6 +1329,7 @@ export class IngestionService {private async getGenerationUsage(params: {projectId: string;pricingMatchAttributeValues?: PricingTierMatchAttributeValues;+ isAiGatewayGeneration?: boolean;observationRecord: Pick<ObservationRecordInsertType,| "project_id"@@ -1344,8 +1353,12 @@ export class IngestionService {| "usage_pricing_tier_name">> {- const { projectId, observationRecord, pricingMatchAttributeValues } =- params;+ const {+ projectId,+ observationRecord,+ pricingMatchAttributeValues,+ isAiGatewayGeneration = false,+ } = params;const { model: internalModel, pricingTiers } =observationRecord.provided_model_name? await findModel({@@ -1357,6 +1370,7 @@ export class IngestionService {const final_usage_details = await this.getUsageUnits(observationRecord,internalModel,+ isAiGatewayGeneration,);// Match pricing tier based on usage_details. Skip when usage is empty@@ -1450,6 +1464,7 @@ export class IngestionService {| "id">,model: Model | null | undefined,+ isAiGatewayGeneration: boolean,): Promise<Pick<ObservationRecordInsertType,@@ -1468,12 +1483,17 @@ export class IngestionService {observationRecord.provided_cost_details ?? {},).some((value) => value != null);+ // The AI gateway forwards provider-reported usage whenever the provider+ // returns it. Without it (failed or cancelled upstream calls), the true+ // usage is unknown: the request body holds tool schemas and encrypted+ // reasoning, so tokenizing it fabricates usage and cost.if (// Manual tokenisation when no user provided usage or cost and generation has not status ERRORmodel &&Object.keys(providedUsageDetails).length === 0 &&!hasProvidedCostDetails &&- observationRecord.level !== ObservationLevel.ERROR+ observationRecord.level !== ObservationLevel.ERROR &&+ !isAiGatewayGeneration) {try {let newInputCount: number | undefined;diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.tsindex 2b2a7ee46e..8a30d5aa05 100644--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts@@ -1362,6 +1362,147 @@ describe("Token Cost Calculation", () => {expect(generation.usage_details.total).toBeUndefined();});+ describe("tokenizer inference on failed and cancelled generations", () => {+ const gatewayScope = { name: "langfuse-ai-gateway", version: "0.1.0" };++ beforeEach(() => {+ tokenisationMocks.tokenCountAsyncOverride = async () => 7;+ });++ afterEach(() => {+ tokenisationMocks.tokenCountAsyncOverride = null;+ });++ it("should tokenize a DEFAULT generation on the events table", async () => {+ const eventRecord = await (mockIngestionService as any).createEventRecord(+ {+ spanId: generationId,+ traceId: uuidv4(),+ projectId,+ startTimeISO: new Date().toISOString(),+ type: "GENERATION",+ modelName,+ input: "hello world",+ output: "hey whassup",+ },+ "testfile.txt",+ );++ expect(eventRecord.usage_details).toEqual({+ input: 7,+ output: 7,+ total: 14,+ });+ });++ it("should skip tokenization for ERROR generations written to the events table", async () => {+ const eventRecord = await (mockIngestionService as any).createEventRecord(+ {+ spanId: generationId,+ traceId: uuidv4(),+ projectId,+ startTimeISO: new Date().toISOString(),+ type: "GENERATION",+ modelName,+ level: "ERROR",+ input: "hello world",+ output: "hey whassup",+ },+ "testfile.txt",+ );++ expect(eventRecord.level).toBe("ERROR");+ expect(eventRecord.model_id).toBe(tokenModelData.id);+ expect(eventRecord.usage_details).toEqual({});+ expect(eventRecord.cost_details).toEqual({});+ });++ it("should not tokenize a cancelled gateway generation on the events table", async () => {+ const eventRecord = await (mockIngestionService as any).createEventRecord(+ {+ spanId: generationId,+ traceId: uuidv4(),+ projectId,+ startTimeISO: new Date().toISOString(),+ type: "GENERATION",+ modelName,+ level: "WARNING",+ statusMessage: "Client cancelled the response",+ scopeName: gatewayScope.name,+ metadata: { scope: gatewayScope },+ input: "hello world",+ output: "[]",+ },+ "testfile.txt",+ );++ expect(eventRecord.model_id).toBe(tokenModelData.id);+ expect(eventRecord.usage_details).toEqual({});+ expect(eventRecord.cost_details).toEqual({});+ });++ it("should cost provider-reported usage on a cancelled gateway generation", async () => {+ const eventRecord = await (mockIngestionService as any).createEventRecord(+ {+ spanId: generationId,+ traceId: uuidv4(),+ projectId,+ startTimeISO: new Date().toISOString(),+ type: "GENERATION",+ modelName,+ level: "WARNING",+ scopeName: gatewayScope.name,+ metadata: { scope: gatewayScope },+ providedUsageDetails: { input: 10, output: 5 },+ input: "hello world",+ output: "[]",+ },+ "testfile.txt",+ );++ expect(eventRecord.usage_details).toEqual({+ input: 10,+ output: 5,+ total: 15,+ });+ expect(eventRecord.cost_details.input).toBe(0.1);+ expect(eventRecord.cost_details.output).toBe(0.1);+ });++ it("should not tokenize a cancelled gateway generation on the observations table", async () => {+ const events = [+ {+ id: uuidv4(),+ type: "generation-create",+ timestamp: new Date().toISOString(),+ body: {+ id: generationId,+ startTime: new Date().toISOString(),+ model: modelName,+ level: "WARNING",+ statusMessage: "Client cancelled the response",+ metadata: { scope: gatewayScope },+ input: "hello world",+ output: "[]",+ },+ },+ ];++ await (mockIngestionService as any).processObservationEventList({+ projectId,+ entityId: generationId,+ createdAtTimestamp: new Date(),+ observationEventList: events,+ });++ const [tableName, generation] = mockAddToClickhouseWriter.mock.calls[0];+ expect(tableName).toBe("observations");+ expect(generation.internal_model_id).toBe(tokenModelData.id);+ expect(generation.usage_details).toEqual({});+ expect(generation.cost_details).toEqual({});+ });+ });+it("should skip tokenization and leave usage details blank when cost details are provided", async () => {const generationUsage1 = {model: modelName,diff --git a/worker/src/services/IngestionService/utils.ts b/worker/src/services/IngestionService/utils.tsindex b6e35c036b..b144076f5f 100644--- a/worker/src/services/IngestionService/utils.ts+++ b/worker/src/services/IngestionService/utils.ts@@ -1,5 +1,6 @@/* eslint-disable no-nested-ternary */import { JsonNested, Prisma } from "@langfuse/shared";+import { AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME } from "@langfuse/shared/src/server";import { mergeWith, merge } from "lodash";// Theoretically this returns Record<string, unknown>, but it would be hard to align the typing accordingly.@@ -61,6 +62,18 @@ export const convertRecordValuesToString = ();};+// OTel-ingested observation events carry the instrumentation scope as+// `metadata.scope` ({ name, version, attributes }).+export const hasAiGatewayScope = (metadata: unknown): boolean => {+ if (!metadata || typeof metadata !== "object") return false;+ const scope = (metadata as { scope?: unknown }).scope;+ return (+ typeof scope === "object" &&+ scope !== null &&+ (scope as { name?: unknown }).name === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME+ );+};+export function overwriteObject(a: {id: string;