Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): stop inferring usage for failed and cancelled AI gateway generations (#17959)

Hassieb Pakzadb7b4c7cc0cb1
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index b0f3f4942f..a61896e639 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -40,6 +40,8 @@ import { OtelIngestionQueue } from "../redis/otelIngestionQueue";
import { isValidDateString, flattenJsonToPathArrays } from "./utils";
import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";
+
// Foreign level vocabularies observed from OTel senders (OTel severity
// names, python logging, loguru, console) mapped onto the Langfuse enum.
// The classic ingestion API keeps its strict enum; only the
@@ -1891,10 +1893,9 @@ export class OtelIngestionProcessor {
delete rawFilteredAttributes[key];
});
- // Gateway spans only use known Langfuse attributes for values that are
- // represented by canonical trace and observation fields. Keep unknown
- // attributes available for diagnostics.
- if (instrumentationScopeName === "langfuse-ai-gateway") {
+ // Gateway observation attributes are represented by canonical fields.
+ // Keep unknown attributes available for diagnostics.
+ if (instrumentationScopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME) {
for (const key of Object.values(LangfuseOtelSpanAttributes)) {
delete rawFilteredAttributes[key];
}
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index a44ba13158..7fc90692df 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -60,6 +60,7 @@ import {
sanitizeSdkMetricTagValue,
type IngestionAttribution,
type PricingTierMatchAttributes,
+ AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
} from "@langfuse/shared/src/server";
import { tokenCountAsync } from "../../features/tokenisation/async-usage";
@@ -68,6 +69,7 @@ import {
convertJsonSchemaToRecord,
convertPostgresJsonToMetadataRecord,
convertRecordValuesToString,
+ hasAiGatewayScope,
overwriteObject,
} from "./utils";
import { randomUUID } from "crypto";
@@ -355,6 +357,8 @@ export class IngestionService {
modelParameters,
metadata,
},
+ isAiGatewayGeneration:
+ eventData.scopeName === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME,
observationRecord: {
id: eventData.spanId,
project_id: eventData.projectId,
@@ -362,6 +366,7 @@ export class IngestionService {
provided_model_name: eventData.modelName,
provided_usage_details: eventData.providedUsageDetails ?? {},
provided_cost_details: eventData.providedCostDetails ?? {},
+ level: eventData.level,
input,
output,
},
@@ -1086,6 +1091,9 @@ export class IngestionService {
const generationUsage = await this.getGenerationUsage({
projectId,
pricingMatchAttributeValues,
+ isAiGatewayGeneration: timeSortedEvents.some((event) =>
+ hasAiGatewayScope(event.body?.metadata),
+ ),
observationRecord: mergedObservationRecord,
});
const finalObservationRecord = {
@@ -1321,6 +1329,7 @@ export class IngestionService {
private async getGenerationUsage(params: {
projectId: string;
pricingMatchAttributeValues?: PricingTierMatchAttributeValues;
+ isAiGatewayGeneration?: boolean;
observationRecord: Pick<
ObservationRecordInsertType,
| "project_id"
@@ -1344,8 +1353,12 @@ export class IngestionService {
| "usage_pricing_tier_name"
>
> {
- const { projectId, observationRecord, pricingMatchAttributeValues } =
- params;
+ const {
+ projectId,
+ observationRecord,
+ pricingMatchAttributeValues,
+ isAiGatewayGeneration = false,
+ } = params;
const { model: internalModel, pricingTiers } =
observationRecord.provided_model_name
? await findModel({
@@ -1357,6 +1370,7 @@ export class IngestionService {
const final_usage_details = await this.getUsageUnits(
observationRecord,
internalModel,
+ isAiGatewayGeneration,
);
// Match pricing tier based on usage_details. Skip when usage is empty
@@ -1450,6 +1464,7 @@ export class IngestionService {
| "id"
>,
model: Model | null | undefined,
+ isAiGatewayGeneration: boolean,
): Promise<
Pick<
ObservationRecordInsertType,
@@ -1468,12 +1483,17 @@ export class IngestionService {
observationRecord.provided_cost_details ?? {},
).some((value) => value != null);
+ // The AI gateway forwards provider-reported usage whenever the provider
+ // returns it. Without it (failed or cancelled upstream calls), the true
+ // usage is unknown: the request body holds tool schemas and encrypted
+ // reasoning, so tokenizing it fabricates usage and cost.
if (
// Manual tokenisation when no user provided usage or cost and generation has not status ERROR
model &&
Object.keys(providedUsageDetails).length === 0 &&
!hasProvidedCostDetails &&
- observationRecord.level !== ObservationLevel.ERROR
+ observationRecord.level !== ObservationLevel.ERROR &&
+ !isAiGatewayGeneration
) {
try {
let newInputCount: number | undefined;
diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
index 2b2a7ee46e..8a30d5aa05 100644
--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
@@ -1362,6 +1362,147 @@ describe("Token Cost Calculation", () => {
expect(generation.usage_details.total).toBeUndefined();
});
+ describe("tokenizer inference on failed and cancelled generations", () => {
+ const gatewayScope = { name: "langfuse-ai-gateway", version: "0.1.0" };
+
+ beforeEach(() => {
+ tokenisationMocks.tokenCountAsyncOverride = async () => 7;
+ });
+
+ afterEach(() => {
+ tokenisationMocks.tokenCountAsyncOverride = null;
+ });
+
+ it("should tokenize a DEFAULT generation on the events table", async () => {
+ const eventRecord = await (mockIngestionService as any).createEventRecord(
+ {
+ spanId: generationId,
+ traceId: uuidv4(),
+ projectId,
+ startTimeISO: new Date().toISOString(),
+ type: "GENERATION",
+ modelName,
+ input: "hello world",
+ output: "hey whassup",
+ },
+ "testfile.txt",
+ );
+
+ expect(eventRecord.usage_details).toEqual({
+ input: 7,
+ output: 7,
+ total: 14,
+ });
+ });
+
+ it("should skip tokenization for ERROR generations written to the events table", async () => {
+ const eventRecord = await (mockIngestionService as any).createEventRecord(
+ {
+ spanId: generationId,
+ traceId: uuidv4(),
+ projectId,
+ startTimeISO: new Date().toISOString(),
+ type: "GENERATION",
+ modelName,
+ level: "ERROR",
+ input: "hello world",
+ output: "hey whassup",
+ },
+ "testfile.txt",
+ );
+
+ expect(eventRecord.level).toBe("ERROR");
+ expect(eventRecord.model_id).toBe(tokenModelData.id);
+ expect(eventRecord.usage_details).toEqual({});
+ expect(eventRecord.cost_details).toEqual({});
+ });
+
+ it("should not tokenize a cancelled gateway generation on the events table", async () => {
+ const eventRecord = await (mockIngestionService as any).createEventRecord(
+ {
+ spanId: generationId,
+ traceId: uuidv4(),
+ projectId,
+ startTimeISO: new Date().toISOString(),
+ type: "GENERATION",
+ modelName,
+ level: "WARNING",
+ statusMessage: "Client cancelled the response",
+ scopeName: gatewayScope.name,
+ metadata: { scope: gatewayScope },
+ input: "hello world",
+ output: "[]",
+ },
+ "testfile.txt",
+ );
+
+ expect(eventRecord.model_id).toBe(tokenModelData.id);
+ expect(eventRecord.usage_details).toEqual({});
+ expect(eventRecord.cost_details).toEqual({});
+ });
+
+ it("should cost provider-reported usage on a cancelled gateway generation", async () => {
+ const eventRecord = await (mockIngestionService as any).createEventRecord(
+ {
+ spanId: generationId,
+ traceId: uuidv4(),
+ projectId,
+ startTimeISO: new Date().toISOString(),
+ type: "GENERATION",
+ modelName,
+ level: "WARNING",
+ scopeName: gatewayScope.name,
+ metadata: { scope: gatewayScope },
+ providedUsageDetails: { input: 10, output: 5 },
+ input: "hello world",
+ output: "[]",
+ },
+ "testfile.txt",
+ );
+
+ expect(eventRecord.usage_details).toEqual({
+ input: 10,
+ output: 5,
+ total: 15,
+ });
+ expect(eventRecord.cost_details.input).toBe(0.1);
+ expect(eventRecord.cost_details.output).toBe(0.1);
+ });
+
+ it("should not tokenize a cancelled gateway generation on the observations table", async () => {
+ const events = [
+ {
+ id: uuidv4(),
+ type: "generation-create",
+ timestamp: new Date().toISOString(),
+ body: {
+ id: generationId,
+ startTime: new Date().toISOString(),
+ model: modelName,
+ level: "WARNING",
+ statusMessage: "Client cancelled the response",
+ metadata: { scope: gatewayScope },
+ input: "hello world",
+ output: "[]",
+ },
+ },
+ ];
+
+ await (mockIngestionService as any).processObservationEventList({
+ projectId,
+ entityId: generationId,
+ createdAtTimestamp: new Date(),
+ observationEventList: events,
+ });
+
+ const [tableName, generation] = mockAddToClickhouseWriter.mock.calls[0];
+ expect(tableName).toBe("observations");
+ expect(generation.internal_model_id).toBe(tokenModelData.id);
+ expect(generation.usage_details).toEqual({});
+ expect(generation.cost_details).toEqual({});
+ });
+ });
+
it("should skip tokenization and leave usage details blank when cost details are provided", async () => {
const generationUsage1 = {
model: modelName,
diff --git a/worker/src/services/IngestionService/utils.ts b/worker/src/services/IngestionService/utils.ts
index b6e35c036b..b144076f5f 100644
--- a/worker/src/services/IngestionService/utils.ts
+++ b/worker/src/services/IngestionService/utils.ts
@@ -1,5 +1,6 @@
/* eslint-disable no-nested-ternary */
import { JsonNested, Prisma } from "@langfuse/shared";
+import { AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME } from "@langfuse/shared/src/server";
import { mergeWith, merge } from "lodash";
// Theoretically this returns Record<string, unknown>, but it would be hard to align the typing accordingly.
@@ -61,6 +62,18 @@ export const convertRecordValuesToString = (
);
};
+// OTel-ingested observation events carry the instrumentation scope as
+// `metadata.scope` ({ name, version, attributes }).
+export const hasAiGatewayScope = (metadata: unknown): boolean => {
+ if (!metadata || typeof metadata !== "object") return false;
+ const scope = (metadata as { scope?: unknown }).scope;
+ return (
+ typeof scope === "object" &&
+ scope !== null &&
+ (scope as { name?: unknown }).name === AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME
+ );
+};
+
export function overwriteObject(
a: {
id: string;