🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): treat a duplicate Stripe meter event as delivered (#16292)
Steffen Schmitz6d995ef14311
diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.tsindex 58263472c7..95c7babaa8 100644--- a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts@@ -103,6 +103,30 @@ const createJob = () =>const identifiersFromCalls = () =>mockMeterEventsCreate.mock.calls.map(([params]) => params.identifier);+const callsForMeter = (eventName: string) =>+ mockMeterEventsCreate.mock.calls.filter(+ ([params]) => params.event_name === eventName,+ );++const closingUpdate = () =>+ mockCronJobsUpdate.mock.calls.find(+ ([args]) => args.data.lastRun !== undefined,+ );++// Shape taken from the prod-eu rejection: a 400 invalid_request_error whose only+// machine-readable marker is the message. Stripe sets stripe-should-retry: false.+const stripeInvalidRequestError = (message: string) =>+ Object.assign(new Error(message), {+ type: "StripeInvalidRequestError",+ rawType: "invalid_request_error",+ statusCode: 400,+ });++const duplicateIdentifierError = (identifier: string) =>+ stripeInvalidRequestError(+ `An event already exists with identifier ${identifier}.`,+ );+describe("handleCloudUsageMeteringJob", () => {beforeEach(() => {vi.clearAllMocks();@@ -213,4 +237,72 @@ describe("handleCloudUsageMeteringJob", () => {expect(mockUsageMeteringQueueAdd).not.toHaveBeenCalled();expect(mockLoggerWarn).toHaveBeenCalled();});++ it("skips a meter event Stripe already holds and still sends the other meter", async () => {+ // The replayed run died between the two meters last time, so Stripe holds+ // tracing_observations for this interval but never received tracing_events.+ mockMeterEventsCreate.mockImplementation(+ async ({ event_name, identifier }) => {+ if (event_name === "tracing_observations") {+ throw duplicateIdentifierError(identifier);+ }+ return {};+ },+ );++ await expect(+ handleCloudUsageMeteringJob(createJob()),+ ).resolves.toBeUndefined();++ // Skipping the whole org on the first duplicate would leave this meter+ // unsent for good, turning the over-bill into an under-bill.+ expect(callsForMeter("tracing_events")).toHaveLength(1);+ // A rejection reaching the caller would strand the interval: lastRun only+ // advances after the loop, so the next tick replays it and fails again.+ expect(closingUpdate()?.[0].data.lastRun).toEqual(+ new Date(INTERVAL_START.getTime() + 3600000),+ );+ expect(mockRecordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",+ 1,+ { unit: "events" },+ );+ expect(mockLoggerInfo).toHaveBeenCalledWith(+ expect.stringContaining("tracing_observations:org-1"),+ );+ });++ it("does not retry a meter event Stripe rejected as a duplicate", async () => {+ mockMeterEventsCreate.mockImplementation(async ({ identifier }) => {+ throw duplicateIdentifierError(identifier);+ });++ await handleCloudUsageMeteringJob(createJob());++ // Stripe answers stripe-should-retry: false, so backOff must not spend its+ // three attempts re-posting a request that can never succeed.+ expect(callsForMeter("tracing_observations")).toHaveLength(1);+ expect(callsForMeter("tracing_events")).toHaveLength(1);+ });++ it("fails the job when Stripe rejects a meter event for another reason", async () => {+ // The >35 day backdating rejection is the same status and rawType, and it+ // means the usage was never delivered - swallowing it would lose billing.+ mockMeterEventsCreate.mockRejectedValue(+ stripeInvalidRequestError(+ "Timestamp must be no more than 35 days in the past.",+ ),+ );++ await expect(handleCloudUsageMeteringJob(createJob())).rejects.toThrow(+ /35 days/,+ );++ expect(closingUpdate()).toBeUndefined();+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",+ expect.anything(),+ expect.anything(),+ );+ });});diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.tsindex dc71149bf9..2b7a282ae0 100644--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts@@ -35,6 +35,64 @@ const meterEventIdentifier = (intervalStart: Date,) => `${eventName}:${orgId}:${intervalStart.getTime()}`;+// Stripe does not silently drop a meter event whose identifier it has already+// seen - it answers with a 400. Left to propagate, the first replayed org ends+// the whole run, and because `lastRun` only advances after the org loop the same+// interval is claimed again on the next tick: the replay the identifier was+// added to make harmless instead stalls metering outright.+const isDuplicateMeterEventError = (error: unknown): boolean => {+ const stripeError = error as+ | { statusCode?: unknown; rawType?: unknown; message?: unknown }+ | null+ | undefined;++ return (+ typeof stripeError === "object" &&+ stripeError !== null &&+ stripeError.statusCode === 400 &&+ stripeError.rawType === "invalid_request_error" &&+ typeof stripeError.message === "string" &&+ /already exists with identifier/i.test(stripeError.message)+ );+};++// Every other rejection still fails the job. Only the duplicate is safe to+// swallow, because the identifier is derived from the tuple that defines the+// billable fact, so Stripe already holding it is proof this usage was delivered.+const sendMeterEvent = async (+ stripe: Stripe,+ params: Stripe.Billing.MeterEventCreateParams,+) => {+ try {+ // retrying the stripe call in case of an HTTP error+ await backOff(async () => await stripe.billing.meterEvents.create(params), {+ numOfAttempts: 3,+ // Stripe returns `stripe-should-retry: false` on a duplicate, so retrying+ // only multiplies the rejection.+ retry: (e) => !isDuplicateMeterEventError(e),+ });+ } catch (e) {+ if (!isDuplicateMeterEventError(e)) {+ throw e;+ }++ // Skip only this meter event, not the rest of the organization. Both meters+ // are sent sequentially, so a run that died between them leaves one+ // delivered and the other missing - skipping the org here would under-bill+ // it for this interval for good.+ logger.info(+ `[CLOUD USAGE METERING] Stripe already holds meter event ${params.identifier}, skipping`,+ );+ recordIncrement(+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",+ 1,+ {+ unit: "events",+ },+ );+ }+};+export const handleCloudUsageMeteringJob = async (job: Job) => {if (!env.STRIPE_SECRET_KEY) {logger.warn("[CLOUD USAGE METERING] Stripe secret key not found");@@ -231,25 +289,19 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {`[CLOUD USAGE METERING] Job for org ${org.id} - ${stripeCustomerId} stripe customer id - ${countObservations} observations`,);if (countObservations > 0) {- await backOff(- async () =>- await stripe.billing.meterEvents.create({- event_name: "tracing_observations",- identifier: meterEventIdentifier(- "tracing_observations",- org.id,- meterIntervalStart,- ),- timestamp: meterIntervalEnd.getTime() / 1000,- payload: {- stripe_customer_id: stripeCustomerId,- value: countObservations.toString(), // value is a string in stripe- },- }),- {- numOfAttempts: 3,+ await sendMeterEvent(stripe, {+ event_name: "tracing_observations",+ identifier: meterEventIdentifier(+ "tracing_observations",+ org.id,+ meterIntervalStart,+ ),+ timestamp: meterIntervalEnd.getTime() / 1000,+ payload: {+ stripe_customer_id: stripeCustomerId,+ value: countObservations.toString(), // value is a string in stripe},- );+ });}// Events@@ -264,26 +316,19 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {`[CLOUD USAGE METERING] Job for org ${org.id} - ${stripeCustomerId} stripe customer id - ${countEvents} events`,);if (countEvents > 0) {- // retrying the stripe call in case of an HTTP error- await backOff(- async () =>- await stripe.billing.meterEvents.create({- event_name: "tracing_events",- identifier: meterEventIdentifier(- "tracing_events",- org.id,- meterIntervalStart,- ),- timestamp: meterIntervalEnd.getTime() / 1000,- payload: {- stripe_customer_id: stripeCustomerId,- value: countEvents.toString(), // value is a string in stripe- },- }),- {- numOfAttempts: 3,+ await sendMeterEvent(stripe, {+ event_name: "tracing_events",+ identifier: meterEventIdentifier(+ "tracing_events",+ org.id,+ meterIntervalStart,+ ),+ timestamp: meterIntervalEnd.getTime() / 1000,+ payload: {+ stripe_customer_id: stripeCustomerId,+ value: countEvents.toString(), // value is a string in stripe},- );+ });}if (countEvents === 0 && countObservations === 0) {