Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): treat a duplicate Stripe meter event as delivered (#16292)

Steffen Schmitz6d995ef14311
diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
index 58263472c7..95c7babaa8 100644
--- a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
@@ -103,6 +103,30 @@ const createJob = () =>
const identifiersFromCalls = () =>
mockMeterEventsCreate.mock.calls.map(([params]) => params.identifier);
+const callsForMeter = (eventName: string) =>
+ mockMeterEventsCreate.mock.calls.filter(
+ ([params]) => params.event_name === eventName,
+ );
+
+const closingUpdate = () =>
+ mockCronJobsUpdate.mock.calls.find(
+ ([args]) => args.data.lastRun !== undefined,
+ );
+
+// Shape taken from the prod-eu rejection: a 400 invalid_request_error whose only
+// machine-readable marker is the message. Stripe sets stripe-should-retry: false.
+const stripeInvalidRequestError = (message: string) =>
+ Object.assign(new Error(message), {
+ type: "StripeInvalidRequestError",
+ rawType: "invalid_request_error",
+ statusCode: 400,
+ });
+
+const duplicateIdentifierError = (identifier: string) =>
+ stripeInvalidRequestError(
+ `An event already exists with identifier ${identifier}.`,
+ );
+
describe("handleCloudUsageMeteringJob", () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -213,4 +237,72 @@ describe("handleCloudUsageMeteringJob", () => {
expect(mockUsageMeteringQueueAdd).not.toHaveBeenCalled();
expect(mockLoggerWarn).toHaveBeenCalled();
});
+
+ it("skips a meter event Stripe already holds and still sends the other meter", async () => {
+ // The replayed run died between the two meters last time, so Stripe holds
+ // tracing_observations for this interval but never received tracing_events.
+ mockMeterEventsCreate.mockImplementation(
+ async ({ event_name, identifier }) => {
+ if (event_name === "tracing_observations") {
+ throw duplicateIdentifierError(identifier);
+ }
+ return {};
+ },
+ );
+
+ await expect(
+ handleCloudUsageMeteringJob(createJob()),
+ ).resolves.toBeUndefined();
+
+ // Skipping the whole org on the first duplicate would leave this meter
+ // unsent for good, turning the over-bill into an under-bill.
+ expect(callsForMeter("tracing_events")).toHaveLength(1);
+ // A rejection reaching the caller would strand the interval: lastRun only
+ // advances after the loop, so the next tick replays it and fails again.
+ expect(closingUpdate()?.[0].data.lastRun).toEqual(
+ new Date(INTERVAL_START.getTime() + 3600000),
+ );
+ expect(mockRecordIncrement).toHaveBeenCalledWith(
+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",
+ 1,
+ { unit: "events" },
+ );
+ expect(mockLoggerInfo).toHaveBeenCalledWith(
+ expect.stringContaining("tracing_observations:org-1"),
+ );
+ });
+
+ it("does not retry a meter event Stripe rejected as a duplicate", async () => {
+ mockMeterEventsCreate.mockImplementation(async ({ identifier }) => {
+ throw duplicateIdentifierError(identifier);
+ });
+
+ await handleCloudUsageMeteringJob(createJob());
+
+ // Stripe answers stripe-should-retry: false, so backOff must not spend its
+ // three attempts re-posting a request that can never succeed.
+ expect(callsForMeter("tracing_observations")).toHaveLength(1);
+ expect(callsForMeter("tracing_events")).toHaveLength(1);
+ });
+
+ it("fails the job when Stripe rejects a meter event for another reason", async () => {
+ // The >35 day backdating rejection is the same status and rawType, and it
+ // means the usage was never delivered - swallowing it would lose billing.
+ mockMeterEventsCreate.mockRejectedValue(
+ stripeInvalidRequestError(
+ "Timestamp must be no more than 35 days in the past.",
+ ),
+ );
+
+ await expect(handleCloudUsageMeteringJob(createJob())).rejects.toThrow(
+ /35 days/,
+ );
+
+ expect(closingUpdate()).toBeUndefined();
+ expect(mockRecordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",
+ expect.anything(),
+ expect.anything(),
+ );
+ });
});
diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
index dc71149bf9..2b7a282ae0 100644
--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
@@ -35,6 +35,64 @@ const meterEventIdentifier = (
intervalStart: Date,
) => `${eventName}:${orgId}:${intervalStart.getTime()}`;
+// Stripe does not silently drop a meter event whose identifier it has already
+// seen - it answers with a 400. Left to propagate, the first replayed org ends
+// the whole run, and because `lastRun` only advances after the org loop the same
+// interval is claimed again on the next tick: the replay the identifier was
+// added to make harmless instead stalls metering outright.
+const isDuplicateMeterEventError = (error: unknown): boolean => {
+ const stripeError = error as
+ | { statusCode?: unknown; rawType?: unknown; message?: unknown }
+ | null
+ | undefined;
+
+ return (
+ typeof stripeError === "object" &&
+ stripeError !== null &&
+ stripeError.statusCode === 400 &&
+ stripeError.rawType === "invalid_request_error" &&
+ typeof stripeError.message === "string" &&
+ /already exists with identifier/i.test(stripeError.message)
+ );
+};
+
+// Every other rejection still fails the job. Only the duplicate is safe to
+// swallow, because the identifier is derived from the tuple that defines the
+// billable fact, so Stripe already holding it is proof this usage was delivered.
+const sendMeterEvent = async (
+ stripe: Stripe,
+ params: Stripe.Billing.MeterEventCreateParams,
+) => {
+ try {
+ // retrying the stripe call in case of an HTTP error
+ await backOff(async () => await stripe.billing.meterEvents.create(params), {
+ numOfAttempts: 3,
+ // Stripe returns `stripe-should-retry: false` on a duplicate, so retrying
+ // only multiplies the rejection.
+ retry: (e) => !isDuplicateMeterEventError(e),
+ });
+ } catch (e) {
+ if (!isDuplicateMeterEventError(e)) {
+ throw e;
+ }
+
+ // Skip only this meter event, not the rest of the organization. Both meters
+ // are sent sequentially, so a run that died between them leaves one
+ // delivered and the other missing - skipping the org here would under-bill
+ // it for this interval for good.
+ logger.info(
+ `[CLOUD USAGE METERING] Stripe already holds meter event ${params.identifier}, skipping`,
+ );
+ recordIncrement(
+ "langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",
+ 1,
+ {
+ unit: "events",
+ },
+ );
+ }
+};
+
export const handleCloudUsageMeteringJob = async (job: Job) => {
if (!env.STRIPE_SECRET_KEY) {
logger.warn("[CLOUD USAGE METERING] Stripe secret key not found");
@@ -231,25 +289,19 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
`[CLOUD USAGE METERING] Job for org ${org.id} - ${stripeCustomerId} stripe customer id - ${countObservations} observations`,
);
if (countObservations > 0) {
- await backOff(
- async () =>
- await stripe.billing.meterEvents.create({
- event_name: "tracing_observations",
- identifier: meterEventIdentifier(
- "tracing_observations",
- org.id,
- meterIntervalStart,
- ),
- timestamp: meterIntervalEnd.getTime() / 1000,
- payload: {
- stripe_customer_id: stripeCustomerId,
- value: countObservations.toString(), // value is a string in stripe
- },
- }),
- {
- numOfAttempts: 3,
+ await sendMeterEvent(stripe, {
+ event_name: "tracing_observations",
+ identifier: meterEventIdentifier(
+ "tracing_observations",
+ org.id,
+ meterIntervalStart,
+ ),
+ timestamp: meterIntervalEnd.getTime() / 1000,
+ payload: {
+ stripe_customer_id: stripeCustomerId,
+ value: countObservations.toString(), // value is a string in stripe
},
- );
+ });
}
// Events
@@ -264,26 +316,19 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
`[CLOUD USAGE METERING] Job for org ${org.id} - ${stripeCustomerId} stripe customer id - ${countEvents} events`,
);
if (countEvents > 0) {
- // retrying the stripe call in case of an HTTP error
- await backOff(
- async () =>
- await stripe.billing.meterEvents.create({
- event_name: "tracing_events",
- identifier: meterEventIdentifier(
- "tracing_events",
- org.id,
- meterIntervalStart,
- ),
- timestamp: meterIntervalEnd.getTime() / 1000,
- payload: {
- stripe_customer_id: stripeCustomerId,
- value: countEvents.toString(), // value is a string in stripe
- },
- }),
- {
- numOfAttempts: 3,
+ await sendMeterEvent(stripe, {
+ event_name: "tracing_events",
+ identifier: meterEventIdentifier(
+ "tracing_events",
+ org.id,
+ meterIntervalStart,
+ ),
+ timestamp: meterIntervalEnd.getTime() / 1000,
+ payload: {
+ stripe_customer_id: stripeCustomerId,
+ value: countEvents.toString(), // value is a string in stripe
},
- );
+ });
}
if (countEvents === 0 && countObservations === 0) {