Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): add identifier to stripe meter events (#16251)

Steffen Schmitza7cf5179585a
diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
new file mode 100644
index 0000000000..58263472c7
--- /dev/null
+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
@@ -0,0 +1,216 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { Job } from "bullmq";
+
+const {
+ mockCronJobsUpsert,
+ mockCronJobsUpdate,
+ mockOrganizationFindMany,
+ mockMeterEventsCreate,
+ mockUsageMeteringQueueAdd,
+ mockSpendAlertQueueAdd,
+ mockObservationCounts,
+ mockTraceCounts,
+ mockScoreCounts,
+ mockLoggerInfo,
+ mockLoggerWarn,
+ mockLoggerError,
+ mockRecordIncrement,
+ mockTraceException,
+} = vi.hoisted(() => ({
+ mockCronJobsUpsert: vi.fn(),
+ mockCronJobsUpdate: vi.fn(),
+ mockOrganizationFindMany: vi.fn(),
+ mockMeterEventsCreate: vi.fn(),
+ mockUsageMeteringQueueAdd: vi.fn(),
+ mockSpendAlertQueueAdd: vi.fn(),
+ mockObservationCounts: vi.fn(),
+ mockTraceCounts: vi.fn(),
+ mockScoreCounts: vi.fn(),
+ mockLoggerInfo: vi.fn(),
+ mockLoggerWarn: vi.fn(),
+ mockLoggerError: vi.fn(),
+ mockRecordIncrement: vi.fn(),
+ mockTraceException: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared", () => ({
+ getBillingProvider: () => "stripe",
+ parseDbOrg: (org: unknown) => org,
+ Prisma: { DbNull: "DbNull" },
+}));
+
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: {
+ cronJobs: {
+ upsert: mockCronJobsUpsert,
+ update: mockCronJobsUpdate,
+ },
+ organization: {
+ findMany: mockOrganizationFindMany,
+ },
+ },
+}));
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ CloudUsageMeteringQueue: {
+ getInstance: () => ({ add: mockUsageMeteringQueueAdd }),
+ },
+ CloudSpendAlertQueue: {
+ getInstance: () => ({ add: mockSpendAlertQueueAdd }),
+ },
+ getObservationCountsByProjectInCreationInterval: mockObservationCounts,
+ getTraceCountsByProjectInCreationInterval: mockTraceCounts,
+ getScoreCountsByProjectInCreationInterval: mockScoreCounts,
+ logger: {
+ info: mockLoggerInfo,
+ warn: mockLoggerWarn,
+ error: mockLoggerError,
+ },
+ QueueJobs: {
+ CloudUsageMeteringJob: "cloud-usage-metering-job",
+ CloudSpendAlertJob: "cloud-spend-alert-job",
+ },
+ recordIncrement: mockRecordIncrement,
+ traceException: mockTraceException,
+}));
+
+vi.mock("stripe", () => ({
+ default: class {
+ billing = { meterEvents: { create: mockMeterEventsCreate } };
+ },
+}));
+
+vi.mock("../../../env", () => ({
+ env: {
+ STRIPE_SECRET_KEY: "sk_test",
+ LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,
+ },
+}));
+
+import { handleCloudUsageMeteringJob } from "../handleCloudUsageMeteringJob";
+import { CloudUsageMeteringDbCronJobStates } from "../constants";
+
+// Mirrors the prod-eu incident: a run claims the 08:00-09:00 interval, dies
+// partway through, and the next hourly tick takes the same interval over.
+const NOW = new Date("2026-08-03T10:30:00.000Z");
+const INTERVAL_START = new Date("2026-08-03T08:00:00.000Z");
+
+const createJob = () =>
+ ({
+ updateProgress: vi.fn(),
+ }) as unknown as Job;
+
+const identifiersFromCalls = () =>
+ mockMeterEventsCreate.mock.calls.map(([params]) => params.identifier);
+
+describe("handleCloudUsageMeteringJob", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.useFakeTimers({ shouldAdvanceTime: true });
+ vi.setSystemTime(NOW);
+
+ mockCronJobsUpsert.mockResolvedValue({
+ name: "cloud_usage_metering",
+ lastRun: INTERVAL_START,
+ state: CloudUsageMeteringDbCronJobStates.Queued,
+ jobStartedAt: null,
+ });
+ mockCronJobsUpdate.mockResolvedValue({});
+ mockOrganizationFindMany.mockResolvedValue([
+ {
+ id: "org-1",
+ cloudConfig: { stripe: { customerId: "cus_1" } },
+ projects: [{ id: "project-1" }],
+ cloudSpendAlerts: [],
+ },
+ ]);
+ mockObservationCounts.mockResolvedValue([
+ { projectId: "project-1", count: 5 },
+ ]);
+ mockTraceCounts.mockResolvedValue([{ projectId: "project-1", count: 3 }]);
+ mockScoreCounts.mockResolvedValue([{ projectId: "project-1", count: 2 }]);
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("resolves the same meter event identifier when an interval is replayed", async () => {
+ await handleCloudUsageMeteringJob(createJob());
+ const firstRun = identifiersFromCalls();
+
+ expect(firstRun).toHaveLength(2);
+ expect(
+ firstRun.every((id) => typeof id === "string" && id.length > 0),
+ ).toBe(true);
+
+ // lastRun never advanced, so the replay recomputes the same interval from
+ // the top. Identical identifiers are what lets Stripe drop the duplicates
+ // instead of summing them into the customer's usage.
+ mockMeterEventsCreate.mockClear();
+ await handleCloudUsageMeteringJob(createJob());
+
+ expect(identifiersFromCalls()).toEqual(firstRun);
+ });
+
+ it("derives the identifier from the meter, the org and the interval", async () => {
+ await handleCloudUsageMeteringJob(createJob());
+
+ const byMeter = new Map(
+ mockMeterEventsCreate.mock.calls.map(([params]) => [
+ params.event_name,
+ params.identifier,
+ ]),
+ );
+
+ // Both meters are reported for the same org and interval, so event_name has
+ // to be part of the key or Stripe would dedup them against each other and
+ // silently drop one.
+ expect(byMeter.get("tracing_observations")).not.toEqual(
+ byMeter.get("tracing_events"),
+ );
+ for (const [eventName, identifier] of byMeter) {
+ expect(identifier).toContain(eventName);
+ expect(identifier).toContain("org-1");
+ expect(identifier).toContain(String(INTERVAL_START.getTime()));
+ }
+ });
+
+ it("advances lastRun under the lease it claimed", async () => {
+ await handleCloudUsageMeteringJob(createJob());
+
+ const closingCall = mockCronJobsUpdate.mock.calls.find(
+ ([args]) => args.data.lastRun !== undefined,
+ );
+ const claimCall = mockCronJobsUpdate.mock.calls.find(
+ ([args]) =>
+ args.data.state === CloudUsageMeteringDbCronJobStates.Processing,
+ );
+
+ expect(claimCall).toBeDefined();
+ expect(closingCall?.[0].where).toMatchObject({
+ state: CloudUsageMeteringDbCronJobStates.Processing,
+ jobStartedAt: claimCall?.[0].data.jobStartedAt,
+ });
+ });
+
+ it("does not advance lastRun or re-enqueue when the interval was taken over", async () => {
+ // The 20 minute stale-job takeover handed this interval to another run, so
+ // the closing update no longer matches the lease this run claimed.
+ mockCronJobsUpdate.mockImplementation(async ({ data }) => {
+ if (data.lastRun !== undefined) {
+ throw new Error("Record to update not found.");
+ }
+ return {};
+ });
+
+ await expect(
+ handleCloudUsageMeteringJob(createJob()),
+ ).resolves.toBeUndefined();
+
+ // Throwing here would make the wrapper reset the state and replay the
+ // interval a third time.
+ expect(mockUsageMeteringQueueAdd).not.toHaveBeenCalled();
+ expect(mockLoggerWarn).toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
index d32c0135d4..dc71149bf9 100644
--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
@@ -24,6 +24,17 @@ import { backOff } from "exponential-backoff";
const delayFromStartOfInterval = 3600000 + 5 * 60 * 1000; // 5 minutes after the end of the interval
+// Stripe deduplicates meter events by identifier, so deriving it from the tuple
+// that defines the billable fact makes a replayed interval idempotent instead of
+// adding a second time to the customer's usage. The meter name is part of the key
+// because both meters are reported for the same org and interval - without it the
+// two would collide and Stripe would drop one of them.
+const meterEventIdentifier = (
+ eventName: string,
+ orgId: string,
+ intervalStart: Date,
+) => `${eventName}:${orgId}:${intervalStart.getTime()}`;
+
export const handleCloudUsageMeteringJob = async (job: Job) => {
if (!env.STRIPE_SECRET_KEY) {
logger.warn("[CLOUD USAGE METERING] Stripe secret key not found");
@@ -71,6 +82,7 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
}
}
+ const jobStartedAt = new Date();
try {
await prisma.cronJobs.update({
where: {
@@ -80,7 +92,7 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
},
data: {
state: CloudUsageMeteringDbCronJobStates.Processing,
- jobStartedAt: new Date(),
+ jobStartedAt,
},
});
} catch (e) {
@@ -223,6 +235,11 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
async () =>
await stripe.billing.meterEvents.create({
event_name: "tracing_observations",
+ identifier: meterEventIdentifier(
+ "tracing_observations",
+ org.id,
+ meterIntervalStart,
+ ),
timestamp: meterIntervalEnd.getTime() / 1000,
payload: {
stripe_customer_id: stripeCustomerId,
@@ -252,6 +269,11 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
async () =>
await stripe.billing.meterEvents.create({
event_name: "tracing_events",
+ identifier: meterEventIdentifier(
+ "tracing_events",
+ org.id,
+ meterIntervalStart,
+ ),
timestamp: meterIntervalEnd.getTime() / 1000,
payload: {
stripe_customer_id: stripeCustomerId,
@@ -322,15 +344,39 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
}
}
- // update cron job
- await prisma.cronJobs.update({
- where: { name: cloudUsageMeteringDbCronJobName },
- data: {
- lastRun: meterIntervalEnd,
- state: CloudUsageMeteringDbCronJobStates.Queued,
- jobStartedAt: null,
- },
- });
+ // Advance the cron job only while still holding the lease claimed above. The
+ // stale-job takeover can hand this interval to another run mid-flight; without
+ // the predicate a superseded run would still move lastRun forward and mask the
+ // fact that another run owns the interval.
+ try {
+ await prisma.cronJobs.update({
+ where: {
+ name: cloudUsageMeteringDbCronJobName,
+ state: CloudUsageMeteringDbCronJobStates.Processing,
+ jobStartedAt,
+ },
+ data: {
+ lastRun: meterIntervalEnd,
+ state: CloudUsageMeteringDbCronJobStates.Queued,
+ jobStartedAt: null,
+ },
+ });
+ } catch (e) {
+ logger.warn(
+ "[CLOUD USAGE METERING] Job lease was taken over before completion, leaving cron state to the run that owns it now",
+ { e },
+ );
+ recordIncrement(
+ "langfuse.queue.cloud_usage_metering_queue.lost_job_lease",
+ 1,
+ {
+ unit: "jobs",
+ },
+ );
+ // Returning rather than throwing keeps the queue wrapper from resetting the
+ // state and replaying this interval on top of the run that took it over.
+ return;
+ }
logger.info(
`[CLOUD USAGE METERING] Job for interval ${meterIntervalStart.toISOString()} - ${meterIntervalEnd.toISOString()} completed`,