🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): add identifier to stripe meter events (#16251)
Steffen Schmitza7cf5179585a
diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.tsnew file mode 100644index 0000000000..58263472c7--- /dev/null+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts@@ -0,0 +1,216 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { Job } from "bullmq";++const {+ mockCronJobsUpsert,+ mockCronJobsUpdate,+ mockOrganizationFindMany,+ mockMeterEventsCreate,+ mockUsageMeteringQueueAdd,+ mockSpendAlertQueueAdd,+ mockObservationCounts,+ mockTraceCounts,+ mockScoreCounts,+ mockLoggerInfo,+ mockLoggerWarn,+ mockLoggerError,+ mockRecordIncrement,+ mockTraceException,+} = vi.hoisted(() => ({+ mockCronJobsUpsert: vi.fn(),+ mockCronJobsUpdate: vi.fn(),+ mockOrganizationFindMany: vi.fn(),+ mockMeterEventsCreate: vi.fn(),+ mockUsageMeteringQueueAdd: vi.fn(),+ mockSpendAlertQueueAdd: vi.fn(),+ mockObservationCounts: vi.fn(),+ mockTraceCounts: vi.fn(),+ mockScoreCounts: vi.fn(),+ mockLoggerInfo: vi.fn(),+ mockLoggerWarn: vi.fn(),+ mockLoggerError: vi.fn(),+ mockRecordIncrement: vi.fn(),+ mockTraceException: vi.fn(),+}));++vi.mock("@langfuse/shared", () => ({+ getBillingProvider: () => "stripe",+ parseDbOrg: (org: unknown) => org,+ Prisma: { DbNull: "DbNull" },+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {+ cronJobs: {+ upsert: mockCronJobsUpsert,+ update: mockCronJobsUpdate,+ },+ organization: {+ findMany: mockOrganizationFindMany,+ },+ },+}));++vi.mock("@langfuse/shared/src/server", () => ({+ CloudUsageMeteringQueue: {+ getInstance: () => ({ add: mockUsageMeteringQueueAdd }),+ },+ CloudSpendAlertQueue: {+ getInstance: () => ({ add: mockSpendAlertQueueAdd }),+ },+ getObservationCountsByProjectInCreationInterval: mockObservationCounts,+ getTraceCountsByProjectInCreationInterval: mockTraceCounts,+ getScoreCountsByProjectInCreationInterval: mockScoreCounts,+ logger: {+ info: mockLoggerInfo,+ warn: mockLoggerWarn,+ error: mockLoggerError,+ },+ QueueJobs: {+ CloudUsageMeteringJob: "cloud-usage-metering-job",+ CloudSpendAlertJob: "cloud-spend-alert-job",+ },+ recordIncrement: mockRecordIncrement,+ traceException: mockTraceException,+}));++vi.mock("stripe", () => ({+ default: class {+ billing = { meterEvents: { create: mockMeterEventsCreate } };+ },+}));++vi.mock("../../../env", () => ({+ env: {+ STRIPE_SECRET_KEY: "sk_test",+ LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,+ },+}));++import { handleCloudUsageMeteringJob } from "../handleCloudUsageMeteringJob";+import { CloudUsageMeteringDbCronJobStates } from "../constants";++// Mirrors the prod-eu incident: a run claims the 08:00-09:00 interval, dies+// partway through, and the next hourly tick takes the same interval over.+const NOW = new Date("2026-08-03T10:30:00.000Z");+const INTERVAL_START = new Date("2026-08-03T08:00:00.000Z");++const createJob = () =>+ ({+ updateProgress: vi.fn(),+ }) as unknown as Job;++const identifiersFromCalls = () =>+ mockMeterEventsCreate.mock.calls.map(([params]) => params.identifier);++describe("handleCloudUsageMeteringJob", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.useFakeTimers({ shouldAdvanceTime: true });+ vi.setSystemTime(NOW);++ mockCronJobsUpsert.mockResolvedValue({+ name: "cloud_usage_metering",+ lastRun: INTERVAL_START,+ state: CloudUsageMeteringDbCronJobStates.Queued,+ jobStartedAt: null,+ });+ mockCronJobsUpdate.mockResolvedValue({});+ mockOrganizationFindMany.mockResolvedValue([+ {+ id: "org-1",+ cloudConfig: { stripe: { customerId: "cus_1" } },+ projects: [{ id: "project-1" }],+ cloudSpendAlerts: [],+ },+ ]);+ mockObservationCounts.mockResolvedValue([+ { projectId: "project-1", count: 5 },+ ]);+ mockTraceCounts.mockResolvedValue([{ projectId: "project-1", count: 3 }]);+ mockScoreCounts.mockResolvedValue([{ projectId: "project-1", count: 2 }]);+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("resolves the same meter event identifier when an interval is replayed", async () => {+ await handleCloudUsageMeteringJob(createJob());+ const firstRun = identifiersFromCalls();++ expect(firstRun).toHaveLength(2);+ expect(+ firstRun.every((id) => typeof id === "string" && id.length > 0),+ ).toBe(true);++ // lastRun never advanced, so the replay recomputes the same interval from+ // the top. Identical identifiers are what lets Stripe drop the duplicates+ // instead of summing them into the customer's usage.+ mockMeterEventsCreate.mockClear();+ await handleCloudUsageMeteringJob(createJob());++ expect(identifiersFromCalls()).toEqual(firstRun);+ });++ it("derives the identifier from the meter, the org and the interval", async () => {+ await handleCloudUsageMeteringJob(createJob());++ const byMeter = new Map(+ mockMeterEventsCreate.mock.calls.map(([params]) => [+ params.event_name,+ params.identifier,+ ]),+ );++ // Both meters are reported for the same org and interval, so event_name has+ // to be part of the key or Stripe would dedup them against each other and+ // silently drop one.+ expect(byMeter.get("tracing_observations")).not.toEqual(+ byMeter.get("tracing_events"),+ );+ for (const [eventName, identifier] of byMeter) {+ expect(identifier).toContain(eventName);+ expect(identifier).toContain("org-1");+ expect(identifier).toContain(String(INTERVAL_START.getTime()));+ }+ });++ it("advances lastRun under the lease it claimed", async () => {+ await handleCloudUsageMeteringJob(createJob());++ const closingCall = mockCronJobsUpdate.mock.calls.find(+ ([args]) => args.data.lastRun !== undefined,+ );+ const claimCall = mockCronJobsUpdate.mock.calls.find(+ ([args]) =>+ args.data.state === CloudUsageMeteringDbCronJobStates.Processing,+ );++ expect(claimCall).toBeDefined();+ expect(closingCall?.[0].where).toMatchObject({+ state: CloudUsageMeteringDbCronJobStates.Processing,+ jobStartedAt: claimCall?.[0].data.jobStartedAt,+ });+ });++ it("does not advance lastRun or re-enqueue when the interval was taken over", async () => {+ // The 20 minute stale-job takeover handed this interval to another run, so+ // the closing update no longer matches the lease this run claimed.+ mockCronJobsUpdate.mockImplementation(async ({ data }) => {+ if (data.lastRun !== undefined) {+ throw new Error("Record to update not found.");+ }+ return {};+ });++ await expect(+ handleCloudUsageMeteringJob(createJob()),+ ).resolves.toBeUndefined();++ // Throwing here would make the wrapper reset the state and replay the+ // interval a third time.+ expect(mockUsageMeteringQueueAdd).not.toHaveBeenCalled();+ expect(mockLoggerWarn).toHaveBeenCalled();+ });+});diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.tsindex d32c0135d4..dc71149bf9 100644--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts@@ -24,6 +24,17 @@ import { backOff } from "exponential-backoff";const delayFromStartOfInterval = 3600000 + 5 * 60 * 1000; // 5 minutes after the end of the interval+// Stripe deduplicates meter events by identifier, so deriving it from the tuple+// that defines the billable fact makes a replayed interval idempotent instead of+// adding a second time to the customer's usage. The meter name is part of the key+// because both meters are reported for the same org and interval - without it the+// two would collide and Stripe would drop one of them.+const meterEventIdentifier = (+ eventName: string,+ orgId: string,+ intervalStart: Date,+) => `${eventName}:${orgId}:${intervalStart.getTime()}`;+export const handleCloudUsageMeteringJob = async (job: Job) => {if (!env.STRIPE_SECRET_KEY) {logger.warn("[CLOUD USAGE METERING] Stripe secret key not found");@@ -71,6 +82,7 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {}}+ const jobStartedAt = new Date();try {await prisma.cronJobs.update({where: {@@ -80,7 +92,7 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {},data: {state: CloudUsageMeteringDbCronJobStates.Processing,- jobStartedAt: new Date(),+ jobStartedAt,},});} catch (e) {@@ -223,6 +235,11 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {async () =>await stripe.billing.meterEvents.create({event_name: "tracing_observations",+ identifier: meterEventIdentifier(+ "tracing_observations",+ org.id,+ meterIntervalStart,+ ),timestamp: meterIntervalEnd.getTime() / 1000,payload: {stripe_customer_id: stripeCustomerId,@@ -252,6 +269,11 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {async () =>await stripe.billing.meterEvents.create({event_name: "tracing_events",+ identifier: meterEventIdentifier(+ "tracing_events",+ org.id,+ meterIntervalStart,+ ),timestamp: meterIntervalEnd.getTime() / 1000,payload: {stripe_customer_id: stripeCustomerId,@@ -322,15 +344,39 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {}}- // update cron job- await prisma.cronJobs.update({- where: { name: cloudUsageMeteringDbCronJobName },- data: {- lastRun: meterIntervalEnd,- state: CloudUsageMeteringDbCronJobStates.Queued,- jobStartedAt: null,- },- });+ // Advance the cron job only while still holding the lease claimed above. The+ // stale-job takeover can hand this interval to another run mid-flight; without+ // the predicate a superseded run would still move lastRun forward and mask the+ // fact that another run owns the interval.+ try {+ await prisma.cronJobs.update({+ where: {+ name: cloudUsageMeteringDbCronJobName,+ state: CloudUsageMeteringDbCronJobStates.Processing,+ jobStartedAt,+ },+ data: {+ lastRun: meterIntervalEnd,+ state: CloudUsageMeteringDbCronJobStates.Queued,+ jobStartedAt: null,+ },+ });+ } catch (e) {+ logger.warn(+ "[CLOUD USAGE METERING] Job lease was taken over before completion, leaving cron state to the run that owns it now",+ { e },+ );+ recordIncrement(+ "langfuse.queue.cloud_usage_metering_queue.lost_job_lease",+ 1,+ {+ unit: "jobs",+ },+ );+ // Returning rather than throwing keeps the queue wrapper from resetting the+ // state and replaying this interval on top of the run that took it over.+ return;+ }logger.info(`[CLOUD USAGE METERING] Job for interval ${meterIntervalStart.toISOString()} - ${meterIntervalEnd.toISOString()} completed`,