Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(billing): longer credit read limit for background checks (#17023)

Abdul78160fd3abad
diff --git a/packages/server/api/src/app/ee/platform/platform-plan/billing-providers/autumn-billing.ts b/packages/server/api/src/app/ee/platform/platform-plan/billing-providers/autumn-billing.ts
index 11083aebbfc..625cf2cb0de 100644
--- a/packages/server/api/src/app/ee/platform/platform-plan/billing-providers/autumn-billing.ts
+++ b/packages/server/api/src/app/ee/platform/platform-plan/billing-providers/autumn-billing.ts
@@ -6,7 +6,7 @@ import { FastifyBaseLogger } from 'fastify'
import { AUTUMN_ENROLL_LOCK_TIMEOUT_SECONDS, getAutumnEnrollLockKey, getBillingEnforcedKey, getBillingOverviewFetchLockKey, getBillingOverviewKey, getCustomerStateFetchLockKey, getCustomerStateMissKey, getCustomerStateRefreshKey } from '../../../../database/redis/keys'
import { distributedLock, distributedStore } from '../../../../database/redis-connections'
import { rejectedPromiseHandler } from '../../../../helper/promise-handler'
-import { ActivateLicenseParams, ApplyAppSumoPlanParams, AppSumoAiCreditsUsage, BillingInfo, BillingOverview, BillingProvider, CreditsAndAppSumoState, CreditsGateState, CreditsUsage, emptyBillingOverview, ShouldBlockOnCreditsParams, TrackFeatureParams } from '../../../../platform/billing-provider'
+import { ActivateLicenseParams, ApplyAppSumoPlanParams, AppSumoAiCreditsUsage, BillingInfo, BillingOverview, BillingProvider, CreditCheckSpeed, CreditsAndAppSumoState, CreditsGateState, CreditsUsage, emptyBillingOverview, ShouldBlockOnCreditsParams, TrackFeatureParams } from '../../../../platform/billing-provider'
import { assertSeatsNotBelowActiveUsers, platformPlanService } from '../platform-plan.service'
import { autumnConsole, autumnUtils, BalanceCacheSnapshot, ConsoleCustomerCall, CreditsBalanceCache } from './autumn-utils'
@@ -14,8 +14,11 @@ const CREDITS_REFETCH_PERIOD_MS = 180 * 1000
const CUSTOMER_STATE_REFRESH_DEBOUNCE_SECONDS = 15
const CUSTOMER_STATE_MISS_DEBOUNCE_SECONDS = 60
const CUSTOMER_STATE_FETCH_LOCK_TIMEOUT_SECONDS = 15
-const CREDITS_CACHE_READ_TIMEOUT_MS = 400
-const FAST_CREDIT_CHECK_READ_TIMEOUT_MS = 25
+const CREDITS_CACHE_READ_TIMEOUT_MS: Record<CreditCheckSpeed, number> = {
+ [CreditCheckSpeed.FAST]: 25,
+ [CreditCheckSpeed.NORMAL]: 400,
+ [CreditCheckSpeed.BACKGROUND]: 5000,
+}
const SLOW_CREDITS_CACHE_READ_LOG_MS = 25
const BILLING_OVERVIEW_TTL_SECONDS = 5 * 60
const TRIAL_DURATION_UNITS: Partial<Record<string, 'day' | 'month' | 'year'>> = { day: 'day', month: 'month', year: 'year' }
@@ -156,12 +159,11 @@ export const autumnBillingProvider = (log: FastifyBaseLogger): BillingProvider =
isBillingEnforced: async (platformId: string) => {
return (await distributedStore.get<boolean>(getBillingEnforcedKey(platformId))) ?? false
},
- shouldBlockOnCredits: async ({ platformId, fastCreditCheck }: ShouldBlockOnCreditsParams) => {
- const readTimeoutMs = fastCreditCheck ? FAST_CREDIT_CHECK_READ_TIMEOUT_MS : CREDITS_CACHE_READ_TIMEOUT_MS
- return (await computeCreditsAndAppSumoState({ log, platformId, readTimeoutMs })).credits.blocked
+ shouldBlockOnCredits: async ({ platformId, creditCheckSpeed }: ShouldBlockOnCreditsParams) => {
+ return (await computeCreditsAndAppSumoState({ log, platformId, readTimeoutMs: CREDITS_CACHE_READ_TIMEOUT_MS[creditCheckSpeed] })).credits.blocked
},
getCreditsAndAppSumoState: async (platformId: string) => {
- return computeCreditsAndAppSumoState({ log, platformId, readTimeoutMs: CREDITS_CACHE_READ_TIMEOUT_MS })
+ return computeCreditsAndAppSumoState({ log, platformId, readTimeoutMs: CREDITS_CACHE_READ_TIMEOUT_MS[CreditCheckSpeed.NORMAL] })
},
getConsumablesUsage: async (platformId: string) => {
const { credits, appSumo } = await resolveCreditsCache(log, platformId)
@@ -313,7 +315,7 @@ async function computeCreditsAndAppSumoState({ log, platformId, readTimeoutMs }:
wideEvent.set({ creditsGate: { readMs, readTimeoutMs, timedOut: error instanceof CreditsReadTimeoutError } })
if (isNil(snapshot)) {
log.warn({ error, platform: { id: platformId }, creditsGate: { readMs, readTimeoutMs } }, 'Credits gate cache read timed out or failed; failing open without gating this request')
- read.then(() => log.info({ platform: { id: platformId }, creditsGate: { readMs: elapsedMs(startedAt), readTimeoutMs } }, 'Credits gate cache read finished after the timeout')).catch(() => undefined)
+ read.then(() => log.info({ platform: { id: platformId }, creditsGate: { lateReadMs: elapsedMs(startedAt), readTimeoutMs } }, 'Credits gate cache read finished after the timeout')).catch(() => undefined)
return {
credits: computeCreditState({ balance: null, enforced: false }),
appSumo: computeCreditState({ balance: null, enforced: true }),
diff --git a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
index 15cca2381fd..307901efcbd 100644
--- a/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
+++ b/packages/server/api/src/app/flows/flow-run/flow-run-service.ts
@@ -13,7 +13,7 @@ import { paginationHelper } from '../../helper/pagination/pagination-utils'
import { Order } from '../../helper/pagination/paginator'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
-import { assertRunCreditsNotExceeded, shouldBlockRunOnCredits } from '../../platform/billing-provider'
+import { assertRunCreditsNotExceeded, CreditCheckSpeed, shouldBlockRunOnCredits } from '../../platform/billing-provider'
import { projectService } from '../../project/project-service'
import { waitpointService } from '../../waitpoints/waitpoint-service'
import { jobQueue, JobType } from '../../workers/job-queue/job-queue'
@@ -158,6 +158,7 @@ export const flowRunService = (log: FastifyBaseLogger) => ({
await assertRunCreditsNotExceeded({
platformId: project.platformId,
environment: oldFlowRun.environment,
+ creditCheckSpeed: CreditCheckSpeed.NORMAL,
log,
})
@@ -432,6 +433,7 @@ export const flowRunService = (log: FastifyBaseLogger) => ({
const creditsExhausted = await shouldBlockRunOnCredits({
platformId,
environment: RunEnvironment.PRODUCTION,
+ creditCheckSpeed: CreditCheckSpeed.NORMAL,
log,
})
if (creditsExhausted) {
diff --git a/packages/server/api/src/app/platform/billing-provider.ts b/packages/server/api/src/app/platform/billing-provider.ts
index 7730f888a59..13ef998be22 100644
--- a/packages/server/api/src/app/platform/billing-provider.ts
+++ b/packages/server/api/src/app/platform/billing-provider.ts
@@ -112,18 +112,18 @@ export async function hasCreditsLeft({ platformId, log }: { platformId: string,
return isNil(left) || left > 0
}
-export async function shouldBlockRunOnCredits({ platformId, environment, fastCreditCheck = false, log }: RunCreditsGateParams): Promise<boolean> {
+export async function shouldBlockRunOnCredits({ platformId, environment, creditCheckSpeed, log }: RunCreditsGateParams): Promise<boolean> {
if (system.getEdition() === ApEdition.ENTERPRISE) {
return false
}
if (environment !== RunEnvironment.PRODUCTION) {
return false
}
- return billingProvider.get(log).shouldBlockOnCredits({ platformId, fastCreditCheck })
+ return billingProvider.get(log).shouldBlockOnCredits({ platformId, creditCheckSpeed })
}
-export async function assertRunCreditsNotExceeded({ platformId, environment, log }: RunCreditsGateParams): Promise<void> {
- const blocked = await shouldBlockRunOnCredits({ platformId, environment, log })
+export async function assertRunCreditsNotExceeded({ platformId, environment, creditCheckSpeed, log }: RunCreditsGateParams): Promise<void> {
+ const blocked = await shouldBlockRunOnCredits({ platformId, environment, creditCheckSpeed, log })
if (!blocked) {
return
}
@@ -186,13 +186,13 @@ type EmptyBillingOverviewParams = {
type RunCreditsGateParams = {
platformId: string
environment: RunEnvironment
- fastCreditCheck?: boolean
+ creditCheckSpeed: CreditCheckSpeed
log: FastifyBaseLogger
}
export type ShouldBlockOnCreditsParams = {
platformId: string
- fastCreditCheck: boolean
+ creditCheckSpeed: CreditCheckSpeed
}
export type CreditEventBreakdownEntry = {
@@ -426,3 +426,9 @@ export type BillingProvider = {
getConsumablesUsage(platformId: string): Promise<ConsumablesUsage>
getCreditUsage(params: CreditUsageByProjectParams): Promise<CreditUsage>
}
+
+export enum CreditCheckSpeed {
+ FAST = 'FAST',
+ NORMAL = 'NORMAL',
+ BACKGROUND = 'BACKGROUND',
+}
diff --git a/packages/server/api/src/app/waitpoints/fan-out-dispatcher-job.ts b/packages/server/api/src/app/waitpoints/fan-out-dispatcher-job.ts
index 2d8094b5ffb..ea979d30ec7 100644
--- a/packages/server/api/src/app/waitpoints/fan-out-dispatcher-job.ts
+++ b/packages/server/api/src/app/waitpoints/fan-out-dispatcher-job.ts
@@ -7,7 +7,7 @@ import { transaction } from '../core/db/transaction'
import { distributedStore } from '../database/redis-connections'
import { ChildDispatchTarget, childRunDispatchService } from '../flows/flow-run/child-run-dispatch-service'
-import { shouldBlockRunOnCredits } from '../platform/billing-provider'
+import { CreditCheckSpeed, shouldBlockRunOnCredits } from '../platform/billing-provider'
import { barrierQueue } from './barrier-queue'
import { BarrierFanOutPayload, BarrierJobData, barrierSourceKey } from './barrier-queue-factory'
import { barrierService, SignalClaim } from './barrier-service'
@@ -80,7 +80,7 @@ export async function handleFanOutDispatchGaveUp({ data, log }: JobParams): Prom
async function dispatchChunks({ data, source, target, log }: DispatchChunksParams): Promise<DispatchOutcome> {
let afterSequence: number | null = START_OF_BARRIER
while (!isNil(afterSequence)) {
- if (await shouldBlockRunOnCredits({ platformId: target.platformId, environment: target.environment, log })) {
+ if (await shouldBlockRunOnCredits({ platformId: target.platformId, environment: target.environment, creditCheckSpeed: CreditCheckSpeed.NORMAL, log })) {
return 'credits-exhausted'
}
const cursor: number = afterSequence
diff --git a/packages/server/api/src/app/webhooks/webhook.service.ts b/packages/server/api/src/app/webhooks/webhook.service.ts
index 448629357da..c5b8e46cca9 100644
--- a/packages/server/api/src/app/webhooks/webhook.service.ts
+++ b/packages/server/api/src/app/webhooks/webhook.service.ts
@@ -10,7 +10,7 @@ import { pinoLogging } from '../helper/logger'
import { rejectedPromiseHandler } from '../helper/promise-handler'
import { system } from '../helper/system/system'
import { AppSystemProp } from '../helper/system/system-props'
-import { shouldBlockRunOnCredits } from '../platform/billing-provider'
+import { CreditCheckSpeed, shouldBlockRunOnCredits } from '../platform/billing-provider'
import { triggerSourceService } from '../trigger/trigger-source/trigger-source-service'
import { engineResponseWatcher } from '../workers/engine-response-watcher'
import { jobQueue, JobType } from '../workers/job-queue/job-queue'
@@ -274,7 +274,7 @@ async function handleSync(params: SyncWebhookParams): Promise<EngineHttpResponse
const creditsExhausted = await shouldBlockRunOnCredits({
platformId,
environment: runEnvironment,
- fastCreditCheck: true,
+ creditCheckSpeed: CreditCheckSpeed.FAST,
log: logger,
})
diff --git a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts b/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
index 9dcbe6517c2..b4b1b683ed0 100644
--- a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
+++ b/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
@@ -22,7 +22,7 @@ import { rejectedPromiseHandler } from '../../helper/promise-handler'
import { system } from '../../helper/system/system'
import { AppSystemProp } from '../../helper/system/system-props'
import { pieceMetadataService } from '../../pieces/metadata/piece-metadata-service'
-import { shouldBlockRunOnCredits } from '../../platform/billing-provider'
+import { CreditCheckSpeed, shouldBlockRunOnCredits } from '../../platform/billing-provider'
import { projectService } from '../../project/project-service'
import { dedupeService } from '../../trigger/dedupe-service'
import { triggerEventService } from '../../trigger/trigger-events/trigger-event.service'
@@ -114,7 +114,8 @@ export function createHandlers(log: FastifyBaseLogger, assignment: WorkerGroupAs
const creditsExhausted = await shouldBlockRunOnCredits({
platformId,
environment,
- log,
+ creditCheckSpeed: CreditCheckSpeed.BACKGROUND,
+ log: log.child({ rpc: 'submitPayloads', flowVersion: { id: flowVersionId } }),
})
const flowRuns = await Promise.all(
diff --git a/packages/server/api/test/unit/app/ee/platform-plan/credits-gate.test.ts b/packages/server/api/test/unit/app/ee/platform-plan/credits-gate.test.ts
index 5af3cc2ae67..4e1a4795c2c 100644
--- a/packages/server/api/test/unit/app/ee/platform-plan/credits-gate.test.ts
+++ b/packages/server/api/test/unit/app/ee/platform-plan/credits-gate.test.ts
@@ -1,8 +1,9 @@
-import { createLogger, wideEvent } from '@activepieces/server-utils'
+import { apLogger, createLogger, wideEvent } from '@activepieces/server-utils'
import { FastifyBaseLogger } from 'fastify'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { autumnBillingProvider, computeCreditState } from '../../../../../src/app/ee/platform/platform-plan/billing-providers/autumn-billing'
import { CreditsBalanceCache } from '../../../../../src/app/ee/platform/platform-plan/billing-providers/autumn-utils'
+import { CreditCheckSpeed } from '../../../../../src/app/platform/billing-provider'
let storedCredits: CreditsBalanceCache | null = null
let autumnCredits: CreditsBalanceCache | null = null
@@ -251,8 +252,17 @@ describe('credits gate decides from the cache and refreshes in the background',
mockBillingEnforced.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve(true), 100)))
const provider = autumnBillingProvider(log)
- await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', fastCreditCheck: true })).resolves.toBe(false)
- await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', fastCreditCheck: false })).resolves.toBe(true)
+ await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.FAST })).resolves.toBe(false)
+ await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.NORMAL })).resolves.toBe(true)
+ })
+
+ it('waits longest for a background check', async () => {
+ storedCredits = fresh({ remaining: 0 })
+ mockBillingEnforced.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve(true), 500)))
+ const provider = autumnBillingProvider(log)
+
+ await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.NORMAL })).resolves.toBe(false)
+ await expect(provider.shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.BACKGROUND })).resolves.toBe(true)
})
it('records the read time on the request row instead of a separate log line', async () => {
@@ -285,7 +295,7 @@ describe('credits gate decides from the cache and refreshes in the background',
mockBillingEnforced.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(true), 100)))
const slowRead = createLogger()
- await wideEvent.run({ logger: slowRead, fn: () => autumnBillingProvider(log).shouldBlockOnCredits({ platformId: 'platform-1', fastCreditCheck: true }) })
+ await wideEvent.run({ logger: slowRead, fn: () => autumnBillingProvider(log).shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.FAST }) })
expect(failedRead.getContext()).toMatchObject({ creditsGate: { timedOut: false } })
expect(slowRead.getContext()).toMatchObject({ creditsGate: { timedOut: true, readTimeoutMs: 25 } })
@@ -303,11 +313,27 @@ describe('credits gate decides from the cache and refreshes in the background',
storedCredits = fresh({ remaining: 0 })
mockBillingEnforced.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve(true), 100)))
- await autumnBillingProvider(log).shouldBlockOnCredits({ platformId: 'platform-1', fastCreditCheck: true })
+ await autumnBillingProvider(log).shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.FAST })
await new Promise((resolve) => setTimeout(resolve, 150))
const lateRead = vi.mocked(log.info).mock.calls.find(([, msg]) => msg === 'Credits gate cache read finished after the timeout')
expect(lateRead?.[0]).toMatchObject({ platform: { id: 'platform-1' }, creditsGate: { readTimeoutMs: 25 } })
- expect(lateRead?.[0]).toHaveProperty('creditsGate.readMs', expect.any(Number))
+ expect(lateRead?.[0]).toHaveProperty('creditsGate.lateReadMs', expect.any(Number))
+ })
+
+ it('keeps the read time on the request row when a timed-out read finishes later in the same request', async () => {
+ storedCredits = fresh({ remaining: 0 })
+ mockBillingEnforced.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve(true), 100)))
+ const requestLogger = createLogger()
+
+ await wideEvent.run({
+ logger: requestLogger,
+ fn: async () => {
+ await autumnBillingProvider(apLogger.create()).shouldBlockOnCredits({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.FAST })
+ await new Promise((resolve) => setTimeout(resolve, 150))
+ },
+ })
+
+ expect(requestLogger.getContext()).toMatchObject({ creditsGate: { readMs: expect.any(Number), lateReadMs: expect.any(Number), timedOut: true } })
})
})
diff --git a/packages/server/api/test/unit/app/platform/run-credits-gate.test.ts b/packages/server/api/test/unit/app/platform/run-credits-gate.test.ts
index 710d569aeb0..c5f9ab1f101 100644
--- a/packages/server/api/test/unit/app/platform/run-credits-gate.test.ts
+++ b/packages/server/api/test/unit/app/platform/run-credits-gate.test.ts
@@ -11,13 +11,13 @@ vi.mock('../../../../src/app/helper/system/system', () => ({
},
}))
-import { billingProvider, shouldBlockRunOnCredits } from '../../../../src/app/platform/billing-provider'
+import { billingProvider, CreditCheckSpeed, shouldBlockRunOnCredits } from '../../../../src/app/platform/billing-provider'
const log = { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } as unknown as FastifyBaseLogger
const noopProvider = billingProvider.get(log)
-function gate(environment: RunEnvironment, fastCreditCheck?: boolean) {
- return shouldBlockRunOnCredits({ platformId: 'platform-1', environment, fastCreditCheck, log })
+function gate(environment: RunEnvironment, creditCheckSpeed = CreditCheckSpeed.NORMAL) {
+ return shouldBlockRunOnCredits({ platformId: 'platform-1', environment, creditCheckSpeed, log })
}
describe('shouldBlockRunOnCredits', () => {
@@ -38,14 +38,14 @@ describe('shouldBlockRunOnCredits', () => {
mockGetEdition.mockReturnValue(ApEdition.CLOUD)
await expect(gate(RunEnvironment.PRODUCTION)).resolves.toBe(true)
- expect(mockShouldBlockOnCredits).toHaveBeenCalledWith({ platformId: 'platform-1', fastCreditCheck: false })
+ expect(mockShouldBlockOnCredits).toHaveBeenCalledWith({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.NORMAL })
})
- it('tells the provider when the check must be fast', async () => {
+ it('passes the caller\'s check speed to the provider', async () => {
mockGetEdition.mockReturnValue(ApEdition.CLOUD)
- await gate(RunEnvironment.PRODUCTION, true)
- expect(mockShouldBlockOnCredits).toHaveBeenCalledWith({ platformId: 'platform-1', fastCreditCheck: true })
+ await gate(RunEnvironment.PRODUCTION, CreditCheckSpeed.FAST)
+ expect(mockShouldBlockOnCredits).toHaveBeenCalledWith({ platformId: 'platform-1', creditCheckSpeed: CreditCheckSpeed.FAST })
})
it('never gates a testing run', async () => {
diff --git a/packages/server/api/test/unit/app/webhooks/webhook-credit-gate.test.ts b/packages/server/api/test/unit/app/webhooks/webhook-credit-gate.test.ts
index 8a7e8bb5c84..ef3063d4d4d 100644
--- a/packages/server/api/test/unit/app/webhooks/webhook-credit-gate.test.ts
+++ b/packages/server/api/test/unit/app/webhooks/webhook-credit-gate.test.ts
@@ -2,6 +2,7 @@ import { FlowStatus, RunEnvironment } from '@activepieces/shared'
import { StatusCodes } from 'http-status-codes'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { WebhookFlowVersionToRun } from '../../../../src/app/webhooks/webhook.service'
+import { CreditCheckSpeed } from '../../../../src/app/platform/billing-provider'
const { mockShouldBlockRunOnCredits, mockStart, mockCreateQuotaExceededRun, mockFindOneBy, mockOneTimeListener } = vi.hoisted(() => ({
mockShouldBlockRunOnCredits: vi.fn(),
@@ -24,7 +25,8 @@ vi.mock('../../../../src/app/helper/system/system', () => ({
},
}))
-vi.mock('../../../../src/app/platform/billing-provider', () => ({
+vi.mock('../../../../src/app/platform/billing-provider', async (importOriginal) => ({
+ ...(await importOriginal<typeof import('../../../../src/app/platform/billing-provider')>()),
shouldBlockRunOnCredits: mockShouldBlockRunOnCredits,
}))
@@ -160,7 +162,7 @@ describe('sync webhook credit gate', () => {
expect(mockShouldBlockRunOnCredits.mock.calls[0][0]).toMatchObject({
platformId: 'plat-1',
environment: RunEnvironment.PRODUCTION,
- fastCreditCheck: true,
+ creditCheckSpeed: CreditCheckSpeed.FAST,
})
})
})