Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(autumn): add dry-run mode for Autumn credit check rollout (#3176)

firecrawl-spring[bot]24b6180f88ba
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 4c334c1fe..01baba795 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -49,6 +49,7 @@ const configSchema = z.object({
STRIPE_SECRET_KEY: z.string().optional(),
AUTUMN_SECRET_KEY: z.string().optional(),
AUTUMN_CHECK_ENABLED: z.string().optional(),
+ AUTUMN_CHECK_DRY_RUN: z.string().optional(),
AUTUMN_CHECK_EXPERIMENT_PERCENT: z.coerce.number().default(100),
AUTUMN_EXPERIMENT: z.string().optional(),
AUTUMN_EXPERIMENT_PERCENT: z.coerce.number().default(100),
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 7fd580aa3..cb32e637f 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -27,6 +27,7 @@ import { supabase_service } from "../services/supabase";
import {
autumnService,
isAutumnCheckEnabled,
+ isAutumnCheckDryRun,
} from "../services/autumn/autumn.service";
export function checkCreditsMiddleware(
@@ -142,6 +143,7 @@ export function checkCreditsMiddleware(
let { success, remainingCredits, chunk } = legacyCheck;
if (autumnAllowed !== null) {
+ const dryRun = isAutumnCheckDryRun();
if (autumnAllowed !== legacyCheck.success) {
logger.warn("Autumn check result diverged from legacy credit gate", {
teamId: req.auth.team_id,
@@ -149,9 +151,20 @@ export function checkCreditsMiddleware(
requestedCredits,
autumnAllowed,
legacyAllowed: legacyCheck.success,
+ dryRun,
});
}
- success = autumnAllowed;
+ if (dryRun) {
+ logger.info("Autumn check dry-run result (not enforced)", {
+ teamId: req.auth.team_id,
+ path: req.path,
+ requestedCredits,
+ autumnAllowed,
+ legacyAllowed: legacyCheck.success,
+ });
+ } else {
+ success = autumnAllowed;
+ }
remainingCredits = legacyCheck.remainingCredits;
}
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index f1096a740..f24c3772e 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -71,6 +71,7 @@ jest.mock("../../supabase", () => ({
jest.mock("../../../config", () => ({
config: {
AUTUMN_CHECK_ENABLED: undefined,
+ AUTUMN_CHECK_DRY_RUN: undefined,
AUTUMN_EXPERIMENT: "true",
AUTUMN_EXPERIMENT_PERCENT: 100,
AUTUMN_REQUEST_TRACK_EXPERIMENT: undefined,
@@ -83,6 +84,7 @@ import {
AutumnService,
BoundedMap,
BoundedSet,
+ isAutumnCheckDryRun,
isAutumnCheckEnabled,
isAutumnEnabled,
isAutumnRequestTrackEnabled,
@@ -102,17 +104,20 @@ function makeEntity(usage: number) {
return { balances: { CREDITS: { usage } } };
}
-function setAutumnConfig(overrides: {
- AUTUMN_CHECK_ENABLED?: string;
- AUTUMN_EXPERIMENT?: string;
- AUTUMN_EXPERIMENT_PERCENT?: number;
- AUTUMN_REQUEST_TRACK_EXPERIMENT?: string;
- AUTUMN_REQUEST_TRACK_EXPERIMENT_PERCENT?: number;
-} = {}) {
+function setAutumnConfig(
+ overrides: {
+ AUTUMN_CHECK_ENABLED?: string;
+ AUTUMN_CHECK_DRY_RUN?: string;
+ AUTUMN_EXPERIMENT?: string;
+ AUTUMN_EXPERIMENT_PERCENT?: number;
+ AUTUMN_REQUEST_TRACK_EXPERIMENT?: string;
+ AUTUMN_REQUEST_TRACK_EXPERIMENT_PERCENT?: number;
+ } = {},
+) {
config.AUTUMN_CHECK_ENABLED = overrides.AUTUMN_CHECK_ENABLED;
+ config.AUTUMN_CHECK_DRY_RUN = overrides.AUTUMN_CHECK_DRY_RUN;
config.AUTUMN_EXPERIMENT = overrides.AUTUMN_EXPERIMENT ?? "true";
- config.AUTUMN_EXPERIMENT_PERCENT =
- overrides.AUTUMN_EXPERIMENT_PERCENT ?? 100;
+ config.AUTUMN_EXPERIMENT_PERCENT = overrides.AUTUMN_EXPERIMENT_PERCENT ?? 100;
config.AUTUMN_REQUEST_TRACK_EXPERIMENT =
overrides.AUTUMN_REQUEST_TRACK_EXPERIMENT;
config.AUTUMN_REQUEST_TRACK_EXPERIMENT_PERCENT =
@@ -459,9 +464,7 @@ describe("trackCredits", () => {
mockTrack.mockRejectedValueOnce(new Error("track failed"));
const svc = makeService();
- expect(await svc.trackCredits({ teamId: "team-1", value: 42 })).toBe(
- false,
- );
+ expect(await svc.trackCredits({ teamId: "team-1", value: 42 })).toBe(false);
});
});
@@ -613,6 +616,27 @@ describe("isAutumnCheckEnabled", () => {
});
});
+describe("isAutumnCheckDryRun", () => {
+ afterEach(() => {
+ config.AUTUMN_CHECK_DRY_RUN = undefined;
+ });
+
+ it("returns false when AUTUMN_CHECK_DRY_RUN is not set", () => {
+ config.AUTUMN_CHECK_DRY_RUN = undefined;
+ expect(isAutumnCheckDryRun()).toBe(false);
+ });
+
+ it("returns false when AUTUMN_CHECK_DRY_RUN is not 'true'", () => {
+ config.AUTUMN_CHECK_DRY_RUN = "false";
+ expect(isAutumnCheckDryRun()).toBe(false);
+ });
+
+ it("returns true when AUTUMN_CHECK_DRY_RUN is 'true'", () => {
+ config.AUTUMN_CHECK_DRY_RUN = "true";
+ expect(isAutumnCheckDryRun()).toBe(true);
+ });
+});
+
describe("isAutumnRequestTrackEnabled", () => {
afterEach(() => {
setAutumnConfig({ AUTUMN_REQUEST_TRACK_EXPERIMENT: undefined });
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index c735b9f04..1fb329bc7 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -58,6 +58,14 @@ export function isAutumnCheckEnabled(orgId?: string): boolean {
return orgBucket(orgId) < percent;
}
+/**
+ * When true, Autumn check results are logged but never used to gate requests.
+ * The legacy credit system remains authoritative.
+ */
+export function isAutumnCheckDryRun(): boolean {
+ return config.AUTUMN_CHECK_DRY_RUN === "true";
+}
+
export function isAutumnRequestTrackEnabled(orgId?: string): boolean {
if (config.AUTUMN_REQUEST_TRACK_EXPERIMENT !== "true") return false;
if (!isAutumnEnabled(orgId)) return false;