Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): fail open to high limits when Autumn errors (#4133)
firecrawl-spring[bot]ab033afd95b6
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 7f79f0a66..246e376f0 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -25,8 +25,9 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;/*** Returns the team's effective concurrency limit from Autumn's CONCURRENCY- * balance. Autumn is authoritative; when it can't answer (unavailable, or the- * entity is missing) we fall back to the default of 2.+ * balance. Autumn is authoritative; when the entity is missing we fall back to+ * the low default of 2. When Autumn errors, getConcurrencyLimit already returns+ * a high fail-open value, so that carries through here.*/export async function getEffectiveConcurrencyLimit(teamId: string,diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex f2a988455..9cc8fa67f 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -602,3 +602,67 @@ describe("featureIdForBillingEndpoint", () => {expect(featureIdForBillingEndpoint(undefined)).toBe("CREDITS");});});++// ---------------------------------------------------------------------------+// getEntityLimits fallback behavior (concurrency + rate-limit multiplier)+// ---------------------------------------------------------------------------++describe("entity limits fallback", () => {+ it("returns the Autumn values on the happy path", async () => {+ const svc = makeService();+ mockEntityGet.mockResolvedValue({+ balances: {+ CONCURRENCY: { remaining: 7 },+ rate_limits: { granted: 25 },+ },+ });++ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(7);+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(25);+ });++ it("falls back LOW (null / multiplier 1) when the entity is missing (404)", async () => {+ const svc = makeService();+ mockEntityGet.mockRejectedValue({ statusCode: 404 });++ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBeNull();+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(1);+ });++ it("falls back LOW when the entity exists but the balances are absent", async () => {+ const svc = makeService();+ mockEntityGet.mockResolvedValue({ balances: {} });++ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBeNull();+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(1);+ });++ it("fails OPEN with high limits when Autumn errors (not a 404)", async () => {+ const svc = makeService();+ mockEntityGet.mockRejectedValue({ statusCode: 500 });++ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(2500);+ });++ it("fails OPEN when Autumn throws a non-HTTP error", async () => {+ const svc = makeService();+ mockEntityGet.mockRejectedValue(new Error("ECONNREFUSED"));++ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(2500);+ });++ it("does NOT cache the error fail-open value β retries Autumn next call", async () => {+ const svc = makeService();+ mockEntityGet.mockRejectedValueOnce({ statusCode: 500 });+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);++ // Autumn recovers on the next call; the earlier error must not be pinned.+ mockEntityGet.mockResolvedValue({+ balances: { CONCURRENCY: { remaining: 3 }, rate_limits: { granted: 10 } },+ });+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(3);+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(10);+ });+});diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex c619ea004..12df559f3 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -540,12 +540,27 @@ export class AutumnService {>(50_000);private static readonly ENTITY_LIMITS_TTL_MS = 60_000;+ // Fail-open fallbacks used ONLY when Autumn itself errors (network / 5xx /+ // unexpected exception) so a billing-API outage doesn't throttle real+ // customers down to the low defaults. A 404 or an absent balance is NOT an+ // error β it legitimately means the team has no elevated entitlement, so+ // those keep falling back low (concurrency 2, multiplier 1). These values are+ // intentionally generous but bounded (the concurrency queue cap still+ // applies).+ private static readonly ERROR_FALLBACK_CONCURRENCY = 200;+ private static readonly ERROR_FALLBACK_RATE_MULTIPLIER = 2500;+/*** Fetches the team's Autumn entity once and derives both the CONCURRENCY* limit and the rate-limit multiplier from it. Each team has its own Autumn* entity, so the entity balances are per-team regardless of whether the org- * has one or many teams. Returns nulls when Autumn is unavailable, the entity- * is missing, or a balance isn't present β callers fall back accordingly.+ * has one or many teams.+ *+ * Returns nulls when Autumn is not configured, the entity is missing (404),+ * or a balance isn't present β these mean "no elevated entitlement", so+ * callers fall back to the low defaults. When Autumn itself errors (network /+ * 5xx / unexpected exception) we instead fail OPEN, returning the high+ * ERROR_FALLBACK_* limits so a billing outage doesn't throttle real teams.*/private async getEntityLimits(teamId: string,@@ -605,20 +620,31 @@ export class AutumnService {return store(concurrency, rateLimitMultiplier);} catch (error) {const status = this.getErrorStatus(error);+ // 404 = the entity genuinely doesn't exist in Autumn (not an error):+ // fall back low, and cache it so we don't re-query for a team we know is+ // absent.if (status === 404) return store(null, null);+ // Any other failure means we couldn't reach Autumn / it errored. Fail+ // OPEN with high limits rather than throttling the team to the low+ // defaults. Deliberately not cached, so we retry Autumn on the next+ // request instead of pinning the team to the fallback for the TTL window.logger.error(- "Autumn getEntityLimits failed β billing API may be unavailable",+ "Autumn getEntityLimits failed β billing API may be unavailable, falling back to high limits",{ teamId, error },);- return { concurrency: null, rateLimitMultiplier: null };+ return {+ concurrency: AutumnService.ERROR_FALLBACK_CONCURRENCY,+ rateLimitMultiplier: AutumnService.ERROR_FALLBACK_RATE_MULTIPLIER,+ };}}/*** Reads the team's allowed concurrent-browser count from Autumn's- * entity-scoped CONCURRENCY balance. Returns null when Autumn is unavailable,- * the entity is missing, or there's no balance β callers fall back to the- * default via getEffectiveConcurrencyLimit.+ * entity-scoped CONCURRENCY balance. Returns null when the entity is missing+ * or there's no balance β callers fall back to the low default via+ * getEffectiveConcurrencyLimit. On an Autumn error it returns the high+ * ERROR_FALLBACK_CONCURRENCY (fail open) rather than null.*/async getConcurrencyLimit(teamId: string,@@ -630,9 +656,10 @@ export class AutumnService {/*** Reads the team's rate-limit multiplier from Autumn's `rate_limits` feature.* Effective rate limits are `base Γ multiplier`. Falls back to a multiplier of- * 1 when Autumn is unavailable or the feature is missing, so callers don't- * have to. Shares a single cached entity fetch with getConcurrencyLimit, so it- * adds no Autumn call.+ * 1 when the feature is missing or the entity doesn't exist, so callers don't+ * have to; on an Autumn error it fails open with the high+ * ERROR_FALLBACK_RATE_MULTIPLIER instead. Shares a single cached entity fetch+ * with getConcurrencyLimit, so it adds no Autumn call.*/async getRateLimitMultiplier(teamId: string,