Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): fail open to high limits when Autumn errors (#4133)

firecrawl-spring[bot]ab033afd95b6
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 7f79f0a66..246e376f0 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -25,8 +25,9 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;
/**
* Returns the team's effective concurrency limit from Autumn's CONCURRENCY
- * balance. Autumn is authoritative; when it can't answer (unavailable, or the
- * entity is missing) we fall back to the default of 2.
+ * balance. Autumn is authoritative; when the entity is missing we fall back to
+ * the low default of 2. When Autumn errors, getConcurrencyLimit already returns
+ * a high fail-open value, so that carries through here.
*/
export async function getEffectiveConcurrencyLimit(
teamId: string,
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index f2a988455..9cc8fa67f 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -602,3 +602,67 @@ describe("featureIdForBillingEndpoint", () => {
expect(featureIdForBillingEndpoint(undefined)).toBe("CREDITS");
});
});
+
+// ---------------------------------------------------------------------------
+// getEntityLimits fallback behavior (concurrency + rate-limit multiplier)
+// ---------------------------------------------------------------------------
+
+describe("entity limits fallback", () => {
+ it("returns the Autumn values on the happy path", async () => {
+ const svc = makeService();
+ mockEntityGet.mockResolvedValue({
+ balances: {
+ CONCURRENCY: { remaining: 7 },
+ rate_limits: { granted: 25 },
+ },
+ });
+
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(7);
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(25);
+ });
+
+ it("falls back LOW (null / multiplier 1) when the entity is missing (404)", async () => {
+ const svc = makeService();
+ mockEntityGet.mockRejectedValue({ statusCode: 404 });
+
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBeNull();
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(1);
+ });
+
+ it("falls back LOW when the entity exists but the balances are absent", async () => {
+ const svc = makeService();
+ mockEntityGet.mockResolvedValue({ balances: {} });
+
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBeNull();
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(1);
+ });
+
+ it("fails OPEN with high limits when Autumn errors (not a 404)", async () => {
+ const svc = makeService();
+ mockEntityGet.mockRejectedValue({ statusCode: 500 });
+
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(2500);
+ });
+
+ it("fails OPEN when Autumn throws a non-HTTP error", async () => {
+ const svc = makeService();
+ mockEntityGet.mockRejectedValue(new Error("ECONNREFUSED"));
+
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(2500);
+ });
+
+ it("does NOT cache the error fail-open value β€” retries Autumn next call", async () => {
+ const svc = makeService();
+ mockEntityGet.mockRejectedValueOnce({ statusCode: 500 });
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(200);
+
+ // Autumn recovers on the next call; the earlier error must not be pinned.
+ mockEntityGet.mockResolvedValue({
+ balances: { CONCURRENCY: { remaining: 3 }, rate_limits: { granted: 10 } },
+ });
+ expect(await svc.getConcurrencyLimit("team-1", "org-1")).toBe(3);
+ expect(await svc.getRateLimitMultiplier("team-1", "org-1")).toBe(10);
+ });
+});
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index c619ea004..12df559f3 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -540,12 +540,27 @@ export class AutumnService {
>(50_000);
private static readonly ENTITY_LIMITS_TTL_MS = 60_000;
+ // Fail-open fallbacks used ONLY when Autumn itself errors (network / 5xx /
+ // unexpected exception) so a billing-API outage doesn't throttle real
+ // customers down to the low defaults. A 404 or an absent balance is NOT an
+ // error β€” it legitimately means the team has no elevated entitlement, so
+ // those keep falling back low (concurrency 2, multiplier 1). These values are
+ // intentionally generous but bounded (the concurrency queue cap still
+ // applies).
+ private static readonly ERROR_FALLBACK_CONCURRENCY = 200;
+ private static readonly ERROR_FALLBACK_RATE_MULTIPLIER = 2500;
+
/**
* Fetches the team's Autumn entity once and derives both the CONCURRENCY
* limit and the rate-limit multiplier from it. Each team has its own Autumn
* entity, so the entity balances are per-team regardless of whether the org
- * has one or many teams. Returns nulls when Autumn is unavailable, the entity
- * is missing, or a balance isn't present β€” callers fall back accordingly.
+ * has one or many teams.
+ *
+ * Returns nulls when Autumn is not configured, the entity is missing (404),
+ * or a balance isn't present β€” these mean "no elevated entitlement", so
+ * callers fall back to the low defaults. When Autumn itself errors (network /
+ * 5xx / unexpected exception) we instead fail OPEN, returning the high
+ * ERROR_FALLBACK_* limits so a billing outage doesn't throttle real teams.
*/
private async getEntityLimits(
teamId: string,
@@ -605,20 +620,31 @@ export class AutumnService {
return store(concurrency, rateLimitMultiplier);
} catch (error) {
const status = this.getErrorStatus(error);
+ // 404 = the entity genuinely doesn't exist in Autumn (not an error):
+ // fall back low, and cache it so we don't re-query for a team we know is
+ // absent.
if (status === 404) return store(null, null);
+ // Any other failure means we couldn't reach Autumn / it errored. Fail
+ // OPEN with high limits rather than throttling the team to the low
+ // defaults. Deliberately not cached, so we retry Autumn on the next
+ // request instead of pinning the team to the fallback for the TTL window.
logger.error(
- "Autumn getEntityLimits failed β€” billing API may be unavailable",
+ "Autumn getEntityLimits failed β€” billing API may be unavailable, falling back to high limits",
{ teamId, error },
);
- return { concurrency: null, rateLimitMultiplier: null };
+ return {
+ concurrency: AutumnService.ERROR_FALLBACK_CONCURRENCY,
+ rateLimitMultiplier: AutumnService.ERROR_FALLBACK_RATE_MULTIPLIER,
+ };
}
}
/**
* Reads the team's allowed concurrent-browser count from Autumn's
- * entity-scoped CONCURRENCY balance. Returns null when Autumn is unavailable,
- * the entity is missing, or there's no balance β€” callers fall back to the
- * default via getEffectiveConcurrencyLimit.
+ * entity-scoped CONCURRENCY balance. Returns null when the entity is missing
+ * or there's no balance β€” callers fall back to the low default via
+ * getEffectiveConcurrencyLimit. On an Autumn error it returns the high
+ * ERROR_FALLBACK_CONCURRENCY (fail open) rather than null.
*/
async getConcurrencyLimit(
teamId: string,
@@ -630,9 +656,10 @@ export class AutumnService {
/**
* Reads the team's rate-limit multiplier from Autumn's `rate_limits` feature.
* Effective rate limits are `base Γ— multiplier`. Falls back to a multiplier of
- * 1 when Autumn is unavailable or the feature is missing, so callers don't
- * have to. Shares a single cached entity fetch with getConcurrencyLimit, so it
- * adds no Autumn call.
+ * 1 when the feature is missing or the entity doesn't exist, so callers don't
+ * have to; on an Autumn error it fails open with the high
+ * ERROR_FALLBACK_RATE_MULTIPLIER instead. Shares a single cached entity fetch
+ * with getConcurrencyLimit, so it adds no Autumn call.
*/
async getRateLimitMultiplier(
teamId: string,