Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(frontend): add jittered rate-limit backoff (#15236)

aivong-openhands1747b39562ff
diff --git a/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx b/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx
index 51e6474802..b04c156302 100644
--- a/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx
+++ b/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx
@@ -1,6 +1,7 @@
-import { renderHook, waitFor } from "@testing-library/react";
+import { act, renderHook, waitFor } from "@testing-library/react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
-import { describe, expect, it, vi, beforeEach } from "vitest";
+import { AxiosError } from "axios";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import React from "react";
import { SandboxService } from "#/api/sandbox-service/sandbox-service.api";
import V1ConversationService from "#/api/conversation-service/v1-conversation-service.api";
@@ -49,6 +50,11 @@ describe("useUnifiedResumeConversationSandbox", () => {
vi.restoreAllMocks();
});
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ });
+
const wrapper = ({ children }: { children: React.ReactNode }) => (
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
);
@@ -85,6 +91,55 @@ describe("useUnifiedResumeConversationSandbox", () => {
});
});
+ it("backs off exponentially when the resume preflight is rate limited", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ const rateLimitError = {
+ response: {
+ status: 429,
+ headers: { "retry-after": "0" },
+ },
+ } as unknown as AxiosError;
+ const batchGetAppConversations = vi
+ .spyOn(V1ConversationService, "batchGetAppConversations")
+ .mockRejectedValueOnce(rateLimitError)
+ .mockRejectedValueOnce(rateLimitError)
+ .mockResolvedValueOnce([createConversation()]);
+ vi.spyOn(SandboxService, "resumeSandbox").mockResolvedValue({
+ success: true,
+ });
+
+ const { result } = renderHook(() => useUnifiedResumeConversationSandbox(), {
+ wrapper,
+ });
+
+ const resume = result.current.mutateAsync({
+ conversationId: "test-conv-id",
+ });
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(0);
+ });
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(1);
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1000);
+ });
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1999);
+ });
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1);
+ });
+ await expect(resume).resolves.toEqual({ success: true });
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(3);
+ });
+
it("invalidates sandbox and vscode_url queries on settled", async () => {
// Mock the API calls in the mutation chain
vi.spyOn(
diff --git a/frontend/__tests__/hooks/query/use-user-conversation.test.tsx b/frontend/__tests__/hooks/query/use-user-conversation.test.tsx
index 913d873e69..02424f336a 100644
--- a/frontend/__tests__/hooks/query/use-user-conversation.test.tsx
+++ b/frontend/__tests__/hooks/query/use-user-conversation.test.tsx
@@ -50,7 +50,7 @@ describe("useUserConversation", () => {
vi.restoreAllMocks();
});
- it("retries rate-limited conversation fetches after the Retry-After delay", async () => {
+ it("backs off exponentially between rate-limited conversation fetches", async () => {
const rateLimitError = {
response: {
status: 429,
@@ -63,7 +63,9 @@ describe("useUserConversation", () => {
const batchGetAppConversations = vi
.spyOn(V1ConversationService, "batchGetAppConversations")
.mockRejectedValueOnce(rateLimitError)
+ .mockRejectedValueOnce(rateLimitError)
.mockResolvedValueOnce([createConversation()]);
+ vi.spyOn(Math, "random").mockReturnValue(0);
const { result } = renderHook(() => useUserConversation("conversation-1"), {
wrapper: createWrapper(),
@@ -81,6 +83,17 @@ describe("useUserConversation", () => {
});
expect(batchGetAppConversations).toHaveBeenCalledTimes(2);
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1999);
+ });
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);
+
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1);
+ });
+
+ expect(batchGetAppConversations).toHaveBeenCalledTimes(3);
await vi.waitFor(() => {
expect(result.current.data?.id).toBe("conversation-1");
});
diff --git a/frontend/__tests__/query-client-config.test.ts b/frontend/__tests__/query-client-config.test.ts
index 0c0e09467b..1103d0514a 100644
--- a/frontend/__tests__/query-client-config.test.ts
+++ b/frontend/__tests__/query-client-config.test.ts
@@ -13,22 +13,33 @@ const make429 = () =>
describe("queryClient mutation defaults", () => {
afterEach(() => {
+ vi.useRealTimers();
vi.restoreAllMocks();
queryClient.clear();
});
- it("should retry a mutation rejected with 429 (request was never processed)", async () => {
+ it("backs off before retrying a rate-limited mutation", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
const mutationFn = vi
.fn()
.mockRejectedValueOnce(make429())
.mockResolvedValueOnce("ok");
- const result = await queryClient
+ const result = queryClient
.getMutationCache()
.build(queryClient, { mutationFn, meta: { disableToast: true } })
.execute(undefined);
- expect(result).toBe("ok");
+ await vi.advanceTimersByTimeAsync(0);
+ expect(mutationFn).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(999);
+ expect(mutationFn).toHaveBeenCalledTimes(1);
+
+ await vi.advanceTimersByTimeAsync(1);
+ await expect(result).resolves.toBe("ok");
expect(mutationFn).toHaveBeenCalledTimes(2);
});
diff --git a/frontend/__tests__/utils/rate-limit-retry.test.ts b/frontend/__tests__/utils/rate-limit-retry.test.ts
index 54ec0f5d44..6ce53fbe02 100644
--- a/frontend/__tests__/utils/rate-limit-retry.test.ts
+++ b/frontend/__tests__/utils/rate-limit-retry.test.ts
@@ -16,6 +16,7 @@ const createAxiosError = (status: number, headers?: unknown): AxiosError =>
describe("rate limit retry helpers", () => {
afterEach(() => {
vi.useRealTimers();
+ vi.restoreAllMocks();
});
it("recognizes response status 429 as a rate-limit error", () => {
@@ -30,7 +31,44 @@ describe("rate limit retry helpers", () => {
headerName.toLowerCase() === "retry-after" ? "2" : undefined,
};
- expect(getRateLimitRetryDelayMs(createAxiosError(429, headers))).toBe(2000);
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(getRateLimitRetryDelayMs(0, createAxiosError(429, headers))).toBe(
+ 2000,
+ );
+ });
+
+ it("accepts numeric Retry-After values from plain headers", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "Retry-After": 2 }),
+ ),
+ ).toBe(2000);
+ });
+
+ it("falls back to backoff when AxiosHeaders returns a non-string value", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { get: () => 2 }),
+ ),
+ ).toBe(1000);
+ });
+
+ it("falls back to backoff for unsupported plain header values", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": { seconds: 2 } }),
+ ),
+ ).toBe(1000);
});
it("parses Retry-After HTTP dates", () => {
@@ -38,9 +76,11 @@ describe("rate limit retry helpers", () => {
vi.setSystemTime(new Date("2026-06-04T20:00:00.000Z"));
const retryAt = new Date("2026-06-04T20:00:03.000Z").toUTCString();
+ vi.spyOn(Math, "random").mockReturnValue(0);
expect(
getRateLimitRetryDelayMs(
+ 0,
createAxiosError(429, {
"retry-after": retryAt,
}),
@@ -49,34 +89,52 @@ describe("rate limit retry helpers", () => {
});
it("falls back to one second when Retry-After is missing", () => {
- expect(getRateLimitRetryDelayMs(createAxiosError(429))).toBe(1000);
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1000);
});
it("falls back to one second for non-numeric Retry-After", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
expect(
getRateLimitRetryDelayMs(
+ 0,
createAxiosError(429, { "retry-after": "invalid" }),
),
).toBe(1000);
});
- it("treats negative Retry-After seconds as immediate retry (already expired)", () => {
- // Date.parse("-5") returns a valid timestamp (epoch - 5000ms), and since
- // that time has passed, Math.max(negative - now, 0) returns 0
+ it("uses the initial backoff when Retry-After is already expired", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
expect(
- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "-5" })),
- ).toBe(0);
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": "-5" }),
+ ),
+ ).toBe(1000);
});
- it("treats zero Retry-After seconds as immediate (0ms delay)", () => {
+ it("uses the initial backoff when Retry-After is zero", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
expect(
- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "0" })),
- ).toBe(0);
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": "0" }),
+ ),
+ ).toBe(1000);
});
it("clamps large Retry-After seconds to max 60 seconds", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
expect(
- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "120" })),
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": "120" }),
+ ),
).toBe(60_000);
});
@@ -85,9 +143,40 @@ describe("rate limit retry helpers", () => {
vi.setSystemTime(new Date("2026-06-04T20:00:00.000Z"));
const farFuture = new Date("2026-06-04T21:00:00.000Z").toUTCString();
+ vi.spyOn(Math, "random").mockReturnValue(0);
expect(
- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": farFuture })),
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": farFuture }),
+ ),
).toBe(60_000);
});
+
+ it("grows the fallback delay exponentially for repeated failures", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+
+ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1000);
+ expect(getRateLimitRetryDelayMs(1, createAxiosError(429))).toBe(2000);
+ expect(getRateLimitRetryDelayMs(2, createAxiosError(429))).toBe(4000);
+ });
+
+ it("adds jitter while never retrying before Retry-After", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0.5);
+
+ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1500);
+ // jitter scales off the 5s Retry-After: 0.5 * min(5000, MAX_JITTER_MS) = 2500
+ expect(
+ getRateLimitRetryDelayMs(
+ 0,
+ createAxiosError(429, { "retry-after": "5" }),
+ ),
+ ).toBe(7500);
+ });
+
+ it("caps jitter to avoid unbounded delays", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0.999);
+
+ expect(getRateLimitRetryDelayMs(10, createAxiosError(429))).toBe(64_995);
+ });
});
diff --git a/frontend/src/hooks/mutation/use-unified-start-conversation.ts b/frontend/src/hooks/mutation/use-unified-start-conversation.ts
index 3c857ea6a0..9067393127 100644
--- a/frontend/src/hooks/mutation/use-unified-start-conversation.ts
+++ b/frontend/src/hooks/mutation/use-unified-start-conversation.ts
@@ -38,7 +38,8 @@ export const useUnifiedResumeConversationSandbox = () => {
providers?: Provider[];
}) => resumeV1ConversationSandbox(variables.conversationId),
retry: (failureCount, error) => isRateLimitError(error) && failureCount < 3,
- retryDelay: (_failureCount, error) => getRateLimitRetryDelayMs(error),
+ retryDelay: (failureCount, error) =>
+ getRateLimitRetryDelayMs(failureCount, error),
onMutate: async (variables) => {
await queryClient.cancelQueries({ queryKey: ["user", "conversations"] });
const previousConversations = queryClient.getQueryData([
diff --git a/frontend/src/hooks/query/use-user-conversation.ts b/frontend/src/hooks/query/use-user-conversation.ts
index afb780b0bd..8becf1e0e1 100644
--- a/frontend/src/hooks/query/use-user-conversation.ts
+++ b/frontend/src/hooks/query/use-user-conversation.ts
@@ -37,7 +37,8 @@ export const useUserConversation = (
},
enabled: !!cid && !cid.startsWith("task-"),
retry: (failureCount, error) => isRateLimitError(error) && failureCount < 3,
- retryDelay: (_failureCount, error) => getRateLimitRetryDelayMs(error),
+ retryDelay: (failureCount, error) =>
+ getRateLimitRetryDelayMs(failureCount, error),
refetchInterval,
staleTime: FIVE_MINUTES,
gcTime: FIFTEEN_MINUTES,
diff --git a/frontend/src/query-client-config.ts b/frontend/src/query-client-config.ts
index bd3cd51a1c..5c3f315ad7 100644
--- a/frontend/src/query-client-config.ts
+++ b/frontend/src/query-client-config.ts
@@ -25,7 +25,8 @@ export const queryClient = new QueryClient({
// would otherwise surface as a spurious error toast.
retry: (failureCount, error) =>
failureCount < 2 && isRateLimitError(error),
- retryDelay: (_, error) => getRateLimitRetryDelayMs(error),
+ retryDelay: (failureCount, error) =>
+ getRateLimitRetryDelayMs(failureCount, error),
},
},
queryCache: new QueryCache({
diff --git a/frontend/src/utils/rate-limit-retry.ts b/frontend/src/utils/rate-limit-retry.ts
index c497076b5c..170c397285 100644
--- a/frontend/src/utils/rate-limit-retry.ts
+++ b/frontend/src/utils/rate-limit-retry.ts
@@ -1,7 +1,8 @@
import { AxiosError } from "axios";
-const DEFAULT_RETRY_AFTER_MS = 1000;
-const MAX_RETRY_DELAY_MS = 60_000;
+const INITIAL_RETRY_DELAY_MS = 1000;
+const MAX_BASE_RETRY_DELAY_MS = 60_000;
+const MAX_JITTER_MS = 5000;
const RETRY_AFTER_HEADER = "retry-after";
function getHeaderValue(headers: unknown, name: string): string | undefined {
@@ -17,6 +18,9 @@ function getHeaderValue(headers: unknown, name: string): string | undefined {
const value = (headers as { get: (headerName: string) => unknown }).get(
name,
);
+ // Axios normalizes Retry-After to a string, so the AxiosHeaders.get() path
+ // only trusts strings; numeric coercion is left to the plain-object branch
+ // below (which handles hand-built/mocked headers).
return typeof value === "string" ? value : undefined;
}
@@ -36,7 +40,7 @@ export function isRateLimitError(error: unknown): boolean {
return axiosError?.response?.status === 429 || axiosError?.status === 429;
}
-export function getRateLimitRetryDelayMs(error: unknown): number {
+function getRetryAfterDelayMs(error: unknown): number | undefined {
const axiosError = error as AxiosError | undefined;
const retryAfter = getHeaderValue(
axiosError?.response?.headers,
@@ -44,18 +48,40 @@ export function getRateLimitRetryDelayMs(error: unknown): number {
);
if (!retryAfter) {
- return DEFAULT_RETRY_AFTER_MS;
+ return undefined;
}
const seconds = Number(retryAfter);
if (Number.isFinite(seconds) && seconds >= 0) {
- return Math.min(seconds * 1000, MAX_RETRY_DELAY_MS);
+ return Math.min(seconds * 1000, MAX_BASE_RETRY_DELAY_MS);
}
const retryAt = Date.parse(retryAfter);
if (Number.isNaN(retryAt)) {
- return DEFAULT_RETRY_AFTER_MS;
+ return undefined;
}
- return Math.min(Math.max(retryAt - Date.now(), 0), MAX_RETRY_DELAY_MS);
+ return Math.min(Math.max(retryAt - Date.now(), 0), MAX_BASE_RETRY_DELAY_MS);
+}
+
+export function getRateLimitRetryDelayMs(
+ failureCount: number,
+ error: unknown,
+): number {
+ // The 60s clamp is defensive: current callers cap at failureCount 2 (4s), so
+ // it only becomes load-bearing if the retry limits are raised.
+ const exponentialDelay = Math.min(
+ INITIAL_RETRY_DELAY_MS * 2 ** failureCount,
+ MAX_BASE_RETRY_DELAY_MS,
+ );
+ const retryAfterDelay = getRetryAfterDelayMs(error);
+ // Retry-After is a server-directed minimum; jitter only delays further.
+ const minimumDelay = Math.max(retryAfterDelay ?? 0, exponentialDelay);
+ // Scale jitter off the delay we actually return so a large Retry-After still
+ // gets a proportional spread instead of being tied to the exponential term.
+ const jitter = Math.floor(
+ Math.random() * Math.min(minimumDelay, MAX_JITTER_MS),
+ );
+
+ return minimumDelay + jitter;
}