π OpenHands: AI-Driven Development
fix(frontend): add jittered rate-limit backoff (#15236)
aivong-openhands1747b39562ff
diff --git a/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx b/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsxindex 51e6474802..b04c156302 100644--- a/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx+++ b/frontend/__tests__/hooks/mutation/use-unified-resume-conversation-sandbox.test.tsx@@ -1,6 +1,7 @@-import { renderHook, waitFor } from "@testing-library/react";+import { act, renderHook, waitFor } from "@testing-library/react";import { QueryClient, QueryClientProvider } from "@tanstack/react-query";-import { describe, expect, it, vi, beforeEach } from "vitest";+import { AxiosError } from "axios";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import React from "react";import { SandboxService } from "#/api/sandbox-service/sandbox-service.api";import V1ConversationService from "#/api/conversation-service/v1-conversation-service.api";@@ -49,6 +50,11 @@ describe("useUnifiedResumeConversationSandbox", () => {vi.restoreAllMocks();});+ afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ });+const wrapper = ({ children }: { children: React.ReactNode }) => (<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>);@@ -85,6 +91,55 @@ describe("useUnifiedResumeConversationSandbox", () => {});});+ it("backs off exponentially when the resume preflight is rate limited", async () => {+ vi.useFakeTimers();+ vi.spyOn(Math, "random").mockReturnValue(0);++ const rateLimitError = {+ response: {+ status: 429,+ headers: { "retry-after": "0" },+ },+ } as unknown as AxiosError;+ const batchGetAppConversations = vi+ .spyOn(V1ConversationService, "batchGetAppConversations")+ .mockRejectedValueOnce(rateLimitError)+ .mockRejectedValueOnce(rateLimitError)+ .mockResolvedValueOnce([createConversation()]);+ vi.spyOn(SandboxService, "resumeSandbox").mockResolvedValue({+ success: true,+ });++ const { result } = renderHook(() => useUnifiedResumeConversationSandbox(), {+ wrapper,+ });++ const resume = result.current.mutateAsync({+ conversationId: "test-conv-id",+ });++ await act(async () => {+ await vi.advanceTimersByTimeAsync(0);+ });+ expect(batchGetAppConversations).toHaveBeenCalledTimes(1);++ await act(async () => {+ await vi.advanceTimersByTimeAsync(1000);+ });+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);++ await act(async () => {+ await vi.advanceTimersByTimeAsync(1999);+ });+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);++ await act(async () => {+ await vi.advanceTimersByTimeAsync(1);+ });+ await expect(resume).resolves.toEqual({ success: true });+ expect(batchGetAppConversations).toHaveBeenCalledTimes(3);+ });+it("invalidates sandbox and vscode_url queries on settled", async () => {// Mock the API calls in the mutation chainvi.spyOn(diff --git a/frontend/__tests__/hooks/query/use-user-conversation.test.tsx b/frontend/__tests__/hooks/query/use-user-conversation.test.tsxindex 913d873e69..02424f336a 100644--- a/frontend/__tests__/hooks/query/use-user-conversation.test.tsx+++ b/frontend/__tests__/hooks/query/use-user-conversation.test.tsx@@ -50,7 +50,7 @@ describe("useUserConversation", () => {vi.restoreAllMocks();});- it("retries rate-limited conversation fetches after the Retry-After delay", async () => {+ it("backs off exponentially between rate-limited conversation fetches", async () => {const rateLimitError = {response: {status: 429,@@ -63,7 +63,9 @@ describe("useUserConversation", () => {const batchGetAppConversations = vi.spyOn(V1ConversationService, "batchGetAppConversations").mockRejectedValueOnce(rateLimitError)+ .mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce([createConversation()]);+ vi.spyOn(Math, "random").mockReturnValue(0);const { result } = renderHook(() => useUserConversation("conversation-1"), {wrapper: createWrapper(),@@ -81,6 +83,17 @@ describe("useUserConversation", () => {});expect(batchGetAppConversations).toHaveBeenCalledTimes(2);++ await act(async () => {+ await vi.advanceTimersByTimeAsync(1999);+ });+ expect(batchGetAppConversations).toHaveBeenCalledTimes(2);++ await act(async () => {+ await vi.advanceTimersByTimeAsync(1);+ });++ expect(batchGetAppConversations).toHaveBeenCalledTimes(3);await vi.waitFor(() => {expect(result.current.data?.id).toBe("conversation-1");});diff --git a/frontend/__tests__/query-client-config.test.ts b/frontend/__tests__/query-client-config.test.tsindex 0c0e09467b..1103d0514a 100644--- a/frontend/__tests__/query-client-config.test.ts+++ b/frontend/__tests__/query-client-config.test.ts@@ -13,22 +13,33 @@ const make429 = () =>describe("queryClient mutation defaults", () => {afterEach(() => {+ vi.useRealTimers();vi.restoreAllMocks();queryClient.clear();});- it("should retry a mutation rejected with 429 (request was never processed)", async () => {+ it("backs off before retrying a rate-limited mutation", async () => {+ vi.useFakeTimers();+ vi.spyOn(Math, "random").mockReturnValue(0);+const mutationFn = vi.fn().mockRejectedValueOnce(make429()).mockResolvedValueOnce("ok");- const result = await queryClient+ const result = queryClient.getMutationCache().build(queryClient, { mutationFn, meta: { disableToast: true } }).execute(undefined);- expect(result).toBe("ok");+ await vi.advanceTimersByTimeAsync(0);+ expect(mutationFn).toHaveBeenCalledTimes(1);++ await vi.advanceTimersByTimeAsync(999);+ expect(mutationFn).toHaveBeenCalledTimes(1);++ await vi.advanceTimersByTimeAsync(1);+ await expect(result).resolves.toBe("ok");expect(mutationFn).toHaveBeenCalledTimes(2);});diff --git a/frontend/__tests__/utils/rate-limit-retry.test.ts b/frontend/__tests__/utils/rate-limit-retry.test.tsindex 54ec0f5d44..6ce53fbe02 100644--- a/frontend/__tests__/utils/rate-limit-retry.test.ts+++ b/frontend/__tests__/utils/rate-limit-retry.test.ts@@ -16,6 +16,7 @@ const createAxiosError = (status: number, headers?: unknown): AxiosError =>describe("rate limit retry helpers", () => {afterEach(() => {vi.useRealTimers();+ vi.restoreAllMocks();});it("recognizes response status 429 as a rate-limit error", () => {@@ -30,7 +31,44 @@ describe("rate limit retry helpers", () => {headerName.toLowerCase() === "retry-after" ? "2" : undefined,};- expect(getRateLimitRetryDelayMs(createAxiosError(429, headers))).toBe(2000);+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(getRateLimitRetryDelayMs(0, createAxiosError(429, headers))).toBe(+ 2000,+ );+ });++ it("accepts numeric Retry-After values from plain headers", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "Retry-After": 2 }),+ ),+ ).toBe(2000);+ });++ it("falls back to backoff when AxiosHeaders returns a non-string value", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { get: () => 2 }),+ ),+ ).toBe(1000);+ });++ it("falls back to backoff for unsupported plain header values", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": { seconds: 2 } }),+ ),+ ).toBe(1000);});it("parses Retry-After HTTP dates", () => {@@ -38,9 +76,11 @@ describe("rate limit retry helpers", () => {vi.setSystemTime(new Date("2026-06-04T20:00:00.000Z"));const retryAt = new Date("2026-06-04T20:00:03.000Z").toUTCString();+ vi.spyOn(Math, "random").mockReturnValue(0);expect(getRateLimitRetryDelayMs(+ 0,createAxiosError(429, {"retry-after": retryAt,}),@@ -49,34 +89,52 @@ describe("rate limit retry helpers", () => {});it("falls back to one second when Retry-After is missing", () => {- expect(getRateLimitRetryDelayMs(createAxiosError(429))).toBe(1000);+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1000);});it("falls back to one second for non-numeric Retry-After", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+expect(getRateLimitRetryDelayMs(+ 0,createAxiosError(429, { "retry-after": "invalid" }),),).toBe(1000);});- it("treats negative Retry-After seconds as immediate retry (already expired)", () => {- // Date.parse("-5") returns a valid timestamp (epoch - 5000ms), and since- // that time has passed, Math.max(negative - now, 0) returns 0+ it("uses the initial backoff when Retry-After is already expired", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+expect(- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "-5" })),- ).toBe(0);+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": "-5" }),+ ),+ ).toBe(1000);});- it("treats zero Retry-After seconds as immediate (0ms delay)", () => {+ it("uses the initial backoff when Retry-After is zero", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+expect(- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "0" })),- ).toBe(0);+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": "0" }),+ ),+ ).toBe(1000);});it("clamps large Retry-After seconds to max 60 seconds", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+expect(- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": "120" })),+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": "120" }),+ ),).toBe(60_000);});@@ -85,9 +143,40 @@ describe("rate limit retry helpers", () => {vi.setSystemTime(new Date("2026-06-04T20:00:00.000Z"));const farFuture = new Date("2026-06-04T21:00:00.000Z").toUTCString();+ vi.spyOn(Math, "random").mockReturnValue(0);expect(- getRateLimitRetryDelayMs(createAxiosError(429, { "retry-after": farFuture })),+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": farFuture }),+ ),).toBe(60_000);});++ it("grows the fallback delay exponentially for repeated failures", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);++ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1000);+ expect(getRateLimitRetryDelayMs(1, createAxiosError(429))).toBe(2000);+ expect(getRateLimitRetryDelayMs(2, createAxiosError(429))).toBe(4000);+ });++ it("adds jitter while never retrying before Retry-After", () => {+ vi.spyOn(Math, "random").mockReturnValue(0.5);++ expect(getRateLimitRetryDelayMs(0, createAxiosError(429))).toBe(1500);+ // jitter scales off the 5s Retry-After: 0.5 * min(5000, MAX_JITTER_MS) = 2500+ expect(+ getRateLimitRetryDelayMs(+ 0,+ createAxiosError(429, { "retry-after": "5" }),+ ),+ ).toBe(7500);+ });++ it("caps jitter to avoid unbounded delays", () => {+ vi.spyOn(Math, "random").mockReturnValue(0.999);++ expect(getRateLimitRetryDelayMs(10, createAxiosError(429))).toBe(64_995);+ });});diff --git a/frontend/src/hooks/mutation/use-unified-start-conversation.ts b/frontend/src/hooks/mutation/use-unified-start-conversation.tsindex 3c857ea6a0..9067393127 100644--- a/frontend/src/hooks/mutation/use-unified-start-conversation.ts+++ b/frontend/src/hooks/mutation/use-unified-start-conversation.ts@@ -38,7 +38,8 @@ export const useUnifiedResumeConversationSandbox = () => {providers?: Provider[];}) => resumeV1ConversationSandbox(variables.conversationId),retry: (failureCount, error) => isRateLimitError(error) && failureCount < 3,- retryDelay: (_failureCount, error) => getRateLimitRetryDelayMs(error),+ retryDelay: (failureCount, error) =>+ getRateLimitRetryDelayMs(failureCount, error),onMutate: async (variables) => {await queryClient.cancelQueries({ queryKey: ["user", "conversations"] });const previousConversations = queryClient.getQueryData([diff --git a/frontend/src/hooks/query/use-user-conversation.ts b/frontend/src/hooks/query/use-user-conversation.tsindex afb780b0bd..8becf1e0e1 100644--- a/frontend/src/hooks/query/use-user-conversation.ts+++ b/frontend/src/hooks/query/use-user-conversation.ts@@ -37,7 +37,8 @@ export const useUserConversation = (},enabled: !!cid && !cid.startsWith("task-"),retry: (failureCount, error) => isRateLimitError(error) && failureCount < 3,- retryDelay: (_failureCount, error) => getRateLimitRetryDelayMs(error),+ retryDelay: (failureCount, error) =>+ getRateLimitRetryDelayMs(failureCount, error),refetchInterval,staleTime: FIVE_MINUTES,gcTime: FIFTEEN_MINUTES,diff --git a/frontend/src/query-client-config.ts b/frontend/src/query-client-config.tsindex bd3cd51a1c..5c3f315ad7 100644--- a/frontend/src/query-client-config.ts+++ b/frontend/src/query-client-config.ts@@ -25,7 +25,8 @@ export const queryClient = new QueryClient({// would otherwise surface as a spurious error toast.retry: (failureCount, error) =>failureCount < 2 && isRateLimitError(error),- retryDelay: (_, error) => getRateLimitRetryDelayMs(error),+ retryDelay: (failureCount, error) =>+ getRateLimitRetryDelayMs(failureCount, error),},},queryCache: new QueryCache({diff --git a/frontend/src/utils/rate-limit-retry.ts b/frontend/src/utils/rate-limit-retry.tsindex c497076b5c..170c397285 100644--- a/frontend/src/utils/rate-limit-retry.ts+++ b/frontend/src/utils/rate-limit-retry.ts@@ -1,7 +1,8 @@import { AxiosError } from "axios";-const DEFAULT_RETRY_AFTER_MS = 1000;-const MAX_RETRY_DELAY_MS = 60_000;+const INITIAL_RETRY_DELAY_MS = 1000;+const MAX_BASE_RETRY_DELAY_MS = 60_000;+const MAX_JITTER_MS = 5000;const RETRY_AFTER_HEADER = "retry-after";function getHeaderValue(headers: unknown, name: string): string | undefined {@@ -17,6 +18,9 @@ function getHeaderValue(headers: unknown, name: string): string | undefined {const value = (headers as { get: (headerName: string) => unknown }).get(name,);+ // Axios normalizes Retry-After to a string, so the AxiosHeaders.get() path+ // only trusts strings; numeric coercion is left to the plain-object branch+ // below (which handles hand-built/mocked headers).return typeof value === "string" ? value : undefined;}@@ -36,7 +40,7 @@ export function isRateLimitError(error: unknown): boolean {return axiosError?.response?.status === 429 || axiosError?.status === 429;}-export function getRateLimitRetryDelayMs(error: unknown): number {+function getRetryAfterDelayMs(error: unknown): number | undefined {const axiosError = error as AxiosError | undefined;const retryAfter = getHeaderValue(axiosError?.response?.headers,@@ -44,18 +48,40 @@ export function getRateLimitRetryDelayMs(error: unknown): number {);if (!retryAfter) {- return DEFAULT_RETRY_AFTER_MS;+ return undefined;}const seconds = Number(retryAfter);if (Number.isFinite(seconds) && seconds >= 0) {- return Math.min(seconds * 1000, MAX_RETRY_DELAY_MS);+ return Math.min(seconds * 1000, MAX_BASE_RETRY_DELAY_MS);}const retryAt = Date.parse(retryAfter);if (Number.isNaN(retryAt)) {- return DEFAULT_RETRY_AFTER_MS;+ return undefined;}- return Math.min(Math.max(retryAt - Date.now(), 0), MAX_RETRY_DELAY_MS);+ return Math.min(Math.max(retryAt - Date.now(), 0), MAX_BASE_RETRY_DELAY_MS);+}++export function getRateLimitRetryDelayMs(+ failureCount: number,+ error: unknown,+): number {+ // The 60s clamp is defensive: current callers cap at failureCount 2 (4s), so+ // it only becomes load-bearing if the retry limits are raised.+ const exponentialDelay = Math.min(+ INITIAL_RETRY_DELAY_MS * 2 ** failureCount,+ MAX_BASE_RETRY_DELAY_MS,+ );+ const retryAfterDelay = getRetryAfterDelayMs(error);+ // Retry-After is a server-directed minimum; jitter only delays further.+ const minimumDelay = Math.max(retryAfterDelay ?? 0, exponentialDelay);+ // Scale jitter off the delay we actually return so a large Retry-After still+ // gets a proportional spread instead of being tied to the exponential term.+ const jitter = Math.floor(+ Math.random() * Math.min(minimumDelay, MAX_JITTER_MS),+ );++ return minimumDelay + jitter;}