Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

add exponential backoff retry on upstream 429

Raine Virta77e2811945be
diff --git a/src/providers/codex/client.ts b/src/providers/codex/client.ts
index 6fbe767..0f2cee0 100644
--- a/src/providers/codex/client.ts
+++ b/src/providers/codex/client.ts
@@ -5,6 +5,7 @@ import { forceRefresh, getAuth } from "./auth/manager.ts"
import type { Logger } from "../../log.ts"
import type { RequestContext } from "../types.ts"
import type { ResponsesRequest } from "./translate/request.ts"
+import { retryOn429 } from "../retry.ts"
export interface CodexResponse {
body: ReadableStream<Uint8Array>
@@ -17,6 +18,21 @@ export async function postCodex(
ctx: RequestContext,
): Promise<CodexResponse> {
const log = ctx.childLogger("codex.client")
+ return retryOn429(() => attemptPostCodex(body, ctx, log), {
+ log,
+ signal: ctx.signal,
+ classify: (err) =>
+ err instanceof CodexError && err.status === 429
+ ? { retryAfter: err.meta?.retryAfter }
+ : undefined,
+ })
+}
+
+async function attemptPostCodex(
+ body: ResponsesRequest,
+ ctx: RequestContext,
+ log: Logger,
+): Promise<CodexResponse> {
let auth = await getAuth()
let resp = await doFetch(auth.access, auth.accountId, body, log, ctx.signal, ctx.sessionId)
diff --git a/src/providers/kimi/client.ts b/src/providers/kimi/client.ts
index 6486a68..07d27b7 100644
--- a/src/providers/kimi/client.ts
+++ b/src/providers/kimi/client.ts
@@ -4,6 +4,7 @@ import { forceRefresh, getAuth, KimiAuthUnauthorizedError } from "./auth/manager
import type { Logger } from "../../log.ts"
import type { RequestContext } from "../types.ts"
import type { KimiChatRequest } from "./translate/request.ts"
+import { retryOn429 } from "../retry.ts"
export interface KimiResponse {
body: ReadableStream<Uint8Array>
@@ -29,6 +30,21 @@ export async function postKimi(
ctx: RequestContext,
): Promise<KimiResponse> {
const log = ctx.childLogger("kimi.client")
+ return retryOn429(() => attemptPostKimi(body, ctx, log), {
+ log,
+ signal: ctx.signal,
+ classify: (err) =>
+ err instanceof KimiError && err.status === 429
+ ? { retryAfter: err.meta?.retryAfter }
+ : undefined,
+ })
+}
+
+async function attemptPostKimi(
+ body: KimiChatRequest,
+ ctx: RequestContext,
+ log: Logger,
+): Promise<KimiResponse> {
let auth = await getAuth()
const requestStartTime = Date.now()
let resp = await doFetch(auth.access, body, log, ctx.signal)
diff --git a/src/providers/retry.test.ts b/src/providers/retry.test.ts
new file mode 100644
index 0000000..4318fd8
--- /dev/null
+++ b/src/providers/retry.test.ts
@@ -0,0 +1,162 @@
+import { describe, expect, it } from "bun:test"
+import {
+ computeBackoffDelay,
+ MAX_RATE_LIMIT_RETRIES,
+ RETRY_INITIAL_DELAY_MS,
+ RETRY_MAX_DELAY_MS,
+ retryOn429,
+ sleep,
+} from "./retry.ts"
+
+const silentLog = {
+ debug: () => {},
+ info: () => {},
+ warn: () => {},
+ error: () => {},
+ child: () => silentLog,
+}
+
+describe("computeBackoffDelay", () => {
+ it("uses exponential backoff without retry-after", () => {
+ expect(computeBackoffDelay(0).waitMs).toBe(RETRY_INITIAL_DELAY_MS)
+ expect(computeBackoffDelay(1).waitMs).toBe(RETRY_INITIAL_DELAY_MS * 2)
+ expect(computeBackoffDelay(2).waitMs).toBe(RETRY_INITIAL_DELAY_MS * 4)
+ })
+
+ it("caps exponential backoff at max delay", () => {
+ expect(computeBackoffDelay(20).waitMs).toBe(RETRY_MAX_DELAY_MS)
+ })
+
+ it("respects numeric retry-after as seconds", () => {
+ expect(computeBackoffDelay(0, "5").waitMs).toBe(5000)
+ })
+
+ it("flags retry-after that exceeds budget", () => {
+ const out = computeBackoffDelay(0, "120")
+ expect(out.waitMs).toBe(RETRY_MAX_DELAY_MS)
+ expect(out.exceedsBudget).toBe(true)
+ })
+
+ it("rejects non-numeric retry-after garbage", () => {
+ expect(computeBackoffDelay(0, "1abc").waitMs).toBe(RETRY_INITIAL_DELAY_MS)
+ })
+
+ it("parses HTTP-date retry-after", () => {
+ const date = new Date(Date.now() + 5000).toUTCString()
+ const out = computeBackoffDelay(0, date)
+ expect(out.waitMs).toBeGreaterThan(3500)
+ expect(out.waitMs).toBeLessThanOrEqual(5000)
+ })
+})
+
+describe("sleep", () => {
+ it("rejects if signal already aborted", async () => {
+ const c = new AbortController()
+ c.abort()
+ await expect(sleep(1000, c.signal)).rejects.toThrow()
+ })
+
+ it("rejects when aborted mid-sleep", async () => {
+ const c = new AbortController()
+ const p = sleep(1000, c.signal)
+ setTimeout(() => c.abort(), 10)
+ await expect(p).rejects.toThrow()
+ })
+})
+
+describe("retryOn429", () => {
+ class FakeRateLimit extends Error {
+ constructor(public retryAfter?: string) {
+ super("rate limited")
+ }
+ }
+
+ it("returns successful result without retry", async () => {
+ let calls = 0
+ const result = await retryOn429(
+ async () => {
+ calls++
+ return "ok"
+ },
+ {
+ log: silentLog,
+ classify: () => undefined,
+ },
+ )
+ expect(result).toBe("ok")
+ expect(calls).toBe(1)
+ })
+
+ it("retries up to MAX_RATE_LIMIT_RETRIES then throws", async () => {
+ let calls = 0
+ const start = Date.now()
+ await expect(
+ retryOn429(
+ async () => {
+ calls++
+ throw new FakeRateLimit("0")
+ },
+ {
+ log: silentLog,
+ classify: (err) =>
+ err instanceof FakeRateLimit ? { retryAfter: err.retryAfter } : undefined,
+ },
+ ),
+ ).rejects.toBeInstanceOf(FakeRateLimit)
+ expect(calls).toBe(MAX_RATE_LIMIT_RETRIES + 1)
+ expect(Date.now() - start).toBeLessThan(2000)
+ })
+
+ it("gives up immediately when retry-after exceeds budget", async () => {
+ let calls = 0
+ await expect(
+ retryOn429(
+ async () => {
+ calls++
+ throw new FakeRateLimit("120")
+ },
+ {
+ log: silentLog,
+ classify: (err) =>
+ err instanceof FakeRateLimit ? { retryAfter: err.retryAfter } : undefined,
+ },
+ ),
+ ).rejects.toBeInstanceOf(FakeRateLimit)
+ expect(calls).toBe(1)
+ })
+
+ it("does not retry non-rate-limit errors", async () => {
+ let calls = 0
+ await expect(
+ retryOn429(
+ async () => {
+ calls++
+ throw new Error("other")
+ },
+ {
+ log: silentLog,
+ classify: () => undefined,
+ },
+ ),
+ ).rejects.toThrow("other")
+ expect(calls).toBe(1)
+ })
+
+ it("succeeds after a transient 429", async () => {
+ let calls = 0
+ const result = await retryOn429(
+ async () => {
+ calls++
+ if (calls === 1) throw new FakeRateLimit("0")
+ return "recovered"
+ },
+ {
+ log: silentLog,
+ classify: (err) =>
+ err instanceof FakeRateLimit ? { retryAfter: err.retryAfter } : undefined,
+ },
+ )
+ expect(result).toBe("recovered")
+ expect(calls).toBe(2)
+ })
+})
diff --git a/src/providers/retry.ts b/src/providers/retry.ts
new file mode 100644
index 0000000..b42cd2b
--- /dev/null
+++ b/src/providers/retry.ts
@@ -0,0 +1,85 @@
+import type { Logger } from "../log.ts"
+
+export const RETRY_INITIAL_DELAY_MS = 2000
+export const RETRY_BACKOFF_FACTOR = 2
+export const RETRY_MAX_DELAY_MS = 30_000
+export const MAX_RATE_LIMIT_RETRIES = 3
+
+const STRICT_NUMERIC = /^\s*\d+(?:\.\d+)?\s*$/
+
+export interface BackoffOutcome {
+ waitMs: number
+ exceedsBudget: boolean
+}
+
+export function computeBackoffDelay(attempt: number, retryAfter?: string): BackoffOutcome {
+ if (retryAfter) {
+ if (STRICT_NUMERIC.test(retryAfter)) {
+ const ms = Math.ceil(Number.parseFloat(retryAfter) * 1000)
+ return { waitMs: Math.min(ms, RETRY_MAX_DELAY_MS), exceedsBudget: ms > RETRY_MAX_DELAY_MS }
+ }
+ const dateMs = Date.parse(retryAfter) - Date.now()
+ if (!Number.isNaN(dateMs) && dateMs > 0) {
+ return {
+ waitMs: Math.min(Math.ceil(dateMs), RETRY_MAX_DELAY_MS),
+ exceedsBudget: dateMs > RETRY_MAX_DELAY_MS,
+ }
+ }
+ }
+ const exp = RETRY_INITIAL_DELAY_MS * Math.pow(RETRY_BACKOFF_FACTOR, attempt)
+ return { waitMs: Math.min(exp, RETRY_MAX_DELAY_MS), exceedsBudget: false }
+}
+
+export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) {
+ reject(new DOMException("Aborted", "AbortError"))
+ return
+ }
+ const onAbort = () => {
+ clearTimeout(timer)
+ reject(new DOMException("Aborted", "AbortError"))
+ }
+ const timer = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort)
+ resolve()
+ }, ms)
+ signal?.addEventListener("abort", onAbort, { once: true })
+ })
+}
+
+export interface RateLimitInfo {
+ retryAfter?: string
+}
+
+export interface RetryOptions {
+ log: Logger
+ signal?: AbortSignal
+ classify: (err: unknown) => RateLimitInfo | undefined
+}
+
+export async function retryOn429<T>(run: () => Promise<T>, opts: RetryOptions): Promise<T> {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await run()
+ } catch (err) {
+ const info = opts.classify(err)
+ if (!info || attempt >= MAX_RATE_LIMIT_RETRIES) throw err
+ const { waitMs, exceedsBudget } = computeBackoffDelay(attempt, info.retryAfter)
+ if (exceedsBudget) {
+ opts.log.warn("upstream 429 retry-after exceeds budget; giving up", {
+ retryAfter: info.retryAfter,
+ maxDelayMs: RETRY_MAX_DELAY_MS,
+ })
+ throw err
+ }
+ opts.log.warn("upstream 429, retrying after backoff", {
+ attempt: attempt + 1,
+ maxRetries: MAX_RATE_LIMIT_RETRIES,
+ waitMs,
+ retryAfter: info.retryAfter,
+ })
+ await sleep(waitMs, opts.signal)
+ }
+ }
+}