TypeScript
fix(server): anchor Claude wake cooldowns to result arrival (#25061)
Julius Marminge371e9451ef27
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.tsindex f3cf3c43c3d..23f61197a56 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts@@ -11254,6 +11254,88 @@ describe("ClaudeAdapterV2 background wake turns", () => {),);+ it.effect(+ "dates a buffered wake gateway limit from its arrival, not the continuation replay",+ () =>+ Effect.scoped(+ Effect.gen(function* () {+ const harness = yield* makeWakeHarness;+ const now = yield* DateTime.now;++ yield* harness.runtime.startTurn(+ makeClaudeTestTurnInput({+ threadId: harness.threadId,+ providerThread: harness.providerThread,+ now,+ attemptId: RunAttemptId.make("attempt-claude-wake-gateway-1"),+ text: "Run the build in the background.",+ attachments: [],+ }),+ );+ yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);+ yield* Queue.offer(harness.sdkMessages, turnOneResult);+ yield* Queue.take(harness.terminalReceipts);++ yield* Queue.offer(harness.sdkMessages, wakeNotification);+ yield* harness.offerAndWait(+ makeAssistantErrorFrame({+ uuid: "00000000-0000-4000-8000-000000000634",+ error: "server_error",+ }),+ );+ assert.lengthOf(harness.continuationRequests, 1);+ const arrivedAt = yield* DateTime.now;+ yield* harness.offerAndWait(+ makeResultFrame({+ uuid: "00000000-0000-4000-8000-000000000635",+ result:+ 'API Error: 503 [cpa/claude-opus-5-5] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials are cooling down"}} (reset after 2h54m21s)',+ isError: true,+ apiErrorStatus: 503,+ terminalReason: "api_error",+ origin: { kind: "task-notification" },+ }),+ );++ yield* TestClock.adjust("1 hour");+ yield* harness.runtime.startTurn(+ makeClaudeTestTurnInput({+ threadId: harness.threadId,+ providerThread: harness.providerThread,+ now: yield* DateTime.now,+ attemptId: RunAttemptId.make("attempt-claude-wake-gateway-2"),+ text: "Background task completed.",+ attachments: [],+ providerTurnOrdinal: 2,+ messageCreatedBy: "agent",+ messageCreationSource: "provider",+ }),+ );+ const terminal = yield* Queue.take(harness.terminalReceipts);+ assert.equal(terminal.status, "failed");+ if (terminal.status !== "failed") return;+ assert.equal(terminal.failure.class, "usage_limit");+ assert.equal(+ terminal.failure.resetAt,+ DateTime.formatIso(+ DateTime.makeUnsafe(+ DateTime.toEpochMillis(arrivedAt) + (2 * 3_600 + 54 * 60 + 21) * 1_000,+ ),+ ),+ );+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ McpProviderSessions.layer,+ NodeServices.layer,+ NodeCrypto.layer,+ ),+ ),+ ),+ ),+ );+it.effect("leaves buffered wake messages for the continuation queued behind a user turn", () =>Effect.scoped(Effect.gen(function* () {diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.tsindex ef5849edb8f..049d2e2cf0e 100644--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts@@ -3800,7 +3800,7 @@ function gatewayCooldownResetAt(function providerFailureFromResult(message: SDKResultMessage,- completedAt: DateTime.Utc,+ receivedAt: DateTime.Utc,failureHint?: string,usageLimited = false,): OrchestrationV2ProviderFailure | null {@@ -3814,7 +3814,7 @@ function providerFailureFromResult(/^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result,),- completedAt,+ receivedAt,): undefined;if (gatewayResetAt !== undefined) {@@ -3858,7 +3858,7 @@ function providerFailureFromResult(apiErrorStatus === 429 && message.result.startsWith("API Error:")? gatewayCooldownResetAt(/\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result),- completedAt,+ receivedAt,): undefined;return makeProviderFailure({@@ -4882,6 +4882,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (const pendingSubagentFramesByToolUseId = yield* Ref.make(new Map<string, ReadonlyArray<SDKMessage>>(),);+ // A gateway's relative reset counts from the result's arrival, not its replay.+ const bufferedResultArrivals = new WeakMap<SDKMessage, DateTime.Utc>();const wakeBuffers = yield* Ref.make(new Map<string,@@ -8065,6 +8067,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (childThreadId: registered?.childThreadId,});}+ if (message.type === "result") {+ bufferedResultArrivals.set(message, yield* DateTime.now);+ }yield* Ref.update(wakeBuffers, (current) => {const existing = current.get(wakeInput.nativeThreadId);const updated = new Map(current);@@ -9535,7 +9540,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (message: limitedCompaction.message ?? failureHint,code: "compaction_failed",})- : providerFailureFromResult(message, completedAt, failureHint, usageLimited);+ : providerFailureFromResult(+ message,+ bufferedResultArrivals.get(message) ?? completedAt,+ failureHint,+ usageLimited,+ );const terminalFailure =resultFailure?.class === "usage_limit"? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null }