Skip to content
TypeScript

fix(server): anchor Claude wake cooldowns to result arrival (#25061)

Julius Marminge371e9451ef27
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
index f3cf3c43c3d..23f61197a56 100644
--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
@@ -11254,6 +11254,88 @@ describe("ClaudeAdapterV2 background wake turns", () => {
),
);
+ it.effect(
+ "dates a buffered wake gateway limit from its arrival, not the continuation replay",
+ () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const harness = yield* makeWakeHarness;
+ const now = yield* DateTime.now;
+
+ yield* harness.runtime.startTurn(
+ makeClaudeTestTurnInput({
+ threadId: harness.threadId,
+ providerThread: harness.providerThread,
+ now,
+ attemptId: RunAttemptId.make("attempt-claude-wake-gateway-1"),
+ text: "Run the build in the background.",
+ attachments: [],
+ }),
+ );
+ yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
+ yield* Queue.offer(harness.sdkMessages, turnOneResult);
+ yield* Queue.take(harness.terminalReceipts);
+
+ yield* Queue.offer(harness.sdkMessages, wakeNotification);
+ yield* harness.offerAndWait(
+ makeAssistantErrorFrame({
+ uuid: "00000000-0000-4000-8000-000000000634",
+ error: "server_error",
+ }),
+ );
+ assert.lengthOf(harness.continuationRequests, 1);
+ const arrivedAt = yield* DateTime.now;
+ yield* harness.offerAndWait(
+ makeResultFrame({
+ uuid: "00000000-0000-4000-8000-000000000635",
+ result:
+ 'API Error: 503 [cpa/claude-opus-5-5] [429]: {"type":"error","error":{"type":"rate_limit_error","message":"All credentials are cooling down"}} (reset after 2h54m21s)',
+ isError: true,
+ apiErrorStatus: 503,
+ terminalReason: "api_error",
+ origin: { kind: "task-notification" },
+ }),
+ );
+
+ yield* TestClock.adjust("1 hour");
+ yield* harness.runtime.startTurn(
+ makeClaudeTestTurnInput({
+ threadId: harness.threadId,
+ providerThread: harness.providerThread,
+ now: yield* DateTime.now,
+ attemptId: RunAttemptId.make("attempt-claude-wake-gateway-2"),
+ text: "Background task completed.",
+ attachments: [],
+ providerTurnOrdinal: 2,
+ messageCreatedBy: "agent",
+ messageCreationSource: "provider",
+ }),
+ );
+ const terminal = yield* Queue.take(harness.terminalReceipts);
+ assert.equal(terminal.status, "failed");
+ if (terminal.status !== "failed") return;
+ assert.equal(terminal.failure.class, "usage_limit");
+ assert.equal(
+ terminal.failure.resetAt,
+ DateTime.formatIso(
+ DateTime.makeUnsafe(
+ DateTime.toEpochMillis(arrivedAt) + (2 * 3_600 + 54 * 60 + 21) * 1_000,
+ ),
+ ),
+ );
+ }).pipe(
+ Effect.provide(
+ Layer.mergeAll(
+ IdAllocator.layer,
+ McpProviderSessions.layer,
+ NodeServices.layer,
+ NodeCrypto.layer,
+ ),
+ ),
+ ),
+ ),
+ );
+
it.effect("leaves buffered wake messages for the continuation queued behind a user turn", () =>
Effect.scoped(
Effect.gen(function* () {
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
index ef5849edb8f..049d2e2cf0e 100644
--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
@@ -3800,7 +3800,7 @@ function gatewayCooldownResetAt(
function providerFailureFromResult(
message: SDKResultMessage,
- completedAt: DateTime.Utc,
+ receivedAt: DateTime.Utc,
failureHint?: string,
usageLimited = false,
): OrchestrationV2ProviderFailure | null {
@@ -3814,7 +3814,7 @@ function providerFailureFromResult(
/^API Error: 503 \[[^\]\r\n]+\] \[429\]: \{"type":"error","error":\{"type":"rate_limit_error",[^\r\n]*\} \(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(
message.result,
),
- completedAt,
+ receivedAt,
)
: undefined;
if (gatewayResetAt !== undefined) {
@@ -3858,7 +3858,7 @@ function providerFailureFromResult(
apiErrorStatus === 429 && message.result.startsWith("API Error:")
? gatewayCooldownResetAt(
/\(reset after (?:(\d+)h ?)?(?:(\d+)m ?)?(?:(\d+)s)?\)/.exec(message.result),
- completedAt,
+ receivedAt,
)
: undefined;
return makeProviderFailure({
@@ -4882,6 +4882,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
const pendingSubagentFramesByToolUseId = yield* Ref.make(
new Map<string, ReadonlyArray<SDKMessage>>(),
);
+ // A gateway's relative reset counts from the result's arrival, not its replay.
+ const bufferedResultArrivals = new WeakMap<SDKMessage, DateTime.Utc>();
const wakeBuffers = yield* Ref.make(
new Map<
string,
@@ -8065,6 +8067,9 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
childThreadId: registered?.childThreadId,
});
}
+ if (message.type === "result") {
+ bufferedResultArrivals.set(message, yield* DateTime.now);
+ }
yield* Ref.update(wakeBuffers, (current) => {
const existing = current.get(wakeInput.nativeThreadId);
const updated = new Map(current);
@@ -9535,7 +9540,12 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
message: limitedCompaction.message ?? failureHint,
code: "compaction_failed",
})
- : providerFailureFromResult(message, completedAt, failureHint, usageLimited);
+ : providerFailureFromResult(
+ message,
+ bufferedResultArrivals.get(message) ?? completedAt,
+ failureHint,
+ usageLimited,
+ );
const terminalFailure =
resultFailure?.class === "usage_limit"
? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null }