Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(pi): recover code-less temporary service availability errors (#5062)
Chris Zhang0e3545f3445e
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.tsindex 5bb7a1db9c..4229ff8224 100644--- a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts+++ b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts@@ -1,6 +1,15 @@import { AUTO_REVIEW_SOURCE_CONTENT, AUTO_REVIEW_USER_INTENT, appendAutoReviewUserIntent } from '@cindy/maker-core';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import { AgentInputCoordinator } from '../agent-input-coordinator.js';+import {+ createPiTranslateContext, disposePiTranslateContext, translatePiEvent,+} from '../../../../../../packages/maker-core/src/agents/pi/translator.js';+import type { AgentEvent } from '../../../../../../packages/maker-core/src/types/events.js';+import type { Logger } from '../../../../../../packages/maker-core/src/interfaces/logger.js';+import {+ InterruptedTurnAutoResumeGuard, INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS,+ isInterruptedTurnError,+} from '../interruptedTurnAutoResume.js';import { createQueuedDispatchReceipts } from '../queuedDispatchReceipts.js';import { createOrcaInterAgentDispatcher } from '../orcaInterAgentDispatcher.js';import {@@ -11091,6 +11100,108 @@ describe('AgentInputCoordinator replaceQueuedMessage(Orca lead 排队消息修});describe('AgentInputCoordinator 中断自动续跑', () => {+ const availabilityError = "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.";++ // Offline provider -> real Pi translator -> real host classifier/guard ->+ // coordinator dispatch. The provider and durable DB progress are fixtures;+ // no model, process or tool is actually invoked.+ function availabilityInput() {+ const item = makeItem('q-first', 'original request with possible side effects');+ return { ...item, model: 'grok-4.7', createOpts: { ...item.createOpts!, agentKind: 'pi' as const, model: 'grok-4.7' } };+ }++ function availabilityHarness() {+ const h = createHarness();+ const guard = new InterruptedTurnAutoResumeGuard({+ isEnabled: () => true, log: mocks.logger, random: () => 0.5,+ });+ h.isResumableTurnErrorCandidate.mockImplementation(isInterruptedTurnError);+ h.onResumableTurnError.mockImplementation((sid, signals) => {+ if (!isInterruptedTurnError(signals)) return null;+ const decision = guard.onInterruptedTurn(sid, Date.now());+ return decision.action === 'resume' ? { ...decision, error: signals.message } : null;+ });+ let durableToolResult = false;+ h.setHasAssistantProgressAfter(async () => durableToolResult);+ const logger: Logger = { ...mocks.logger, trace: vi.fn(), fatal: vi.fn(), child: () => logger };+ async function fail(sid: string, opts: { toolResult?: boolean; nativeExhausted?: boolean } = {}) {+ const ctx = createPiTranslateContext(logger);+ const events: AgentEvent[] = [];+ const queue = { push: (event: AgentEvent) => { events.push(event); }, end: () => {} } as unknown as Parameters<typeof translatePiEvent>[1];+ const emit = (event: Record<string, unknown>) => translatePiEvent(event as Parameters<typeof translatePiEvent>[0], queue, ctx);+ try {+ emit({ type: 'agent_start' });+ if (opts.toolResult) {+ emit({ type: 'tool_execution_start', toolCallId: 'completed-tool', toolName: 'read', args: { path: 'fixture.md' } });+ emit({ type: 'tool_execution_end', toolCallId: 'completed-tool', toolName: 'read', result: { content: [{ type: 'text', text: 'fixture result' }] }, isError: false });+ expect(events.some(event => event.type === 'tool_result')).toBe(true);+ durableToolResult = true;+ }+ emit({ type: 'message_end', message: { role: 'assistant', content: [], stopReason: 'error', errorMessage: availabilityError } });+ emit({ type: 'agent_end', messages: [] });+ expect(events.filter(event => event.type === 'error' || event.type === 'done')).toHaveLength(0);+ if (opts.nativeExhausted) emit({ type: 'auto_retry_end', success: false, finalError: availabilityError });+ emit({ type: 'agent_settled' });+ h.setRunning(false);+ for (const event of events) {+ if (event.type === 'error') {+ const data = event.data as { message: string; isTerminal?: boolean; sdkError?: string; reason?: string; errorStatus?: number };+ if (data.isTerminal) h.coordinator.onTurnEvent(sid, 'error', data.message, data);+ } else if (event.type === 'done') h.coordinator.onTurnEvent(sid, 'done');+ }+ await flush();+ } finally { disposePiTranslateContext(ctx); }+ }+ return { h, guard, fail };+ }++ it('continues after a completed tool and stops at the existing host retry budget', async () => {+ const { h, guard, fail } = availabilityHarness();+ const sid = 'pi-availability-budget';+ h.coordinator.enqueue(sid, availabilityInput());+ await flush();+ for (let attempt = 1; attempt <= INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS; attempt += 1) {+ await fail(sid, { toolResult: attempt === 1 });+ expect(latestProjection(h.projections).error).toBeNull();+ expect(await h.coordinator.autoRetryLastError(sid, attempt)).toBe('resumed');+ await flush();+ expect(h.sendToAgent.mock.calls[attempt]?.[1]).toEqual({ type: 'user', content: CONTINUE_AFTER_ERROR_PROMPT });+ expect(h.sendToAgent.mock.calls[attempt]?.[3]?.persistUserMessage?.autoResume).toBe(true);+ // Production retires the pending token on the first provider event.+ expect(guard.noteAttemptEvent(sid, attempt)).toBe(true);+ }+ await fail(sid);+ expect(latestProjection(h.projections).error).not.toBeNull();+ expect(h.coordinator.isAutoResumePending(sid)).toBe(false);+ expect(h.sendToAgent).toHaveBeenCalledTimes(1 + INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS);+ });++ it('does not resume an availability error after the user stops during backoff', async () => {+ const { h, guard, fail } = availabilityHarness();+ const sid = 'pi-availability-stop';+ h.coordinator.enqueue(sid, availabilityInput());+ await flush();+ await fail(sid, { toolResult: true });+ h.coordinator.stop(sid);+ guard.noteSessionReset(sid);+ expect(await h.coordinator.autoRetryLastError(sid, 1)).toBe('superseded');+ await flush();+ expect(h.sendToAgent).toHaveBeenCalledTimes(1);+ expect(guard.isCurrentAttempt(sid, 1)).toBe(false);+ });++ it('does not add host retries after Pi reports native availability retry exhaustion', async () => {+ const { h, fail } = availabilityHarness();+ const sid = 'pi-availability-native-exhausted';+ h.coordinator.enqueue(sid, availabilityInput());+ await flush();+ await fail(sid, { toolResult: true, nativeExhausted: true });+ expect(h.onResumableTurnError).toHaveReturnedWith(null);+ expect(latestProjection(h.projections).error).not.toBeNull();+ expect(h.coordinator.isAutoResumePending(sid)).toBe(false);+ expect(h.sendToAgent).toHaveBeenCalledTimes(1);+ });+// 上游把「已经干到一半」的 turn 打断时,main 守卫自动替用户点一次「继续」。// coordinator 这一侧只负责两件事:把带结构化信号的失败告知 host(判据不在这里),// 以及提供一条**带 autoResume 标记**的补发路径(标记是额度不自我充值的判据)。diff --git a/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.tsindex 0aa555fd18..22489c47dc 100644--- a/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts+++ b/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts@@ -15,6 +15,42 @@ import {// 判定单测的核心是**白名单收紧**:自动重试一个确定性失败(认证过期、协议错)会反复// 烧额度并反复报错,所以只有识别得了的"上游把 turn 打断了"才允许自愈。describe('isInterruptedTurnError', () => {+ const availabilityError = "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.";++ it('recovers code-less service availability errors and opaque HTTP 503 errors', () => {+ expect(isInterruptedTurnError({ message: availabilityError })).toBe(true);+ expect(isInterruptedTurnError({ message: 'opaque', errorStatus: 503 })).toBe(true);+ });++ it.each([400, 401, 402, 403, 404, 413, 429])(+ 'does not override explicit HTTP %s with availability wording', (errorStatus) => {+ expect(isInterruptedTurnError({ message: availabilityError, errorStatus })).toBe(false);+ },+ );++ it.each(+ ['authentication_failed', 'authentication_error', 'billing_error', 'rate_limit', 'invalid_request', 'permission_error', 'insufficient_quota', 'context_length_exceeded']+ .flatMap(sdkError => [undefined, 502, 503, 504, 529].map(errorStatus => ({ sdkError, errorStatus }))),+ )(+ 'does not override $sdkError with availability wording or HTTP $errorStatus', ({ sdkError, errorStatus }) => {+ expect(isInterruptedTurnError({ message: availabilityError, sdkError, errorStatus })).toBe(false);+ },+ );++ it.each([502, 503, 504, 529])('still recovers HTTP %s without a rejection category', (errorStatus) => {+ expect(isInterruptedTurnError({ message: 'opaque', sdkError: 'server_error', errorStatus })).toBe(true);+ });++ it('preserves text compatibility for unclassified server errors', () => {+ expect(isInterruptedTurnError({ message: availabilityError, errorStatus: 500 })).toBe(true);+ expect(isInterruptedTurnError({ message: 'Selected model is at capacity.', errorStatus: 500 })).toBe(true);+ expect(isInterruptedTurnError({ message: 'opaque', errorStatus: 500 })).toBe(false);+ });++ it('does not layer host retries over exhausted Pi retries', () => {+ expect(isInterruptedTurnError({ message: availabilityError, reason: 'pi-gateway-drop' })).toBe(false);+ });+it('never automatically resumes a detected tool loop, even with network-like text', () => {expect(isInterruptedTurnError({reason: 'tool_use_loop_detected', message: 'Request timed out',diff --git a/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts b/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.tsindex 5071f8157c..cfba40d1ed 100644--- a/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts+++ b/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts@@ -119,7 +119,8 @@ function isStreamTruncationError(signals: InterruptedTurnErrorSignals): boolean* 是 #844 主动交回用户的情形(重投会重复已产生的副作用),正是本份该接的。** 认这三类刻意**不**要求 `server_error` tag、也**允许**带状态码——502 / 529 本身就带- * 状态码,网络 errno 也没有 SDK tag。收紧只对第 1 类成立。+ * 状态码,网络 errno 也没有 SDK tag。已分类 reason 优先;其余候选先排除明确的+ * 4xx/鉴权/额度等拒绝信号,再走临时网络/过载文案兼容。*//*** 这类 reason 表示 turn **已经被上游 / daemon accept** 之后卡死,不是 admission 失败。@@ -199,6 +200,16 @@ export function isInterruptedTurnError(signals: InterruptedTurnErrorSignals): boreturn true;}if (reason.length > 0) return false;+ // Prefer provider status/tags to text compatibility. In particular, an auth,+ // quota or invalid-request rejection must not become retryable just because+ // its explanation mentions temporary unavailability or also carries a 5xx.+ const status = signals.errorStatus;+ if (status !== undefined && status >= 400 && status < 500) return false;+ if ([+ 'authentication_failed', 'authentication_error', 'billing_error', 'rate_limit',+ 'invalid_request', 'permission_error', 'insufficient_quota', 'context_length_exceeded',+ ].includes(signals.sdkError ?? '')) return false;+ if ([502, 503, 504, 529].includes(status ?? 0)) return true;if (isStreamTruncationError(signals)) return true;const message = signals.message;if (typeof message !== 'string' || message.length === 0) return false;diff --git a/apps/desktop/src/renderer/__tests__/networkError.test.ts b/apps/desktop/src/renderer/__tests__/networkError.test.tsindex e61c5878cb..0fbca98527 100644--- a/apps/desktop/src/renderer/__tests__/networkError.test.ts+++ b/apps/desktop/src/renderer/__tests__/networkError.test.ts@@ -14,6 +14,12 @@ import { isNetworkishErrorMessage, parseReconnectAttemptMessage } from '@/utils/describe('isNetworkishErrorMessage', () => {it.each([// Anthropic SDK 重试耗尽后透传的终止型错误原文+ 'Error Code null: Service temporarily unavailable. The model\'s availability is currently degraded.',+ 'SERVICE TEMPORARILY UNAVAILABLE',+ 'Service is temporarily unavailable.',+ 'Service Unavailable',+ "The model's availability is currently degraded.",+ 'Model availability is degraded.','Request timed out.','API Error: The operation timed out.','Connection error.',@@ -36,6 +42,11 @@ describe('isNetworkishErrorMessage', () => {it.each(['Invalid API key',+ 'Permission denied',+ 'Insufficient quota',+ 'Model unavailable for this account',+ 'Tool availability is currently degraded',+ 'Degraded performance','thread not found','context window exceeded','Local tool operation timed out.',diff --git a/apps/desktop/src/renderer/utils/networkError.ts b/apps/desktop/src/renderer/utils/networkError.tsindex fc542a64fc..826c4cd4e0 100644--- a/apps/desktop/src/renderer/utils/networkError.ts+++ b/apps/desktop/src/renderer/utils/networkError.ts@@ -40,9 +40,10 @@ export function parseReconnectAttemptMessage(message: string): ReconnectAttempt}export function isNetworkishErrorMessage(message: string): boolean {+ // Keep code-less Responses availability errors aligned with maker-core.return (parseReconnectAttemptMessage(message) !== null ||- /\b50[234]\b|Bad Gateway|Service Unavailable|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(+ /\b50[234]\b|Bad Gateway|\bService(?:\s+is)?(?:\s+temporarily)?\s+Unavailable\b|\bmodel(?:'s)?\s+availability\s+is\s+(?:currently\s+)?degraded\b|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(message,));diff --git a/docs/dev-rules/pi-harness.md b/docs/dev-rules/pi-harness.mdindex 78791f5a73..aa9eddb842 100644--- a/docs/dev-rules/pi-harness.md+++ b/docs/dev-rules/pi-harness.md@@ -281,6 +281,12 @@ Pi CLI 管理入口、内核自更新与旧工具兼容的执行边界见改判成失败,provider continuation claim 也不能被当作最终结束。- Pi 的 `Request was aborted` 只在无当前 generation 的 Host Stop 时归入请求断流失败;无错误正文的 bare abort 仍保持取消。复用既有错误收口及重试预算,不重放包命令或工具。+- Pi 未归类的缺码临时服务故障(如 `Service temporarily unavailable` 或明确的模型+ availability degraded)由 Cindy 的既有有界续跑守卫兜底;结构化状态/错误类别优先,+ 不把裸 `unavailable`/`degraded` 当重试依据,不伪造 HTTP 503。已有正文或工具结果时+ 发送续跑指令,保留已有结果;用户 Stop 取消待续跑。Pi `auto_retry_end(success=false)`+ 已用尽原生预算时保持 `pi-gateway-drop` 终态,不再叠加 Host 重试。本规则不修改受管+ Pi 二进制、版本或原生重试上限;原生分类兼容需由 Pi 上游独立修正。- SDK 成功与正文入库/交付分开取证。只见 JSONL 成功但 SQLite 缺正文时,不能自动重跑已成功的工作;应沿 RPC → translator → Session → persistence 查丢失边界。diff --git a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.tsindex dd9d43344c..0b0e25f2a0 100644--- a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts+++ b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts@@ -980,7 +980,10 @@ describe('pi translator', () => {expect(events.filter((event) => event.type === 'error')).toHaveLength(0);});- it('hands exhausted network retries back to the user instead of host auto-resume', () => {+ it.each([+ 'The operation timed out.',+ "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.",+ ])('hands exhausted network retries back to the user instead of host auto-resume: %s', (finalError) => {const ctx = createPiTranslateContext(noopLogger);const { queue, events } = makeQueue();@@ -989,7 +992,7 @@ describe('pi translator', () => {ev({type: 'auto_retry_end',success: false,- finalError: 'The operation timed out.',+ finalError,}),queue,ctx,@@ -1003,7 +1006,7 @@ describe('pi translator', () => {);expect(terminalErrors).toHaveLength(1);expect(terminalErrors[0]?.data).toMatchObject({- message: 'The operation timed out.',+ message: finalError,reason: 'pi-gateway-drop',});expect(events.find((event) => event.type === 'done')?.data)diff --git a/packages/maker-core/src/agents/shared/network-error.test.ts b/packages/maker-core/src/agents/shared/network-error.test.tsindex c944dc6b74..0a0d35093d 100644--- a/packages/maker-core/src/agents/shared/network-error.test.ts+++ b/packages/maker-core/src/agents/shared/network-error.test.ts@@ -17,6 +17,12 @@ import {describe('isNetworkishErrorMessage', () => {it.each([// Anthropic SDK 重试耗尽后透传的终止型错误原文+ 'Error Code null: Service temporarily unavailable. The model\'s availability is currently degraded.',+ 'SERVICE TEMPORARILY UNAVAILABLE',+ 'Service is temporarily unavailable.',+ 'Service Unavailable',+ "The model's availability is currently degraded.",+ 'Model availability is degraded.','Request timed out.','API Error: The operation timed out.','Connection error.',@@ -39,6 +45,11 @@ describe('isNetworkishErrorMessage', () => {it.each(['Invalid API key',+ 'Permission denied',+ 'Insufficient quota',+ 'Model unavailable for this account',+ 'Tool availability is currently degraded',+ 'Degraded performance','thread not found','context window exceeded','Local tool operation timed out.',diff --git a/packages/maker-core/src/agents/shared/network-error.ts b/packages/maker-core/src/agents/shared/network-error.tsindex e47e20fe06..3beb294495 100644--- a/packages/maker-core/src/agents/shared/network-error.ts+++ b/packages/maker-core/src/agents/shared/network-error.ts@@ -54,9 +54,11 @@ export function parseReconnectAttemptMessage(message: string): ReconnectAttempt}export function isNetworkishErrorMessage(message: string): boolean {+ // Some Responses providers omit the error code and only report temporary+ // service/model unavailability. Do not match bare "unavailable" or "degraded".return (parseReconnectAttemptMessage(message) !== null ||- /\b50[234]\b|Bad Gateway|Service Unavailable|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(+ /\b50[234]\b|Bad Gateway|\bService(?:\s+is)?(?:\s+temporarily)?\s+Unavailable\b|\bmodel(?:'s)?\s+availability\s+is\s+(?:currently\s+)?degraded\b|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(message,));