Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(pi): recover code-less temporary service availability errors (#5062)

Chris Zhang0e3545f3445e
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
index 5bb7a1db9c..4229ff8224 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
@@ -1,6 +1,15 @@
import { AUTO_REVIEW_SOURCE_CONTENT, AUTO_REVIEW_USER_INTENT, appendAutoReviewUserIntent } from '@cindy/maker-core';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { AgentInputCoordinator } from '../agent-input-coordinator.js';
+import {
+ createPiTranslateContext, disposePiTranslateContext, translatePiEvent,
+} from '../../../../../../packages/maker-core/src/agents/pi/translator.js';
+import type { AgentEvent } from '../../../../../../packages/maker-core/src/types/events.js';
+import type { Logger } from '../../../../../../packages/maker-core/src/interfaces/logger.js';
+import {
+ InterruptedTurnAutoResumeGuard, INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS,
+ isInterruptedTurnError,
+} from '../interruptedTurnAutoResume.js';
import { createQueuedDispatchReceipts } from '../queuedDispatchReceipts.js';
import { createOrcaInterAgentDispatcher } from '../orcaInterAgentDispatcher.js';
import {
@@ -11091,6 +11100,108 @@ describe('AgentInputCoordinator replaceQueuedMessage(Orca lead 排队消息修
});
describe('AgentInputCoordinator 中断自动续跑', () => {
+ const availabilityError = "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.";
+
+ // Offline provider -> real Pi translator -> real host classifier/guard ->
+ // coordinator dispatch. The provider and durable DB progress are fixtures;
+ // no model, process or tool is actually invoked.
+ function availabilityInput() {
+ const item = makeItem('q-first', 'original request with possible side effects');
+ return { ...item, model: 'grok-4.7', createOpts: { ...item.createOpts!, agentKind: 'pi' as const, model: 'grok-4.7' } };
+ }
+
+ function availabilityHarness() {
+ const h = createHarness();
+ const guard = new InterruptedTurnAutoResumeGuard({
+ isEnabled: () => true, log: mocks.logger, random: () => 0.5,
+ });
+ h.isResumableTurnErrorCandidate.mockImplementation(isInterruptedTurnError);
+ h.onResumableTurnError.mockImplementation((sid, signals) => {
+ if (!isInterruptedTurnError(signals)) return null;
+ const decision = guard.onInterruptedTurn(sid, Date.now());
+ return decision.action === 'resume' ? { ...decision, error: signals.message } : null;
+ });
+ let durableToolResult = false;
+ h.setHasAssistantProgressAfter(async () => durableToolResult);
+ const logger: Logger = { ...mocks.logger, trace: vi.fn(), fatal: vi.fn(), child: () => logger };
+ async function fail(sid: string, opts: { toolResult?: boolean; nativeExhausted?: boolean } = {}) {
+ const ctx = createPiTranslateContext(logger);
+ const events: AgentEvent[] = [];
+ const queue = { push: (event: AgentEvent) => { events.push(event); }, end: () => {} } as unknown as Parameters<typeof translatePiEvent>[1];
+ const emit = (event: Record<string, unknown>) => translatePiEvent(event as Parameters<typeof translatePiEvent>[0], queue, ctx);
+ try {
+ emit({ type: 'agent_start' });
+ if (opts.toolResult) {
+ emit({ type: 'tool_execution_start', toolCallId: 'completed-tool', toolName: 'read', args: { path: 'fixture.md' } });
+ emit({ type: 'tool_execution_end', toolCallId: 'completed-tool', toolName: 'read', result: { content: [{ type: 'text', text: 'fixture result' }] }, isError: false });
+ expect(events.some(event => event.type === 'tool_result')).toBe(true);
+ durableToolResult = true;
+ }
+ emit({ type: 'message_end', message: { role: 'assistant', content: [], stopReason: 'error', errorMessage: availabilityError } });
+ emit({ type: 'agent_end', messages: [] });
+ expect(events.filter(event => event.type === 'error' || event.type === 'done')).toHaveLength(0);
+ if (opts.nativeExhausted) emit({ type: 'auto_retry_end', success: false, finalError: availabilityError });
+ emit({ type: 'agent_settled' });
+ h.setRunning(false);
+ for (const event of events) {
+ if (event.type === 'error') {
+ const data = event.data as { message: string; isTerminal?: boolean; sdkError?: string; reason?: string; errorStatus?: number };
+ if (data.isTerminal) h.coordinator.onTurnEvent(sid, 'error', data.message, data);
+ } else if (event.type === 'done') h.coordinator.onTurnEvent(sid, 'done');
+ }
+ await flush();
+ } finally { disposePiTranslateContext(ctx); }
+ }
+ return { h, guard, fail };
+ }
+
+ it('continues after a completed tool and stops at the existing host retry budget', async () => {
+ const { h, guard, fail } = availabilityHarness();
+ const sid = 'pi-availability-budget';
+ h.coordinator.enqueue(sid, availabilityInput());
+ await flush();
+ for (let attempt = 1; attempt <= INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS; attempt += 1) {
+ await fail(sid, { toolResult: attempt === 1 });
+ expect(latestProjection(h.projections).error).toBeNull();
+ expect(await h.coordinator.autoRetryLastError(sid, attempt)).toBe('resumed');
+ await flush();
+ expect(h.sendToAgent.mock.calls[attempt]?.[1]).toEqual({ type: 'user', content: CONTINUE_AFTER_ERROR_PROMPT });
+ expect(h.sendToAgent.mock.calls[attempt]?.[3]?.persistUserMessage?.autoResume).toBe(true);
+ // Production retires the pending token on the first provider event.
+ expect(guard.noteAttemptEvent(sid, attempt)).toBe(true);
+ }
+ await fail(sid);
+ expect(latestProjection(h.projections).error).not.toBeNull();
+ expect(h.coordinator.isAutoResumePending(sid)).toBe(false);
+ expect(h.sendToAgent).toHaveBeenCalledTimes(1 + INTERRUPTED_TURN_MAX_CONSECUTIVE_ATTEMPTS);
+ });
+
+ it('does not resume an availability error after the user stops during backoff', async () => {
+ const { h, guard, fail } = availabilityHarness();
+ const sid = 'pi-availability-stop';
+ h.coordinator.enqueue(sid, availabilityInput());
+ await flush();
+ await fail(sid, { toolResult: true });
+ h.coordinator.stop(sid);
+ guard.noteSessionReset(sid);
+ expect(await h.coordinator.autoRetryLastError(sid, 1)).toBe('superseded');
+ await flush();
+ expect(h.sendToAgent).toHaveBeenCalledTimes(1);
+ expect(guard.isCurrentAttempt(sid, 1)).toBe(false);
+ });
+
+ it('does not add host retries after Pi reports native availability retry exhaustion', async () => {
+ const { h, fail } = availabilityHarness();
+ const sid = 'pi-availability-native-exhausted';
+ h.coordinator.enqueue(sid, availabilityInput());
+ await flush();
+ await fail(sid, { toolResult: true, nativeExhausted: true });
+ expect(h.onResumableTurnError).toHaveReturnedWith(null);
+ expect(latestProjection(h.projections).error).not.toBeNull();
+ expect(h.coordinator.isAutoResumePending(sid)).toBe(false);
+ expect(h.sendToAgent).toHaveBeenCalledTimes(1);
+ });
+
// 上游把「已经干到一半」的 turn 打断时,main 守卫自动替用户点一次「继续」。
// coordinator 这一侧只负责两件事:把带结构化信号的失败告知 host(判据不在这里),
// 以及提供一条**带 autoResume 标记**的补发路径(标记是额度不自我充值的判据)。
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts
index 0aa555fd18..22489c47dc 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/interruptedTurnAutoResume.test.ts
@@ -15,6 +15,42 @@ import {
// 判定单测的核心是**白名单收紧**:自动重试一个确定性失败(认证过期、协议错)会反复
// 烧额度并反复报错,所以只有识别得了的"上游把 turn 打断了"才允许自愈。
describe('isInterruptedTurnError', () => {
+ const availabilityError = "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.";
+
+ it('recovers code-less service availability errors and opaque HTTP 503 errors', () => {
+ expect(isInterruptedTurnError({ message: availabilityError })).toBe(true);
+ expect(isInterruptedTurnError({ message: 'opaque', errorStatus: 503 })).toBe(true);
+ });
+
+ it.each([400, 401, 402, 403, 404, 413, 429])(
+ 'does not override explicit HTTP %s with availability wording', (errorStatus) => {
+ expect(isInterruptedTurnError({ message: availabilityError, errorStatus })).toBe(false);
+ },
+ );
+
+ it.each(
+ ['authentication_failed', 'authentication_error', 'billing_error', 'rate_limit', 'invalid_request', 'permission_error', 'insufficient_quota', 'context_length_exceeded']
+ .flatMap(sdkError => [undefined, 502, 503, 504, 529].map(errorStatus => ({ sdkError, errorStatus }))),
+ )(
+ 'does not override $sdkError with availability wording or HTTP $errorStatus', ({ sdkError, errorStatus }) => {
+ expect(isInterruptedTurnError({ message: availabilityError, sdkError, errorStatus })).toBe(false);
+ },
+ );
+
+ it.each([502, 503, 504, 529])('still recovers HTTP %s without a rejection category', (errorStatus) => {
+ expect(isInterruptedTurnError({ message: 'opaque', sdkError: 'server_error', errorStatus })).toBe(true);
+ });
+
+ it('preserves text compatibility for unclassified server errors', () => {
+ expect(isInterruptedTurnError({ message: availabilityError, errorStatus: 500 })).toBe(true);
+ expect(isInterruptedTurnError({ message: 'Selected model is at capacity.', errorStatus: 500 })).toBe(true);
+ expect(isInterruptedTurnError({ message: 'opaque', errorStatus: 500 })).toBe(false);
+ });
+
+ it('does not layer host retries over exhausted Pi retries', () => {
+ expect(isInterruptedTurnError({ message: availabilityError, reason: 'pi-gateway-drop' })).toBe(false);
+ });
+
it('never automatically resumes a detected tool loop, even with network-like text', () => {
expect(isInterruptedTurnError({
reason: 'tool_use_loop_detected', message: 'Request timed out',
diff --git a/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts b/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts
index 5071f8157c..cfba40d1ed 100644
--- a/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts
+++ b/apps/desktop/src/main/maker-ipc/interruptedTurnAutoResume.ts
@@ -119,7 +119,8 @@ function isStreamTruncationError(signals: InterruptedTurnErrorSignals): boolean
* 是 #844 主动交回用户的情形(重投会重复已产生的副作用),正是本份该接的。
*
* 认这三类刻意**不**要求 `server_error` tag、也**允许**带状态码——502 / 529 本身就带
- * 状态码,网络 errno 也没有 SDK tag。收紧只对第 1 类成立。
+ * 状态码,网络 errno 也没有 SDK tag。已分类 reason 优先;其余候选先排除明确的
+ * 4xx/鉴权/额度等拒绝信号,再走临时网络/过载文案兼容。
*/
/**
* 这类 reason 表示 turn **已经被上游 / daemon accept** 之后卡死,不是 admission 失败。
@@ -199,6 +200,16 @@ export function isInterruptedTurnError(signals: InterruptedTurnErrorSignals): bo
return true;
}
if (reason.length > 0) return false;
+ // Prefer provider status/tags to text compatibility. In particular, an auth,
+ // quota or invalid-request rejection must not become retryable just because
+ // its explanation mentions temporary unavailability or also carries a 5xx.
+ const status = signals.errorStatus;
+ if (status !== undefined && status >= 400 && status < 500) return false;
+ if ([
+ 'authentication_failed', 'authentication_error', 'billing_error', 'rate_limit',
+ 'invalid_request', 'permission_error', 'insufficient_quota', 'context_length_exceeded',
+ ].includes(signals.sdkError ?? '')) return false;
+ if ([502, 503, 504, 529].includes(status ?? 0)) return true;
if (isStreamTruncationError(signals)) return true;
const message = signals.message;
if (typeof message !== 'string' || message.length === 0) return false;
diff --git a/apps/desktop/src/renderer/__tests__/networkError.test.ts b/apps/desktop/src/renderer/__tests__/networkError.test.ts
index e61c5878cb..0fbca98527 100644
--- a/apps/desktop/src/renderer/__tests__/networkError.test.ts
+++ b/apps/desktop/src/renderer/__tests__/networkError.test.ts
@@ -14,6 +14,12 @@ import { isNetworkishErrorMessage, parseReconnectAttemptMessage } from '@/utils/
describe('isNetworkishErrorMessage', () => {
it.each([
// Anthropic SDK 重试耗尽后透传的终止型错误原文
+ 'Error Code null: Service temporarily unavailable. The model\'s availability is currently degraded.',
+ 'SERVICE TEMPORARILY UNAVAILABLE',
+ 'Service is temporarily unavailable.',
+ 'Service Unavailable',
+ "The model's availability is currently degraded.",
+ 'Model availability is degraded.',
'Request timed out.',
'API Error: The operation timed out.',
'Connection error.',
@@ -36,6 +42,11 @@ describe('isNetworkishErrorMessage', () => {
it.each([
'Invalid API key',
+ 'Permission denied',
+ 'Insufficient quota',
+ 'Model unavailable for this account',
+ 'Tool availability is currently degraded',
+ 'Degraded performance',
'thread not found',
'context window exceeded',
'Local tool operation timed out.',
diff --git a/apps/desktop/src/renderer/utils/networkError.ts b/apps/desktop/src/renderer/utils/networkError.ts
index fc542a64fc..826c4cd4e0 100644
--- a/apps/desktop/src/renderer/utils/networkError.ts
+++ b/apps/desktop/src/renderer/utils/networkError.ts
@@ -40,9 +40,10 @@ export function parseReconnectAttemptMessage(message: string): ReconnectAttempt
}
export function isNetworkishErrorMessage(message: string): boolean {
+ // Keep code-less Responses availability errors aligned with maker-core.
return (
parseReconnectAttemptMessage(message) !== null ||
- /\b50[234]\b|Bad Gateway|Service Unavailable|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(
+ /\b50[234]\b|Bad Gateway|\bService(?:\s+is)?(?:\s+temporarily)?\s+Unavailable\b|\bmodel(?:'s)?\s+availability\s+is\s+(?:currently\s+)?degraded\b|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(
message,
)
);
diff --git a/docs/dev-rules/pi-harness.md b/docs/dev-rules/pi-harness.md
index 78791f5a73..aa9eddb842 100644
--- a/docs/dev-rules/pi-harness.md
+++ b/docs/dev-rules/pi-harness.md
@@ -281,6 +281,12 @@ Pi CLI 管理入口、内核自更新与旧工具兼容的执行边界见
改判成失败,provider continuation claim 也不能被当作最终结束。
- Pi 的 `Request was aborted` 只在无当前 generation 的 Host Stop 时归入请求断流失败;
无错误正文的 bare abort 仍保持取消。复用既有错误收口及重试预算,不重放包命令或工具。
+- Pi 未归类的缺码临时服务故障(如 `Service temporarily unavailable` 或明确的模型
+ availability degraded)由 Cindy 的既有有界续跑守卫兜底;结构化状态/错误类别优先,
+ 不把裸 `unavailable`/`degraded` 当重试依据,不伪造 HTTP 503。已有正文或工具结果时
+ 发送续跑指令,保留已有结果;用户 Stop 取消待续跑。Pi `auto_retry_end(success=false)`
+ 已用尽原生预算时保持 `pi-gateway-drop` 终态,不再叠加 Host 重试。本规则不修改受管
+ Pi 二进制、版本或原生重试上限;原生分类兼容需由 Pi 上游独立修正。
- SDK 成功与正文入库/交付分开取证。只见 JSONL 成功但 SQLite 缺正文时,不能自动重跑
已成功的工作;应沿 RPC → translator → Session → persistence 查丢失边界。
diff --git a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
index dd9d43344c..0b0e25f2a0 100644
--- a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
+++ b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
@@ -980,7 +980,10 @@ describe('pi translator', () => {
expect(events.filter((event) => event.type === 'error')).toHaveLength(0);
});
- it('hands exhausted network retries back to the user instead of host auto-resume', () => {
+ it.each([
+ 'The operation timed out.',
+ "Error Code null: Service temporarily unavailable. The model's availability is currently degraded.",
+ ])('hands exhausted network retries back to the user instead of host auto-resume: %s', (finalError) => {
const ctx = createPiTranslateContext(noopLogger);
const { queue, events } = makeQueue();
@@ -989,7 +992,7 @@ describe('pi translator', () => {
ev({
type: 'auto_retry_end',
success: false,
- finalError: 'The operation timed out.',
+ finalError,
}),
queue,
ctx,
@@ -1003,7 +1006,7 @@ describe('pi translator', () => {
);
expect(terminalErrors).toHaveLength(1);
expect(terminalErrors[0]?.data).toMatchObject({
- message: 'The operation timed out.',
+ message: finalError,
reason: 'pi-gateway-drop',
});
expect(events.find((event) => event.type === 'done')?.data)
diff --git a/packages/maker-core/src/agents/shared/network-error.test.ts b/packages/maker-core/src/agents/shared/network-error.test.ts
index c944dc6b74..0a0d35093d 100644
--- a/packages/maker-core/src/agents/shared/network-error.test.ts
+++ b/packages/maker-core/src/agents/shared/network-error.test.ts
@@ -17,6 +17,12 @@ import {
describe('isNetworkishErrorMessage', () => {
it.each([
// Anthropic SDK 重试耗尽后透传的终止型错误原文
+ 'Error Code null: Service temporarily unavailable. The model\'s availability is currently degraded.',
+ 'SERVICE TEMPORARILY UNAVAILABLE',
+ 'Service is temporarily unavailable.',
+ 'Service Unavailable',
+ "The model's availability is currently degraded.",
+ 'Model availability is degraded.',
'Request timed out.',
'API Error: The operation timed out.',
'Connection error.',
@@ -39,6 +45,11 @@ describe('isNetworkishErrorMessage', () => {
it.each([
'Invalid API key',
+ 'Permission denied',
+ 'Insufficient quota',
+ 'Model unavailable for this account',
+ 'Tool availability is currently degraded',
+ 'Degraded performance',
'thread not found',
'context window exceeded',
'Local tool operation timed out.',
diff --git a/packages/maker-core/src/agents/shared/network-error.ts b/packages/maker-core/src/agents/shared/network-error.ts
index e47e20fe06..3beb294495 100644
--- a/packages/maker-core/src/agents/shared/network-error.ts
+++ b/packages/maker-core/src/agents/shared/network-error.ts
@@ -54,9 +54,11 @@ export function parseReconnectAttemptMessage(message: string): ReconnectAttempt
}
export function isNetworkishErrorMessage(message: string): boolean {
+ // Some Responses providers omit the error code and only report temporary
+ // service/model unavailability. Do not match bare "unavailable" or "degraded".
return (
parseReconnectAttemptMessage(message) !== null ||
- /\b50[234]\b|Bad Gateway|Service Unavailable|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(
+ /\b50[234]\b|Bad Gateway|\bService(?:\s+is)?(?:\s+temporarily)?\s+Unavailable\b|\bmodel(?:'s)?\s+availability\s+is\s+(?:currently\s+)?degraded\b|Gateway Time-?out|upstream unreachable|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|EPIPE|EAI_AGAIN|fetch failed|network error|socket hang up|AggregateError|Request timed out|^API Error:\s*The operation timed out|^The operation timed out|stream ended before a terminal|Connection error|upstream stream error/i.test(
message,
)
);