Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(auto-review): 网关超时后在兜底候选之后原位重试 (#5108)

Dasha51ded8cc045
diff --git a/apps/desktop/src/main/maker-host/__tests__/autoReviewModelRouter.test.ts b/apps/desktop/src/main/maker-host/__tests__/autoReviewModelRouter.test.ts
index 367e8f59f2..76f7b823df 100644
--- a/apps/desktop/src/main/maker-host/__tests__/autoReviewModelRouter.test.ts
+++ b/apps/desktop/src/main/maker-host/__tests__/autoReviewModelRouter.test.ts
@@ -143,6 +143,80 @@ describe('dedicated Auto-review candidate policy', () => {
expect(calls).toEqual(['cindy-gateway', 'chatgpt-nano']);
});
+ it('retries a timed-out gateway after unavailable fallbacks skip instantly', async () => {
+ vi.useFakeTimers();
+ const log = logger();
+ const calls: string[] = [];
+ const requestCandidate = vi.fn((_prompt, candidate) => {
+ calls.push(candidate.id);
+ if (candidate.id === 'cindy-gateway') {
+ return calls.length === 1
+ ? new Promise<UtilityTextResult>(() => undefined)
+ : Promise.resolve(succeeded(candidate, '{"verdict":"allow"}'));
+ }
+ // 本机没有可用 OpenAI 凭证:立即返回,不占预算。
+ return Promise.resolve<UtilityTextResult>({ ok: false, reason: 'no_candidate', attempts: [] });
+ });
+ const route = createAutoReviewModelRouter({ logger: log, requestCandidate });
+
+ const pending = route('classify');
+ await vi.advanceTimersByTimeAsync(AUTO_REVIEW_CANDIDATE_TIMEOUT_MS);
+
+ await expect(pending).resolves.toBe('{"verdict":"allow"}');
+ expect(calls).toEqual(['cindy-gateway', 'chatgpt-nano', 'chatgpt-luna', 'cindy-gateway']);
+ expect(log.warn).toHaveBeenCalledWith('auto-review model candidate failed', expect.objectContaining({
+ candidateId: 'cindy-gateway',
+ reason: 'timeout',
+ retrying: 'deferred',
+ }));
+ });
+
+ it('keeps an earlier deferred gateway retry ahead of a later fallback retry', async () => {
+ vi.useFakeTimers();
+ const calls: string[] = [];
+ const requestCandidate = vi.fn((_prompt, candidate) => {
+ calls.push(candidate.id);
+ if (candidate.id === 'chatgpt-nano') {
+ return Promise.resolve<UtilityTextResult>({ ok: false, reason: 'no_candidate', attempts: [] });
+ }
+ if (candidate.id === 'cindy-gateway' && calls.length > 1) {
+ return Promise.resolve(succeeded(candidate, '{"verdict":"allow"}'));
+ }
+ // 首次网关与 luna 都耗满超时。
+ return new Promise<UtilityTextResult>(() => undefined);
+ });
+ const route = createAutoReviewModelRouter({ logger: logger(), requestCandidate });
+
+ const pending = route('classify');
+ await vi.advanceTimersByTimeAsync(AUTO_REVIEW_CANDIDATE_TIMEOUT_MS * 2);
+
+ await expect(pending).resolves.toBe('{"verdict":"allow"}');
+ // luna 不得原位重试插到已排队的网关重试前面。
+ expect(calls).toEqual(['cindy-gateway', 'chatgpt-nano', 'chatgpt-luna', 'cindy-gateway']);
+ });
+
+ it('does not retry a timed-out candidate without a full timeout left', async () => {
+ vi.useFakeTimers();
+ const calls: string[] = [];
+ const requestCandidate = vi.fn((_prompt, candidate) => {
+ calls.push(candidate.id);
+ return new Promise<UtilityTextResult>(() => undefined);
+ });
+ const log = logger();
+ const route = createAutoReviewModelRouter({ logger: log, requestCandidate });
+
+ const pending = route('classify');
+ await vi.advanceTimersByTimeAsync(AUTO_REVIEW_CHAIN_TIMEOUT_MS);
+
+ await expect(pending).resolves.toBeNull();
+ expect(calls).toEqual(['cindy-gateway', 'chatgpt-nano', 'chatgpt-luna']);
+ // 已排队的延后重试因预算不足被跳过时要留下记录,不能只剩一条 deferred。
+ const skipped = log.warn.mock.calls
+ .filter(([message]) => message === 'auto-review model candidate retry skipped')
+ .map(([, fields]) => fields.candidateId);
+ expect(skipped).toEqual(['cindy-gateway', 'chatgpt-nano', 'chatgpt-luna']);
+ });
+
it('aborts the in-flight request at the total deadline without starting another chain', async () => {
vi.useFakeTimers();
const observedSignals: AbortSignal[] = [];
diff --git a/apps/desktop/src/main/maker-host/auto-review-model-router.ts b/apps/desktop/src/main/maker-host/auto-review-model-router.ts
index 8c4d92d7f9..e228532d2f 100644
--- a/apps/desktop/src/main/maker-host/auto-review-model-router.ts
+++ b/apps/desktop/src/main/maker-host/auto-review-model-router.ts
@@ -10,8 +10,9 @@ import { parseAutoPermissionReviewDecision } from './auto-permission-reviewer.js
export const AUTO_REVIEW_CANDIDATE_TIMEOUT_MS = 12_000;
/**
- * 整条链的预算 = 每个候选一次完整超时 + 4s 余量。按候选数推导:预算只够「逐个兜底」,
- * 一个候选整段超时后直接让给下一个,而不是在原位重试吃掉后续候选的时间。
+ * 整条链的预算 = 每个候选一次完整超时 + 4s 余量。按候选数推导:一个候选整段超时后先让给
+ * 下一个,而不是在原位重试吃掉后续候选的时间;兜底候选本机不可用(立即 no_candidate)时,
+ * 省下的预算留给超时候选在所有兜底之后再试一次。
*/
export const AUTO_REVIEW_CHAIN_TIMEOUT_MS =
DEDICATED_AUTO_REVIEW_CANDIDATES.length * AUTO_REVIEW_CANDIDATE_TIMEOUT_MS + 4_000;
@@ -141,12 +142,21 @@ function safeFailureReason(result: UtilityTextResult): string {
return failed?.reason ?? result.reason;
}
+interface PendingCandidateAttempt {
+ candidate: DedicatedAutoReviewCandidate;
+ attempt: number;
+}
+
/**
* Runs the dedicated Auto-review model chain once.
*
- * Each candidate owns its HTTP timeout and receives the chain AbortSignal. Only a quick,
- * infrastructure-shaped failure is retried in place; a full timeout immediately yields to
- * the next provider so one outage cannot consume the entire fallback budget.
+ * Each candidate owns its HTTP timeout and receives the chain AbortSignal, and gets at most
+ * AUTO_REVIEW_TRANSIENT_RETRY_ATTEMPTS attempts. A transient failure is retried in place only
+ * when the budget still covers one full timeout per pending attempt (untried candidates and
+ * earlier deferred retries); otherwise the retry joins the end of the queue. A full timeout
+ * therefore cannot starve fallback or an earlier, higher-priority retry, and fallbacks that
+ * are unavailable locally (`no_candidate` returns immediately) do not waste the budget
+ * reserved for them.
*/
export function createAutoReviewModelRouter(
deps: AutoReviewModelRouterDeps,
@@ -167,74 +177,90 @@ export function createAutoReviewModelRouter(
const deadline = setTimeout(() => controller.abort(), AUTO_REVIEW_CHAIN_TIMEOUT_MS);
try {
- for (const [candidateIndex, candidate] of candidates.entries()) {
- if (controller.signal.aborted) break;
-
- for (let attempt = 1; attempt <= AUTO_REVIEW_TRANSIENT_RETRY_ATTEMPTS; attempt++) {
- const remainingMs = deadlineAt - now();
- if (remainingMs <= 0 || controller.signal.aborted) break;
- const attemptStartedAt = now();
- let result: UtilityTextResult;
- try {
- result = await requestCandidateWithinTimeout(
- requestCandidate,
- prompt,
- candidate,
- Math.min(AUTO_REVIEW_CANDIDATE_TIMEOUT_MS, remainingMs),
- controller.signal,
- );
- } catch {
- // Credential refresh and catalog probes are runtime boundaries too. A thrown
- // candidate must not skip the remaining controlled providers or leak details.
- result = candidateRequestFailure(candidate);
- }
- const durationMs = now() - attemptStartedAt;
-
- if (result.ok) {
- const decision = parseDecision(result.text);
- if (decision) {
- deps.logger.debug('auto-review model candidate completed', {
- candidateId: candidate.id,
- providerId: candidate.providerId,
- model: candidate.model,
- attempt,
- verdict: decision.verdict,
- durationMs: now() - startedAt,
- });
- return JSON.stringify(decision);
- }
- deps.logger.warn('auto-review model candidate returned malformed decision', {
+ const queue: PendingCandidateAttempt[] = candidates.map((candidate) => ({ candidate, attempt: 1 }));
+ while (queue.length > 0 && !controller.signal.aborted) {
+ const { candidate, attempt } = queue.shift()!;
+ const remainingMs = deadlineAt - now();
+ if (remainingMs <= 0) break;
+ // A retry always gets a full candidate timeout; a truncated one mostly just delays
+ // the confirmation card.
+ if (attempt > 1 && remainingMs < AUTO_REVIEW_CANDIDATE_TIMEOUT_MS) {
+ deps.logger.warn('auto-review model candidate retry skipped', {
+ candidateId: candidate.id,
+ providerId: candidate.providerId,
+ model: candidate.model,
+ attempt,
+ reason: 'insufficient_budget',
+ remainingMs,
+ });
+ continue;
+ }
+ const attemptStartedAt = now();
+ let result: UtilityTextResult;
+ try {
+ result = await requestCandidateWithinTimeout(
+ requestCandidate,
+ prompt,
+ candidate,
+ Math.min(AUTO_REVIEW_CANDIDATE_TIMEOUT_MS, remainingMs),
+ controller.signal,
+ );
+ } catch {
+ // Credential refresh and catalog probes are runtime boundaries too. A thrown
+ // candidate must not skip the remaining controlled providers or leak details.
+ result = candidateRequestFailure(candidate);
+ }
+ const durationMs = now() - attemptStartedAt;
+
+ if (result.ok) {
+ const decision = parseDecision(result.text);
+ if (decision) {
+ deps.logger.debug('auto-review model candidate completed', {
candidateId: candidate.id,
providerId: candidate.providerId,
model: candidate.model,
attempt,
- durationMs,
+ verdict: decision.verdict,
+ durationMs: now() - startedAt,
});
- break;
+ return JSON.stringify(decision);
}
-
- const remainingAfterAttemptMs = deadlineAt - now();
- const laterCandidateReserveMs =
- (candidates.length - candidateIndex - 1) * AUTO_REVIEW_CANDIDATE_TIMEOUT_MS;
- const canRetry = attempt < AUTO_REVIEW_TRANSIENT_RETRY_ATTEMPTS
- && transientCandidateFailure(result)
- && remainingAfterAttemptMs >= (
- AUTO_REVIEW_CANDIDATE_TIMEOUT_MS
- + AUTO_REVIEW_TRANSIENT_RETRY_BACKOFF_MS
- + laterCandidateReserveMs
- )
- && !controller.signal.aborted;
- deps.logger.warn('auto-review model candidate failed', {
+ deps.logger.warn('auto-review model candidate returned malformed decision', {
candidateId: candidate.id,
providerId: candidate.providerId,
model: candidate.model,
attempt,
- reason: safeFailureReason(result),
- retrying: canRetry,
durationMs,
});
- if (!canRetry) break;
+ continue;
+ }
+
+ // Queued work keeps its order: a retry may only jump ahead when the budget still covers
+ // one full timeout for every pending attempt, including earlier deferred retries.
+ const pendingAttempts = queue.length;
+ const retriable = attempt < AUTO_REVIEW_TRANSIENT_RETRY_ATTEMPTS
+ && transientCandidateFailure(result)
+ && !controller.signal.aborted;
+ const retryInPlace = retriable && deadlineAt - now() >= (
+ AUTO_REVIEW_CANDIDATE_TIMEOUT_MS
+ + AUTO_REVIEW_TRANSIENT_RETRY_BACKOFF_MS
+ + pendingAttempts * AUTO_REVIEW_CANDIDATE_TIMEOUT_MS
+ );
+ const retryDeferred = retriable && !retryInPlace && pendingAttempts > 0;
+ deps.logger.warn('auto-review model candidate failed', {
+ candidateId: candidate.id,
+ providerId: candidate.providerId,
+ model: candidate.model,
+ attempt,
+ reason: safeFailureReason(result),
+ retrying: retryInPlace ? 'in_place' : retryDeferred ? 'deferred' : false,
+ durationMs,
+ });
+ if (retryInPlace) {
await sleep(AUTO_REVIEW_TRANSIENT_RETRY_BACKOFF_MS, controller.signal);
+ queue.unshift({ candidate, attempt: attempt + 1 });
+ } else if (retryDeferred) {
+ queue.push({ candidate, attempt: attempt + 1 });
}
}
} finally {