Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(provider-group): 保留 done 与关闭期间的产品回执

guyongc9a34f952d77
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
index 67f0835cbc..49ee5876dd 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/agent-input-coordinator.test.ts
@@ -11,6 +11,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { admitTaskAction, bindTaskActionInput, taskActionInputGuard } from '../taskActionAuthorization.js';
import { AgentInputCoordinator } from '../agent-input-coordinator.js';
import { ProviderGroupUsageLimitError } from '../../provider-group/failurePolicy.js';
+import type { Session } from '@cindy/maker-core';
+import { createSessionBindingLifecycle } from '../sessionBindingLifecycle.js';
+import { captureProductTurnFailureOwner, createDeferredProductTurnFailureGate } from '../productTurnFailureOwner.js';
+import { createProviderGroupHeldRecovery, createProviderGroupHeldTurnErrors } from '../../provider-group/heldTurnErrors.js';
+import { createProviderGroupHeldWorkerTerminals } from '../../provider-group/heldWorkerTerminals.js';
+import { UpstreamMergeController, type UpstreamMergeDependencies } from '../../cindy-make/upstreamMergeController.js';
import { createProviderGroupService, type ProviderGroupServiceDeps } from '../../provider-group/service.js';
import { UsageLimitAutoResume, USAGE_LIMIT_MAX_CONSECUTIVE_RESUMES } from '../usageLimitAutoResume.js';
import { MCP_APP_MESSAGE_CLIENT_PREFIX, MCP_APP_MESSAGE_GUARD } from '../../mcp-apps/message.js';
@@ -14982,6 +14988,129 @@ describe('provider group computer switch hold', () => {
sessionTotal: 0,
};
+ // #6671/r4339380528, unit: queue acceptance releases the original hold, so a
+ // runtime close after onPersisted used to erase the product owner before the
+ // real coordinator's onDiscarded outcome. Prior tests manually cleared the
+ // gate and missed the upstream waiter. Execute register's actual close adapter
+ // with the real lifecycle/coordinator/controller and a paused send transaction.
+ it.each(['failed', 'closed-failed', 'closed-dispatched'] as const)(
+ 'finishes the upstream waiter after a queued group continuation: %s', async outcome => {
+ const sid = 'group-close-upstream';
+ const h = createHarness({ providerGroupSwitchCandidate: () => 'overload' });
+ const gate = createDeferredProductTurnFailureGate();
+ let close!: () => void;
+ const session = {
+ id: sid, instanceId: 'original', getTurnGeneration: () => 0,
+ setInteractionListener: vi.fn(),
+ onStatusChange: (listener: (status: string) => void) => {
+ close = () => listener('closed'); return () => {};
+ },
+ } as unknown as Session;
+ const upstream = new UpstreamMergeController({
+ read: () => ({ state: {
+ id: 'group-close-operation', status: 'conflict', ref: 'main',
+ upstreamCommit: 'a'.repeat(40), hasWorkspace: true, sessionId: sid,
+ }, sessionOwner: 'account' }),
+ write: vi.fn(), publish: vi.fn(), owner: () => 'account', hasWorkspace: () => true,
+ exclusive: async run => run(),
+ latest: async () => ({ ref: 'main', commit: 'a'.repeat(40) }),
+ prepare: async state => state, cancel: async () => {},
+ session: async (_state, _opts, bind) => { bind(sid); return sid; },
+ running: () => false, refresh: async () => {}, cleanup: async () => true,
+ apply: async state => ({ ...state, status: 'merged', commit: 'b'.repeat(40) }),
+ } satisfies UpstreamMergeDependencies);
+ await upstream.resolve(undefined, 'group-close-operation');
+ const abort = new AbortController();
+ let waiter: unknown = 'pending';
+ const completion = upstream.waitForCompletion('group-close-operation', abort.signal)
+ .then(state => { waiter = state.status; }, error => { waiter = error.code; });
+ const settleFailure = () => gate.settle(sid,
+ owner => owner.session === session && owner.generation === session.getTurnGeneration(),
+ () => { void upstream.interrupt(sid); });
+ const errors = createProviderGroupHeldTurnErrors({
+ capture: () => null, persist: vi.fn(), surface: vi.fn(), settleFailure,
+ ownerKey: () => 'account',
+ releaseHold: (id, hold) => h.coordinator.releaseProviderGroupSwitchHold(id, hold),
+ });
+ const delivered = vi.fn();
+ const terminals = createProviderGroupHeldWorkerTerminals({ deliver: delivered, ownerKey: () => 'account' });
+ const recovery = createProviderGroupHeldRecovery({ errors, terminals, settleFailure, ownerKey: () => 'account' });
+ // Run the production adapter rather than duplicating its close policy.
+ const source = readFileSync(new URL('../register.ts', import.meta.url), 'utf8');
+ const start = source.indexOf(' beforeCloseTeardown: (session: WiredSession) => {');
+ const end = source.indexOf(' finalizeClosedSession:', start);
+ const adapter = transpileModule('return ({' + source.slice(start, end) + '}).beforeCloseTeardown;', {
+ compilerOptions: { target: ScriptTarget.ES2022 },
+ }).outputText;
+ const beforeCloseTeardown = new Function(
+ 'botCompactRuntimeRefreshCoordinator', 'cancelDirectAbortReconciliation',
+ 'pendingFailedTurnAssistantPersistId', 'deferredProductTurnFailureGate',
+ 'rehydrateCloseSuppression', 'agentInputCoordinatorHolder', 'hasPendingProviderGroupContinuation', adapter,
+ )({ clearForClosedSession: vi.fn() }, vi.fn(), new Map(), gate,
+ { isSuppressed: () => false }, h.coordinator,
+ recovery.hasPendingContinuation,
+ ) as (runtime: Session) => void;
+ const binding = createSessionBindingLifecycle({
+ log: { warn: vi.fn() }, restoreInteractionListener: vi.fn(), beforeReplace: vi.fn(),
+ onBind: vi.fn(), broadcastStatus: vi.fn(), captureCloseContext: () => null,
+ cleanupClosedSession: () => h.coordinator.onSessionClosed(sid),
+ beforeCloseTeardown, finalizeClosedSession: vi.fn(),
+ });
+ const registration = binding.beginBinding(session)!;
+ binding.attachStatusListener(registration);
+ const entered = deferred<void>(), finishSend = deferred<void>();
+ h.onDiscardedQueuedMessage.mockImplementation(() => recovery.settleContinuation(sid, false));
+ h.onDispatchedUserTurn.mockImplementation(() => {
+ gate.clearSession(sid); recovery.settleContinuation(sid, true);
+ });
+ try {
+ h.coordinator.enqueue(sid, makeItem('original-input', 'resolve upstream'));
+ await flush();
+ h.setRunning(false);
+ const failure = { message: 'model at capacity', reason: 'upstream-overload' };
+ h.coordinator.onTurnEvent(sid, 'error', failure.message, failure);
+ const hold = h.coordinator.getProviderGroupSwitchHoldId(sid)!;
+ expect(hold).toEqual(expect.any(Number));
+ gate.defer(captureProductTurnFailureOwner(session, { type: 'error', source: 'codex', data: failure }));
+ errors.stash(sid, hold, failure);
+ terminals.stash(sid, hold, 'held-terminal');
+ const token = h.onUsageLimitedTurnError.mock.calls.at(-1)![3];
+ const lease = h.coordinator.leaseUsageLimitRecovery(sid, token)!;
+ const rearmed = h.coordinator.rearmUsageLimitWait(sid, lease, 1)!;
+ h.sendToAgent.mockImplementationOnce(async (id, _message, _opts, sendOpts) => {
+ await persistQueuedUserMessage(id, sendOpts);
+ entered.resolve();
+ await finishSend.promise;
+ return outcome === 'closed-dispatched' ? sendSuccess() : sessionDispatchFailure('group-close');
+ });
+ expect(await h.coordinator.continueAfterUsageLimitReset(sid, rearmed, INFO)).toBe('resumed');
+ recovery.discard(sid, hold);
+ recovery.release(sid, hold);
+ await entered.promise;
+ expect(h.coordinator.getProviderGroupSwitchHoldId(sid)).toBeNull();
+ expect(h.coordinator.hasUsageLimitRecovery(sid)).toBe(false);
+ if (outcome.startsWith('closed-')) close();
+ expect(gate.pending(sid)).toBeDefined();
+ finishSend.resolve();
+ await flush();
+ if (outcome === 'closed-dispatched') {
+ await upstream.finish(sid);
+ expect(delivered).not.toHaveBeenCalled();
+ expect(upstream.status()?.status).toBe('merged');
+ } else {
+ expect(delivered).toHaveBeenCalledExactlyOnceWith(sid, 'held-terminal');
+ expect(upstream.status()).toMatchObject({ status: 'failed', error: 'interrupted' });
+ }
+ await completion;
+ expect(waiter).toBe(outcome === 'closed-dispatched' ? 'merged' : 'interrupted');
+ expect(gate.pending(sid)).toBeUndefined();
+ } finally {
+ finishSend.resolve(); abort.abort(); await completion;
+ h.coordinator.onSessionClosed(sid);
+ }
+ },
+ );
+
// Unit regression through onTurnEvent: an overload and a switch candidate used
// to spend the generic retry budget instead of handing off (including a guest's
// short-lived switch offer). Existing hold cases only used non-resumable limits.
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/sessionEventPipeline.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/sessionEventPipeline.test.ts
index 6e71555b21..ae7f62424a 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/sessionEventPipeline.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/sessionEventPipeline.test.ts
@@ -586,8 +586,13 @@ describe('production Session event pipeline', () => {
// capacity error with only a group hold must retain its real product owner;
// success discards failure, abandonment settles once, and async decline hands
// ownership back to generic recovery. Drive handleSessionEvent + real stores.
- it.each(['continued', 'rejected', 'generic-continued', 'generic-rejected', 'unheld'] as const)(
- 'retains persisted product callers through a group capacity hold: %s', async outcome => {
+ it.each([
+ ...(['continued', 'rejected', 'generic-continued', 'generic-rejected', 'unheld'] as const)
+ .map(outcome => ({ outcome, terminal: 'error' as const })),
+ ...(['continued', 'rejected', 'generic-continued', 'generic-rejected'] as const)
+ .flatMap(outcome => (['done', 'failed-done'] as const).map(terminal => ({ outcome, terminal }))),
+ ])(
+ 'retains persisted product callers through a group capacity hold: $terminal / $outcome', async ({ outcome, terminal }) => {
const h = harness();
const gate = createDeferredProductTurnFailureGate();
const successful = h.deps.onSuccessfulProductTurn;
@@ -634,7 +639,16 @@ describe('production Session event pipeline', () => {
const execution = { sessionInstanceId: h.session.instanceId, sessionTurnGeneration: generation };
try {
h.deps.agentInputCoordinatorHolder.getProviderGroupSwitchHoldId.mockReturnValue(3);
- h.emit(event('error', failure, execution));
+ if (terminal !== 'error') {
+ // Coordinator can synthesize its error from this same-generation
+ // snapshot while the outer pipeline receives only the paired done.
+ vi.spyOn(h.session, 'getObservedCurrentTurnTerminal').mockReturnValue({
+ kind: 'error', generation, ...failure,
+ });
+ (h.deps as { stashProviderGroupHeldWorkerTerminal?: unknown }).stashProviderGroupHeldWorkerTerminal = vi.fn();
+ }
+ h.emit(terminal === 'error' ? event('error', failure, execution)
+ : event('done', terminal === 'failed-done' ? { status: 'failed' } : {}, execution));
if (outcome === 'unheld') {
expect(deferOwner).not.toHaveBeenCalled();
expect(unsuccessful).toHaveBeenCalledOnce();
@@ -643,15 +657,18 @@ describe('production Session event pipeline', () => {
}
expect(h.deps.onPluginTaskTerminal.mock.calls.length).toBe(0);
expect(unsuccessful).not.toHaveBeenCalled();
+ expect(successful).not.toHaveBeenCalled();
expect(deferOwner).toHaveBeenCalledOnce();
expect(gate.pending('task')).toMatchObject({
session: h.session, instanceId: execution.sessionInstanceId, generation,
});
expect(effects.fn('onTurnErrorEvent')).not.toHaveBeenCalled();
- h.deps.autoResumeBookkeeping.consumeFailedTurnCompletionTail.mockReturnValueOnce(true);
- h.emit(event('done', {}, execution));
- expect(deferOwner).toHaveBeenCalledOnce();
- expect(h.deps.onPluginTaskTerminal).not.toHaveBeenCalled();
+ if (terminal === 'error') {
+ h.deps.autoResumeBookkeeping.consumeFailedTurnCompletionTail.mockReturnValueOnce(true);
+ h.emit(event('done', {}, execution));
+ expect(deferOwner).toHaveBeenCalledOnce();
+ expect(h.deps.onPluginTaskTerminal).not.toHaveBeenCalled();
+ }
if (outcome.startsWith('generic-')) {
book.stashSuppressedError('task', held.takeForRecovery('task', 3));
diff --git a/apps/desktop/src/main/maker-ipc/register.ts b/apps/desktop/src/main/maker-ipc/register.ts
index f910cf9435..1c7063f289 100644
--- a/apps/desktop/src/main/maker-ipc/register.ts
+++ b/apps/desktop/src/main/maker-ipc/register.ts
@@ -2038,6 +2038,7 @@ const {
discard: discardProviderGroupHeld,
release: releaseProviderGroupHeld,
settleContinuation: settleProviderGroupContinuation,
+ hasPendingContinuation: hasPendingProviderGroupContinuation,
} = createProviderGroupHeldRecovery({
errors: providerGroupHeldErrors,
terminals: providerGroupHeldTerminals,
@@ -5819,8 +5820,9 @@ const sessionBindings = createSessionBindingLifecycle<WiredSession, WiredSession
// Keep it through the exact close/bootstrap transaction, including a failed
// bootstrap. User stop and a new dispatch still clear all deferred receipts.
deferredProductTurnFailureGate.clearSession(session.id, session, {
- preserveRecovery: rehydrateCloseSuppression.isSuppressed(session.id) &&
- agentInputCoordinatorHolder?.getProviderGroupSwitchHoldId(session.id) != null,
+ preserveRecovery: (rehydrateCloseSuppression.isSuppressed(session.id) &&
+ agentInputCoordinatorHolder?.getProviderGroupSwitchHoldId(session.id) != null) ||
+ hasPendingProviderGroupContinuation(session.id),
});
},
finalizeClosedSession: (session: WiredSession, context) => {
diff --git a/apps/desktop/src/main/maker-ipc/sessionEventTerminal.ts b/apps/desktop/src/main/maker-ipc/sessionEventTerminal.ts
index 827532f11e..69ac7c619c 100644
--- a/apps/desktop/src/main/maker-ipc/sessionEventTerminal.ts
+++ b/apps/desktop/src/main/maker-ipc/sessionEventTerminal.ts
@@ -604,12 +604,43 @@ export function finishSessionTerminalEvent(
session.id,
event.sessionTurnGeneration,
);
+ const skipWorkerTerminal = shouldSkipOrcaWorkerTerminal({
+ isContinuationBoundary,
+ stashedThisErrorEvent: deferredOrcaWorkerTerminal,
+ eventType: event.type,
+ isPairedFailedTurnDone,
+ isFailedTurnCompletionTail,
+ hasSuppressedError: deps.autoResumeBookkeeping.hasSuppressedError(session.id),
+ isAutoResumePending:
+ deps.agentInputCoordinatorHolder?.isAutoResumePending(session.id) === true,
+ isAutoResumeDeferred:
+ deps.agentInputCoordinatorHolder?.isAutoResumeDeferred(session.id) === true,
+ });
+ if (!skipWorkerTerminal && providerGroupDoneHoldId !== null && deps.stashProviderGroupHeldWorkerTerminal) {
+ // A missed error callback can be reconstructed by the coordinator from
+ // Session's terminal snapshot while this outer event is still done. Keep
+ // its failure detail on the same hold, so abandonment and generic handback
+ // retain the product receipt even when no error event reached this pipeline.
+ const observed = session.getObservedCurrentTurnTerminal();
+ const detail = observed.kind === 'error' && observed.generation === terminalOwner?.generation
+ ? observed : {};
+ if (deps.stashProviderGroupHeldError?.(session.id, providerGroupDoneHoldId, detail, eventAgentMeta) !== false) {
+ deps.stashProviderGroupHeldWorkerTerminal(session.id, providerGroupDoneHoldId, {
+ status: 'done', finalText: workerTerminalFinalText,
+ diagnostic: workerTerminalDiagnostic, capture: workerTerminalCapture,
+ }, groupTerminal);
+ providerGroupHeldWorkerTerminal = true;
+ recoveryOwnsUnsuccessfulBoundary = true;
+ deferUnsuccessfulProductTurn();
+ }
+ }
if (
event.type === 'done' &&
!isContinuationBoundary &&
!isPairedFailedTurnDone &&
!isFailedTurnCompletionTail &&
!deferredOrcaWorkerTerminal &&
+ !recoveryOwnsUnsuccessfulBoundary &&
!isTerminalTurnErrorEvent(event) &&
isSuccessfulAssistantReplyDoneData(event.data) &&
!deps.autoResumeBookkeeping.hasSuppressedError(session.id) &&
@@ -628,20 +659,7 @@ export function finishSessionTerminalEvent(
);
}
}
- if (
- !shouldSkipOrcaWorkerTerminal({
- isContinuationBoundary,
- stashedThisErrorEvent: deferredOrcaWorkerTerminal,
- eventType: event.type,
- isPairedFailedTurnDone,
- isFailedTurnCompletionTail,
- hasSuppressedError: deps.autoResumeBookkeeping.hasSuppressedError(session.id),
- isAutoResumePending:
- deps.agentInputCoordinatorHolder?.isAutoResumePending(session.id) === true,
- isAutoResumeDeferred:
- deps.agentInputCoordinatorHolder?.isAutoResumeDeferred(session.id) === true,
- })
- ) {
+ if (!skipWorkerTerminal) {
// Capture before queue-drain microtasks can promote the follow-up turn.
// Recovery-owned terminals are settled by their owner (deferred auth /
// gateway persistence, overflow surface, or auto-resume abandonment).
@@ -709,8 +727,7 @@ export function finishSessionTerminalEvent(
})();
if (
groupTerminal && deps.botGroupChatServiceHolder &&
- !providerGroupHeldWorkerTerminal &&
- !(providerGroupDoneHoldId !== null && deps.stashProviderGroupHeldWorkerTerminal)
+ !providerGroupHeldWorkerTerminal
) {
const groupService = deps.botGroupChatServiceHolder;
void settleBotGroupLaneTerminal(groupService, groupTerminal, drainPersistQueue).catch((error) => {
@@ -721,15 +738,7 @@ export function finishSessionTerminalEvent(
});
}
if (providerGroupHeldWorkerTerminal) {
- // 已与 error 行一起暂存,由那次登记结算。
- } else if (providerGroupDoneHoldId !== null && deps.stashProviderGroupHeldWorkerTerminal) {
- // 换电脑期间到达的 done(只以 done 收尾的失败):同样暂存,没换成时才回报;同一次登记只回报一次。
- deps.stashProviderGroupHeldWorkerTerminal(session.id, providerGroupDoneHoldId, {
- status: isTerminalTurnErrorEvent(event) ? 'error' : 'done',
- finalText: workerTerminalFinalText,
- diagnostic: workerTerminalDiagnostic,
- capture: workerTerminalCapture,
- }, groupTerminal);
+ // 已与产品终态一起暂存,由同一次恢复登记结算。
} else {
void (async () => {
try {
diff --git a/apps/desktop/src/main/provider-group/__tests__/heldTurnErrors.test.ts b/apps/desktop/src/main/provider-group/__tests__/heldTurnErrors.test.ts
index 4c35fb01a6..381508be64 100644
--- a/apps/desktop/src/main/provider-group/__tests__/heldTurnErrors.test.ts
+++ b/apps/desktop/src/main/provider-group/__tests__/heldTurnErrors.test.ts
@@ -105,6 +105,7 @@ describe('provider group held turn errors', () => {
if (worker) terminals.stash('s1', 1, 'original-terminal');
recovery.discard('s1', 1);
+ expect(recovery.hasPendingContinuation('s1')).toBe(true);
recovery.release('s1', 1); // The failover hook's finally runs before dispatch.
expect(productFailed).not.toHaveBeenCalled();
expect(delivered).not.toHaveBeenCalled();
@@ -112,9 +113,11 @@ describe('provider group held turn errors', () => {
if (dispatched || takeover === 'stop' || takeover === 'new-turn') gate.clearSession('s1');
if (takeover === 'new-turn') generation++;
if (takeover === 'account-change') h.switchOwner('other-account');
+ expect(recovery.hasPendingContinuation('s1')).toBe(takeover !== 'account-change');
recovery.settleContinuation('s1', dispatched);
recovery.settleContinuation('s1', false);
+ expect(recovery.hasPendingContinuation('s1')).toBe(false);
recovery.release('s1', 1);
if (!dispatched && takeover === 'none') {
expect(productFailed).toHaveBeenCalledExactlyOnceWith(owner);
diff --git a/apps/desktop/src/main/provider-group/heldTurnErrors.ts b/apps/desktop/src/main/provider-group/heldTurnErrors.ts
index 2082282e76..fbc74b0f9c 100644
--- a/apps/desktop/src/main/provider-group/heldTurnErrors.ts
+++ b/apps/desktop/src/main/provider-group/heldTurnErrors.ts
@@ -147,6 +147,11 @@ export function createProviderGroupHeldRecovery(deps: {
}) {
const continuationHolds = new Map<string, { holdId: number; owner: string }>();
return {
+ /** A runtime close must wait for this queued send's actual dispatch outcome. */
+ hasPendingContinuation(sessionId: string): boolean {
+ const continuation = continuationHolds.get(sessionId);
+ return continuation !== undefined && continuation.owner === deps.ownerKey();
+ },
discard(sessionId: string, holdId: number): void {
deps.errors.discard(sessionId, holdId);
// Ordinary product callers also own a receipt, even without a Worker terminal.