Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Tests for the provider-sidelining finalizeAgent does on a failed run.
*
* The bug this covers: an `agy` (Antigravity) TUI agent that hits the
* "We're finishing verifying your account eligibility" banner fails in ~3s —
* correctly — but nothing marked the provider unavailable, so the very next
* dequeued task resolved onto the same provider and died on the same banner.
* A run of queued tasks would all fail in sequence. The eligibility signal now
* carries `benchMs`, and finalizeAgent honors it via the generic
* `markProviderUnavailable` marker so `resolveAgentProviderAndModel` routes the
* retry to a fallback until the deadline auto-recovers the provider.
*
* Every case here builds its `errorAnalysis` by running a real transcript through
* `detectImmediateFallbackSignal` / `analyzeAgentFailure` rather than hand-stamping
* `{ category, origin }`. A hand-stamped origin is how a real regression shipped
* green (#3635): the detector was rewritten so Claude Code's genuine usage-limit
* banners classified as `output-scan`, the gate stopped benching, and this suite
* never noticed because it had asserted against a shape it had written itself.
*/
// The goal-fidelity gate (#5994) reaches a local model at completion. Pinned OFF
// here so these tests exercise the path they are about without depending on the
// developer's own reviewer settings — and so a machine that HAS a local reviewer
// configured never has its suite dispatch a real review request.
vi.mock('./codeReview.js', async (importOriginal) => ({
...(await importOriginal()),
getGoalFidelityConfig: vi.fn(async () => null),
}));
import { describe, it, expect, vi, beforeEach } from 'vitest';
vi.mock('../lib/execGit.js', () => ({
execGit: vi.fn(async () => ({ stdout: 'main\n', stderr: '', exitCode: 0 })),
}));
vi.mock('./github.js', () => ({
findPullRequestForBranch: vi.fn(async () => ({ status: 'found', number: 1, url: 'u' })),
ensureForgeReachable: vi.fn(async () => ({ ok: true, status: 'ok' })),
}));
vi.mock('./gitlab.js', () => ({ findMergeRequestForBranch: vi.fn() }));
vi.mock('./git.js', () => ({ resolveForgeForRepo: vi.fn(async () => ({ cli: 'gh' })) }));
vi.mock('./cosEvents.js', () => ({ emitLog: vi.fn() }));
vi.mock('./cosAgentLifecycle.js', () => ({
getAgent: vi.fn(async () => null),
getAgentRecord: vi.fn(async () => null),
updateAgent: vi.fn(async () => null),
completeAgent: vi.fn(async () => null),
}));
vi.mock('./cosTaskStore.js', () => ({ updateTask: vi.fn(async () => ({})) }));
const getActiveProviderMock = vi.fn(async () => null);
vi.mock('./providers.js', () => ({ getActiveProvider: (...a) => getActiveProviderMock(...a) }));
const markProviderUnavailableMock = vi.fn(async () => null);
const markProviderUsageLimitMock = vi.fn(async () => null);
vi.mock('./providerStatus.js', () => ({
markProviderUsageLimit: (...a) => markProviderUsageLimitMock(...a),
markProviderRateLimited: vi.fn(async () => null),
markProviderUnavailable: (...a) => markProviderUnavailableMock(...a),
}));
vi.mock('./executionLanes.js', () => ({ release: vi.fn() }));
vi.mock('./toolStateMachine.js', () => ({ completeExecution: vi.fn(), errorExecution: vi.fn() }));
// Partial mock: the two finalize-path helpers are stubbed, but `analyzeAgentFailure`
// passes through to the REAL detector so the banner tests below exercise what ships.
vi.mock('./agentErrorAnalysis.js', async (importOriginal) => ({
...(await importOriginal()),
resolveFailedTaskUpdate: vi.fn(async () => ({ status: 'pending', metadata: {} })),
resolveTypeFailureSignal: vi.fn(() => ({ record: 'skip' })),
}));
vi.mock('../lib/gitCommitProbe.js', () => ({
committedDuringRun: vi.fn(async () => false),
}));
vi.mock('./agentRunTracking.js', () => ({
createAgentRun: vi.fn(),
completeAgentRun: vi.fn(async () => null),
}));
vi.mock('./taskTypeHooks.js', () => ({
canRunTaskOutputHookWithoutPayload: vi.fn(() => false),
isProgrammaticIoTaskType: vi.fn(() => false),
resolveTaskHookType: vi.fn(() => null),
declaresNoCommitCriterion: vi.fn(() => false),
getTaskOutputHook: vi.fn(async () => null),
getTaskOutputPayloadPredicate: vi.fn(async () => null),
}));
vi.mock('./agentCompletion.js', () => ({ processAgentCompletion: vi.fn(async () => null) }));
vi.mock('./agentSummaryExtraction.js', () => ({ extractSimplifySummaries: vi.fn(() => null), extractFinalSummary: vi.fn(() => null) }));
import { finalizeAgent } from './agentFinalization.js';
// The real detector and the real cooldown table supply the analysis shape and
// the window, so the assertions below can't drift from what actually ships.
import { detectImmediateFallbackSignal } from '../lib/aiToolkit/errorDetection.js';
import { analyzeAgentFailure } from './agentErrorAnalysis.js';
import { COOLDOWN_MS_BY_CATEGORY } from '../lib/providerCooldown.js';
import { processAgentCompletion } from './agentCompletion.js';
const failedRun = (errorAnalysis, providerId = 'antigravity-tui') => finalizeAgent({
agentId: 'agent-1',
task: { id: 'task-1', taskType: 'internal', description: 'do a thing', metadata: {} },
runId: 'run-1',
providerId,
success: false,
exitCode: 1,
duration: 3000,
outputBuffer: '',
errorAnalysis,
terminatedByUser: false,
error: errorAnalysis?.message,
completionReason: 'fallback-signal',
workspacePath: '/w',
prExpected: false,
});
beforeEach(() => vi.clearAllMocks());
describe('finalizeAgent provider sidelining', () => {
it('resolves a missing provider lazily and still completes when its cooldown cannot persist', async () => {
const analysis = detectImmediateFallbackSignal(
"We're finishing verifying your account eligibility. This usually takes a moment. Please try again shortly."
);
getActiveProviderMock.mockResolvedValueOnce({ id: 'fallback-provider' });
markProviderUnavailableMock.mockRejectedValueOnce(new Error('cooldown write failed'));
const outcome = await failedRun(analysis, null);
expect(getActiveProviderMock).toHaveBeenCalledOnce();
expect(markProviderUnavailableMock).toHaveBeenCalledWith('fallback-provider', expect.objectContaining({ reason: 'auth-error' }));
expect(outcome.success).toBe(false);
expect(processAgentCompletion).toHaveBeenCalledWith('agent-1', expect.any(Object), false, '');
});
it('benches the provider on the agy eligibility signal', async () => {
const analysis = detectImmediateFallbackSignal(
"We're finishing verifying your account eligibility. This usually takes a moment. Please try again shortly."
);
await failedRun(analysis);
expect(markProviderUnavailableMock).toHaveBeenCalledWith('antigravity-tui', {
reason: 'auth-error',
message: analysis.message,
waitTimeMs: COOLDOWN_MS_BY_CATEGORY['auth-error'],
});
// Before this fix only usage-limit/rate-limit benched, so an auth-error left
// the provider healthy and the next dequeued task died on the same banner.
expect(markProviderUsageLimitMock).not.toHaveBeenCalled();
});
// #3631, the broad half of the same gate: `detectImmediateFallbackSignal` used to
// stamp `origin: 'provider'` unconditionally, so an agent that merely PRINTED one
// of these banners benched a healthy provider for every subsequently dequeued
// task. These drive the REAL detector — hand-stamping `origin` is exactly how the
// sibling regression shipped green.
it.each([
['prose quoting the banner', "The known failure mode is: We're finishing verifying your account eligibility. This usually takes a moment. Please try again shortly. — see errorDetection.js"],
['a grep hit over a prior run\'s transcript', "data/cos/agents/agent-1/output.txt:412: ⎿ We're finishing verifying your account eligibility. This usually takes a moment. Please try again shortly."],
// The extra-usage status line has no loose alternative — quoting it inline
// does not even register as a signal, so there is nothing to bench on.
['a quoted extra-usage status line', 'the tail showed "Now using extra usage" in a prior run'],
])('does not bench when the agent merely printed the banner (%s)', async (_label, transcript) => {
const analysis = detectImmediateFallbackSignal(transcript);
expect(analysis?.origin ?? null).not.toBe('provider');
await failedRun(analysis);
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
expect(markProviderUsageLimitMock).not.toHaveBeenCalled();
});
it('still benches when the eligibility banner arrives as the run\'s own terminal output', async () => {
const analysis = detectImmediateFallbackSignal(
"starting the task…\n ⎿ We're finishing verifying your account eligibility.\n This usually takes a moment. Please try again shortly.\r\n"
);
expect(analysis).toMatchObject({ category: 'auth-error', origin: 'provider' });
await failedRun(analysis);
expect(markProviderUnavailableMock).toHaveBeenCalledWith('antigravity-tui', expect.objectContaining({
reason: 'auth-error',
}));
});
it('still benches on the real extra-usage status line', async () => {
const analysis = detectImmediateFallbackSignal('Now using extra usage\n');
expect(analysis).toMatchObject({ category: 'usage-limit', origin: 'provider' });
await failedRun(analysis, 'claude-code-tui');
expect(markProviderUsageLimitMock).toHaveBeenCalledWith('claude-code-tui', expect.objectContaining({
category: 'usage-limit',
}));
});
// The provenance gate is only as good as what `analyzeAgentFailure` promotes to
// `origin: 'provider'`. The case above hand-stamps that origin, which is exactly
// how a real regression shipped green: Claude Code's actual banners classified as
// `origin: 'output-scan'`, so a genuine 5-hour window never benched the provider
// and every subsequent dequeue re-picked it. These drive the REAL detector.
it.each([
['Claude usage limit reached. Your limit will reset at 5pm.'],
['5-hour limit reached · resets 3am'],
])('benches the provider on the real usage-limit banner %#', async (banner) => {
const analysis = analyzeAgentFailure(
`${banner}\nthe transcript tail continues past the banner.`,
{ id: 'task-1' },
'claude',
{}
);
expect(analysis).toMatchObject({ category: 'usage-limit', origin: 'provider', requiresFallback: true });
await failedRun(analysis, 'claude-code-tui');
expect(markProviderUsageLimitMock).toHaveBeenCalledWith('claude-code-tui', expect.objectContaining({
category: 'usage-limit',
}));
});
// The other half of the same gate: a generic limit phrasing a task's own output
// can print must stay `output-scan` and leave the provider healthy.
it('does not bench on a generic limit phrasing from the agent transcript', async () => {
const analysis = analyzeAgentFailure(
'Error: the daily limit for widgets was exceeded in our test fixture.\nmore output.',
{ id: 'task-1' },
'claude',
{}
);
expect(analysis).toMatchObject({ category: 'usage-limit', origin: 'output-scan' });
await failedRun(analysis, 'claude-code-tui');
expect(markProviderUsageLimitMock).not.toHaveBeenCalled();
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
});
it('leaves the usage-limit marker owning its own cooldown', async () => {
const analysis = analyzeAgentFailure(
'hit your usage limit · resets 6am\nthe transcript tail continues past the banner.',
{ id: 'task-1' },
'claude',
{}
);
expect(analysis).toMatchObject({ category: 'usage-limit', origin: 'provider', requiresFallback: true });
await failedRun(analysis, 'claude-code-tui');
// markUsageLimit parses its own window out of the provider's message, so it
// keeps the dedicated marker rather than the flat per-category cooldown.
expect(markProviderUsageLimitMock).toHaveBeenCalledWith('claude-code-tui', expect.objectContaining({
category: 'usage-limit',
}));
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
});
// The provenance gate (#2642): a repainted TUI transcript is a whole session
// of text the agent itself wrote, so a loose keyword match must never bench a
// healthy provider — only structured provider chrome does.
// `rate-limit` and `usage-limit` are the load-bearing cases: their patterns have
// loose alternatives (a bare "rate limit" / "quota exceeded" a failing test in
// the agent's own workspace can print), and this gate used to key on the
// category alone — so an agent's transcript could bench a healthy provider.
// Each transcript below is text a task's OWN run can print.
it.each([
['auth-error', 'Error: unauthorized while writing the snapshot fixture in the widget suite.\nmore output follows.'],
['rate-limit', 'Error: the fixture asserted a rate limit banner renders for the widget list.\nmore output follows.'],
['usage-limit', 'Error: quota exceeded for the fixture bucket while uploading the report artifact.\nmore output.'],
])(
'does not bench an output-scan %s that merely looks provider-ish',
async (category, transcript) => {
const analysis = analyzeAgentFailure(transcript, { id: 'task-1' }, 'claude', {});
expect(analysis).toMatchObject({ category, origin: 'output-scan' });
await failedRun(analysis);
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
expect(markProviderUsageLimitMock).not.toHaveBeenCalled();
// No marker fired, so the lazy active-provider lookup must stay unread.
expect(getActiveProviderMock).not.toHaveBeenCalled();
}
);
it('does not bench an ordinary agent-work failure', async () => {
const analysis = analyzeAgentFailure(
'The suite reported a test failure in the widget list module. Review the assertions and retry.',
{ id: 'task-1' },
'claude',
{}
);
expect(analysis).toMatchObject({ category: 'test-failure', origin: 'output-scan' });
await failedRun(analysis);
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
expect(markProviderUsageLimitMock).not.toHaveBeenCalled();
});
// A bad model id is REQUEST-specific: benching would take the provider's other
// working models offline over one wrong id. Driven through the real detector so
// the `origin: 'provider'` half of the case is the one the classifier assigns.
it('does not bench a provider-origin model-not-found', async () => {
const analysis = analyzeAgentFailure(
'API Error: 404 Not Found model: example-model-v1\nthe run stopped here without retrying.',
{ id: 'task-1' },
'claude',
{}
);
expect(analysis).toMatchObject({ category: 'model-not-found', origin: 'provider' });
await failedRun(analysis);
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
});
it('does not bench when the user terminated the run', async () => {
const analysis = detectImmediateFallbackSignal(
"We're finishing verifying your account eligibility. This usually takes a moment. Please try again shortly."
);
await finalizeAgent({
agentId: 'agent-2',
task: { id: 'task-2', taskType: 'internal', description: 'x', metadata: {} },
runId: 'run-2',
providerId: 'antigravity-tui',
success: false,
exitCode: 130,
duration: 10,
outputBuffer: '',
errorAnalysis: analysis,
terminatedByUser: true,
completionReason: 'terminated',
workspacePath: '/w',
prExpected: false,
});
expect(markProviderUnavailableMock).not.toHaveBeenCalled();
});
});
describe('finalizeAgent — Creative Director scratch cleanup (#4650)', () => {
it('removes the per-agent scratch cwd when the CD run finishes', async () => {
const { mkdirSync, writeFileSync, existsSync } = await import('fs');
const { join } = await import('path');
const { creativeDirectorScratchCwd } = await import('../lib/spawnCwd.js');
const dir = creativeDirectorScratchCwd('agent-cd-fin');
mkdirSync(dir, { recursive: true });
writeFileSync(join(dir, 'leftover.txt'), 'x');
await finalizeAgent({
agentId: 'agent-cd-fin',
task: {
id: 'task-cd',
taskType: 'internal',
description: 'CD plan',
metadata: { creativeDirector: { projectId: 'p', kind: 'plan' }, useWorktree: false },
},
runId: 'run-cd',
providerId: 'codex',
success: true,
exitCode: 0,
duration: 1000,
outputBuffer: '',
terminatedByUser: false,
workspacePath: dir,
prExpected: false,
});
expect(existsSync(dir)).toBe(false);
// finalizeAgent's real cleanup only removes the per-agent leaf — in
// production the shared `portos-cd-cwd` container is meant to persist and
// be reused across runs, so this test leaves it in place rather than
// removing it itself (which would race spawnCwd.test.js's own use of the
// same literal path if Vitest runs them concurrently). The run-scoped
// temp root's teardown (#9032, server/test/runTempRoot.js) treats an
// empty leftover container as a non-leak, so the now-empty
// `portos-cd-cwd` needs no cleanup at all.
});
});