Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Tests for `evaluateSuccessCriteria` (issue #2344) — the success-criteria
* validation verdict finalizeAgent stamps onto every completion, distinct from
* the runner's exit-code `success`. The run-window commit probe is mocked so
* these run without git; the focus is the null-sentinel gating (no criterion
* declared vs declared-and-checked).
*/
// The goal-fidelity gate (#5994) reaches a local model at completion. Pinned OFF
// here so these tests exercise the path they are about without depending on the
// developer's own reviewer settings — and so a machine that HAS a local reviewer
// configured never has its suite dispatch a real review request.
vi.mock('./codeReview.js', async (importOriginal) => ({
...(await importOriginal()),
getGoalFidelityConfig: vi.fn(async () => null),
}));
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
vi.mock('./agentRunTracking.js', () => ({
// finalizeAgent's imports from this module — stubbed so the graph loads.
createAgentRun: vi.fn(),
completeAgentRun: vi.fn(),
}));
vi.mock('../lib/gitCommitProbe.js', () => ({
committedDuringRun: vi.fn(),
}));
import { evaluateSuccessCriteria, resolveProgrammaticIoVerdict, withOutputHookTimeout } from './agentFinalization.js';
import { committedDuringRun } from '../lib/gitCommitProbe.js';
import { SKIP_LEARNING_VERDICT } from '../lib/learningVerdict.js';
// The run window every commit-criterion assertion below is evaluated against
// (#3637). A criterion-declaring call MUST pass one — without a window there is
// no way to attribute a commit to this run, and the verdict is the null sentinel.
const STARTED_AT = Date.parse('2026-08-09T00:00:00.000Z');
describe('evaluateSuccessCriteria (#2344)', () => {
beforeEach(() => vi.clearAllMocks());
it('returns null (no declared criterion) for interactive/user tasks', async () => {
expect(await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'user' }, workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('returns null for a user-terminated run — no criterion was evaluated', async () => {
const out = await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'internal' }, terminatedByUser: true, workspacePath: '/w' });
expect(out).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('returns null when there is no task id or no workspace to validate against', async () => {
expect(await evaluateSuccessCriteria({ task: { taskType: 'internal' }, workspacePath: '/w' })).toBeNull();
expect(await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'internal' } })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('returns null for pipeline/media tasks (they deliver artifacts, not a commit)', async () => {
expect(await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'internal', metadata: { pipeline: true } }, workspacePath: '/w' })).toBeNull();
expect(await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'internal', metadata: { mediaJob: true } }, workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('never applies the commit criterion to a programmatic-I/O task (#2700)', async () => {
// A layered-intelligence run is explicitly told NOT to commit or open a PR: it
// writes `.agent-done` and its output hook does the filing. Checking for a
// commit would stamp validationPassed:false on every correct run —
// and since a declared verdict OVERRIDES the runner's exit code in task-learning,
// that recorded successful LI runs as failures and drove the type's success rate
// to ~0.
const task = { id: 't1', taskType: 'internal', metadata: { analysisType: 'layered-intelligence', selfImprovement: true } };
await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true, hookResult: { ran: true, outcome: { action: 'filed' } } });
expect(committedDuringRun).not.toHaveBeenCalled();
});
});
/**
* Tracker-filing types (#3273). `reference-watch` and `ux` deliver their findings
* as tracker items, not as a commit — but only on a FORGE tracker: on a
* `plan`-tracker app the same type appends + commits PLAN.md checklist items. The
* static NON_COMMITTING_COORDINATOR_TASK_TYPES set cannot express "sometimes", so
* the criterion keys on the per-task `worktreeChangesExpected` flag the generator
* already derives from the resolved tracker.
*/
describe('evaluateSuccessCriteria — tracker-filing tasks (#3273)', () => {
beforeEach(() => vi.clearAllMocks());
const uxTask = (worktreeChangesExpected) => ({
id: 't1', taskType: 'internal',
metadata: { analysisType: 'ux', ...(worktreeChangesExpected === undefined ? {} : { worktreeChangesExpected }) },
});
it('declares NO criterion when worktreeChangesExpected is false (forge tracker)', async () => {
expect(await evaluateSuccessCriteria({ task: uxTask(false), workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('accepts the string form the TASKS.md round-trip produces', async () => {
expect(await evaluateSuccessCriteria({ task: uxTask('false'), workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('still applies the commit criterion when the flag is TRUE (plan tracker)', async () => {
committedDuringRun.mockResolvedValue(true);
expect(await evaluateSuccessCriteria({ task: uxTask(true), workspacePath: '/w', startedAt: STARTED_AT })).toBe(true);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('still applies the commit criterion when the flag is ABSENT', async () => {
committedDuringRun.mockResolvedValue(false);
expect(await evaluateSuccessCriteria({ task: uxTask(undefined), workspacePath: '/w', startedAt: STARTED_AT })).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('retro-fixes reference-watch on a forge tracker (the same latent artifact)', async () => {
const task = { id: 't2', taskType: 'internal', metadata: { analysisType: 'reference-watch', worktreeChangesExpected: false } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('resolves the archived-agent projection the same way as the live task', async () => {
const task = { id: 't3', taskType: 'internal', metadata: { taskAnalysisType: 'ux', worktreeChangesExpected: false } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w' })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('does NOT let the flag exempt a non-tracker-filing type from its commit criterion', async () => {
// `worktreeChangesExpected` is a user-settable per-app taskMetadata override
// accepted for EVERY task type — it exists to opt a run out of the TUI
// idle-complete clean-tree gate, not to disable success validation. Ungated,
// a `security` run that exited 0 having committed nothing would be recorded
// as a pass instead of the honest miss it is.
committedDuringRun.mockResolvedValue(false);
const task = { id: 't4', taskType: 'internal', metadata: { analysisType: 'security', worktreeChangesExpected: false } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', startedAt: STARTED_AT })).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
});
/**
* The programmatic-I/O criterion (#2727): these tasks declare their OWN success
* criterion — "the sentinel parsed and the output hook accepted it" — instead of
* declaring none and falling through to the runner's exit code, which recorded an
* exit-0 run that produced nothing usable as a success.
*/
describe('evaluateSuccessCriteria — programmatic-I/O criterion (#2727)', () => {
beforeEach(() => vi.clearAllMocks());
const liTask = { id: 't1', taskType: 'internal', metadata: { analysisType: 'layered-intelligence' } };
it('records an exit-0 run with a missing/malformed sentinel as a FAILURE', async () => {
// The hook reports `unparseable-response` when the `.agent-done` payload is
// absent or unparseable — the run exited clean but produced nothing usable.
const hookResult = { ran: true, outcome: { action: 'no-op', reason: 'unparseable-response' } };
expect(await evaluateSuccessCriteria({ task: liTask, workspacePath: '/w', success: true, hookResult })).toBe(false);
});
it('records an exit-0 run whose output hook THREW as a FAILURE', async () => {
expect(await evaluateSuccessCriteria({
task: liTask, workspacePath: '/w', success: true, hookResult: { ran: true, threw: true }
})).toBe(false);
});
it('records an exit-0 run whose hook accepted the payload as a SUCCESS — no commit required', async () => {
const hookResult = { ran: true, outcome: { app: 'a1', action: 'filed', reason: null } };
expect(await evaluateSuccessCriteria({ task: liTask, workspacePath: '/w', success: true, hookResult })).toBe(true);
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('treats benign hook reasons (no-proposal, duplicate, scope-suppressed) as a SUCCESS', async () => {
// The agent did its job; the deterministic step simply had nothing to file.
for (const reason of ['no-proposal', 'duplicate', 'semantic-duplicate', 'scope-suppressed', 'tracker-read-failed']) {
expect(await evaluateSuccessCriteria({
task: liTask, workspacePath: '/w', success: true, hookResult: { ran: true, outcome: { action: 'no-op', reason } }
})).toBe(true);
}
});
it('records a non-zero exit as a FAILURE regardless of the hook outcome', async () => {
expect(await evaluateSuccessCriteria({
task: liTask, workspacePath: '/w', success: false, hookResult: { ran: true, outcome: { action: 'no-op', reason: 'agent-failed' } }
})).toBe(false);
});
it('declares NO verdict (null) when no hook ran — "not evaluated" must not become "accepted"', async () => {
// Sentinel discipline: a registered type whose module exports no
// processTaskOutput yields `{ ran: false }`. Nothing judged the output, so the
// verdict is undeclared and task-learning falls back to the exit code.
expect(await evaluateSuccessCriteria({ task: liTask, workspacePath: '/w', success: true, hookResult: { ran: false } })).toBeNull();
expect(await evaluateSuccessCriteria({ task: liTask, workspacePath: '/w', success: true })).toBeNull();
});
it('declares NO verdict for a user-terminated programmatic-I/O run', async () => {
expect(await evaluateSuccessCriteria({
task: liTask, workspacePath: '/w', success: false, terminatedByUser: true, hookResult: { ran: true, threw: true }
})).toBeNull();
});
it('judges the hook result even with no workspace to validate against', async () => {
// The commit criterion needs a workspace; the programmatic-I/O criterion does
// not — a hook that already ran is a real verdict even if the worktree is gone.
expect(await evaluateSuccessCriteria({
task: liTask, success: true, hookResult: { ran: true, threw: true }
})).toBe(false);
});
it('applies the criterion to a task typed on taskType alone, not just metadata.analysisType', async () => {
// The criterion gate and the hook-dispatch gate share one resolver
// (resolveTaskHookType), so a task shaped with the scheduled type at the top
// level can't run a hook AND still get commit-checked (the #2700 bug, one shape
// over).
const task = { id: 't9', taskType: 'layered-intelligence' };
expect(await evaluateSuccessCriteria({
task, workspacePath: '/w', success: true, hookResult: { ran: true, threw: true }
})).toBe(false);
expect(committedDuringRun).not.toHaveBeenCalled();
});
});
/**
* `resolveProgrammaticIoVerdict` is the pure criterion behind the branch above.
* Tested directly so the three-way sentinel (accepted / rejected / undeclared) is
* pinned without routing every case through evaluateSuccessCriteria.
*/
describe('resolveProgrammaticIoVerdict (#2727)', () => {
beforeEach(() => vi.clearAllMocks());
it('rejects a run whose hook threw, and one whose output was unparseable', () => {
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: true, threw: true } })).toBe(false);
expect(resolveProgrammaticIoVerdict({
success: true, hookResult: { ran: true, outcome: { reason: 'unparseable-response' } }
})).toBe(false);
});
it('accepts a run whose hook processed the output', () => {
expect(resolveProgrammaticIoVerdict({
success: true, hookResult: { ran: true, outcome: { action: 'filed', reason: null } }
})).toBe(true);
});
it('declares no verdict when nothing evaluated the output', () => {
// No hook ran / no hook result at all / the dispatch timed out — none of these
// are a rejection, so task-learning falls back to the exit code.
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: false } })).toBeNull();
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: null })).toBeNull();
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: false, timedOut: true } })).toBeNull();
});
it('declares no verdict when the hook ran but handed back no structured outcome', () => {
// `ran: true` with a missing/non-object outcome must NOT optional-chain its way
// into the success default — nothing evaluated the output.
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: true } })).toBeNull();
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: true, outcome: undefined } })).toBeNull();
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: true, outcome: 'nope' } })).toBeNull();
});
it('declares no verdict when the exit-code result is absent/non-boolean', () => {
// "Not supplied" must not silently mean "the run failed".
expect(resolveProgrammaticIoVerdict({ hookResult: { ran: true, outcome: { reason: null } } })).toBeNull();
expect(resolveProgrammaticIoVerdict({ success: undefined, hookResult: { ran: true, outcome: { reason: null } } })).toBeNull();
});
it('treats a downstream tracker failure as a SUCCESS — the output was accepted', () => {
// `file-failed` / `tracker-read-failed` mean the reasoning landed but the forge
// was unreachable. That is environmental: blaming the run would tank the type's
// success rate (and auto-park it) every time `gh` has a bad afternoon. Raised in
// review on #2727 and deliberately kept.
for (const reason of ['file-failed', 'tracker-read-failed']) {
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: { ran: true, outcome: { reason } } })).toBe(true);
}
});
it('asks task-learning to SKIP a run whose hook aborted before it could look at the output (#4107)', () => {
// `no-app` / `app-not-found` return before the payload is validated (and before
// the hook records anything). Nothing evaluated the agent's output, so neither
// recordable answer is honest: `false` blames the model for a user deleting an
// app mid-run, and the undeclared `null` this used to return still recorded the
// run against its EXIT CODE — banking a free win for the type on every exit-0.
for (const reason of ['no-app', 'app-not-found']) {
const verdict = resolveProgrammaticIoVerdict({
success: true, hookResult: { ran: true, outcome: { action: 'no-op', reason } }
});
expect(verdict).toBe(SKIP_LEARNING_VERDICT);
// Explicitly NOT collapsed into any of the three recordable verdicts.
expect(verdict).not.toBeNull();
expect(typeof verdict).not.toBe('boolean');
}
});
it('still skips the run when the exit code says it FAILED', () => {
// The skip is about "nothing evaluated this run", not about the exit code —
// an aborted hook has no verdict either way, so a non-zero exit must not be
// banked as a failure for the type either.
expect(resolveProgrammaticIoVerdict({
success: false, hookResult: { ran: true, outcome: { action: 'no-op', reason: 'no-app' } }
})).toBe(SKIP_LEARNING_VERDICT);
});
it('propagates the skip verdict out through evaluateSuccessCriteria', async () => {
// finalizeAgent stamps whatever this returns onto `result.validationPassed`,
// which is the only channel the learning writer reads — so the sentinel has to
// survive the wrapper, not just the pure criterion.
const task = { id: 't10', taskType: 'layered-intelligence' };
expect(await evaluateSuccessCriteria({
task, workspacePath: '/w', success: true,
hookResult: { ran: true, outcome: { action: 'no-op', reason: 'app-not-found' } }
})).toBe(SKIP_LEARNING_VERDICT);
expect(committedDuringRun).not.toHaveBeenCalled();
});
});
/**
* The hard bound that keeps a hung output hook from pinning a CoS concurrency slot
* until restart — the mitigation matters enough to pin directly (#2727).
*/
describe('withOutputHookTimeout (#2727)', () => {
beforeEach(() => vi.useRealTimers());
afterEach(() => vi.useRealTimers());
it('resolves a hung dispatch to the undeclared sentinel rather than hanging finalize', async () => {
vi.useFakeTimers();
// A hook that never settles — the wedge case.
const settled = withOutputHookTimeout(new Promise(() => {}), { agentId: 'a1', timeoutMs: 1000 });
await vi.advanceTimersByTimeAsync(1000);
// `ran: false` → resolveProgrammaticIoVerdict returns null → task-learning falls
// back to the exit code. A timeout is "we never got a verdict", not a rejection.
expect(await settled).toEqual({ ran: false, timedOut: true });
expect(resolveProgrammaticIoVerdict({ success: true, hookResult: await settled })).toBeNull();
});
it('passes a hook that settles in time straight through, and clears its timer', async () => {
vi.useFakeTimers();
const outcome = { ran: true, outcome: { action: 'filed', reason: null } };
const settled = withOutputHookTimeout(Promise.resolve(outcome), { agentId: 'a1', timeoutMs: 1000 });
expect(await settled).toBe(outcome);
// Timer cleared on the resolve path — nothing left pending to fire.
expect(vi.getTimerCount()).toBe(0);
});
it('propagates a hook rejection (finalizeAgent maps it to the thrown-hook verdict)', async () => {
await expect(withOutputHookTimeout(Promise.reject(new Error('boom')), { agentId: 'a1', timeoutMs: 1000 }))
.rejects.toThrow('boom');
});
});
/**
* The commit criterion (#2344), rebuilt on the run-window probe (#3637). The old
* criterion grepped for a task-id commit subject that NOTHING ever emitted,
* so it was unsatisfiable: every ordinary code-editing run recorded
* `validationPassed: false` regardless of what it did, and — because a declared
* verdict overrides the exit code in task-learning — its bucket filled with
* fabricated failures. These two cases are the whole fix: a run that committed
* inside its own window passes, a run that committed nothing fails.
*/
describe('evaluateSuccessCriteria — commit criterion (#2344, #3637)', () => {
beforeEach(() => vi.clearAllMocks());
it('accepts a verified no-change result for an explicitly marked autonomous audit', async () => {
const task = {
id: 'catalog-audit-1',
taskType: 'internal',
metadata: { autonomousJob: true, noChangeSuccess: true }
};
expect(await evaluateSuccessCriteria({
task,
workspacePath: '/w',
success: true,
branchProvenEmpty: true,
startedAt: STARTED_AT
})).toBe(true);
// The no-change proof came from verifyPrClaim's forge + branch checks; it
// must not trigger a second commit probe or turn a clean audit into a miss.
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('does not let the marker bypass validation without the verified no-change proof', async () => {
const task = {
id: 'catalog-audit-2',
taskType: 'internal',
metadata: { autonomousJob: true, noChangeSuccess: true }
};
committedDuringRun.mockResolvedValueOnce(false);
expect(await evaluateSuccessCriteria({
task,
workspacePath: '/w',
success: true,
branchProvenEmpty: false,
startedAt: STARTED_AT
})).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
// dependency-updates is not a catalog audit, but "every dependency is already
// current" is its ordinary outcome, so its schedule default ships
// `noChangeSuccess: true` and lib/noChangeCompletion.js admits the type by name.
it('accepts a verified no-change result for a marked dependency-updates run', async () => {
const task = {
id: 'deps-1',
taskType: 'internal',
metadata: { analysisType: 'dependency-updates', useWorktree: true, openPR: true, noChangeSuccess: true }
};
expect(await evaluateSuccessCriteria({
task, workspacePath: '/w', success: true, branchProvenEmpty: true, startedAt: STARTED_AT
})).toBe(true);
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('does not extend the no-change exemption to other marked non-audit scheduled types', async () => {
committedDuringRun.mockResolvedValueOnce(false);
expect(await evaluateSuccessCriteria({
task: { id: 'ideas-1', taskType: 'internal', metadata: { analysisType: 'feature-ideas', noChangeSuccess: true } },
workspacePath: '/w', success: true, branchProvenEmpty: true, startedAt: STARTED_AT
})).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('does not let an unmarked task claim the no-change exemption', async () => {
committedDuringRun.mockResolvedValueOnce(false);
expect(await evaluateSuccessCriteria({
task: { id: 'ordinary-audit-1', taskType: 'internal', metadata: { autonomousJob: true } },
workspacePath: '/w',
success: true,
branchProvenEmpty: true,
startedAt: STARTED_AT
})).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('accepts persisted string markers after the task markdown round-trip', async () => {
const task = {
id: 'catalog-audit-3',
taskType: 'internal',
metadata: { autonomousJob: 'true', noChangeSuccess: 'true' }
};
expect(await evaluateSuccessCriteria({
task, workspacePath: '/w', success: true, branchProvenEmpty: true, startedAt: STARTED_AT
})).toBe(true);
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('passes an autonomous code run that COMMITTED during its window', async () => {
committedDuringRun.mockResolvedValueOnce(true);
expect(await evaluateSuccessCriteria({ task: { id: 't1', taskType: 'internal' }, workspacePath: '/w', startedAt: STARTED_AT })).toBe(true);
// Probed against the run's own window, NOT the whole repo history — a commit
// another agent pushed before this run began is not this run's work.
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('fails an autonomous code run that committed NOTHING (an honest miss, not null)', async () => {
committedDuringRun.mockResolvedValueOnce(false);
expect(await evaluateSuccessCriteria({ task: { id: 't2', taskType: 'internal' }, workspacePath: '/w', startedAt: STARTED_AT })).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('still applies the commit criterion to a NON-programmatic self-improvement task', async () => {
// The exemption is keyed on the taskTypeHooks registry, not on selfImprovement —
// an ordinary self-improve task still commits and must still be checked.
committedDuringRun.mockResolvedValueOnce(true);
const task = { id: 't1', taskType: 'internal', metadata: { analysisType: 'ui', selfImprovement: true } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', startedAt: STARTED_AT })).toBe(true);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
it('declares NO criterion (null) when the run window is missing or unusable', async () => {
// Without a window there is no way to attribute a commit to THIS run. The
// sentinel — never a manufactured `false`, which task-learning would treat as
// a real failure and let override the exit code.
for (const startedAt of [undefined, null, NaN, 'not-a-date']) {
expect(await evaluateSuccessCriteria({ task: { id: 't3', taskType: 'internal' }, workspacePath: '/w', startedAt })).toBeNull();
}
expect(committedDuringRun).not.toHaveBeenCalled();
});
});
/**
* gh/git COORDINATOR task types (#2696): branch-reconcile / issue-reconcile drive
* their work through git+gh in the app's LIVE checkout (workspacePath IS set) and never
* produce a commit at all, so the commit criterion scored every successful run a
* failure and drove their learning bucket to ~0% — the same artifact #2700 fixed for the
* programmatic-I/O reasoning run. They must declare NO commit criterion (fall back to the
* exit code), exactly like pipeline/media jobs.
*/
describe('evaluateSuccessCriteria — gh/git coordinator exemption (#2696)', () => {
beforeEach(() => vi.clearAllMocks());
it('declares NO commit criterion for every non-committing coordinator type', async () => {
// The structurally-no-commit coordinators: they run in the live checkout and deliver a
// git/gh/external side effect, never a commit.
for (const analysisType of ['app-checkout-recovery', 'branch-reconcile', 'issue-reconcile', 'branch-cleanup', 'jira-status-report']) {
const task = { id: 't1', taskType: 'internal', metadata: { analysisType } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true })).toBeNull();
}
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('declares NO commit criterion for a PR follow-up (review-loop or merge-only)', async () => {
// The happy path makes no commit at all: a merge-only follow-up on an already-green
// PR just merges it, and a review follow-up commits nothing when every reviewer is
// clean. Commit-checking them would score every successful run a failure (#2696 again).
for (const metadata of [
{ reviewLoopFollowUp: true },
{ reviewLoopFollowUp: true, reviewLoopMergeOnly: true },
{ reviewLoopFollowUp: 'true' },
]) {
const task = { id: 'sys-rl-1', taskType: 'internal', metadata };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true })).toBeNull();
}
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('STILL commit-checks committing self-improve types (jira-sprint-manager, do-replan)', async () => {
// jira-sprint-manager commits + opens MRs; do-replan commits PLAN.md edits — their commit
// criterion is real, so exempting them would MASK genuine failures. Must stay checked.
for (const analysisType of ['jira-sprint-manager', 'do-replan']) {
committedDuringRun.mockResolvedValueOnce(true);
const task = { id: 't1', taskType: 'internal', metadata: { analysisType, selfImprovement: true } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', startedAt: STARTED_AT, success: true })).toBe(true);
}
expect(committedDuringRun).toHaveBeenCalledTimes(2);
});
it('exempts the ARCHIVED coordinator shape (metadata.taskAnalysisType) (#2696 codex)', async () => {
// Matches extractTaskType's bucket resolution so the criterion agrees with the bucket on
// the archived agent shape too (agentLifecycle stamps taskAnalysisType).
const task = { id: 't4', taskType: 'internal', metadata: { taskAnalysisType: 'branch-cleanup' } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('exempts a coordinator typed on taskType alone, not just metadata.analysisType', async () => {
// Same resolver as the programmatic-I/O gate (resolveTaskHookType), so a task shaped
// with the scheduled type at the top level is exempted the same way.
const task = { id: 't2', taskType: 'branch-reconcile' };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('does NOT exempt accessibility — it is a fixing task that DOES commit (#2696 scope)', async () => {
// accessibility's prompt ends "Test and commit changes": it makes code changes in a
// worktree and commits, so its commit criterion is real and its 0% (if any) is a
// genuine agent failure, NOT the coordinator artifact. Must stay commit-checked.
committedDuringRun.mockResolvedValueOnce(false);
const task = { id: 't3', taskType: 'internal', metadata: { analysisType: 'accessibility', selfImprovement: true } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', startedAt: STARTED_AT, success: true })).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
});
/**
* CLAIM flows (plan-task / claim-issue / claim-issue-gitlab / claim-issue-jira /
* claim-work): the agent cuts its OWN claim/<item> worktree and opens/merges the PR
* there, while CoS keeps `useWorktree: false` so the workspace this criterion probes
* is the app's live checkout — which the claim run never commits to. The probe is
* workspace-scoped, so it stamped `validationPassed: false` on every successful run
* (the boolean OVERRIDES the exit code in task-learning), pinning the type's bucket
* at ~0-25% while the runs visibly succeed. The exemption falls back to the exit
* code; the goal-fidelity gate still judges the run against the claimed issue's
* requirements, so nothing is lost but the misattributed probe.
*/
describe('evaluateSuccessCriteria — claim-flow exemption', () => {
beforeEach(() => vi.clearAllMocks());
it('declares NO commit criterion for every claim-flow type', async () => {
for (const analysisType of ['plan-task', 'claim-issue', 'claim-issue-gitlab', 'claim-issue-jira', 'claim-work']) {
const task = { id: 't1', taskType: 'internal', metadata: { analysisType } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true, startedAt: STARTED_AT })).toBeNull();
}
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('accepts the explicit claimFlow marker on a task of any type', async () => {
// A manually marked claim (e.g. a dependency task also marked claimFlow) or a
// schedule queued before the marker existed carries only the flag.
for (const claimFlow of [true, 'true']) {
const task = { id: 't2', taskType: 'internal', metadata: { claimFlow } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true, startedAt: STARTED_AT })).toBeNull();
}
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('exempts the archived-agent projection (metadata.taskAnalysisType) too', async () => {
// Same analysisType → taskAnalysisType → taskType chain the learning bucket
// uses, so the criterion agrees with the bucket on the archived shape.
const task = { id: 't3', taskType: 'internal', metadata: { taskAnalysisType: 'claim-issue' } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true, startedAt: STARTED_AT })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('exempts a claim task typed on taskType alone', async () => {
const task = { id: 't4', taskType: 'claim-work' };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', success: true, startedAt: STARTED_AT })).toBeNull();
expect(committedDuringRun).not.toHaveBeenCalled();
});
it('does NOT exempt a task whose claimFlow flag is falsy or absent', async () => {
// The flag is the marker of a claim dispatch; a task that merely carries the
// string "false" (or nothing) is an ordinary committing run.
committedDuringRun.mockResolvedValueOnce(false);
const task = { id: 't5', taskType: 'internal', metadata: { analysisType: 'security', claimFlow: 'false' } };
expect(await evaluateSuccessCriteria({ task, workspacePath: '/w', startedAt: STARTED_AT, success: true })).toBe(false);
expect(committedDuringRun).toHaveBeenCalledWith('/w', STARTED_AT);
});
});
describe('no-change quality audit delivery', () => {
it('requires both verified empty branch and saved assessment, without waiving changed-branch commit checks', async () => {
const args = { task: { id: 'audit', taskType: 'internal', metadata: { analysisType: 'security', noChangeSuccess: true } }, workspacePath: '/worktree', startedAt: 1000, success: true };
expect(await evaluateSuccessCriteria({ ...args, branchProvenEmpty: true, hookResult: { auditAssessment: { status: 'recorded' } } })).toBe(true);
expect(await evaluateSuccessCriteria({ ...args, branchProvenEmpty: true, hookResult: { auditAssessment: { status: 'not-recorded' } } })).toBe(false);
committedDuringRun.mockResolvedValue(false);
expect(await evaluateSuccessCriteria({ ...args, branchProvenEmpty: false, hookResult: { auditAssessment: { status: 'recorded' } } })).toBe(false);
});
});