Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Preserve recovered audit workspace evidence through finalization (#11578)

[._.]/2d79a6485c14
diff --git a/server/services/agentCompletionCleanup.js b/server/services/agentCompletionCleanup.js
index 62baf3f166..0933c88161 100644
--- a/server/services/agentCompletionCleanup.js
+++ b/server/services/agentCompletionCleanup.js
@@ -350,7 +350,13 @@ async function runCompletionCleanupSteps(context, { onStepError }) {
await runStep('Plan question notification', () => notifyPlanQuestionIfNeeded(context));
await runStep('Pipeline progression', () => handlePipelineProgression(context.task, context.agentId, context.effectiveSuccess));
await runStep('Creative Director completion', () => advanceCreativeDirectorIfNeeded(context));
- const cleanupWarnings = await runStep('Worktree cleanup', () => completeWorktreeCleanup(context));
+ // A terminal rejection is not proof that its audit artifacts were consumed.
+ // Keep the checkout and original signal available for explicit recovery.
+ const preserveAuditEvidence = auditEvidencePending(context);
+ if (preserveAuditEvidence) {
+ emitLog('warn', `Audit disposition incomplete for ${context.agentId}; preserving worktree and completion evidence`, { agentId: context.agentId });
+ }
+ const cleanupWarnings = preserveAuditEvidence ? [] : await runStep('Worktree cleanup', () => completeWorktreeCleanup(context));
await runStep('Cleanup warning reporting', () => reportWorktreeCleanupWarnings({ ...context, cleanupWarnings }));
// Last, and never fatal under either policy: a janitorial `rm` that hits EPERM
// must not be able to abort the worktree cleanup and pipeline hand-off above.
@@ -520,6 +526,11 @@ async function advanceCreativeDirectorIfNeeded({ agentId, task, effectiveSuccess
}
}
+function auditEvidencePending({ task, agentState }) {
+ return Number(task?.metadata?.auditDispositionVersion ?? agentState?.metadata?.auditDispositionVersion) === 1
+ && agentState?.metadata?.auditFindings?.status !== 'complete';
+}
+
/**
* Delete the run's own `.agent-done-<agentId>` completion sentinel.
*
@@ -538,7 +549,8 @@ async function advanceCreativeDirectorIfNeeded({ agentId, task, effectiveSuccess
* which is the directory the run wrote its sentinel into (a worktree run
* records its worktree here).
*/
-export async function removeCompletionSentinel({ agentId, agent, agentState }) {
+export async function removeCompletionSentinel({ agentId, agent, agentState, task }) {
+ if (auditEvidencePending({ task, agentState })) return;
// An id that yields no usable slug makes `doneSentinelName` fall back to the
// bare, unscoped `.agent-done` — which belongs to no run and may be a legacy
// agent's live signal (the sweep protects it with an age floor for exactly
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 2139bcb521..454f74900f 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1464,6 +1464,7 @@ export async function finalizeAgent({
error,
completionReason,
workspacePath = null,
+ recovery = false,
prExpected = false,
startedAt = null,
}) {
@@ -1681,7 +1682,7 @@ export async function finalizeAgent({
// of awaiting here is that the agent still counts against the CoS concurrency
// gate for the hook's duration, so the dispatch is hard-bounded — see
// withOutputHookTimeout.
- let hookResult = await dispatchTaskOutputHookOnce({ agentId, task, success: judgedVerdict.success, workspacePath });
+ let hookResult = await dispatchTaskOutputHookOnce({ agentId, task, success: judgedVerdict.success, workspacePath, recovery });
// A private assessment's deliverable is the validated, persisted report.
// A skipped, thrown or timed-out hook cannot establish that deliverable,
// even when the CLI exited zero. Keep other task types' existing semantics.
diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.js
index 4d77957d94..64497cd1ca 100644
--- a/server/services/agentLifecycle.js
+++ b/server/services/agentLifecycle.js
@@ -59,7 +59,6 @@ import { finalizeAgent, releaseAgentLane, retireDeadAgent, stampLiExecutionVerdi
import { extractFinalSummary } from './agentSummaryExtraction.js';
import { handleOrphanedTask } from './agentManagement.js';
-const ROOT_DIR = PATHS.root;
const AGENTS_DIR = PATHS.cosAgents;
/**
@@ -540,6 +539,9 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
// `registerAgent` writes those fields into metadata precisely so this
// question survives a restart, and nothing mutates them mid-run.
const persistedAgent = await getAgentRecord(agentId).catch(() => null);
+ // Re-adoption used to omit the cwd. Prefer the durable registration even
+ // for an already-recovered entry, which may carry a missing or stale path.
+ const workspacePath = persistedAgent?.metadata?.workspacePath || agent.workspacePath || null;
// Already-finalized backstop, mirroring the untracked branch above. Another
// owner may have written the terminal record before this event arrived —
@@ -589,8 +591,7 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
const runStartedAt = toEpochMs(agent.startedAt);
let effectiveSuccess = success;
if (!effectiveSuccess && task?.id) {
- const workspacePath = agent.workspacePath || ROOT_DIR;
- const commitFound = await committedDuringRun(workspacePath, runStartedAt);
+ const commitFound = workspacePath && await committedDuringRun(workspacePath, runStartedAt);
if (commitFound) {
emitLog('warn', `Agent ${agentId} reported failure (exit ${exitCode}) but work completed - commit found for task ${task.id}`, { agentId, taskId: task.id, exitCode });
effectiveSuccess = true;
@@ -634,7 +635,6 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
// Extract pipeline output summary before completion writes metadata to disk
if (task?.metadata?.pipeline && effectiveSuccess) {
- const workspacePath = agent.workspacePath || ROOT_DIR;
const summary = await extractPipelineOutputSummary(task, workspacePath, outputBuffer).catch(err => {
console.log(`⚠️ Failed to extract pipeline summary for ${agentId}: ${err.message}`);
return null;
@@ -679,7 +679,10 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
duration,
outputBuffer,
errorAnalysis,
- workspacePath: agent.workspacePath || null,
+ workspacePath,
+ // A recovered run with no known cwd must not borrow the primary
+ // checkout's sentinel or checkpoint through the normal hook fallback.
+ recovery: agent.recoveredFromRunner === true || !workspacePath,
prExpected: runnerPrClaimExpected,
// The run window the commit criterion is evaluated against (#3637).
startedAt: Number.isFinite(runStartedAt) ? runStartedAt : null,
diff --git a/server/services/agentLifecycle.recoveredAudit.test.js b/server/services/agentLifecycle.recoveredAudit.test.js
new file mode 100644
index 0000000000..c1c3175cc2
--- /dev/null
+++ b/server/services/agentLifecycle.recoveredAudit.test.js
@@ -0,0 +1,212 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+const fixture = vi.hoisted(() => ({ records: new Map(), tasks: [], trace: [], quality: [], serial: 0 }));
+vi.mock('../lib/fileUtils.js', async importOriginal => {
+ const { makePathsProxy, lazyTempDataRoot } = await import('../lib/mockPathsDataRoot.js');
+ return makePathsProxy(await importOriginal(), {
+ dataRoot: () => lazyTempDataRoot('portos-recovered-audit-'),
+ extraOverrides: root => ({ root: join(root, 'primary') }),
+ });
+});
+vi.mock('../lib/maintenanceAdmission.js', () => ({ maintenance: {
+ recoverOwned: vi.fn(), withResource: (_kind, _id, fn) => fn(), finishResource: vi.fn(), markResourceUnsettled: vi.fn(),
+} }));
+vi.mock('./cosRunnerClient.js', () => ({
+ getActiveAgentsFromRunner: vi.fn(), connectTuiSessionViaRunner: vi.fn(),
+ spawnAgentViaRunner: vi.fn(), getRunnerHealth: vi.fn(), classifyRunnerSpawnFailure: vi.fn(),
+}));
+vi.mock('./shell.js', () => ({ getSession: vi.fn(() => ({})) }));
+vi.mock('./cosAgentLifecycle.js', async importOriginal => ({
+ ...(await importOriginal()),
+ readAgentRecordOrUnreadable: vi.fn(),
+ getAgent: vi.fn(async id => fixture.records.get(id)),
+ getAgentRecord: vi.fn(async id => fixture.records.get(id)),
+ updateAgent: vi.fn(async (id, update) => {
+ const record = fixture.records.get(id);
+ Object.assign(record, { ...update, metadata: { ...record.metadata, ...update.metadata } });
+ if (update.metadata?.auditFindings) fixture.trace.push('findings');
+ return record;
+ }),
+ completeAgent: vi.fn(async (id, result) => {
+ fixture.trace.push('complete');
+ Object.assign(fixture.records.get(id), { status: 'completed', result });
+ }),
+}));
+vi.mock('./cos.js', () => ({
+ getAllTasks: vi.fn(async () => ({ user: { grouped: {} }, cos: { grouped: { active: fixture.tasks } } })),
+ getAgent: vi.fn(async id => fixture.records.get(id)),
+ getAgentRecord: vi.fn(async id => fixture.records.get(id)),
+ getTaskById: vi.fn(async id => fixture.tasks.find(t => t.id === id)),
+ updateTask: vi.fn(async (id, update) => Object.assign(fixture.tasks.find(t => t.id === id), update)),
+ addTask: vi.fn(), reviveBlockedTask: vi.fn(), checkStagePrecondition: vi.fn(),
+}));
+vi.mock('./agentWorktreeCleanup.js', () => ({
+ cleanupAgentWorktree: vi.fn(async id => {
+ fixture.trace.push('cleanup');
+ const record = fixture.records.get(id);
+ // Model the destructive boundary only; real finalization must save both
+ // receipts before the checkout disappears.
+ expect(record.metadata.auditFindings.status).toBe('complete');
+ expect(record.metadata.auditAssessment.status).toBe('recorded');
+ await rm(record.metadata.workspacePath, { recursive: true, force: true });
+ return [];
+ }),
+ releaseRetryHold: vi.fn(async () => ({})), spawnMergeRecoveryTask: vi.fn(),
+}));
+vi.mock('../lib/db.js', () => ({ ensureSchema: vi.fn(), query: vi.fn(async (_sql, values) => {
+ fixture.trace.push('quality');
+ fixture.quality.push({ category: values[1], agentId: values[2], report: JSON.parse(values[4]) });
+ return { rows: [] };
+}) }));
+vi.mock('./appQualityFederation.js', () => ({
+ collectAppQuality: vi.fn(), qualityRecord: vi.fn(), readQualityRecords: vi.fn(), readReleaseQuality: vi.fn(),
+}));
+vi.mock('./apps.js', () => ({ getAppById: vi.fn(async () => null) }));
+vi.mock('./codeReview.js', () => ({
+ getGoalFidelityConfig: vi.fn(async () => null), runLocalGoalFidelityReview: vi.fn(),
+ resolveReviewLoopOptions: vi.fn(async () => ({})),
+}));
+vi.mock('../lib/primaryCheckoutGuard.js', async importOriginal => ({
+ ...(await importOriginal()), detectPrimaryCheckoutDrift: vi.fn(async () => ({})),
+}));
+vi.mock('../lib/gitCommitProbe.js', async importOriginal => ({
+ ...(await importOriginal()), committedDuringRun: vi.fn(async () => false),
+}));
+vi.mock('./agentErrorAnalysis.js', async importOriginal => ({
+ ...(await importOriginal()), resolveFailedTaskUpdate: vi.fn(async () => ({ status: 'blocked' })),
+}));
+vi.mock('./cosEvents.js', () => ({ emitLog: vi.fn(), cosEvents: { emit: vi.fn(), on: vi.fn() } }));
+vi.mock('./agentRunEventLog.js', () => ({ appendRunEvent: vi.fn() }));
+vi.mock('./agentRunTracking.js', () => ({ completeAgentRun: vi.fn() }));
+vi.mock('./agentCompletion.js', () => ({ processAgentCompletion: vi.fn() }));
+vi.mock('./taskSchedule.js', () => ({ recordTaskTypeSuccess: vi.fn(async () => null), recordTaskTypeFailure: vi.fn(async () => null) }));
+vi.mock('./agentSpawnPreparation.js', () => ({ createAgentSpawnContext: vi.fn(), prepareAgentSpawn: vi.fn() }));
+vi.mock('./agentSpawnDispatch.js', () => ({ dispatchAgentRun: vi.fn() }));
+vi.mock('./agentManagement.js', () => ({ handleOrphanedTask: vi.fn() }));
+vi.mock('./agentCliSpawning.js', () => ({ isClaudeCliProvider: vi.fn(), getClaudeSettingsEnv: vi.fn() }));
+vi.mock('./forgeAuth.js', () => ({ resolveForgeTokenEnv: vi.fn() }));
+vi.mock('./agentApiAuth.js', () => ({ resolveAgentApiEnv: vi.fn() }));
+vi.mock('./providers.js', () => ({ getActiveProvider: vi.fn() }));
+vi.mock('./providerStatus.js', () => ({ markProviderUsageLimit: vi.fn(), markProviderUnavailable: vi.fn() }));
+vi.mock('./toolStateMachine.js', () => ({ completeExecution: vi.fn(), errorExecution: vi.fn() }));
+vi.mock('./executionLanes.js', () => ({ release: vi.fn() }));
+vi.mock('./updateChecker.js', () => ({ isUpdateInProgress: vi.fn(() => false) }));
+vi.mock('./jira.js', () => ({}));
+vi.mock('./git.js', () => ({}));
+vi.mock('./brainTaskThreads.js', () => ({ ensureTaskThread: vi.fn() }));
+vi.mock('./issueReconcile.js', () => ({ issueNumberFromRef: vi.fn() }));
+
+import { PATHS } from '../lib/fileUtils.js';
+import { cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';
+import { AGENT_RECORD_UNREADABLE, readAgentRecordOrUnreadable } from './cosAgentLifecycle.js';
+import { getActiveAgentsFromRunner } from './cosRunnerClient.js';
+import { cleanupAgentWorktree } from './agentWorktreeCleanup.js';
+import { activeAgents, runnerAgents } from './agentState.js';
+import { syncRunnerAgents } from './agentRunnerSync.js';
+import { handleAgentCompletion } from './agentLifecycle.js';
+
+const report = { version: 1, findings: [{ key: 'example-candidate', title: 'Reviewed example candidate',
+ path: 'server/example.js', severity: 2, confidence: 'high', evidence: 'Synthetic caller already handles this case.',
+ classification: 'dismissed', disposition: { kind: 'not-actionable', reason: 'Existing caller preserves the contract.' } }] };
+const quality = score => ({ version: 1, category: 'observability', score, worstSeverity: 0,
+ coverage: 'partial', confidence: 'high', summary: 'Synthetic category review.', scannedFiles: 1, totalFiles: 2 });
+const sentinel = (findings = report, score = 82) => JSON.stringify({ summary:
+ `Reviewed synthetic source.\nAUDIT_FINDINGS_JSON: ${JSON.stringify(findings)}\nQUALITY_AUDIT_JSON: ${JSON.stringify(quality(score))}` });
+
+beforeEach(async () => {
+ vi.clearAllMocks();
+ fixture.records.clear();
+ fixture.tasks.length = 0;
+ fixture.trace.length = 0;
+ fixture.quality.length = 0;
+ runnerAgents.clear();
+ activeAgents.clear();
+ await mkdir(PATHS.cosAgents, { recursive: true });
+});
+afterAll(cleanupTempDataRoots);
+
+async function recoveredRun({ primary = false, missingPath = false, unreadable = false } = {}) {
+ const agentId = `agent-example-${++fixture.serial}`;
+ const primaryPath = join(PATHS.data, `primary-${fixture.serial}`);
+ const workspacePath = primary ? primaryPath : join(PATHS.data, `worktree-${fixture.serial}`);
+ await mkdir(primaryPath, { recursive: true });
+ await mkdir(workspacePath, { recursive: true });
+ const task = { id: `app-improve-example-${fixture.serial}`, taskType: 'internal', status: 'in_progress',
+ metadata: { analysisType: 'observability', auditDispositionVersion: '1', app: 'example-app', repoPath: primaryPath, openPR: false } };
+ const record = { id: agentId, taskId: task.id, status: 'running', startedAt: '2026-10-01T12:00:00.000Z',
+ metadata: { auditDispositionVersion: 1, runId: `run-example-${fixture.serial}`,
+ ...(!missingPath && !unreadable ? { workspacePath } : {}) } };
+ fixture.records.set(agentId, record);
+ fixture.tasks.push(task);
+ readAgentRecordOrUnreadable.mockResolvedValue(unreadable ? AGENT_RECORD_UNREADABLE : record);
+ getActiveAgentsFromRunner.mockResolvedValue([{ id: agentId, taskId: task.id, kind: 'tui',
+ processActive: true, liveness: 'pty', startedAt: Date.parse(record.startedAt),
+ ...(!missingPath ? { workspacePath } : {}) }]);
+ await writeFile(join(primaryPath, `.agent-done-${agentId}`), sentinel({ version: 1, findings: [] }, 12));
+ await writeFile(join(workspacePath, `.agent-done-${agentId}`), sentinel());
+ await writeFile(join(workspacePath, `.agent-audit-findings-${agentId}.json`), JSON.stringify(report));
+ await syncRunnerAgents();
+ return { agentId, task, record, primaryPath, workspacePath };
+}
+
+describe('recovered runner audit completion (#11526)', () => {
+ // This drives the real recovery -> finalizer -> audit/quality -> cleanup
+ // sequence. The primary checkout's valid but different report is a decoy.
+ it.each(['rehydrated', 'old entry without cwd', 'stale primary cwd'])('records worktree evidence before cleanup for a %s entry', async mode => {
+ const run = await recoveredRun();
+ if (mode === 'old entry without cwd') delete runnerAgents.get(run.agentId).workspacePath;
+ if (mode === 'stale primary cwd') runnerAgents.get(run.agentId).workspacePath = run.primaryPath;
+ await handleAgentCompletion(run.agentId, 0, true, 1000);
+ expect(run.record.metadata.auditFindings).toMatchObject({ status: 'complete', report });
+ expect(run.record.result).toMatchObject({ success: true, auditAssessment: { status: 'recorded' } });
+ expect(fixture.quality).toEqual([{ category: 'observability', agentId: run.agentId, report: quality(82) }]);
+ expect(fixture.trace.indexOf('findings')).toBeLessThan(fixture.trace.indexOf('quality'));
+ expect(fixture.trace.indexOf('quality')).toBeLessThan(fixture.trace.indexOf('complete'));
+ expect(fixture.trace.indexOf('complete')).toBeLessThan(fixture.trace.indexOf('cleanup'));
+ expect(runnerAgents.has(run.agentId)).toBe(false);
+ await expect(readFile(join(run.workspacePath, `.agent-done-${run.agentId}`))).rejects.toMatchObject({ code: 'ENOENT' });
+ expect(await readFile(join(run.primaryPath, `.agent-done-${run.agentId}`), 'utf8')).toBe(sentinel({ version: 1, findings: [] }, 12));
+ });
+
+ it('also records an explicitly registered primary-checkout audit', async () => {
+ const run = await recoveredRun({ primary: true });
+ await handleAgentCompletion(run.agentId, 0, true, 1000);
+ expect(run.record.metadata.auditFindings).toMatchObject({ status: 'complete', report });
+ expect(fixture.quality[0].report.score).toBe(82);
+ });
+
+ it('retains the runner cwd when the registration read is temporarily unavailable', async () => {
+ const run = await recoveredRun({ unreadable: true });
+ expect(runnerAgents.get(run.agentId).workspacePath).toBe(run.workspacePath);
+ // The durable registration becomes readable again before completion.
+ run.record.metadata.workspacePath = run.workspacePath;
+ await handleAgentCompletion(run.agentId, 0, true, 1000);
+ expect(run.record.metadata.auditFindings).toMatchObject({ status: 'complete', report });
+ });
+
+ it('does not borrow primary artifacts when neither registration nor runner knows the workspace', async () => {
+ const run = await recoveredRun({ missingPath: true });
+ await handleAgentCompletion(run.agentId, 0, true, 1000);
+ expect(run.record.result).toMatchObject({ success: false, completionReason: 'audit-disposition-incomplete' });
+ expect(run.record.metadata.auditFindings).toMatchObject({ status: 'incomplete', finalReportReceived: false, report: { version: 1, findings: [] } });
+ expect(fixture.quality).toEqual([]);
+ expect(cleanupAgentWorktree).not.toHaveBeenCalled();
+ expect(await readFile(join(run.workspacePath, `.agent-done-${run.agentId}`), 'utf8')).toBe(sentinel());
+ expect(await readFile(join(run.primaryPath, `.agent-done-${run.agentId}`), 'utf8')).toBe(sentinel({ version: 1, findings: [] }, 12));
+ });
+
+ it.each(['malformed', 'missing'])('keeps the checkout and checkpoint after a %s final report', async kind => {
+ const run = await recoveredRun();
+ const donePath = join(run.workspacePath, `.agent-done-${run.agentId}`);
+ if (kind === 'missing') await rm(donePath);
+ else await writeFile(donePath, sentinel({ version: 1, findings: [{ invalid: true }] }));
+ await handleAgentCompletion(run.agentId, 0, true, 1000);
+ expect(run.record.result).toMatchObject({ success: false, completionReason: 'audit-disposition-incomplete' });
+ expect(run.record.metadata.auditFindings).toMatchObject({ status: 'incomplete', finalReportReceived: false, report });
+ expect(cleanupAgentWorktree).not.toHaveBeenCalled();
+ expect(await readFile(join(run.workspacePath, `.agent-audit-findings-${run.agentId}.json`), 'utf8')).toBe(JSON.stringify(report));
+ if (kind === 'malformed') expect(await readFile(donePath, 'utf8')).toBe(sentinel({ version: 1, findings: [{ invalid: true }] }));
+ });
+});
diff --git a/server/services/agentRunnerSync.js b/server/services/agentRunnerSync.js
index 8bc6a1f51a..a377b619e6 100644
--- a/server/services/agentRunnerSync.js
+++ b/server/services/agentRunnerSync.js
@@ -113,6 +113,10 @@ export async function syncRunnerAgents() {
task: task || { id: agent.taskId, taskType: inferredType, description: 'Recovered from runner' },
runId: recoveredRunId,
model: persisted?.metadata?.model || null,
+ // Completion hooks must read the registered worktree, not the task's
+ // primary checkout. The runner's cwd also survives an unreadable record.
+ workspacePath: persisted?.metadata?.workspacePath || agent.workspacePath || null,
+ recoveredFromRunner: true,
hasStartedWorking: true,
startedAt: agent.startedAt
});