Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

15.8 KB
Raw Blame
/** No-inventory completion must retain one exact obligation through execution release and in-place removal. */
import { afterEach, afterAll, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdtemp, mkdir, readFile, writeFile, rm, realpath } from 'node:fs/promises';
import { join, sep, resolve } from 'node:path';
import { tmpdir } from 'node:os';
import { materializeGitRepo } from '../lib/gitTestRepo.js';
import { execGit } from '../lib/execGit.js';
import { makePathsProxy, lazyTempDataRoot, cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';
import { storedForm } from '../lib/cosTaskRows.js';
const fixture = vi.hoisted(() => ({ root: null, app: null, forge: null, task: null, agents: {}, schedule: null, raceTaskEdit: false, inventoryAlias: false }));
async function persistTask() {
  fixture.task = { ...storedForm(fixture.task, 'user'), taskType: 'user' };
  await writeFile(join(fixture.root, 'task.json'), JSON.stringify(fixture.task));
}
vi.mock('../lib/paths.js', async original => makePathsProxy(await original(), { dataRoot: () => lazyTempDataRoot('portos-completion-retirement-state-') }));
// Keep real Git and teardown; vary only the lexical spelling returned by its
// inventory adapter, like Git-forward versus native Windows separators.
vi.mock('./worktreeManager.js', async original => {
  const actual = await original();
  return { ...actual, listWorktrees: async (...args) => (await actual.listWorktrees(...args)).map(tree =>
    fixture.inventoryAlias && tree.branch === 'refs/heads/claim/issue-42' ? { ...tree, path: `${tree.path}${sep}.` } : tree) };
});
vi.mock('./cos.js', () => ({}));
vi.mock('./domainUsage.js', () => ({ recordDomainUsage: async () => {} }));
vi.mock('./cosEvents.js', () => ({ emitLog: vi.fn(), cosEvents: { emit: vi.fn() } }));
vi.mock('./apps.js', () => ({ getAppById: async id => id === fixture.app?.id ? fixture.app : null }));
vi.mock('./branchReconcileRetirement.js', async original => ({ ...await original(), assertAutomaticRetirementAllowed: async () => {} }));
vi.mock('./branchReconcile.js', async original => ({ ...await original(), listRemoteHeads: async () => new Map() }));
vi.mock('./heldCheckoutReview.js', async original => ({ ...await original(), readRetirementForgeEvidence: async () => fixture.forge }));
vi.mock('./prProbe.js', async original => ({ ...await original(), probePrForBranch: async () => ({ readable: true,
  prState: fixture.forge ? 'MERGED' : 'OPEN', prUrl: 'https://github.com/example/repo/pull/42', prNumber: 42, cli: 'gh' }) }));
vi.mock('./prWatcher.js', () => ({ readPendingMergePrs: () => [] }));
vi.mock('./cosTaskStore.js', () => ({
  getTaskById: async () => JSON.parse(await readFile(join(fixture.root, 'task.json'), 'utf8')),
  getAllTasks: async () => ({ user: { tasks: [fixture.task] }, cos: { tasks: [] } }),
  updateTask: async (_id, patch, _type, { expectedStatus, expectedMetadata } = {}) => {
    const current = JSON.parse(await readFile(join(fixture.root, 'task.json'), 'utf8'));
    if (fixture.raceTaskEdit && patch.metadata?.deliveryRecovery === 'cleanup') {
      fixture.raceTaskEdit = false; current.metadata.updatedAt = 'concurrent-human-edit';
      await writeFile(join(fixture.root, 'task.json'), JSON.stringify(current));
    }
    if ((expectedStatus && current.status !== expectedStatus)
      || Object.entries(expectedMetadata || {}).some(([key, value]) => current.metadata?.[key] !== value)) return { statusChanged: true, task: current };
    fixture.task = current;
    fixture.task = { ...fixture.task, ...patch, metadata: { ...fixture.task.metadata, ...patch.metadata } };
    await persistTask(); return fixture.task;
  }, addTask: async () => { throw new Error('unexpected cognitive recovery'); },
}));
vi.mock('./cosState.js', async () => ({
  AGENTS_DIR: join((await import('../lib/paths.js')).PATHS.cos, 'agents'),
  loadState: async () => ({ agents: fixture.agents, stats: {} }),
  saveState: async state => { fixture.agents = state.agents; await writeFile(join(fixture.root, 'agents-state.json'), JSON.stringify(state)); },
  withStateLock: work => work(), readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: fixture.agents }),
}));
vi.mock('./branchCompletionPolicy.js', async original => ({ ...await original(), readBranchCompletionPolicies: async () => Object.values(fixture.agents) }));
vi.mock('./taskScheduleStore.js', async original => ({ ...await original(), loadSchedule: async () => fixture.schedule,
  updateSchedule: async mutate => { const output = await mutate(fixture.schedule);
    await writeFile(join(fixture.root, 'schedule.json'), JSON.stringify(fixture.schedule)); return output.result; },
}));
const { verifyAgentRepoState, sweepRepoStateVerifications } = await import('./agentRepoStateVerification.js');
const { bindCompletionTaskDispatch, prepareCompletionTaskHandoff } = await import('./agentCompletionTaskHandoff.js');
const { completeAgent, getAgentRecord } = await import('./cosAgentLifecycle.js');
const { getBranchReconcileOutcome } = await import('./taskSchedule.js');
const { activeAgents } = await import('./agentState.js');
const { PATHS } = await import('../lib/paths.js');
const { cosEvents } = await import('./cosEvents.js');
afterAll(cleanupTempDataRoots);
let scratch;
afterEach(async () => { fixture.inventoryAlias = false; activeAgents.clear(); if (scratch) await rm(scratch, { recursive: true, force: true, maxRetries: 3 }); });
describe('completion to archive retirement (real Git and persisted service state)', () => {
  it('drains context-less legacy handoffs instead of sweeping nonretryable failures forever', async () => {
    scratch = await mkdtemp(join(tmpdir(), 'portos-completion-sweep-'));
    fixture.root = PATHS.data; await mkdir(fixture.root, { recursive: true });
    fixture.task = { id: 'sweep-task', taskType: 'user', status: 'completed', metadata: {} };
    await writeFile(join(fixture.root, 'task.json'), JSON.stringify(fixture.task));
    fixture.agents = Object.fromEntries(Array.from({ length: 11 }, (_, index) => {
      const id = `sweep-agent-${index}`;
      return [id, { id, taskId: fixture.task.id, status: 'completed', startedAt: '2020-01-01T00:00:00.000Z',
        result: { success: false }, metadata: { repoStateVerification: { status: 'completed', checkedAt: '2020-01-01T00:00:01.000Z' },
          completionCleanup: { version: 1, status: 'pending', updatedAt: new Date(1600000000000 + index).toISOString(),
            stages: { pipeline: { status: 'failed', retryable: false, reason: 'external-state-unknown' } } } } }];
    }));
    const waiting = fixture.agents['sweep-agent-10'].metadata.completionCleanup.updatedAt;
    expect(await sweepRepoStateVerifications()).toMatchObject({ checked: 10, pendingAtStart: 11 });
    fixture.agents = JSON.parse(await readFile(join(fixture.root, 'agents-state.json'), 'utf8')).agents;
    expect(fixture.agents['sweep-agent-10'].metadata.completionCleanup.updatedAt).toBe(waiting);
    expect(await sweepRepoStateVerifications()).toMatchObject({ checked: 1, pendingAtStart: 1 });
    fixture.agents = JSON.parse(await readFile(join(fixture.root, 'agents-state.json'), 'utf8')).agents;
    expect(fixture.agents['sweep-agent-10'].metadata.completionCleanup.lastAttemptAt).toEqual(expect.any(String));
    expect(fixture.agents['sweep-agent-10'].metadata.completionCleanup.stages.pipeline).toMatchObject({ status: 'failed', retryable: false });
    expect(fixture.agents['sweep-agent-10'].metadata.completionCleanup.status).toBe('abandoned');
    expect(await sweepRepoStateVerifications()).toMatchObject({ checked: 0, pendingAtStart: 0 });
    expect(fixture.task.status).toBe('completed');
  });

  it.each(['unchanged', 'lexical-workspace', 'changed-dispatch', 'concurrent-task-edit', 'claim-reacquired'])('recovers the missing task handoff after restart with exact task CAS (%s)', async scenario => {
    const changed = !['unchanged', 'lexical-workspace'].includes(scenario);
    fixture.raceTaskEdit = false;
    fixture.inventoryAlias = scenario === 'lexical-workspace';
    cosEvents.emit.mockClear();
    scratch = await mkdtemp(join(tmpdir(), 'portos-completion-retirement-'));
    fixture.root = PATHS.data; await mkdir(fixture.root, { recursive: true });
    await materializeGitRepo(join(scratch, 'repo'));
    const repo = await realpath(join(scratch, 'repo'));
    const path = join(repo, '.claude', 'worktrees', 'completion'); const branch = 'claim/issue-42';
    await execGit(['worktree', 'add', '-b', branch, path, 'main'], repo);
    await writeFile(join(path, 'feature.txt'), 'delivered original\n');
    await execGit(['add', 'feature.txt'], path); await execGit(['commit', '-m', 'deliver original'], path);
    const head = (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim();
    const startedAt = new Date(Date.now() - 1000).toISOString();
    const running = { id: `agent-completion-${scenario}`, taskId: `task-completion-${scenario}`, status: 'running', startedAt, output: [],
      metadata: { isWorktree: true, sourceWorkspace: repo, workspacePath: path, worktreeBranch: branch, prCompletionPolicy: 'review-then-merge' } };
    await execGit(['merge', '--ff-only', branch], repo);
    const defaultHead = (await execGit(['rev-parse', 'HEAD'], repo)).stdout.trim();
    await writeFile(join(repo, '.git', 'info', 'exclude'), 'ignored.txt\n');
    await writeFile(join(path, 'ignored.txt'), 'private preserved bytes\n');
    fixture.forge = { repository: 'github.com/example/repo', prUrl: 'https://github.com/example/repo/pull/42', prHead: head, changeBase: (await execGit(['rev-parse', `${head}^`], repo)).stdout.trim(), firstCommit: head,
      mergeCommit: defaultHead, defaultBranch: 'main', defaultHead, remoteHead: null, changedPaths: ['feature.txt'] };
    fixture.app = { id: 'app-completion', repoPath: repo }; fixture.schedule = { tasks: {}, executions: {} };
    fixture.task = { id: running.taskId, taskType: 'user', status: 'in_progress', priority: 'MEDIUM', description: 'Complete example delivery', metadata: { app: fixture.app.id, openPR: true,
      prCompletion: 'review-then-merge', claimedBy: 'fixture-instance', claimedAt: startedAt, totalSpawnCount: 1,
      lastSpawnedAt: startedAt, updatedAt: startedAt } };
    fixture.agents = { [running.id]: running };
    activeAgents.set(running.id, { task: fixture.task });
    if (scenario === 'lexical-workspace') await persistTask();
    expect(await bindCompletionTaskDispatch(running.id, fixture.task)).toBe(true);
    // The dispatcher binds the numeric write result, but completion reads the
    // persisted scalar text. Losing this handoff misclassifies a clean run as
    // an orphan and exhausts its retry budget after successful retirement.
    await persistTask();
    expect(fixture.task.metadata.totalSpawnCount).toBe('1');
    expect(await prepareCompletionTaskHandoff(await getAgentRecord(running.id), { hookDelivery: { selectedPaths: ['feature.txt'] } })).toBe(true);
    if (scenario === 'unchanged') {
      // Receipts from a numeric task snapshot must replay against stored text,
      // while the final compare-and-swap still uses the exact current values.
      const intent = fixture.agents[running.id].metadata.completionCleanup.taskHandoff;
      intent.expected = intent.expected.map(([key, value]) => [key, key === 'totalSpawnCount' ? Number(value) : value]);
    }
    const agent = await completeAgent(running.id, { success: true, executionSuccess: true, requireCheckoutSettlement: true });
    expect(agent.result).toMatchObject({ success: false, cleanupPending: true, executionSuccess: true });
    expect(agent.metadata.completedCheckouts).toHaveLength(1);
    await completeAgent(running.id, { success: true, executionSuccess: true, requireCheckoutSettlement: true });
    expect(cosEvents.emit.mock.calls.filter(([event]) => event === 'agent:completed')).toHaveLength(1);
    // Crash after terminal agent durability, BEFORE the normal task update.
    expect(JSON.parse(await readFile(join(fixture.root, 'task.json'), 'utf8')).metadata.deliveryRecovery).toBeUndefined();
    fixture.agents = JSON.parse(await readFile(join(fixture.root, 'agents-state.json'), 'utf8')).agents;
    fixture.task = JSON.parse(await readFile(join(fixture.root, 'task.json'), 'utf8'));
    if (scenario === 'concurrent-task-edit') fixture.raceTaskEdit = true;
    if (scenario === 'claim-reacquired') {
      // prepareAgentSpawn reclaims before registration or a fresh spawn stamp.
      const { buildClaim } = await import('./cosTaskClaim.js');
      Object.assign(fixture.task.metadata, buildClaim('fixture-instance'));
      await writeFile(join(fixture.root, 'task.json'), JSON.stringify(fixture.task));
    }
    if (scenario === 'changed-dispatch') {
      fixture.task.metadata.totalSpawnCount++;
      await writeFile(join(fixture.root, 'task.json'), JSON.stringify(fixture.task));
    }
    const verify = () => verifyAgentRepoState({ agentId: agent.id, agentState: fixture.agents[agent.id], task: fixture.task, success: false, prExpected: true });
    expect(await getBranchReconcileOutcome(fixture.app.id)).toBeNull(); activeAgents.set(agent.id, { task: fixture.task });
    if (changed) {
      activeAgents.delete(agent.id); // No competing registered owner exists yet.
      expect(await verify()).toMatchObject({ verified: false, skipReason: 'cleanup-owner-changed' });
      expect(fixture.task.metadata.deliveryRecovery).toBeUndefined();
      expect((await getAgentRecord(agent.id)).metadata.repoStateVerification.status).toBe('pending');
      expect((await execGit(['rev-parse', 'HEAD'], path)).stdout.trim()).toBe(head);
      expect(await readFile(join(path, 'ignored.txt'), 'utf8')).toBe('private preserved bytes\n');
      expect(await getBranchReconcileOutcome(fixture.app.id)).toBeNull();
      return;
    }
    expect(await verify()).toMatchObject({ verified: false, skipReason: 'completion-execution-active' });
    expect(fixture.task.metadata).toMatchObject({ deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: running.id,
      hookDelivery: { selectedPaths: ['feature.txt'] } });
    const pending = await getBranchReconcileOutcome(fixture.app.id); expect(pending.retirementObligations).toHaveLength(1);
    const obligation = pending.retirementObligations[0]; expect(obligation).toMatchObject({ path, branch, head });
    expect(fixture.task.status).toBe('in_progress');
    fixture.schedule = JSON.parse(await readFile(join(fixture.root, 'schedule.json'), 'utf8')); activeAgents.delete(agent.id);
    const { buildRenewal } = await import('./cosTaskClaim.js');
    Object.assign(fixture.task.metadata, buildRenewal(fixture.task.metadata, 'fixture-instance'));
    await writeFile(join(fixture.root, 'task.json'), JSON.stringify(fixture.task));
    const results = await Promise.all([verify(), verify()]); expect(results).toMatchObject([{ verified: true, skipReason: null }, { verified: true, skipReason: null }]);
    const outcome = await getBranchReconcileOutcome(fixture.app.id); expect(outcome.retirementObligations).toEqual([]);
    expect(outcome.retired).toHaveLength(1); const receipt = outcome.retired[0];
    expect(receipt).toMatchObject({ path, branch, head, branchDeleted: true, archiveDir: null });
    expect(existsSync(join(PATHS.cos, 'held-checkout-archives'))).toBe(false);
    const { listWorktrees } = await import('./worktreeManager.js');
    expect((await listWorktrees(repo)).some(tree => resolve(tree.path) === resolve(path))).toBe(false);
    expect((await execGit(['rev-parse', '--verify', `refs/heads/${branch}`], repo, { ignoreExitCode: true })).exitCode).not.toBe(0);
    expect(JSON.parse(await readFile(join(fixture.root, 'task.json'), 'utf8')).status).toBe('completed');
    await verify(); expect((await getBranchReconcileOutcome(fixture.app.id)).retired).toHaveLength(1);
  // This public-boundary scenario makes 182 real Git calls across capture,
  // fresh reviews, retirement and independent restore.
  // Measured wall time reaches 29s before host contention; keep its own budget
  // separate from the default 30s unit budget without weakening any proof.
  }, 60_000);
});