Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* The dispatch HOLDS at the `task:ready` chokepoint, and what releases them.
*
* Each leaves the task queued for a condition that clears on its own, rather
* than failing it:
*
* - Runner down. `portos-cos` is a separate PM2 app the user can stop from the
* Apps page, and in runner mode it owns every agent process. Dispatching into
* a stopped runner is not a task failure, but both spawn arms recorded it as
* one — see the `task:ready` listener in subAgentSpawner.js for the two
* failure modes.
* - Self-update in progress (issue #4124). `/api/update/execute` refuses to
* start while an agent is live, but `update.sh` then spends seconds in git
* pull / submodule update / npm install before `pm2 delete`. An agent spawned
* in that window is severed by the restart.
*
* - Forge unreachable (issue #5110). A task that promises a change request cannot
* finish with the forge down: the agent works, `git push` fails, and finalize
* records the non-actionable `forge-unreachable` — which retries, so one VPN
* drop re-ran the same agent three times before the task reached `blocked`.
*
* - Local inference endpoint at capacity (issue #4834). A CoS agent runs a
* vendor CLI that talks to the local model server directly, so promptRunner's
* in-flight gate never sees it; two agents at one GPU takes an accelerator
* OOM. The slot is reserved across the spawn window and released after.
*
* The holds live in that listener, not inside `runAgentSpawn`, because a hold
* below the spawn body's entry returns past `releaseAppReviewMarker` and strands
* the synthetic "in review" marker for the whole outage (issue #989). The side
* effects the dequeue tiers already committed — that marker and the scheduler's
* `spawningJobIds` reservation — are released here instead.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const runnerHandlers = new Map();
vi.mock('./cosRunnerClient.js', () => ({
onCosRunnerEvent: vi.fn((event, handler) => { runnerHandlers.set(event, handler); }),
initCosRunnerConnection: vi.fn(),
isRunnerAvailable: vi.fn().mockResolvedValue(true),
isRunnerReachable: vi.fn().mockResolvedValue(true),
}));
vi.mock('./cosEvents.js', () => ({
emitLog: vi.fn(),
cosEvents: { emit: vi.fn(), on: vi.fn((event, handler) => { taskHandlers.set(event, handler); }) },
}));
vi.mock('./providerStatus.js', () => ({ initProviderStatus: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./agentRunnerSync.js', () => ({ syncRunnerAgents: vi.fn().mockResolvedValue(0) }));
vi.mock('./cosAgentLifecycle.js', () => ({ updateAgent: vi.fn() }));
vi.mock('./agentRunnerOutputBatchers.js', () => ({
getRunnerOutputBatcher: vi.fn(),
flushRunnerOutputBatcher: vi.fn().mockResolvedValue(undefined),
}));
vi.mock('./agentLifecycle.js', () => ({ handleAgentCompletion: vi.fn() }));
vi.mock('./agentManagement.js', () => ({ cleanupOrphanedAgents: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./agentRunTracking.js', () => ({ completeAgentRun: vi.fn() }));
vi.mock('./appActivity.js', () => ({ releaseAppReviewMarker: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./updateChecker.js', () => ({ isUpdateInProgress: vi.fn().mockReturnValue(false) }));
vi.mock('./cosState.js', () => ({ loadState: vi.fn().mockResolvedValue({ agents: {} }) }));
// The real slot module is covered end-to-end in cos.test.js (resolvers, running
// tally, reservations). Mocked here so this suite stays about the LISTENER's
// wiring — hold vs spawn vs release — and doesn't drag promptRunner's provider
// graph into a file that mocks providerStatus.
vi.mock('./cosLocalEndpointSlots.js', () => ({
acquireLocalEndpointSpawnSlot: vi.fn().mockResolvedValue({ ok: true, release: vi.fn() }),
}));
// Same reason: the real gate reaches the app registry through agentPromptBuilder,
// whose graph reads files at module load and pulls in cos.js. Its own decisions
// (which statuses hold, which hosts are probed, the expiry) are covered directly
// in cosForgeSpawnGate.test.js; here it is a seam for the LISTENER's wiring.
vi.mock('./cosForgeSpawnGate.js', () => ({
forgeSpawnHoldReason: vi.fn().mockResolvedValue(null),
}));
vi.mock('./agentOrchestrator.js', () => ({
completeAgent: vi.fn(),
spawnAgentForTask: vi.fn().mockResolvedValue('agent-1'),
terminateAgent: vi.fn(),
}));
vi.mock('fs', () => ({ existsSync: vi.fn().mockReturnValue(false) }));
const taskHandlers = new Map();
import { initSpawner } from './subAgentSpawner.js';
import { cosEvents, emitLog } from './cosEvents.js';
import { isRunnerReachable } from './cosRunnerClient.js';
import { spawnAgentForTask } from './agentOrchestrator.js';
import { releaseAppReviewMarker } from './appActivity.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { isUpdateInProgress } from './updateChecker.js';
import { setUseRunner } from './agentState.js';
import { acquireLocalEndpointSpawnSlot } from './cosLocalEndpointSlots.js';
import { forgeSpawnHoldReason } from './cosForgeSpawnGate.js';
const dispatch = (task) => taskHandlers.get('task:ready')(task);
describe('subAgentSpawner — runner-down hold', () => {
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
vi.useRealTimers();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
});
it('keeps task reservations recoverable when maintenance wins during spawn preparation', async () => {
spawnAgentForTask.mockRejectedValueOnce(Object.assign(new Error('Held'), { code: 'MAINTENANCE_HELD' }));
await dispatch({ id: 'held-task', metadata: { app: 'example-app', jobId: 'example-job' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('example-app');
expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'example-job', reason: 'held' });
});
it('holds a manual/automatic task-ready event before runner preparation', async () => {
const { hold } = maintenance.begin({ reason: 'Work', owner: 'Operator' });
try {
await dispatch({ id: 'held-task', metadata: {} });
expect(spawnAgentForTask).not.toHaveBeenCalled();
expect(isRunnerReachable).not.toHaveBeenCalled();
} finally { maintenance.resume({ id: hold.id, revision: hold.revision }); }
});
it('spawns normally while the runner is up', async () => {
await dispatch({ id: 'cos-1', metadata: {} });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
it('holds the task instead of dispatching it while the runner is down', async () => {
isRunnerReachable.mockResolvedValue(false);
await dispatch({ id: 'cos-2', metadata: {} });
// Never reaches the spawn body, so the task keeps its `pending` record: no
// claim, no agent, no retry charged, no `spawn-rejected` finalization.
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
// Issue #989: the priority-0 tier binds a synthetic "in review" marker before
// emitting `task:ready`. A hold that skipped this would leave the app reading
// "in review" until the next daemon restart.
it('releases the app-review marker the dequeue tier already bound', async () => {
isRunnerReachable.mockResolvedValue(false);
await dispatch({ id: 'cos-3', metadata: { app: 'some-app' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
});
// cosJobScheduler reserves `spawningJobIds` before the emit and clears it on
// `job:spawned` / `job:spawn-failed`. Without this the job stays wedged until
// the scheduler's 5-minute spawn timeout — per job, per outage.
it('frees an autonomous job\'s spawn reservation so its schedule re-registers', async () => {
isRunnerReachable.mockResolvedValue(false);
await dispatch({ id: 'cos-4', metadata: { jobId: 'job-7' } });
expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-7', reason: 'held' });
});
it('does not consult the runner in direct mode', async () => {
setUseRunner(false);
isRunnerReachable.mockResolvedValue(false);
await dispatch({ id: 'cos-5', metadata: {} });
expect(isRunnerReachable).not.toHaveBeenCalled();
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
});
describe('subAgentSpawner — approval hold', () => {
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
vi.useRealTimers();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
});
it('honors an approval-required task before any runner or agent dispatch', async () => {
await dispatch({ id: 'cos-approval-1', approvalRequired: true, metadata: {} });
expect(isRunnerReachable).not.toHaveBeenCalled();
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
});
describe('subAgentSpawner — self-update hold (#4124)', () => {
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
vi.useRealTimers();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
});
it('holds the task instead of spawning into a process update.sh is about to restart', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-u1', metadata: {} });
// No agent is created, and the task record is untouched — it stays
// `pending` and is picked up by the first dequeue after the restart.
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
it('holds BEFORE the awaited runner probe, so nothing can race between check and spawn', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-u2', metadata: {} });
expect(isRunnerReachable).not.toHaveBeenCalled();
});
it('releases the app-review marker and the job spawn reservation, same as the runner hold', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-u3', metadata: { app: 'some-app', jobId: 'job-9' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-9', reason: 'held' });
});
it('resumes spawning once the update settles — the hold is not sticky', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-u4', metadata: {} });
expect(spawnAgentForTask).not.toHaveBeenCalled();
isUpdateInProgress.mockReturnValue(false);
await dispatch({ id: 'cos-u4', metadata: {} });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
it('holds in direct mode too, where there is no runner probe to hide behind', async () => {
setUseRunner(false);
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-u5', metadata: {} });
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
});
describe('subAgentSpawner — local-endpoint capacity hold (#4834)', () => {
// A CoS agent runs a vendor CLI that opens its own connection to the local
// model server, so promptRunner's in-flight gate never sees it. The hold lives
// at this listener rather than in `dequeueNextTask` because six other emitters
// (evaluateTasks, forceSpawnTask, the job scheduler, the Creative Director
// bridge, …) reach the spawner without passing through the scheduler at all.
let release;
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
vi.useRealTimers();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
release = vi.fn();
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: true, release });
});
it('spawns and releases the reserved slot when one is available', async () => {
await dispatch({ id: 'cos-l1', metadata: {} });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
expect(release).toHaveBeenCalledTimes(1);
});
it('holds the task instead of dispatching a second agent at a saturated GPU', async () => {
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: false, reason: 'local endpoint http://localhost:1234/v1 is at capacity (1/1)' });
await dispatch({ id: 'cos-l2', metadata: {} });
// Held, not failed: the task record stays `pending`, so the next dequeue
// tick re-picks it once the running agent frees the endpoint.
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
it('releases the app-review marker and the job reservation, same as the other holds', async () => {
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: false, reason: 'at capacity' });
await dispatch({ id: 'cos-l3', metadata: { app: 'some-app', jobId: 'job-11' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-11', reason: 'held' });
});
it('releases the slot even when the spawn throws', async () => {
// Without the `finally`, a failed spawn would leak the reservation and wedge
// the endpoint until the process restarts.
spawnAgentForTask.mockRejectedValueOnce(new Error('boom'));
await dispatch({ id: 'cos-l4', metadata: {} });
expect(release).toHaveBeenCalledTimes(1);
});
it('checks capacity AFTER the update and runner holds, which are cheaper', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-l5', metadata: {} });
expect(acquireLocalEndpointSpawnSlot).not.toHaveBeenCalled();
});
it('resumes spawning once the endpoint frees up — the hold is not sticky', async () => {
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: false, reason: 'at capacity' });
await dispatch({ id: 'cos-l6', metadata: {} });
expect(spawnAgentForTask).not.toHaveBeenCalled();
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: true, release });
await dispatch({ id: 'cos-l6', metadata: {} });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
});
describe('subAgentSpawner — forge-unreachable hold (#5110)', () => {
// The pre-gate behavior: an agent whose task promises a change request did the
// work, failed to push, and finalized the non-actionable `forge-unreachable` —
// which retries, so one VPN drop re-ran the same 20-to-100-minute agent three
// times before the task reached `blocked`.
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
vi.useRealTimers();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
acquireLocalEndpointSpawnSlot.mockResolvedValue({ ok: true, release: vi.fn() });
forgeSpawnHoldReason.mockResolvedValue(null);
});
it('spawns normally when the gate declines to hold', async () => {
await dispatch({ id: 'cos-f1', metadata: { openPR: true } });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
it('holds instead of dispatching an agent that cannot push what it builds', async () => {
forgeSpawnHoldReason.mockResolvedValue('forge.example.com is unreachable');
await dispatch({ id: 'cos-f2', metadata: { openPR: true } });
// Held, not failed: no agent, no retry charged, so the task keeps its full
// retry budget for a failure that is actually about the task.
expect(spawnAgentForTask).not.toHaveBeenCalled();
});
it('releases the app-review marker and the job reservation, same as the other holds', async () => {
forgeSpawnHoldReason.mockResolvedValue('forge.example.com is unreachable');
await dispatch({ id: 'cos-f3', metadata: { openPR: true, app: 'some-app', jobId: 'job-13' } });
expect(releaseAppReviewMarker).toHaveBeenCalledWith('some-app');
expect(cosEvents.emit).toHaveBeenCalledWith('job:spawn-failed', { jobId: 'job-13', reason: 'held' });
});
it('holds BEFORE reserving a capacity slot, so a held task never books one', async () => {
forgeSpawnHoldReason.mockResolvedValue('forge.example.com is unreachable');
await dispatch({ id: 'cos-f4', metadata: { openPR: true } });
expect(acquireLocalEndpointSpawnSlot).not.toHaveBeenCalled();
});
it('probes the forge AFTER the update and runner holds, which are cheaper', async () => {
isUpdateInProgress.mockReturnValue(true);
await dispatch({ id: 'cos-f5', metadata: { openPR: true } });
expect(forgeSpawnHoldReason).not.toHaveBeenCalled();
});
it('resumes spawning once the forge returns — the hold is not sticky', async () => {
forgeSpawnHoldReason.mockResolvedValue('forge.example.com is unreachable');
await dispatch({ id: 'cos-f6', metadata: { openPR: true } });
expect(spawnAgentForTask).not.toHaveBeenCalled();
forgeSpawnHoldReason.mockResolvedValue(null);
await dispatch({ id: 'cos-f6', metadata: { openPR: true } });
expect(spawnAgentForTask).toHaveBeenCalledTimes(1);
});
});
describe('subAgentSpawner — runner connection events', () => {
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
});
// A runner restart is routine, so the hold is announced on stdout (info), once.
it('announces once when the runner drops, rather than once per held task', () => {
runnerHandlers.get('connection:lost')();
const notices = emitLog.mock.calls.filter(([, message]) => /CoS Runner disconnected/.test(message));
expect(notices.map(([level]) => level)).toEqual(['info']);
expect(emitLog.mock.calls.filter(([level]) => level === 'warn' || level === 'error')).toEqual([]);
});
it('re-runs the dequeue when the runner returns, so held tasks spawn', async () => {
vi.useFakeTimers();
runnerHandlers.get('connection:ready')();
await vi.advanceTimersByTimeAsync(1000);
vi.useRealTimers();
expect(cosEvents.emit).toHaveBeenCalledWith('cos:dequeue-requested');
});
// `reconnectionAttempts` is unbounded, so a crash-looping runner emits
// `connect` repeatedly; each un-debounced edge would drive a full five-tier
// dequeue cycle.
it('coalesces a reconnect storm into one dequeue', async () => {
vi.useFakeTimers();
runnerHandlers.get('connection:ready')();
runnerHandlers.get('connection:ready')();
runnerHandlers.get('connection:ready')();
await vi.advanceTimersByTimeAsync(1000);
vi.useRealTimers();
const dequeues = cosEvents.emit.mock.calls.filter(([event]) => event === 'cos:dequeue-requested');
expect(dequeues).toHaveLength(1);
});
});
describe('subAgentSpawner — release signals for unpersisted tasks', () => {
beforeEach(async () => {
await initSpawner();
vi.clearAllMocks();
setUseRunner(true);
isRunnerReachable.mockResolvedValue(true);
isUpdateInProgress.mockReturnValue(false);
});
it('emits task:held when a hold drops the task', async () => {
isRunnerReachable.mockResolvedValue(false);
await dispatch({ id: 'fa-run-1', metadata: {} });
expect(cosEvents.emit).toHaveBeenCalledWith('task:held', { taskId: 'fa-run-1', reason: 'CoS Runner is down' });
});
it('emits agent:error when spawnAgentForTask throws', async () => {
spawnAgentForTask.mockRejectedValueOnce(new Error('worktree failed'));
await dispatch({ id: 'fa-run-2', metadata: {} });
expect(cosEvents.emit).toHaveBeenCalledWith('agent:error', { taskId: 'fa-run-2', error: 'worktree failed' });
});
});