Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { createDefaultPersistentMindState, normalizePersistentMindState, PERSISTENT_MIND_LIMITS } from '../lib/persistentMind.js';
import { buildPersistentMindCallDenial } from '../lib/persistentMindTrajectory.js';
import { PROVIDER_USAGE_LIMIT_PAUSE_REASON } from '../lib/persistentMindUsageLimit.js';
import { contextBudgetPauseReasonFrom } from '../lib/persistentMindContextBudget.js';
import {
__resetCosAdmissionReservations,
acquireCosActionReservation,
acquireCosGlobalSlot,
} from './cosAdmissionReservations.js';
const mock = vi.hoisted(() => ({
root: null,
inferenceLedger: null,
scheduled: new Map(),
events: new Map(),
emitted: [],
budget: { withinBudget: true, exceeded: null },
budgetError: null,
recordUsage: vi.fn(async () => {}),
acquireSlot: vi.fn(async () => ({ ok: true, release: vi.fn() })),
localEndpointOfProvider: vi.fn(() => null),
appendMindEvent: vi.fn(async (event) => ({ appended: true, event })),
prepareContext: vi.fn(async () => ({ text: 'bounded context', chars: 15, summaryState: 'not-needed' })),
updateInProgress: false,
daemonRunning: true,
profile: {
ok: true,
provider: { id: 'example-cloud' },
model: 'example-model',
effort: 'high',
thinkingInterface: 'text',
},
imageCapability: { status: 'supported', reason: 'Supported.' },
thinkingSession: null,
useActualThinkingSession: false,
providerOverride: null,
providerAvailable: true,
}));
vi.mock('./cosState.js', () => ({
loadState: vi.fn(async () => mock.root),
saveState: vi.fn(async (state) => { mock.root = state; }),
withStateLock: vi.fn(async (fn) => fn()),
isDaemonRunning: vi.fn(() => mock.daemonRunning),
}));
vi.mock('./cosEvents.js', () => ({
cosEvents: { emit: vi.fn((...args) => mock.emitted.push(args)) },
emitLog: vi.fn(),
}));
vi.mock('./eventScheduler.js', () => ({
schedule: vi.fn((config) => {
mock.scheduled.set(config.id, config);
return config;
}),
cancel: vi.fn((id) => mock.scheduled.delete(id)),
getEvent: vi.fn((id) => mock.events.get(id) || null),
}));
vi.mock('./domainUsage.js', () => ({
getDomainBudgetStatus: vi.fn(async () => {
if (mock.budgetError) throw mock.budgetError;
return mock.budget;
}),
recordDomainUsage: (...args) => mock.recordUsage(...args),
}));
vi.mock('./cosLocalEndpointSlots.js', () => ({
acquireLocalEndpointProviderSlot: (...args) => mock.acquireSlot(...args),
localEndpointOfProvider: (...args) => mock.localEndpointOfProvider(...args),
providerBaseUrl: (provider) => provider?.endpoint || null,
}));
vi.mock('./agentRunEventLog.js', () => ({
appendMindEvent: (...args) => mock.appendMindEvent(...args),
}));
vi.mock('./persistentMindContext.js', () => ({
preparePersistentMindContext: (...args) => mock.prepareContext(...args),
}));
vi.mock('./persistentMindProfile.js', async (importOriginal) => ({
resolvePersistentMindProfile: vi.fn(async () => mock.profile),
resolvePersistentMindThinkingSession: vi.fn(async (input) => mock.useActualThinkingSession
? (await importOriginal()).resolvePersistentMindThinkingSession(input)
: mock.thinkingSession),
}));
vi.mock('./providers.js', () => ({ getProviderById: vi.fn(async () => mock.providerOverride || mock.profile.provider) }));
vi.mock('./providerStatus.js', () => ({
isProviderAvailable: () => mock.providerAvailable,
getProviderStatus: () => ({ message: 'Endpoint is temporarily unavailable' }),
markProviderUsageLimit: vi.fn(async () => {
mock.providerAvailable = false;
return { available: false, reason: 'usage-limit' };
}),
}));
vi.mock('./persistentMindImageCapability.js', () => ({
resolvePersistentMindImageCapability: vi.fn(async () => mock.imageCapability),
imageCapabilityAllowsAttempt: (capability, provider) => capability?.status === 'supported'
|| (capability?.status === 'unknown' && provider?.type === 'api'),
}));
vi.mock('./updateChecker.js', () => ({
isUpdateInProgress: vi.fn(() => mock.updateInProgress),
}));
// Keep real reservation/accounting with an isolated in-memory ledger; daemon
// route probes are outside this supervisor workflow contract.
vi.mock('./persistentMindMaintainerInference.js', async (importOriginal) => {
const actual = await importOriginal();
return { ...actual,
inspectMaintainerInferenceRoute: async ({ role }) => ({ ok: true, enforced: true,
lane: 'local-curation', policy: role.inference }),
reserveMaintainerInference: args => actual.reserveMaintainerInference({ ...args,
read: async () => mock.inferenceLedger,
write: async value => { mock.inferenceLedger = structuredClone(value); } }),
};
});
const { maintenance } = await import('../lib/maintenanceAdmission.js');
const supervisor = await import('./persistentMindSupervisor.js');
const makeRoot = () => ({
paused: false,
config: {
domainAutonomy: { cos: 'execute' },
maxConcurrentAgents: 3,
persistentMindProfile: { enabled: true, providerId: 'example-cloud', model: 'example-model', effort: 'high', thinkingInterface: 'text' },
},
agents: {},
persistentMind: createDefaultPersistentMindState(),
});
const deferred = () => {
let resolve;
let reject;
const promise = new Promise((res, rej) => { resolve = res; reject = rej; });
return { promise, resolve, reject };
};
describe('persistent mind supervisor', () => {
beforeEach(() => {
mock.root = makeRoot();
mock.inferenceLedger = null;
mock.scheduled.clear();
mock.emitted.length = 0;
mock.budget = { withinBudget: true, exceeded: null };
mock.budgetError = null;
mock.daemonRunning = true;
mock.updateInProgress = false;
mock.useActualThinkingSession = false;
mock.providerOverride = null;
mock.providerAvailable = true;
mock.recordUsage.mockClear();
mock.localEndpointOfProvider.mockReset();
mock.localEndpointOfProvider.mockReturnValue(null);
mock.profile = {
ok: true,
provider: { id: 'example-cloud', type: 'api' },
model: 'example-model',
effort: 'high',
thinkingInterface: 'text',
};
mock.imageCapability = { status: 'supported', reason: 'Supported.' };
mock.thinkingSession = {
ok: true,
temporary: true,
presetId: 'deep',
presetLabel: 'Deep pass',
provider: { id: 'example-alt', type: 'api' },
model: 'alt-model',
effort: 'max',
thinkingInterface: 'text',
};
mock.acquireSlot.mockReset();
mock.acquireSlot.mockResolvedValue({ ok: true, release: vi.fn() });
mock.appendMindEvent.mockClear();
mock.prepareContext.mockClear();
__resetCosAdmissionReservations();
supervisor.__resetPersistentMindSupervisorForTests();
});
it('maintenance rejects manual wakes without changing the saved paused policy', async () => {
const prepare = vi.fn(); const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.startPersistentMind();
await supervisor.pausePersistentMind();
const saved = structuredClone(mock.root.persistentMind);
const { hold } = maintenance.begin({ reason: 'Work', owner: 'Operator' });
try {
await expect(supervisor.wakePersistentMind()).rejects.toMatchObject({ code: 'MAINTENANCE_HELD' });
await expect(supervisor.resumePersistentMind()).rejects.toMatchObject({ code: 'MAINTENANCE_HELD' });
expect(mock.root.persistentMind).toEqual(saved);
expect(run).not.toHaveBeenCalled();
} finally { maintenance.resume({ id: hold.id, revision: hold.revision }); }
await supervisor.recheckPersistentMindSchedule();
expect(mock.root.persistentMind).toEqual(saved);
});
it('is silent on boot and enables the runtime only when explicitly started', async () => {
const prepare = vi.fn();
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.initializePersistentMindSupervisor();
expect(mock.scheduled.size).toBe(0);
expect(await supervisor.startPersistentMind()).toEqual({ success: true, alreadyStarted: false });
expect(mock.root.persistentMind).toMatchObject({ enabled: true, started: true, status: 'waiting' });
expect(prepare).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
});
it('releases a dead mind-turn after restart recovery and leaves an unproved provider', async () => {
const deadPid = 2147483646;
const turn = maintenance.admit('mind-turn', 'persistent-mind');
const live = maintenance.admit('mind-turn', 'persistent-mind');
const provider = maintenance.admit('provider', 'fixture-source');
const file = join(maintenance.directory, 'state.json');
const journal = JSON.parse(readFileSync(file, 'utf8'));
for (const id of [turn.id, provider.id]) journal.operations.find(op => op.id === id).pid = deadPid;
writeFileSync(file, JSON.stringify(journal));
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'thinking',
activeTurn: {
id: 'fixture-turn',
wake: { kind: 'message', message: { id: 'fixture-message', text: 'Ordinary.', createdAt: new Date(1).toISOString() } },
startedAt: new Date(1).toISOString(),
},
};
try {
const recovered = await supervisor.initializePersistentMindSupervisor();
expect(recovered.activeTurn).toBeNull();
const left = maintenance.operations().filter(op => [turn.id, live.id, provider.id].includes(op.id));
expect(left.map(op => op.id).sort()).toEqual([live.id, provider.id].sort());
} finally {
await maintenance.finishResource('mind-turn', 'persistent-mind');
await maintenance.finishResource('provider', 'fixture-source');
}
});
afterEach(() => vi.useRealTimers());
it.each([
['no adapter', 'degraded', PERSISTENT_MIND_LIMITS.BACKOFF_MAX_MS, 2, 'Persistent mind provider is not configured'],
['global capacity', 'waiting', PERSISTENT_MIND_LIMITS.BACKOFF_BASE_MS, 2, 'CoS agent capacity exhausted (1/1)'],
['budget error', 'degraded', PERSISTENT_MIND_LIMITS.BACKOFF_BASE_MS * 4, 3, 'Persistent mind budget check failed: unavailable'],
['budget exhausted', 'waiting', PERSISTENT_MIND_LIMITS.BACKOFF_MAX_MS, 2, 'CoS actions budget exhausted'],
['action reserved', 'waiting', PERSISTENT_MIND_LIMITS.BACKOFF_MAX_MS, 2, 'CoS actions budget exhausted'],
])('defers an unclaimed wake for %s with its original retry policy', async (gate, status, backoffMs, failureCount, reason) => {
vi.useFakeTimers();
const prepare = vi.fn();
const run = vi.fn();
if (gate !== 'no adapter') await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'deferred-message', text: 'Wait until admitted.' });
mock.root.persistentMind.failureCount = 2;
mock.root.persistentMind.lastError = 'previous error';
if (gate === 'global capacity') {
mock.root.config.maxConcurrentAgents = 1;
mock.root.agents = { running: { status: 'running' } };
}
if (gate === 'budget error') mock.budgetError = new Error('unavailable');
if (gate === 'budget exhausted') mock.budget = { withinBudget: false, exceeded: 'actions' };
if (gate === 'action reserved') {
mock.budget = { withinBudget: true, budget: { maxActionsPerDay: 1 }, usage: { actions: 0, ms: 0 } };
acquireCosActionReservation({ ...mock.budget, reservationId: 'ordinary-agent' });
}
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind).toMatchObject({
status, pauseReason: reason, failureCount, activeTurn: null,
lastError: status === 'degraded' ? reason : 'previous error',
nextEligibleWakeAt: new Date(Date.now() + backoffMs).toISOString(),
});
expect(mock.root.persistentMind.queuedMessages.map(({ id }) => id)).toEqual(['deferred-message']);
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBe(backoffMs);
expect(prepare).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
expect(mock.recordUsage).not.toHaveBeenCalled();
expect(mock.appendMindEvent.mock.calls.some(([event]) => event.kind === 'mind.wake')).toBe(false);
});
it('wakes immediately on demand and leaves an active turn intact', async () => {
const pending = deferred();
const run = vi.fn(() => pending.promise);
await supervisor.registerPersistentMindTurnAdapter({ prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model', effort: 'high' })), run });
await supervisor.startPersistentMind();
mock.root.persistentMind.selfWake.notBefore = new Date(Date.now() + 60_000).toISOString();
mock.root.persistentMind.nextEligibleWakeAt = new Date(Date.now() + 120_000).toISOString();
await supervisor.pausePersistentMind();
await supervisor.wakePersistentMind();
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBe(1);
const running = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
const activeTurn = mock.root.persistentMind.activeTurn;
expect(await supervisor.wakePersistentMind()).toMatchObject({ alreadyRunning: true });
expect(mock.root.persistentMind.activeTurn).toEqual(activeTurn);
pending.resolve({});
await running;
});
it('passes a local-window context bound from prepare through to context assembly', async () => {
const run = vi.fn(async () => ({}));
const prepare = vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model', contextMaxChars: 12_000 }));
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.startPersistentMind();
await supervisor.wakePersistentMind();
await supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
expect(mock.prepareContext).toHaveBeenCalledWith(expect.objectContaining({ maxChars: 12_000 }));
});
it('leaves the default context cap when prepare declares no bound', async () => {
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model' })), run });
await supervisor.startPersistentMind();
await supervisor.wakePersistentMind();
await supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
expect(mock.prepareContext.mock.calls.at(-1)[0]).not.toHaveProperty('maxChars');
});
it('starts and resumes for new messages without allowing retries to undo a later pause', async () => {
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model', effort: 'high' })), run });
await supervisor.enqueuePersistentMindMessage({ id: 'first', text: 'Wake up.' });
expect(mock.root.persistentMind).toMatchObject({ enabled: true, started: true, status: 'waiting' });
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBe(1);
await supervisor.drainPersistentMind();
await supervisor.pausePersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'first', text: 'Wake up.' });
expect(mock.root.persistentMind.status).toBe('paused');
mock.root.persistentMind.nextEligibleWakeAt = new Date(Date.now() + 60_000).toISOString();
await supervisor.enqueuePersistentMindMessage({ id: 'second', text: 'Continue.' });
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBe(1);
await supervisor.drainPersistentMind();
expect(run.mock.calls.map(([turn]) => turn.wake.message.id)).toEqual(['first', 'second']);
});
it('keeps explicit wakes behind the global CoS pause', async () => {
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model', effort: 'high' })), run });
mock.root.paused = true;
await supervisor.wakePersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'held', text: 'Wait for CoS.' });
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
expect(mock.root.persistentMind.queuedMessages).toHaveLength(1);
});
it('accepts a message durably, deduplicates retries, and runs only one turn', async () => {
const pending = deferred();
const prepare = vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' }, model: 'example-model', effort: 'high' }));
const run = vi.fn(() => pending.promise);
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Please review this.' }))
.toEqual({ success: true, duplicate: false, messageId: 'message-1' });
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Please review this.' }))
.toEqual({ success: true, duplicate: true, messageId: 'message-1' });
const firstDrain = supervisor.drainPersistentMind();
const secondDrain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
expect(run.mock.calls[0][0].wake).toMatchObject({ kind: 'message', message: { id: 'message-1' } });
pending.resolve({});
await Promise.all([firstDrain, secondDrain]);
expect(mock.prepareContext).toHaveBeenCalledWith(expect.objectContaining({
mindId: 'cos-persistent-mind',
providerId: 'example-cloud',
model: 'example-model',
}));
expect(run.mock.calls[0][0].context).toMatchObject({ text: 'bounded context', summaryState: 'not-needed' });
expect(mock.appendMindEvent.mock.calls.map(([event]) => event.kind)).toEqual(expect.arrayContaining([
'mind.message.accepted',
'mind.wake',
'mind.model.request',
'mind.model.result',
'mind.turn.completed',
]));
expect(prepare).toHaveBeenCalledTimes(1);
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({
profile: expect.objectContaining({ provider: expect.objectContaining({ id: 'example-cloud' }), model: 'example-model', effort: 'high' }),
}));
expect(mock.acquireSlot).toHaveBeenCalledTimes(1);
expect(mock.root.persistentMind.activeTurn).toBeNull();
expect(mock.root.persistentMind.recentMessageIds).toContain('message-1');
expect(mock.root.persistentMind.recentMessageFingerprints).toEqual([
{ id: 'message-1', fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/) },
]);
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Changed after completion.' }))
.toMatchObject({ success: false, code: 'IDEMPOTENCY_CONFLICT', status: 409 });
expect(mock.recordUsage).toHaveBeenCalledWith('cos', expect.objectContaining({ actions: 1 }));
});
it('caps model-requested follow-ups at the saved maximum quiet period', async () => {
mock.root.config.persistentMindProfile.wakeIntervalMinutes = 15;
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run: vi.fn(async () => ({
selfWake: {
reason: 'Check again later',
notBefore: new Date(Date.now() + 2 * 60 * 60_000).toISOString(),
},
})),
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.drainPersistentMind();
const { lastCompletedAt, selfWake } = mock.root.persistentMind;
expect(selfWake.reason).toBe('Check again later');
expect(selfWake.scheduleKind).toBe('requested');
expect(Date.parse(selfWake.notBefore) - Date.parse(lastCompletedAt)).toBe(15 * 60_000);
});
it('re-arms an idle automatic wake when its saved cadence changes', async () => {
const lastCompletedAt = new Date(Date.now() - 10 * 60_000).toISOString();
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'idle',
lastCompletedTurnId: 'turn-1',
lastCompletedAt,
selfWake: {
id: 'wake-old',
kind: 'self',
scheduleKind: 'quiet',
reason: 'maximum quiet period elapsed',
sourceTurnId: 'turn-1',
createdAt: lastCompletedAt,
notBefore: new Date(Date.parse(lastCompletedAt) + 30 * 60_000).toISOString(),
},
};
mock.root.config.persistentMindProfile.wakeIntervalMinutes = 60;
await supervisor.refreshPersistentMindWakeCadence();
expect(Date.parse(mock.root.persistentMind.selfWake.notBefore) - Date.parse(lastCompletedAt)).toBe(60 * 60_000);
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBeGreaterThan(40 * 60_000);
});
it('does not postpone a requested wake whose reason matches the automatic wake text', async () => {
const lastCompletedAt = new Date(Date.now() - 10 * 60_000).toISOString();
const requestedAt = new Date(Date.now() + 5 * 60_000).toISOString();
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'waiting',
lastCompletedTurnId: 'turn-1',
lastCompletedAt,
selfWake: {
id: 'wake-requested',
kind: 'self',
scheduleKind: 'requested',
reason: 'maximum quiet period elapsed',
sourceTurnId: 'turn-1',
createdAt: lastCompletedAt,
notBefore: requestedAt,
},
};
mock.root.config.persistentMindProfile.wakeIntervalMinutes = 60;
await supervisor.refreshPersistentMindWakeCadence();
expect(mock.root.persistentMind.selfWake.notBefore).toBe(requestedAt);
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID).delayMs).toBeLessThanOrEqual(5 * 60_000);
});
it('fails closed when a legacy completed message has no retry fingerprint', async () => {
mock.root.persistentMind.recentMessageIds = ['legacy-message'];
await expect(supervisor.enqueuePersistentMindMessage({
id: 'legacy-message',
text: 'A changed retry cannot be verified.',
})).resolves.toMatchObject({
success: false,
code: 'IDEMPOTENCY_CONFLICT',
status: 409,
});
});
it('rejects new image messages while a source transition is in progress', async () => {
mock.updateInProgress = true;
await expect(supervisor.enqueuePersistentMindMessage({
id: 'message-during-update',
images: ['attachment-example'],
})).resolves.toMatchObject({
success: false,
code: 'UPDATE_IN_PROGRESS',
status: 409,
});
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
});
it('pauses before adapter preparation when the pinned profile cannot resolve', async () => {
const prepare = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run: vi.fn() });
mock.profile = { ok: false, error: 'Pinned provider unavailable' };
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.drainPersistentMind();
expect(prepare).not.toHaveBeenCalled();
expect(mock.root.persistentMind).toMatchObject({ status: 'degraded', pauseReason: 'Pinned provider unavailable' });
});
it('recovers an orphaned image turn without losing images or duplicating acceptance', async () => {
const message = {
id: 'message-1',
text: 'Do not lose this image.',
images: [{
attachmentId: 'attachment-example',
filename: 'mind-attachment-example.png',
path: '/api/screenshots/mind-attachment-example.png',
originalName: 'diagram.png',
mimeType: 'image/png',
size: 128,
uploadedAt: new Date(1).toISOString(),
}],
createdAt: new Date(1).toISOString(),
};
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'thinking',
activeTurn: {
id: 'turn-orphan',
wake: { kind: 'message', message },
startedAt: new Date(1).toISOString(),
heartbeatAt: new Date(1).toISOString(),
providerId: 'example-cloud',
model: 'example-model',
effort: 'high',
},
};
const recovered = await supervisor.initializePersistentMindSupervisor();
expect(recovered.activeTurn).toBeNull();
expect(recovered.status).toBe('interrupted');
expect(recovered.queuedMessages).toEqual([message]);
expect(recovered.nextEligibleWakeAt).not.toBeNull();
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WATCHDOG_EVENT_ID)).toBe(true);
expect(mock.appendMindEvent).toHaveBeenCalledWith(expect.objectContaining({
kind: 'mind.failed',
turnId: 'turn-orphan',
data: expect.objectContaining({ status: 'interrupted' }),
}));
expect(mock.appendMindEvent.mock.calls.map(([event]) => event.kind))
.not.toContain('mind.message.accepted');
});
it('keeps the last wake failure separate from a later pause reason', async () => {
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
mock.root.persistentMind.lastError = 'Provider process was killed (SIGKILL)';
await supervisor.pausePersistentMind('Agent API pause: OOM safety');
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: 'Agent API pause: OOM safety',
lastError: 'Provider process was killed (SIGKILL)',
});
});
it('records pause, stop, and disable boundaries even without an active provider turn', async () => {
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.pausePersistentMind('Pause for inspection');
expect(mock.appendMindEvent).toHaveBeenLastCalledWith(expect.objectContaining({
kind: 'mind.paused',
data: expect.objectContaining({ status: 'paused', error: 'Pause for inspection' }),
}));
await supervisor.resumePersistentMind();
await supervisor.stopPersistentMind();
expect(mock.appendMindEvent).toHaveBeenLastCalledWith(expect.objectContaining({
kind: 'mind.paused',
data: expect.objectContaining({ status: 'idle', error: 'Persistent mind stopped' }),
}));
await supervisor.setPersistentMindEnabled(false);
expect(mock.appendMindEvent).toHaveBeenLastCalledWith(expect.objectContaining({
kind: 'mind.paused',
data: expect.objectContaining({ status: 'disabled', error: 'Persistent mind disabled' }),
}));
});
it('can wait for an aborted active turn to settle before cleanup starts', async () => {
const pending = deferred();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run: vi.fn(() => pending.promise),
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-cleanup', text: 'This turn will be interrupted.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(mock.root.persistentMind.activeTurn).not.toBeNull());
let stopped = false;
const stopping = supervisor.stopPersistentMind({ waitForTurn: true }).then(() => {
stopped = true;
});
await vi.waitFor(() => expect(mock.root.persistentMind.activeTurn).toBeNull());
expect(stopped).toBe(false);
pending.resolve({ events: [{ kind: 'mind.thought', id: 'late-result', data: {} }] });
await Promise.all([drain, stopping]);
expect(stopped).toBe(true);
expect(mock.root.persistentMind).toMatchObject({ started: false, status: 'idle' });
});
it('requeues a message and degrades visibly when the pinned provider is unavailable', async () => {
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: false, error: 'Pinned provider unavailable' })),
run: vi.fn(),
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Wait for the configured provider.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('degraded');
expect(mock.root.persistentMind.pauseReason).toBe('Pinned provider unavailable');
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
expect(mock.root.persistentMind.nextEligibleWakeAt).not.toBeNull();
expect(mock.acquireSlot).not.toHaveBeenCalled();
});
it('revalidates image capability before inference and retains the claimed message', async () => {
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: mock.profile.provider })),
run,
});
mock.root.persistentMind = {
...mock.root.persistentMind,
enabled: true,
started: true,
status: 'waiting',
queuedMessages: [{
id: 'message-image', text: '', createdAt: new Date().toISOString(),
images: [{ attachmentId: 'attachment-1', filename: 'mind-example.png', originalName: 'example.png', mimeType: 'image/png', size: 10, uploadedAt: new Date().toISOString() }],
}],
};
mock.imageCapability = { status: 'unsupported', reason: 'The pinned model is now text-only.' };
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.status).toBe('degraded');
expect(mock.root.persistentMind.pauseReason).toBe('The pinned model is now text-only.');
expect(mock.root.persistentMind.queuedMessages.map((message) => message.id)).toEqual(['message-image']);
});
it('holds queued work durably when the CoS budget is exhausted', async () => {
const prepare = vi.fn();
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Wait for tomorrow.' });
mock.budget = { withinBudget: false, exceeded: 'actions' };
await supervisor.drainPersistentMind();
expect(prepare).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
expect(mock.root.persistentMind.pauseReason).toBe('CoS actions budget exhausted');
expect(mock.root.persistentMind.nextEligibleWakeAt).not.toBeNull();
});
it('holds queued work durably when ordinary CoS agents fill the global capacity', async () => {
const prepare = vi.fn();
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Wait for a global slot.' });
mock.root.config.maxConcurrentAgents = 1;
mock.root.agents = { 'agent-1': { status: 'running' } };
await supervisor.drainPersistentMind();
expect(prepare).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
expect(mock.root.persistentMind.pauseReason).toBe('CoS agent capacity exhausted (1/1)');
expect(mock.root.persistentMind.nextEligibleWakeAt).not.toBeNull();
});
it('holds a shared global slot throughout provider preparation', async () => {
const prepared = deferred();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(() => prepared.promise),
run: vi.fn(async () => ({})),
});
mock.root.config.maxConcurrentAgents = 1;
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(mock.root.persistentMind.activeTurn).not.toBeNull());
expect(acquireCosGlobalSlot({ agents: {}, limit: 1, reservationId: 'ordinary-task' })).toMatchObject({ ok: false });
await supervisor.pausePersistentMind();
prepared.resolve({ ok: true, provider: { id: 'example-cloud' } });
await drain;
const released = acquireCosGlobalSlot({ agents: {}, limit: 1, reservationId: 'ordinary-task' });
expect(released.ok).toBe(true);
released.release();
});
it('releases the shared global slot early for a local-model turn, once its endpoint slot is held', async () => {
const running = deferred();
mock.localEndpointOfProvider.mockReturnValue('localhost:11434');
mock.profile = { ...mock.profile, provider: { id: 'local-ollama', type: 'api' } };
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'local-ollama', type: 'api' } })),
run: vi.fn(() => running.promise),
});
mock.root.config.maxConcurrentAgents = 1;
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Think locally.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(mock.root.persistentMind.activeTurn).not.toBeNull());
// The endpoint-scoped local slot now owns GPU serialization for this turn,
// so an unrelated agent spawn (e.g. agy, gemini) must be able to claim the
// generic global slot while local inference is still running.
await vi.waitFor(() => {
const probe = acquireCosGlobalSlot({ agents: {}, limit: 1, reservationId: 'ordinary-task' });
expect(probe.ok).toBe(true);
probe.release();
});
running.resolve({});
await drain;
});
it('does not claim a turn when another admission reserved the final daily action', async () => {
mock.budget = {
withinBudget: true,
exceeded: null,
budget: { maxActionsPerDay: 1, maxMinutesPerDay: null },
usage: { actions: 0, ms: 0 },
};
const existing = acquireCosActionReservation({
budget: mock.budget.budget,
usage: mock.budget.usage,
reservationId: 'ordinary-agent',
});
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.activeTurn).toBeNull();
expect(mock.root.persistentMind.pauseReason).toBe('CoS actions budget exhausted');
existing.release();
});
it('requeues a claimed message when the shared local endpoint has no slot', async () => {
const run = vi.fn();
mock.profile = {
ok: true,
provider: { id: 'example-local', endpoint: 'http://127.0.0.1:1234' },
model: 'example-model',
effort: 'high',
thinkingInterface: 'text',
};
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-local', endpoint: 'http://127.0.0.1:1234' } })),
run,
});
mock.acquireSlot.mockResolvedValueOnce({ ok: false, reason: 'local endpoint localhost:1234 is at capacity (1/1)' });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Wait for capacity.' });
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.prepareContext).not.toHaveBeenCalled();
expect(mock.root.persistentMind.status).toBe('waiting');
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
expect(mock.root.persistentMind.pauseReason).toContain('at capacity');
});
it('holds one local endpoint slot and one usage span across context summarization and the turn', async () => {
const contextReady = deferred();
const release = vi.fn();
mock.acquireSlot.mockResolvedValue({ ok: true, release });
mock.prepareContext.mockImplementationOnce(() => contextReady.promise);
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
summarize: vi.fn(async () => 'summary'),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(mock.prepareContext).toHaveBeenCalled());
expect(mock.acquireSlot).toHaveBeenCalledTimes(1);
expect(release).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
contextReady.resolve({ text: 'bounded context', chars: 15, summaryState: 'ready' });
await drain;
expect(run).toHaveBeenCalledTimes(1);
expect(release).toHaveBeenCalledTimes(1);
expect(mock.recordUsage).toHaveBeenCalledWith('cos', expect.objectContaining({ actions: 1 }));
});
it('does not schedule or run a wake while the CoS lifecycle gate is closed', async () => {
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
mock.scheduled.clear();
mock.root.paused = true;
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Wait for resume.' });
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
mock.root.paused = false;
mock.daemonRunning = false;
await supervisor.handlePersistentMindGlobalResume();
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
});
it('re-arms a fired one-shot only after the scheduler can finish cleaning it up', async () => {
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run: vi.fn(async () => ({})),
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const firstWake = mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID);
await firstWake.handler();
// Mirrors eventScheduler's one-shot cleanup immediately after await handler().
mock.scheduled.delete(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
await new Promise((resolve) => setImmediate(resolve));
const replacement = mock.scheduled.get(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID);
expect(replacement).toBeDefined();
expect(replacement).not.toBe(firstWake);
});
it('does not start a turn when cancellation lands while provider preparation is pending', async () => {
const prepared = deferred();
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(() => prepared.promise),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel before inference.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(mock.root.persistentMind.activeTurn).not.toBeNull());
await supervisor.pausePersistentMind();
prepared.resolve({ ok: true, provider: { id: 'example-cloud' } });
await drain;
expect(run).not.toHaveBeenCalled();
expect(mock.acquireSlot).not.toHaveBeenCalled();
expect(mock.root.persistentMind.status).toBe('paused');
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
});
// Each row pauses the turn while one awaited admission/run step is pending
// and names the step that must never start afterwards, so dropping the
// liveness re-check after that await fails exactly its own row.
it.each([
// A refusal decided after the pause would otherwise park over it.
['provider preparation', async (gate, prepare) => {
prepare.mockImplementationOnce(() => gate.promise.then(() => ({ ok: false, error: 'Provider went offline' })));
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel during preparation.' });
return { reached: prepare };
}],
['the image capability check', async (gate) => {
const { resolvePersistentMindImageCapability } = await import('./persistentMindImageCapability.js');
resolvePersistentMindImageCapability.mockImplementationOnce(() => gate.promise.then(() => mock.imageCapability));
mock.root.persistentMind.queuedMessages = [{
id: 'message-image', text: '', createdAt: new Date().toISOString(),
images: [{ attachmentId: 'attachment-1', filename: 'mind-example.png', originalName: 'example.png', mimeType: 'image/png', size: 10, uploadedAt: new Date().toISOString() }],
}];
return { reached: resolvePersistentMindImageCapability, skipped: mock.acquireSlot };
}],
['the local endpoint slot', async (gate) => {
mock.acquireSlot.mockImplementationOnce(() => gate.promise.then(() => ({ ok: true, release: vi.fn() })));
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel during slot wait.' });
return { reached: mock.acquireSlot, skipped: mock.prepareContext };
}],
['the temporary preset re-check', async (gate) => {
withDeepPreset();
const { resolvePersistentMindThinkingSession } = await import('./persistentMindProfile.js');
const resolveSession = resolvePersistentMindThinkingSession.getMockImplementation();
const recheck = vi.fn(() => gate.promise.then(() => mock.thinkingSession));
resolvePersistentMindThinkingSession.mockImplementation((input) => (
mock.acquireSlot.mock.calls.length > 0 ? recheck(input) : resolveSession(input)
));
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel during re-check.', thinkingPresetId: 'deep' });
return {
reached: recheck,
skipped: mock.prepareContext,
restore: () => resolvePersistentMindThinkingSession.mockImplementation(resolveSession),
};
}],
['context preparation', async (gate) => {
mock.prepareContext.mockImplementationOnce(() => gate.promise.then(() => ({ text: 'bounded context', chars: 15, summaryState: 'ready' })));
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel during context.' });
return { reached: mock.prepareContext };
}],
['the model request event', async (gate) => {
const append = mock.appendMindEvent.getMockImplementation();
mock.appendMindEvent.mockImplementation((event) => (
event.kind === 'mind.model.request' ? gate.promise.then(() => append(event)) : append(event)
));
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Cancel before inference.' });
return { reached: mock.appendMindEvent, requestEvents: 1, restore: () => mock.appendMindEvent.mockImplementation(append) };
}],
])('never reaches the provider when cancellation lands during %s', async (_step, arrange) => {
const gate = deferred();
const run = vi.fn(async () => ({}));
const prepare = vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const { reached, skipped = null, requestEvents = 0, restore = () => {} } = await arrange(gate, prepare);
try {
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(reached).toHaveBeenCalled());
if (requestEvents) {
await vi.waitFor(() => expect(mock.appendMindEvent).toHaveBeenCalledWith(expect.objectContaining({ kind: 'mind.model.request' })));
}
await supervisor.pausePersistentMind();
gate.resolve();
await drain;
} finally {
restore();
}
const kinds = mock.appendMindEvent.mock.calls.map(([event]) => event.kind);
expect(run).not.toHaveBeenCalled();
if (skipped) expect(skipped).not.toHaveBeenCalled();
expect(kinds.filter((kind) => kind === 'mind.model.request')).toHaveLength(requestEvents);
expect(kinds).not.toContain('mind.turn.completed');
expect(kinds).not.toContain('mind.failed');
expect(mock.root.persistentMind).toMatchObject({ status: 'paused', activeTurn: null, lastCompletedTurnId: null });
});
it('watchdog interruption aborts and requeues a stale turn without starting a second copy', async () => {
const run = vi.fn(({ signal }) => new Promise((resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
}));
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Watch this turn.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
mock.root.persistentMind.activeTurn.heartbeatAt = new Date(
Date.now() - PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS
).toISOString();
await expect(supervisor.checkPersistentMindWatchdog()).resolves.toEqual({ interrupted: true });
await drain;
expect(run).toHaveBeenCalledTimes(1);
expect(mock.root.persistentMind.activeTurn).toBeNull();
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
expect(mock.root.persistentMind.status).toBe('interrupted');
});
it.each([
['alive', () => Promise.resolve(new Response(JSON.stringify({ models: [{ name: 'example-model' }] }))), false],
['unloaded', () => Promise.resolve(new Response(JSON.stringify({ models: [] }))), true],
['gone', () => Promise.reject(new Error('ECONNREFUSED')), true],
])('watchdog with an Ollama call in flight and the runner %s', async (_label, psResponse, interrupts) => {
let callStarted = false;
const run = vi.fn(({ signal, callBoundary }) => callBoundary({ purpose: 'turn', round: 0 }, () => new Promise((resolve, reject) => {
callStarted = true;
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
})));
const probe = vi.fn(psResponse);
vi.stubGlobal('fetch', probe);
try {
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud', ollamaBacked: true, endpoint: 'http://127.0.0.1:11434/v1' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Think for a long time.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(callStarted).toBe(true));
// Far past the stale window: a long thought, not a suspend.
mock.root.persistentMind.activeTurn.heartbeatAt = new Date(
Date.now() - 10 * PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS
).toISOString();
const result = await supervisor.checkPersistentMindWatchdog();
expect(probe).toHaveBeenCalledWith('http://127.0.0.1:11434/api/ps', expect.anything());
if (interrupts) {
expect(result).toEqual({ interrupted: true });
await drain;
expect(mock.root.persistentMind.activeTurn).toBeNull();
} else {
expect(result).toEqual({ interrupted: false, inFlight: true });
expect(mock.root.persistentMind.activeTurn).not.toBeNull();
await supervisor.interruptPersistentMindTurn?.();
await supervisor.stopPersistentMind?.();
await drain.catch(() => {});
}
} finally {
vi.unstubAllGlobals();
}
});
it('watchdog restarts the stale window after a host suspend instead of killing the in-flight turn', async () => {
const run = vi.fn(({ signal }) => new Promise((resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
}));
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Survive a host suspend.' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
const beforeSuspend = Date.now();
await expect(supervisor.checkPersistentMindWatchdog(beforeSuspend)).resolves.toEqual({ interrupted: false });
// The host (and this process with it) froze for 20 minutes mid-call: the
// heartbeat is far past the stale window only because nothing ran.
const resumedAt = beforeSuspend + 20 * 60_000;
mock.root.persistentMind.activeTurn.heartbeatAt = new Date(beforeSuspend).toISOString();
await expect(supervisor.checkPersistentMindWatchdog(resumedAt)).resolves.toEqual({ interrupted: false, resumeGrace: true });
expect(mock.root.persistentMind.activeTurn).not.toBeNull();
// Still inside the grace window on the next regular tick.
await expect(supervisor.checkPersistentMindWatchdog(resumedAt + supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS))
.resolves.toEqual({ interrupted: false, resumeGrace: true });
// A turn that truly stops beating after the resume is still interrupted
// once a full stale window of running time has passed.
let tick = resumedAt;
while (tick < resumedAt + PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS - supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS) {
tick += supervisor.PERSISTENT_MIND_WATCHDOG_INTERVAL_MS;
await supervisor.checkPersistentMindWatchdog(tick);
}
await expect(supervisor.checkPersistentMindWatchdog(resumedAt + PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS))
.resolves.toEqual({ interrupted: true });
await drain;
expect(mock.root.persistentMind.activeTurn).toBeNull();
expect(mock.root.persistentMind.status).toBe('interrupted');
});
it('does not interrupt a replacement turn when a stale watchdog snapshot loses the race', async () => {
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'thinking',
activeTurn: {
id: 'turn-stale',
wake: { kind: 'self', id: 'wake-stale', reason: 'stale', sourceTurnId: 'turn-0', createdAt: new Date(1).toISOString(), notBefore: null },
startedAt: new Date(1).toISOString(),
heartbeatAt: new Date(1).toISOString(),
providerId: 'example-cloud',
model: null,
effort: null,
},
};
const originalWithStateLock = (await import('./cosState.js')).withStateLock;
let replaced = false;
originalWithStateLock.mockImplementationOnce(async (fn) => {
mock.root.persistentMind.activeTurn = {
...mock.root.persistentMind.activeTurn,
id: 'turn-replacement',
heartbeatAt: new Date().toISOString(),
};
replaced = true;
return fn();
});
await expect(supervisor.checkPersistentMindWatchdog()).resolves.toEqual({ interrupted: false });
expect(replaced).toBe(true);
expect(mock.root.persistentMind.activeTurn.id).toBe('turn-replacement');
expect(mock.root.persistentMind.status).toBe('thinking');
});
it('coalesces self-wakes and rejects a recursive wake without a completed source turn', async () => {
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'idle',
lastCompletedTurnId: 'turn-1',
};
expect(await supervisor.requestPersistentMindWake({ sourceTurnId: 'turn-0', reason: 'stale' }))
.toEqual({ success: false, error: 'Self-wake must reference the last completed turn' });
const first = await supervisor.requestPersistentMindWake({ sourceTurnId: 'turn-1', reason: 'first' });
const second = await supervisor.requestPersistentMindWake({ sourceTurnId: 'turn-1', reason: 'newest' });
expect(first.success).toBe(true);
expect(second.success).toBe(true);
expect(mock.root.persistentMind.selfWake.reason).toBe('newest');
});
const withDeepPreset = () => {
mock.root.config.persistentMindThinkingPresets = {
presets: [{ id: 'deep', label: 'Deep pass', providerId: 'example-alt', model: 'alt-model', effort: 'max' }],
};
};
// Adapters may only prepare the transport for the route they are handed.
const echoProfileAdapter = () => vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
async function approveSelfThinking() {
withDeepPreset();
mock.thinkingSession.provider = { id: 'example-alt', name: 'Example local', type: 'api', endpoint: 'http://localhost:1234/v1' };
const { approvePersistentMindThinkingPresets } = await import('./persistentMindThinkingRequests.js');
mock.root.config.persistentMindCapabilities = {
chooseThinkingPreset: true, thinkingPresetAllowlist: ['deep'],
thinkingPresetGrants: await approvePersistentMindThinkingPresets(mock.root.config, ['deep']),
};
}
it('borrows a self-selected preset only at the next self wake, refuses extension, and resumes the default', async () => {
await approveSelfThinking();
const { requestPersistentMindThinkingPreset } = await import('./persistentMindThinkingRequests.js');
const outcomes = [];
const run = vi.fn(async ({ turnId }) => {
outcomes.push(await requestPersistentMindThinkingPreset({ presetId: 'deep', reason: 'Try a focused pass' }, { turnId, requestId: `request-${turnId}` }));
return {};
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'default-message', text: 'Think about this.' });
await supervisor.drainPersistentMind();
expect(outcomes[0].ok).toBe(true);
expect(mock.root.persistentMind.thinkingRequests.pending).not.toBeNull();
mock.root.persistentMind.selfWake.notBefore = new Date(0).toISOString();
await supervisor.drainPersistentMind();
expect(outcomes[1]).toMatchObject({ ok: false, error: expect.stringContaining('active') });
expect(mock.root.persistentMind.thinkingRequests).toMatchObject({ pending: null, history: [{ outcome: 'completed' }] });
await supervisor.enqueuePersistentMindMessage({ id: 'ordinary', text: 'Continue.' });
await supervisor.drainPersistentMind();
expect(run.mock.calls.map(([input]) => input.provider.id)).toEqual(['example-cloud', 'example-alt', 'example-cloud']);
expect(mock.root.config.persistentMindProfile.model).toBe('example-model');
});
it('consumes failed self-selected admission and restart recovery without replaying the alternate', async () => {
await approveSelfThinking();
const { requestPersistentMindThinkingPreset } = await import('./persistentMindThinkingRequests.js');
const run = vi.fn(async ({ turnId }) => {
await requestPersistentMindThinkingPreset({ presetId: 'deep', reason: 'Try once' }, { turnId, requestId: 'one-shot' });
return {};
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'requesting', text: 'Think.' });
await supervisor.drainPersistentMind();
const accepted = structuredClone(mock.root.persistentMind.thinkingRequests.pending);
mock.root.persistentMind.selfWake.notBefore = new Date(0).toISOString();
mock.thinkingSession = { ok: false, error: 'Model unavailable' };
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
expect(mock.root.persistentMind.thinkingRequests).toMatchObject({ pending: null, history: [{ outcome: 'failed' }] });
expect(mock.root.persistentMind.selfWake?.thinkingRequest).toBeUndefined();
mock.root.persistentMind.activeTurn = { id: 'orphan', startedAt: new Date().toISOString(), wake: {
kind: 'self', id: 'wake-orphan', sourceTurnId: 'prior', reason: 'Try once', thinkingRequest: accepted,
} };
supervisor.__resetPersistentMindSupervisorForTests();
await supervisor.initializePersistentMindSupervisor();
expect(mock.root.persistentMind.activeTurn).toBeNull();
expect(mock.root.persistentMind.selfWake?.thinkingRequest).toBeUndefined();
expect(mock.root.persistentMind.thinkingRequests.history).toHaveLength(1);
});
it('runs one selected message on its preset and returns the very next turn to the default', async () => {
withDeepPreset();
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Deep pass please.', thinkingPresetId: 'deep' }))
.toEqual({ success: true, duplicate: false, messageId: 'message-1' });
await supervisor.enqueuePersistentMindMessage({ id: 'message-2', text: 'Back to normal.' });
await supervisor.drainPersistentMind();
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(2);
expect(run.mock.calls[0][0]).toMatchObject({ provider: { id: 'example-alt' }, model: 'alt-model', effort: 'max' });
expect(run.mock.calls[1][0]).toMatchObject({ provider: { id: 'example-cloud' }, model: 'example-model', effort: 'high' });
// The selection lives on the message, so the stored default is untouched.
expect(mock.root.config.persistentMindProfile).toMatchObject({ providerId: 'example-cloud', model: 'example-model' });
const modelRequests = mock.appendMindEvent.mock.calls
.map(([event]) => event)
.filter((event) => event.kind === 'mind.model.request');
expect(modelRequests.map((event) => event.data.thinkingPresetId)).toEqual(['deep', null]);
});
it('refuses a message naming a preset the user removed, and will not let a retry swap models', async () => {
withDeepPreset();
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run: vi.fn() });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Deep pass please.', thinkingPresetId: 'removed' }))
.toMatchObject({ success: false, code: 'THINKING_PRESET_UNAVAILABLE', status: 422 });
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
await supervisor.enqueuePersistentMindMessage({ id: 'message-2', text: 'Deep pass please.', thinkingPresetId: 'deep' });
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-2', text: 'Deep pass please.', thinkingPresetId: 'deep' }))
.toEqual({ success: true, duplicate: true, messageId: 'message-2' });
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-2', text: 'Deep pass please.' }))
.toMatchObject({ success: false, code: 'IDEMPOTENCY_CONFLICT', status: 409 });
});
it('degrades visibly rather than falling back when a temporary preset no longer resolves', async () => {
withDeepPreset();
mock.thinkingSession = { ok: false, error: 'Temporary thinking preset "Deep pass" model "alt-model" is not available from provider "example-alt"' };
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Deep pass please.', thinkingPresetId: 'deep' });
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.status).toBe('degraded');
expect(mock.root.persistentMind.pauseReason).toMatch(/Temporary thinking preset/);
// Nothing was spent and consent is intact; keep the accepted message while
// its configured provider/model becomes available.
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-1']);
});
it('pins acceptance through persistence and refuses a later preset edit without reviving a matching retry', async () => {
withDeepPreset();
mock.root.config.persistentMindThinkingPresets.presets[0].effort = '';
const selection = structuredClone(mock.root.config.persistentMindThinkingPresets.presets[0]);
mock.useActualThinkingSession = true;
mock.providerOverride = { id: 'example-alt', type: 'api', models: ['alt-model', 'other-model'] };
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.pausePersistentMind();
const input = { id: 'accepted-route', text: 'Use this exact route.', thinkingPresetId: 'deep', thinkingPreset: selection };
expect(await supervisor.enqueuePersistentMindMessage(input)).toMatchObject({ success: true, duplicate: false });
expect(mock.root.persistentMind.status).toBe('waiting');
await supervisor.pausePersistentMind();
expect(await supervisor.enqueuePersistentMindMessage(input)).toMatchObject({ success: true, duplicate: true });
expect(mock.root.persistentMind.status).toBe('paused');
expect(run).not.toHaveBeenCalled();
mock.root = JSON.parse(JSON.stringify(mock.root));
mock.root.persistentMind = normalizePersistentMindState(mock.root.persistentMind);
expect(mock.root.persistentMind.queuedMessages[0].thinkingPreset).toEqual(selection);
mock.root.config.persistentMindThinkingPresets.presets[0].model = 'other-model';
expect(await supervisor.enqueuePersistentMindMessage({ ...input, id: 'stale-display' }))
.toMatchObject({ success: false, code: 'THINKING_PRESET_CHANGED' });
expect(await supervisor.enqueuePersistentMindMessage({ ...input, thinkingPreset: { ...selection, model: 'other-model' } }))
.toMatchObject({ success: false, code: 'IDEMPOTENCY_CONFLICT' });
await supervisor.resumePersistentMind();
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind).toMatchObject({ status: 'degraded', queuedMessages: [] });
expect(mock.appendMindEvent).toHaveBeenCalledWith(expect.objectContaining({
kind: 'mind.failed', data: expect.objectContaining({ requiresResubmission: true, consumedAttempt: false }),
}));
mock.root.config.persistentMindThinkingPresets.presets = [];
expect(await supervisor.enqueuePersistentMindMessage(input)).toMatchObject({ success: true, duplicate: true });
// Old clients send only the id. Their retry reads the accepted receipt,
// rather than revalidating a preset that no longer exists.
const { thinkingPreset: _selection, ...legacyInput } = input;
expect(await supervisor.enqueuePersistentMindMessage(legacyInput)).toMatchObject({ success: true, duplicate: true });
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.config.persistentMindProfile).toMatchObject({ providerId: 'example-cloud', model: 'example-model', effort: 'high' });
});
it('lets an id-only client retry after completion and a label-only edit while retaining the accepted route', async () => {
withDeepPreset();
mock.root.config.persistentMindThinkingPresets.presets[0].effort = '';
mock.useActualThinkingSession = true;
mock.providerOverride = { id: 'example-alt', type: 'api', models: ['alt-model'] };
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const input = { id: 'id-only', text: 'Keep the accepted model.', thinkingPresetId: 'deep' };
await supervisor.enqueuePersistentMindMessage(input);
const accepted = structuredClone(mock.root.persistentMind.queuedMessages[0].thinkingPreset);
mock.root.config.persistentMindThinkingPresets.presets[0].label = 'Renamed';
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ model: 'alt-model', effort: null }));
expect(mock.root.persistentMind.recentMessageFingerprints[0].thinkingPreset).toEqual(accepted);
mock.root.config.persistentMindThinkingPresets.presets = [];
expect(await supervisor.enqueuePersistentMindMessage(input)).toMatchObject({ success: true, duplicate: true });
expect(await supervisor.enqueuePersistentMindMessage({ ...input, thinkingPreset: { ...accepted, effort: 'high' } }))
.toMatchObject({ success: false, code: 'IDEMPOTENCY_CONFLICT' });
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
});
it('retains legacy temporary input but refuses to derive a route from current config', async () => {
withDeepPreset();
mock.useActualThinkingSession = true;
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
mock.root.persistentMind.queuedMessages = [{ id: 'legacy', text: 'Saved before snapshots.', thinkingPresetId: 'deep' }];
await supervisor.drainPersistentMind();
expect(run).not.toHaveBeenCalled();
expect(mock.root.persistentMind.pauseReason).toMatch(/no valid accepted route/);
expect(mock.root.persistentMind.recentMessageIds).toContain('legacy');
});
it.each(['preparation', 'slot wait'])('refuses revocation during %s before summary or turn inference can begin', async (stage) => {
withDeepPreset();
mock.root.config.persistentMindThinkingPresets.presets[0].effort = '';
mock.useActualThinkingSession = true;
mock.providerOverride = { id: 'example-alt', type: 'api', models: ['alt-model'] };
const preparing = deferred();
const prepare = vi.fn(async ({ profile }) => {
if (stage === 'preparation') await preparing.promise;
return { ok: true, provider: profile.provider };
});
const release = vi.fn();
if (stage === 'slot wait') mock.acquireSlot.mockImplementation(async () => {
await preparing.promise;
return { ok: true, release };
});
const run = vi.fn();
const summarize = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({ prepare, run, summarize });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'revoke-during-prepare', text: 'Use the alternate.', thinkingPresetId: 'deep' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(stage === 'slot wait' ? mock.acquireSlot : prepare).toHaveBeenCalledTimes(1));
mock.root.config.persistentMindThinkingPresets.presets = [];
preparing.resolve();
await drain;
expect(mock.prepareContext).not.toHaveBeenCalled();
expect(summarize).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
expect(mock.recordUsage).not.toHaveBeenCalled();
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toContain('revoke-during-prepare');
expect(mock.root.persistentMind).toMatchObject({ status: 'degraded', activeTurn: null });
if (stage === 'slot wait') expect(release).toHaveBeenCalledTimes(1);
});
it.each(['before preparation', 'during preparation'])('keeps unspent temporary messages when a provider goes offline %s', async (stage) => {
withDeepPreset();
mock.root.config.persistentMindThinkingPresets.presets[0].effort = '';
mock.useActualThinkingSession = true;
mock.providerOverride = { id: 'example-alt', type: 'api', models: ['alt-model'] };
let outage = true;
const prepare = vi.fn(async ({ profile }) => {
if (outage && stage === 'during preparation') mock.providerAvailable = false;
return { ok: true, provider: profile.provider };
});
const run = vi.fn(async () => ({}));
await supervisor.registerPersistentMindTurnAdapter({ prepare, run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
const input = { id: 'temporary-outage', text: 'Keep this accepted request.', thinkingPresetId: 'deep' };
await supervisor.enqueuePersistentMindMessage(input);
const accepted = structuredClone(mock.root.persistentMind.queuedMessages[0]);
if (stage === 'before preparation') mock.providerAvailable = false;
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.queuedMessages).toEqual([accepted]);
expect(mock.root.persistentMind.recentMessageIds).not.toContain(input.id);
expect(mock.prepareContext).not.toHaveBeenCalled();
expect(run).not.toHaveBeenCalled();
expect(await supervisor.enqueuePersistentMindMessage(input)).toMatchObject({ success: true, duplicate: true });
outage = false;
mock.providerAvailable = true;
mock.root.persistentMind.nextEligibleWakeAt = null;
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ model: 'alt-model' }));
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
});
it('never auto-replays a temporary session interrupted after its provider span opened', async () => {
withDeepPreset();
const run = vi.fn(({ signal }) => new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
}));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Deep pass please.', thinkingPresetId: 'deep' });
const drain = supervisor.drainPersistentMind();
await vi.waitFor(() => expect(run).toHaveBeenCalledTimes(1));
mock.root.persistentMind.activeTurn.heartbeatAt = new Date(
Date.now() - PERSISTENT_MIND_LIMITS.WATCHDOG_STALE_MS
).toISOString();
await expect(supervisor.checkPersistentMindWatchdog()).resolves.toEqual({ interrupted: true });
await drain;
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toEqual(['message-1']);
// An idempotent client retry reads as a completed duplicate, so a second
// drain cannot repeat work the provider may already have billed.
expect(await supervisor.enqueuePersistentMindMessage({ id: 'message-1', text: 'Deep pass please.', thinkingPresetId: 'deep' }))
.toEqual({ success: true, duplicate: true, messageId: 'message-1' });
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
});
it('retires, never replays, a temporary session abandoned by a restart, a stop, or a disable', async () => {
withDeepPreset();
const temporaryTurn = (turnId, messageId) => ({
id: turnId,
wake: {
kind: 'message',
message: { id: messageId, text: 'Deep pass please.', thinkingPresetId: 'deep', createdAt: new Date().toISOString() },
},
startedAt: new Date().toISOString(),
heartbeatAt: new Date().toISOString(),
});
const startedWith = (turn) => {
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
enabled: true,
started: true,
status: 'thinking',
activeTurn: turn,
};
};
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run: vi.fn() });
// A hard crash leaves started:true on disk, so boot recovery — not the
// state normalizer — owns this one.
startedWith(temporaryTurn('mind-turn-crash', 'paid-crash'));
await supervisor.initializePersistentMindSupervisor();
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toEqual(['paid-crash']);
startedWith(temporaryTurn('mind-turn-stop', 'paid-stop'));
await supervisor.stopPersistentMind();
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toEqual(['paid-stop']);
startedWith(temporaryTurn('mind-turn-disable', 'paid-disable'));
await supervisor.setPersistentMindEnabled(false);
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toEqual(['paid-disable']);
// An ordinary message keeps its free automatic recovery on every path.
const ordinaryTurn = {
id: 'mind-turn-plain',
wake: { kind: 'message', message: { id: 'plain-1', text: 'Ordinary.', createdAt: new Date().toISOString() } },
startedAt: new Date().toISOString(),
heartbeatAt: new Date().toISOString(),
};
startedWith(ordinaryTurn);
await supervisor.stopPersistentMind();
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['plain-1']);
expect(mock.root.persistentMind.recentMessageIds).toEqual([]);
});
describe('per-call provider boundary', () => {
const mindCallReceipts = () => mock.appendMindEvent.mock.calls
.map(([event]) => event)
.filter((event) => event.kind === 'mind.model.call');
it('admits and accounts each provider call of a turn instead of the whole span', async () => {
mock.prepareContext.mockImplementationOnce(async ({ summarize }) => {
await summarize({ events: [], previousSummary: null });
return { text: 'bounded context', chars: 15, summaryState: 'ready' };
});
const summarize = vi.fn(async ({ callBoundary }) => {
await callBoundary({ purpose: 'summary' }, async ({ reportRunId }) => {
reportRunId('run-summary');
return { text: 'An earlier stretch of my life.', runId: 'run-summary' };
});
return 'An earlier stretch of my life.';
});
const run = vi.fn(async ({ callBoundary }) => {
for (const round of [0, 1]) {
await callBoundary({ purpose: round === 0 ? 'turn' : 'tool-round', round }, async ({ reportRunId }) => {
reportRunId(`run-${round}`);
return { text: '{}', runId: `run-${round}`, usage: { inputTokens: 10, outputTokens: 2 } };
});
}
return {};
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run, summarize });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-calls', text: 'Do the work.' });
await supervisor.drainPersistentMind();
expect(summarize).toHaveBeenCalledTimes(1);
// Three calls, three accounted actions — and no extra action for the turn
// itself, which would double-count the span the receipts already cover.
expect(mock.recordUsage).toHaveBeenCalledTimes(3);
expect(mock.recordUsage.mock.calls.every(([domain, delta]) => domain === 'cos' && delta.actions === 1)).toBe(true);
expect(mindCallReceipts().map((event) => event.data.purpose)).toEqual(['summary', 'turn', 'tool-round']);
expect(mindCallReceipts().map((event) => event.data.runId)).toEqual(['run-summary', 'run-0', 'run-1']);
expect(mindCallReceipts()[1].data).toMatchObject({
providerId: 'example-cloud',
model: 'example-model',
effort: 'high',
outcome: 'completed',
temporaryRoute: false,
});
expect(mindCallReceipts()[1].data.elapsedMs).toBeGreaterThanOrEqual(0);
expect(mindCallReceipts()[1].data.usage).toMatchObject({ state: 'reported', totalTokens: 12 });
expect(mock.root.persistentMind.status).toBe('idle');
});
it('denies a later round when the budget empties mid-turn and parks the turn as waiting', async () => {
const run = vi.fn(async ({ callBoundary }) => {
await callBoundary({ purpose: 'turn', round: 0 }, async () => ({ text: '{}', runId: 'run-0' }));
mock.budget = { withinBudget: false, exceeded: 'actions' };
await callBoundary({ purpose: 'tool-round', round: 1 }, async () => {
throw new Error('this second provider call must never start');
});
return {};
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-budget', text: 'Keep going.' });
await supervisor.drainPersistentMind();
expect(mock.recordUsage).toHaveBeenCalledTimes(1);
expect(mindCallReceipts().map((event) => event.data.outcome)).toEqual(['completed', 'denied']);
expect(mindCallReceipts()[1].data.reason).toBe('CoS actions budget exhausted');
expect(mock.root.persistentMind).toMatchObject({
status: 'waiting',
pauseReason: 'CoS actions budget exhausted',
});
});
it('completes a slow maintainer wake without a legacy runtime pause', async () => {
vi.useFakeTimers();
mock.root.config.persistentMindMaintainer = { enabled: true, inference: { maxCallMs: 1000 } };
const run = vi.fn(async ({ callBoundary, heartbeat }) => {
await callBoundary({ purpose: 'turn' }, async ({ timeoutMs }) => {
expect(timeoutMs).toBe(0);
await heartbeat();
await vi.advanceTimersByTimeAsync(3_600_000);
await heartbeat();
return { text: 'Finished after an hour' };
});
return { output: 'Finished after an hour' };
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind).toMatchObject({ status: 'idle', activeTurn: null, pauseReason: null });
expect(mock.inferenceLedger.day).toMatchObject({ calls: 1, reservedMs: 1000 });
expect(mock.recordUsage).toHaveBeenCalledWith('cos', { actions: 1, ms: 3_600_000 });
expect(mindCallReceipts()[0].data).toMatchObject({ outcome: 'completed', elapsedMs: 3_600_000 });
});
it('records a failed attempt and still leaves its route and elapsed time readable', async () => {
const run = vi.fn(async ({ callBoundary }) => callBoundary(
{ purpose: 'turn', round: 0 },
async ({ reportRunId }) => {
reportRunId('run-0');
throw new Error('provider stream ended without a response');
},
));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-failed', text: 'Try this.' });
await supervisor.drainPersistentMind();
expect(mock.recordUsage).toHaveBeenCalledTimes(1);
expect(mindCallReceipts()[0].data).toMatchObject({
outcome: 'failed',
runId: 'run-0',
providerId: 'example-cloud',
model: 'example-model',
reason: 'provider stream ended without a response',
});
expect(mindCallReceipts()[0].data.usage.state).toBe('unknown');
expect(mock.root.persistentMind.status).toBe('interrupted');
});
it('refuses the next call and retires the wake when a temporary preset is revoked mid-turn', async () => {
withDeepPreset();
const run = vi.fn(async ({ callBoundary }) => {
await callBoundary({ purpose: 'turn', round: 0 }, async () => ({ text: '{}', runId: 'run-0' }));
mock.thinkingSession = {
ok: false,
requiresResubmission: true,
error: 'Temporary thinking preset "Deep pass" is no longer available',
};
await callBoundary({ purpose: 'tool-round', round: 1 }, async () => {
throw new Error('this second provider call must never start');
});
return {};
});
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-revoked', text: 'Deep pass please.', thinkingPresetId: 'deep' });
await supervisor.drainPersistentMind();
const receipts = mindCallReceipts();
expect(receipts.map((event) => event.data.outcome)).toEqual(['completed', 'denied']);
expect(receipts[0].data).toMatchObject({
providerId: 'example-alt',
model: 'alt-model',
effort: 'max',
thinkingPresetId: 'deep',
temporaryRoute: true,
});
expect(mock.root.persistentMind.status).toBe('degraded');
// A spent temporary session is never replayed automatically.
expect(mock.root.persistentMind.queuedMessages).toEqual([]);
expect(mock.root.persistentMind.recentMessageIds).toContain('message-revoked');
});
});
describe('provider usage-limit autopause', () => {
const echoProfileAdapter = () => vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
it('autopauses on a hard Cursor usage-limit error instead of climbing failureCount', async () => {
const priorFailures = 2;
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
failureCount: priorFailures,
};
const run = vi.fn(async () => {
throw new Error("You've hit your usage limit Get Cursor Pro to keep going.");
});
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-quota', text: 'Think.' });
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: PROVIDER_USAGE_LIMIT_PAUSE_REASON,
lastError: PROVIDER_USAGE_LIMIT_PAUSE_REASON,
failureCount: priorFailures,
nextEligibleWakeAt: null,
activeTurn: null,
});
// Queued work is requeued, not burned; ordinary wakes stay cancelled while
// a usage-limit readiness probe is armed for auto-recovery.
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-quota']);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID)).toBe(true);
expect(mock.providerAvailable).toBe(false);
expect(mock.appendMindEvent.mock.calls.map(([event]) => event.kind)).toEqual(
expect.arrayContaining(['mind.paused']),
);
expect(mock.appendMindEvent.mock.calls.map(([event]) => event.kind)).not.toEqual(
expect.arrayContaining(['mind.failed']),
);
// A further drain must not claim another turn while paused.
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
});
it('keeps the interrupted + backoff path for transient rate limits', async () => {
const run = vi.fn(async () => {
throw Object.assign(new Error('API Error: 429 Too Many Requests'), {
category: 'rate-limit',
});
});
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-429', text: 'Retry me.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('interrupted');
expect(mock.root.persistentMind.pauseReason).toBe('API Error: 429 Too Many Requests');
expect(mock.root.persistentMind.failureCount).toBeGreaterThan(0);
expect(mock.root.persistentMind.nextEligibleWakeAt).not.toBeNull();
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-429']);
});
it('keeps the interrupted path for ordinary provider failures', async () => {
const run = vi.fn(async ({ callBoundary }) => callBoundary(
{ purpose: 'turn', round: 0 },
async ({ reportRunId }) => {
reportRunId('run-0');
throw new Error('provider stream ended without a response');
},
));
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-soft', text: 'Try this.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('interrupted');
expect(mock.root.persistentMind.pauseReason).not.toBe(PROVIDER_USAGE_LIMIT_PAUSE_REASON);
});
it('stays paused while the usage-limit probe still reports the provider limited', async () => {
const run = vi.fn(async () => {
throw new Error("You've hit your usage limit Get Cursor Pro to keep going.");
});
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-still-limited', text: 'Think.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('paused');
expect(mock.providerAvailable).toBe(false);
const probe = mock.scheduled.get(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID);
expect(probe).toBeTruthy();
await probe.handler();
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: PROVIDER_USAGE_LIMIT_PAUSE_REASON,
});
// Still limited → next probe armed with backoff; no ordinary wake.
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID)).toBe(true);
expect(mock.scheduled.get(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID).metadata.attempt).toBe(1);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
expect(run).toHaveBeenCalledTimes(1);
});
it('auto-resumes when the usage-limit probe says the pinned provider is usable again', async () => {
const run = vi.fn(async () => {
throw new Error("You've hit your usage limit Get Cursor Pro to keep going.");
});
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-recover', text: 'Think.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('paused');
mock.providerAvailable = true;
mock.profile = {
ok: true,
provider: { id: 'example-cloud', type: 'api' },
model: 'example-model',
effort: 'high',
thinkingInterface: 'text',
};
const probe = mock.scheduled.get(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID);
expect(probe).toBeTruthy();
const result = await supervisor.checkPersistentMindUsageLimitRecovery();
expect(result).toMatchObject({ recovered: true, ignored: false });
expect(mock.root.persistentMind).toMatchObject({
status: 'waiting',
pauseReason: null,
lastError: null,
});
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-recover']);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID)).toBe(false);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(true);
});
it('does not auto-resume pauses that are not usage-limit autopauses', async () => {
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.pausePersistentMind('Paused by user');
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: 'Paused by user',
});
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID)).toBe(false);
mock.providerAvailable = true;
const result = await supervisor.checkPersistentMindUsageLimitRecovery();
expect(result).toMatchObject({ recovered: false, ignored: true });
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: 'Paused by user',
});
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
});
});
describe('local context-budget autopause', () => {
const echoProfileAdapter = () => vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
it('autopauses on a known-context-below-request-budget error instead of climbing failureCount', async () => {
const priorFailures = 2;
mock.root.persistentMind = {
...createDefaultPersistentMindState(),
failureCount: priorFailures,
};
const budgetMessage = 'Ollama (qwen3:8b): known 12288-token context is below the 15104-token request budget';
const run = vi.fn(async () => {
throw Object.assign(new Error(budgetMessage), {
errorAnalysis: { category: 'context-length', message: budgetMessage },
});
});
await supervisor.registerPersistentMindTurnAdapter({
prepare: echoProfileAdapter(),
run,
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-budget', text: 'Think.' });
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
const expectedReason = contextBudgetPauseReasonFrom(budgetMessage);
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: expectedReason,
lastError: expectedReason,
failureCount: priorFailures,
nextEligibleWakeAt: null,
activeTurn: null,
});
expect(mock.root.persistentMind.queuedMessages.map((item) => item.id)).toEqual(['message-budget']);
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_WAKE_EVENT_ID)).toBe(false);
// No usage-limit probe — recovery is an operator numCtx / context change.
expect(mock.scheduled.has(supervisor.PERSISTENT_MIND_USAGE_LIMIT_PROBE_EVENT_ID)).toBe(false);
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
});
});
describe('refused provider call parking', () => {
const echoProfileAdapter = () => vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
const drainDeniedTurn = async (denial) => {
const run = vi.fn(async () => { throw denial; });
await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'message-denied', text: 'Think.' });
await supervisor.drainPersistentMind();
return run;
};
it('pauses a held refusal until an operator resumes it, with no backoff gate', async () => {
const run = await drainDeniedTurn(buildPersistentMindCallDenial({
reason: 'Maintenance hold is active', status: 'waiting', disposition: 'hold',
}));
expect(mock.root.persistentMind).toMatchObject({
status: 'paused',
pauseReason: 'Maintenance hold is active',
nextEligibleWakeAt: null,
activeTurn: null,
});
await supervisor.drainPersistentMind();
expect(run).toHaveBeenCalledTimes(1);
});
it('parks a reset-wait refusal under its own status and wakes at the reset time', async () => {
const retryAt = new Date(Date.now() + 3_600_000).toISOString();
await drainDeniedTurn(buildPersistentMindCallDenial({
reason: 'Daily allowance exhausted', status: 'waiting', disposition: 'reset-wait', retryAt,
}));
expect(mock.root.persistentMind).toMatchObject({
status: 'waiting',
pauseReason: 'Daily allowance exhausted',
nextEligibleWakeAt: retryAt,
activeTurn: null,
});
});
});
describe('usage-limit retry time', () => {
const PROBE_ID = 'cos-persistent-mind-usage-limit-probe';
beforeEach(() => mock.events.clear());
it('reports the pending probe as the retry time', () => {
const nextRunAt = Date.now() + 15 * 60_000;
mock.events.set(PROBE_ID, { id: PROBE_ID, active: true, nextRunAt });
expect(supervisor.persistentMindUsageLimitRetryAt()).toBe(new Date(nextRunAt).toISOString());
});
it('reports nothing rather than a retry that is not coming', () => {
expect(supervisor.persistentMindUsageLimitRetryAt()).toBeNull();
// A fired one-shot: the scheduler nulls its own next run.
mock.events.set(PROBE_ID, { id: PROBE_ID, active: true, nextRunAt: null });
expect(supervisor.persistentMindUsageLimitRetryAt()).toBeNull();
// Paused: it keeps a future time it will never honour.
mock.events.set(PROBE_ID, { id: PROBE_ID, active: false, nextRunAt: Date.now() + 60_000 });
expect(supervisor.persistentMindUsageLimitRetryAt()).toBeNull();
// Stale: scheduled before a clock jump, so the time already passed.
mock.events.set(PROBE_ID, { id: PROBE_ID, active: true, nextRunAt: Date.now() - 60_000 });
expect(supervisor.persistentMindUsageLimitRetryAt()).toBeNull();
});
});
describe('degraded park logging', () => {
let warnSpy;
beforeEach(() => {
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
});
afterEach(() => {
warnSpy.mockRestore();
});
it('scrubs credentials and truncates a 21K reason in the degraded park log', async () => {
const secret = 'sk-ant-api03-abcdefghijklmnopqrstuvwxyz123456';
const longReason = `Preparation failed with token ${secret} ` + 'x'.repeat(21_000);
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: false, error: longReason })),
run: vi.fn(),
});
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'msg-degraded', text: 'Degraded test' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('degraded');
const degradedCalls = warnSpy.mock.calls.filter(([msg]) =>
typeof msg === 'string' && msg.startsWith('⚠️ Persistent mind turn degraded:')
);
expect(degradedCalls).toHaveLength(1);
const [logged] = degradedCalls[0];
expect(logged).toContain('[REDACTED]');
expect(logged).not.toContain(secret);
expect(logged.length).toBe('⚠️ Persistent mind turn degraded: '.length + 300);
});
it('does not log when a turn is parked as waiting or paused', async () => {
// 1. Waiting park (local endpoint at capacity)
mock.localEndpointOfProvider.mockReturnValue('http://localhost:1234');
const run = vi.fn();
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: mock.profile.provider })),
run,
});
mock.acquireSlot.mockResolvedValueOnce({ ok: false, reason: 'local endpoint localhost:1234 is at capacity (1/1)' });
await supervisor.setPersistentMindEnabled(true);
await supervisor.startPersistentMind();
await supervisor.enqueuePersistentMindMessage({ id: 'msg-waiting', text: 'Wait for capacity.' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('waiting');
expect(warnSpy.mock.calls.filter(([msg]) =>
typeof msg === 'string' && msg.startsWith('⚠️ Persistent mind turn degraded:')
)).toHaveLength(0);
// 2. Paused park (provider usage limit refusal)
mock.acquireSlot.mockResolvedValue({ ok: true, release: vi.fn() });
await supervisor.registerPersistentMindTurnAdapter({
prepare: vi.fn(async () => ({ ok: true, provider: { id: 'example-cloud' } })),
run: vi.fn(async () => {
throw new Error("You've hit your usage limit Get Cursor Pro to keep going.");
}),
});
await supervisor.enqueuePersistentMindMessage({ id: 'msg-paused', text: 'Quota test' });
await supervisor.drainPersistentMind();
expect(mock.root.persistentMind.status).toBe('paused');
expect(warnSpy.mock.calls.filter(([msg]) =>
typeof msg === 'string' && msg.startsWith('⚠️ Persistent mind turn degraded:')
)).toHaveLength(0);
});
});
});