Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: make CoS stop wait for an in-flight start and roll back a failed start (#11860)
Adam Eivye1a941488570
diff --git a/server/services/cos.js b/server/services/cos.jsindex a023cfc22e..312f3eeb36 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -363,6 +363,20 @@ async function runStart() {setDaemonRunning(true);+ try {+ await bootDaemon(state);+ } catch (err) {+ // A failed start must not leave the flag reading "Already running" with+ // half the timers armed — tear down whatever was scheduled, then rethrow.+ console.error(`❌ CoS daemon start failed: ${err?.message || err}`);+ await teardownDaemon().catch(teardownErr =>+ console.error(`❌ CoS teardown after failed start failed: ${teardownErr?.message || teardownErr}`));+ throw err;+ }+ return { success: true };+}++async function bootDaemon(state) {// First clean up orphaned agents (agents marked running but no live process)const { cleanupOrphanedAgents } = await import('./agentManagement.js');const cleanedAgents = await cleanupOrphanedAgents();@@ -430,6 +444,7 @@ async function runStart() {type: 'interval',intervalMs: state.config.healthCheckIntervalMs,handler: async () => {+ if (!isDaemonRunning()) return;await runHealthCheck();const { runDevelopmentWatchdog } = await import('./developmentWatchdog.js');await runDevelopmentWatchdog().catch(err => console.error(`❌ Development watchdog failed: ${err.message}`));@@ -551,6 +566,7 @@ async function runStart() {type: 'interval',intervalMs: PENDING_MERGE_SWEEP_INTERVAL_MS,handler: async () => {+ if (!isDaemonRunning()) return;const s = await loadState();if (s.paused || getDomainMode(s.config, 'cos') !== 'execute') return;const prWatcher = await import('./prWatcher.js');@@ -596,18 +612,25 @@ async function runStart() {scheduleDequeue();}).catch(err => emitLog('warn', `Post-startup improvement queuing failed: ${err.message}`));}, POST_STARTUP_QUEUE_DELAY_MS);-- return { success: true };}/*** Stop the CoS daemon*/export async function stop() {+ // A Stop issued mid-start would otherwise see the flag already true, cancel+ // timers that don't exist yet, and then watch the resumed boot arm them all+ // (#11860). Let the start settle so Stop tears down a fully started daemon.+ if (daemonStartPromise) await daemonStartPromise.catch(() => {});if (!isDaemonRunning()) {return { success: false, error: 'Not running' };}+ await teardownDaemon();+ return { success: true };+}++async function teardownDaemon() {await shutdownPersistentMindSupervisor();unregisterPersistentMindTurnAdapter();@@ -629,7 +652,6 @@ export async function stop() {setDaemonRunning(false);cosEvents.emit('status', { running: false });- return { success: true };}/**diff --git a/server/services/cos.startStop.test.js b/server/services/cos.startStop.test.jsnew file mode 100644index 0000000000..2bccdc50a5--- /dev/null+++ b/server/services/cos.startStop.test.js@@ -0,0 +1,151 @@+/**+ * #11860 — the CoS daemon start/stop lifecycle. `start()` holds for the Creative+ * Director recovery gate (up to 60 s) after setting the running flag, so a Stop+ * landing in that window used to tear down timers that did not exist yet and+ * then watch the resumed boot arm every interval — including the PR merge sweep.+ *+ * Drives the real start()/stop() with the boot dependencies doubled and the+ * event scheduler replaced by a registry, so "what is still scheduled" is+ * observable. Lives in its own file because the module-wide mocks would+ * replace helpers cos.test.js exercises for real.+ */++import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';++const scheduled = new Set();+vi.mock('./eventScheduler.js', async (importOriginal) => ({+ ...(await importOriginal()),+ schedule: vi.fn(({ id }) => { scheduled.add(id); return id; }),+ cancel: vi.fn((id) => scheduled.delete(id)),+}));++let persisted = { running: false, paused: false, agents: {}, config: { healthCheckIntervalMs: 900000 } };+vi.mock('./cosState.js', async (importOriginal) => ({+ ...(await importOriginal()),+ ensureDirectories: vi.fn(async () => {}),+ loadState: vi.fn(async () => structuredClone(persisted)),+ saveState: vi.fn(async (s) => { persisted = structuredClone(s); }),+ withStateLock: vi.fn(async (fn) => fn()),+}));++const boot = { cleanupOrphanedAgents: vi.fn(async () => 0) };+vi.mock('./agentManagement.js', () => ({+ cleanupOrphanedAgents: (...args) => boot.cleanupOrphanedAgents(...args),+ handleOrphanedTask: vi.fn(async () => {}),+}));++let releaseRecovery;+vi.mock('./creativeDirector/recovery.js', () => ({+ get cdRecoveryDone() {+ return new Promise((resolve) => { releaseRecovery = resolve; });+ },+}));++vi.mock('./cosTaskStore.js', async (importOriginal) => ({+ ...(await importOriginal()),+ getAllTasks: vi.fn(async () => ({ user: { tasks: [] }, cos: { tasks: [] } })),+ getCosTasks: vi.fn(async () => ({ tasks: [] })),+}));+vi.mock('./instanceIdentity.js', async (importOriginal) => ({+ ...(await importOriginal()),+ ensureInstanceId: vi.fn(async () => 'instance-test'),+}));+vi.mock('./appActivity.js', async (importOriginal) => ({+ ...(await importOriginal()),+ clearStaleActiveAgents: vi.fn(async () => ({ cleared: [] })),+}));+vi.mock('./cosAgentArchive.js', async (importOriginal) => ({+ ...(await importOriginal()),+ archiveStaleAgents: vi.fn(async () => ({ archived: 0 })),+}));+vi.mock('./cosJobScheduler.js', async (importOriginal) => ({+ ...(await importOriginal()),+ registerJobSchedules: vi.fn(async () => { scheduled.add('job:test'); }),+ unregisterJobSchedules: vi.fn(async () => { scheduled.delete('job:test'); }),+ scheduleNextImprovementCheck: vi.fn(async () => { scheduled.add('cos-improvement-check'); }),+}));+vi.mock('./cosTaskGenerator.js', async (importOriginal) => ({+ ...(await importOriginal()),+ evaluateTasks: vi.fn(async () => {}),+}));+vi.mock('./cosHealthMonitor.js', async (importOriginal) => ({+ ...(await importOriginal()),+ runHealthCheck: vi.fn(async () => {}),+}));+vi.mock('./persistentMindAdapter.js', () => ({ createPersistentMindTurnAdapter: () => ({}) }));+vi.mock('./persistentMindSupervisor.js', async (importOriginal) => ({+ ...(await importOriginal()),+ initializePersistentMindSupervisor: vi.fn(async () => {}),+ shutdownPersistentMindSupervisor: vi.fn(async () => {}),+ registerPersistentMindTurnAdapter: vi.fn(async () => {}),+}));+vi.mock('./prWatcher.js', () => ({+ PENDING_MERGE_SWEEP_INTERVAL_MS: 1800000,+ sweepPendingMergePrs: vi.fn(async () => null),+}));++const { start, stop } = await import('./cos.js');+const { cosEvents } = await import('./cosEvents.js');+const { isDaemonRunning } = await import('./cosState.js');++const cosEventIds = () => [...scheduled].filter(id => id.startsWith('cos-') || id.startsWith('job:'));+const waitFor = async (predicate) => {+ for (let i = 0; i < 200 && !predicate(); i++) await new Promise(r => setImmediate(r));+ expect(predicate()).toBe(true);+};++let statuses;+const onStatus = (s) => statuses.push(s);++beforeEach(() => {+ // Fake only setTimeout: boot's post-start improvement queue (30 s) and the+ // recovery ceiling (60 s) would otherwise keep the worker alive past the run.+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });+ statuses = [];+ releaseRecovery = null;+ cosEvents.on('status', onStatus);+ vi.spyOn(console, 'error').mockImplementation(() => {});+});++afterEach(async () => {+ cosEvents.off('status', onStatus);+ if (isDaemonRunning()) await stop();+ vi.clearAllTimers();+ vi.useRealTimers();+ vi.restoreAllMocks();+ boot.cleanupOrphanedAgents = vi.fn(async () => 0);+});++describe('CoS daemon start/stop lifecycle (#11860)', () => {+ it('a Stop issued while start() is mid-boot leaves no CoS timer armed', async () => {+ const starting = start();+ await waitFor(() => typeof releaseRecovery === 'function');+ expect(isDaemonRunning()).toBe(true);++ const stopping = stop();+ releaseRecovery();+ await expect(starting).resolves.toEqual({ success: true });+ await expect(stopping).resolves.toEqual({ success: true });++ expect(cosEventIds()).toEqual([]);+ expect(isDaemonRunning()).toBe(false);+ expect(persisted.running).toBe(false);+ expect(statuses.at(-1)).toEqual({ running: false });+ });++ it('a boot step that throws rolls the daemon back so the next start() succeeds', async () => {+ boot.cleanupOrphanedAgents = vi.fn(async () => { throw new Error('agent state unreadable'); });+ await expect(start()).rejects.toThrow('agent state unreadable');+ expect(isDaemonRunning()).toBe(false);+ expect(persisted.running).toBe(false);+ expect(cosEventIds()).toEqual([]);++ boot.cleanupOrphanedAgents = vi.fn(async () => 0);+ const restarting = start();+ await waitFor(() => typeof releaseRecovery === 'function');+ releaseRecovery();+ await expect(restarting).resolves.toEqual({ success: true });+ expect(isDaemonRunning()).toBe(true);+ expect(cosEventIds()).toEqual(expect.arrayContaining(['cos-health-check', 'cos-pending-merge-sweep']));+ });+});diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex 22570328dc..fc98dde6e9 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -2555,13 +2555,6 @@ describe('pending-merge sweep — own timer, not the evaluation cadence (#3630)'expect(sweepBlock).toMatch(/s\.paused\s*\|\|\s*getDomainMode\(s\.config,\s*'cos'\)\s*!==\s*'execute'/);});- it('cancels the timer with the other CoS jobs on stop()', () => {- const stopFn = extractFnBody(COS_SRC, COS_SRC.indexOf('export async function stop'));- expect(stopFn).toMatch(/cancelEvent\('cos-health-check'\)/);- expect(stopFn).toMatch(/cancelEvent\('cos-recording-maintenance'\)/);- expect(stopFn).toMatch(/cancelEvent\('cos-pending-merge-sweep'\)/);- });-it('keeps the opportunistic drain in evaluateTasks so the call site cannot be deleted silently', () => {const evalFn = extractFnBody(GEN_SRC, GEN_SRC.indexOf('export async function evaluateTasks'));expect(evalFn).toMatch(/sweepPendingMergePrs\(\)/);