Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: make CoS stop wait for an in-flight start and roll back a failed start (#11860)

Adam Eivye1a941488570
diff --git a/server/services/cos.js b/server/services/cos.js
index a023cfc22e..312f3eeb36 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -363,6 +363,20 @@ async function runStart() {
setDaemonRunning(true);
+ try {
+ await bootDaemon(state);
+ } catch (err) {
+ // A failed start must not leave the flag reading "Already running" with
+ // half the timers armed — tear down whatever was scheduled, then rethrow.
+ console.error(`❌ CoS daemon start failed: ${err?.message || err}`);
+ await teardownDaemon().catch(teardownErr =>
+ console.error(`❌ CoS teardown after failed start failed: ${teardownErr?.message || teardownErr}`));
+ throw err;
+ }
+ return { success: true };
+}
+
+async function bootDaemon(state) {
// First clean up orphaned agents (agents marked running but no live process)
const { cleanupOrphanedAgents } = await import('./agentManagement.js');
const cleanedAgents = await cleanupOrphanedAgents();
@@ -430,6 +444,7 @@ async function runStart() {
type: 'interval',
intervalMs: state.config.healthCheckIntervalMs,
handler: async () => {
+ if (!isDaemonRunning()) return;
await runHealthCheck();
const { runDevelopmentWatchdog } = await import('./developmentWatchdog.js');
await runDevelopmentWatchdog().catch(err => console.error(`❌ Development watchdog failed: ${err.message}`));
@@ -551,6 +566,7 @@ async function runStart() {
type: 'interval',
intervalMs: PENDING_MERGE_SWEEP_INTERVAL_MS,
handler: async () => {
+ if (!isDaemonRunning()) return;
const s = await loadState();
if (s.paused || getDomainMode(s.config, 'cos') !== 'execute') return;
const prWatcher = await import('./prWatcher.js');
@@ -596,18 +612,25 @@ async function runStart() {
scheduleDequeue();
}).catch(err => emitLog('warn', `Post-startup improvement queuing failed: ${err.message}`));
}, POST_STARTUP_QUEUE_DELAY_MS);
-
- return { success: true };
}
/**
* Stop the CoS daemon
*/
export async function stop() {
+ // A Stop issued mid-start would otherwise see the flag already true, cancel
+ // timers that don't exist yet, and then watch the resumed boot arm them all
+ // (#11860). Let the start settle so Stop tears down a fully started daemon.
+ if (daemonStartPromise) await daemonStartPromise.catch(() => {});
if (!isDaemonRunning()) {
return { success: false, error: 'Not running' };
}
+ await teardownDaemon();
+ return { success: true };
+}
+
+async function teardownDaemon() {
await shutdownPersistentMindSupervisor();
unregisterPersistentMindTurnAdapter();
@@ -629,7 +652,6 @@ export async function stop() {
setDaemonRunning(false);
cosEvents.emit('status', { running: false });
- return { success: true };
}
/**
diff --git a/server/services/cos.startStop.test.js b/server/services/cos.startStop.test.js
new file mode 100644
index 0000000000..2bccdc50a5
--- /dev/null
+++ b/server/services/cos.startStop.test.js
@@ -0,0 +1,151 @@
+/**
+ * #11860 — the CoS daemon start/stop lifecycle. `start()` holds for the Creative
+ * Director recovery gate (up to 60 s) after setting the running flag, so a Stop
+ * landing in that window used to tear down timers that did not exist yet and
+ * then watch the resumed boot arm every interval — including the PR merge sweep.
+ *
+ * Drives the real start()/stop() with the boot dependencies doubled and the
+ * event scheduler replaced by a registry, so "what is still scheduled" is
+ * observable. Lives in its own file because the module-wide mocks would
+ * replace helpers cos.test.js exercises for real.
+ */
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+const scheduled = new Set();
+vi.mock('./eventScheduler.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ schedule: vi.fn(({ id }) => { scheduled.add(id); return id; }),
+ cancel: vi.fn((id) => scheduled.delete(id)),
+}));
+
+let persisted = { running: false, paused: false, agents: {}, config: { healthCheckIntervalMs: 900000 } };
+vi.mock('./cosState.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ ensureDirectories: vi.fn(async () => {}),
+ loadState: vi.fn(async () => structuredClone(persisted)),
+ saveState: vi.fn(async (s) => { persisted = structuredClone(s); }),
+ withStateLock: vi.fn(async (fn) => fn()),
+}));
+
+const boot = { cleanupOrphanedAgents: vi.fn(async () => 0) };
+vi.mock('./agentManagement.js', () => ({
+ cleanupOrphanedAgents: (...args) => boot.cleanupOrphanedAgents(...args),
+ handleOrphanedTask: vi.fn(async () => {}),
+}));
+
+let releaseRecovery;
+vi.mock('./creativeDirector/recovery.js', () => ({
+ get cdRecoveryDone() {
+ return new Promise((resolve) => { releaseRecovery = resolve; });
+ },
+}));
+
+vi.mock('./cosTaskStore.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ getAllTasks: vi.fn(async () => ({ user: { tasks: [] }, cos: { tasks: [] } })),
+ getCosTasks: vi.fn(async () => ({ tasks: [] })),
+}));
+vi.mock('./instanceIdentity.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ ensureInstanceId: vi.fn(async () => 'instance-test'),
+}));
+vi.mock('./appActivity.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ clearStaleActiveAgents: vi.fn(async () => ({ cleared: [] })),
+}));
+vi.mock('./cosAgentArchive.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ archiveStaleAgents: vi.fn(async () => ({ archived: 0 })),
+}));
+vi.mock('./cosJobScheduler.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ registerJobSchedules: vi.fn(async () => { scheduled.add('job:test'); }),
+ unregisterJobSchedules: vi.fn(async () => { scheduled.delete('job:test'); }),
+ scheduleNextImprovementCheck: vi.fn(async () => { scheduled.add('cos-improvement-check'); }),
+}));
+vi.mock('./cosTaskGenerator.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ evaluateTasks: vi.fn(async () => {}),
+}));
+vi.mock('./cosHealthMonitor.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ runHealthCheck: vi.fn(async () => {}),
+}));
+vi.mock('./persistentMindAdapter.js', () => ({ createPersistentMindTurnAdapter: () => ({}) }));
+vi.mock('./persistentMindSupervisor.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ initializePersistentMindSupervisor: vi.fn(async () => {}),
+ shutdownPersistentMindSupervisor: vi.fn(async () => {}),
+ registerPersistentMindTurnAdapter: vi.fn(async () => {}),
+}));
+vi.mock('./prWatcher.js', () => ({
+ PENDING_MERGE_SWEEP_INTERVAL_MS: 1800000,
+ sweepPendingMergePrs: vi.fn(async () => null),
+}));
+
+const { start, stop } = await import('./cos.js');
+const { cosEvents } = await import('./cosEvents.js');
+const { isDaemonRunning } = await import('./cosState.js');
+
+const cosEventIds = () => [...scheduled].filter(id => id.startsWith('cos-') || id.startsWith('job:'));
+const waitFor = async (predicate) => {
+ for (let i = 0; i < 200 && !predicate(); i++) await new Promise(r => setImmediate(r));
+ expect(predicate()).toBe(true);
+};
+
+let statuses;
+const onStatus = (s) => statuses.push(s);
+
+beforeEach(() => {
+ // Fake only setTimeout: boot's post-start improvement queue (30 s) and the
+ // recovery ceiling (60 s) would otherwise keep the worker alive past the run.
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ statuses = [];
+ releaseRecovery = null;
+ cosEvents.on('status', onStatus);
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+});
+
+afterEach(async () => {
+ cosEvents.off('status', onStatus);
+ if (isDaemonRunning()) await stop();
+ vi.clearAllTimers();
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ boot.cleanupOrphanedAgents = vi.fn(async () => 0);
+});
+
+describe('CoS daemon start/stop lifecycle (#11860)', () => {
+ it('a Stop issued while start() is mid-boot leaves no CoS timer armed', async () => {
+ const starting = start();
+ await waitFor(() => typeof releaseRecovery === 'function');
+ expect(isDaemonRunning()).toBe(true);
+
+ const stopping = stop();
+ releaseRecovery();
+ await expect(starting).resolves.toEqual({ success: true });
+ await expect(stopping).resolves.toEqual({ success: true });
+
+ expect(cosEventIds()).toEqual([]);
+ expect(isDaemonRunning()).toBe(false);
+ expect(persisted.running).toBe(false);
+ expect(statuses.at(-1)).toEqual({ running: false });
+ });
+
+ it('a boot step that throws rolls the daemon back so the next start() succeeds', async () => {
+ boot.cleanupOrphanedAgents = vi.fn(async () => { throw new Error('agent state unreadable'); });
+ await expect(start()).rejects.toThrow('agent state unreadable');
+ expect(isDaemonRunning()).toBe(false);
+ expect(persisted.running).toBe(false);
+ expect(cosEventIds()).toEqual([]);
+
+ boot.cleanupOrphanedAgents = vi.fn(async () => 0);
+ const restarting = start();
+ await waitFor(() => typeof releaseRecovery === 'function');
+ releaseRecovery();
+ await expect(restarting).resolves.toEqual({ success: true });
+ expect(isDaemonRunning()).toBe(true);
+ expect(cosEventIds()).toEqual(expect.arrayContaining(['cos-health-check', 'cos-pending-merge-sweep']));
+ });
+});
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index 22570328dc..fc98dde6e9 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -2555,13 +2555,6 @@ describe('pending-merge sweep — own timer, not the evaluation cadence (#3630)'
expect(sweepBlock).toMatch(/s\.paused\s*\|\|\s*getDomainMode\(s\.config,\s*'cos'\)\s*!==\s*'execute'/);
});
- it('cancels the timer with the other CoS jobs on stop()', () => {
- const stopFn = extractFnBody(COS_SRC, COS_SRC.indexOf('export async function stop'));
- expect(stopFn).toMatch(/cancelEvent\('cos-health-check'\)/);
- expect(stopFn).toMatch(/cancelEvent\('cos-recording-maintenance'\)/);
- expect(stopFn).toMatch(/cancelEvent\('cos-pending-merge-sweep'\)/);
- });
-
it('keeps the opportunistic drain in evaluateTasks so the call site cannot be deleted silently', () => {
const evalFn = extractFnBody(GEN_SRC, GEN_SRC.indexOf('export async function evaluateTasks'));
expect(evalFn).toMatch(/sweepPendingMergePrs\(\)/);