Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): keep rejected state changes out of later saves (#12185)
Adam Eivyfb19cff8458b
diff --git a/server/services/cosState.failedWrite.test.js b/server/services/cosState.failedWrite.test.jsnew file mode 100644index 0000000000..afa2348b76--- /dev/null+++ b/server/services/cosState.failedWrite.test.js@@ -0,0 +1,209 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';++const { tempRoot, makeProxy, spies, cleanup } = mockPathsDataRoot({+ prefix: 'cos-failed-write-', wrapExports: ['atomicWrite', 'ensureDirs', 'readJSONFileStrict'], makeSpy: vi.fn,+});+vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));++const cosDir = join(tempRoot, 'cos');+const statePath = join(cosDir, 'state.json');+const configPath = join(cosDir, 'config.json');+const binding = { branch: 'claim/issue-example', repository: 'github.com/example/project' };+const writeJson = (path, value) => writeFileSync(path, JSON.stringify(value));+const readJson = path => JSON.parse(readFileSync(path, 'utf8'));+const writeFailure = () => Object.assign(new Error('Synthetic persistence failure'), { code: 'EIO' });+let store;+let realWrite;++beforeEach(async () => {+ vi.resetModules();+ rmSync(cosDir, { force: true, recursive: true });+ mkdirSync(cosDir, { recursive: true });+ writeJson(configPath, { maxConcurrentAgents: 3 });+ writeJson(statePath, { agents: { example: { id: 'example', status: 'running', metadata: { claimBranches: [] } } } });+ store = await import('./cosState.js');+ const actual = await vi.importActual('../lib/fileUtils.js');+ realWrite = actual.atomicWrite;+ for (const [name, spy] of Object.entries(spies)) spy.mockReset().mockImplementation(actual[name]);+});+afterAll(cleanup);++async function saveHealthCheck() {+ await store.withStateLock(async () => {+ const state = await store.loadState();+ state.lastHealthCheck = '2026-01-01T00:00:00.000Z';+ await store.saveState(state);+ });+}++describe('CoS failed persistence recovery', () => {+ it.each(['bind', 'release'])('does not carry a rejected %s into the next queued write or a stale retry', async action => {+ const durable = action === 'bind' ? [] : [binding];+ writeJson(statePath, { agents: { example: { status: 'running', metadata: { claimBranches: durable } } } });+ let rejectedState;+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());+ await expect(store.withStateLock(async () => {+ const state = await store.loadState();+ rejectedState = { ...state, agents: { ...state.agents, example: {+ ...state.agents.example, metadata: { claimBranches: action === 'bind' ? [binding] : [] },+ } } };+ await store.saveState(rejectedState);+ })).rejects.toMatchObject({ code: 'EIO' });+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual(durable);+ await saveHealthCheck();+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual(durable);+ expect(readJson(statePath).lastHealthCheck).toBe('2026-01-01T00:00:00.000Z');+ await expect(store.saveState({ ...rejectedState })).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });+ });++ it('discards an existing agent lifecycle caller\'s in-place mutation after rejection', async () => {+ const { updateAgent } = await import('./cosAgentLifecycle.js');+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());+ await expect(updateAgent('example', { metadata: { claimBranches: [binding] } })).rejects.toMatchObject({ code: 'EIO' });+ await updateAgent('example', { status: 'paused' });+ expect(readJson(statePath).agents.example).toMatchObject({ status: 'paused', metadata: { claimBranches: [] } });+ });++ it('reloads rejected in-place config changes and re-anchors a live runtime read independently', async () => {+ const state = await store.loadState();+ const failedConfig = state.config;+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());+ await expect(store.withConfigLock(async () => {+ failedConfig.maxConcurrentAgents = 12;+ await store.saveConfig(failedConfig);+ })).rejects.toMatchObject({ code: 'EIO' });+ // An already-started runtime transaction remains valid after a config error.+ await store.withStateLock(async () => {+ state.paused = true;+ await store.saveState(state);+ });+ expect(state.config.maxConcurrentAgents).toBe(3);+ expect(readJson(statePath)).toMatchObject({ paused: true });+ expect(readJson(statePath)).not.toHaveProperty('config');+ await store.withConfigLock(async () => {+ const config = await store.loadConfig();+ expect((await store.loadState()).config).toBe(config);+ await store.saveConfig({ ...config, maxTotalProcesses: 20 });+ });+ expect(readJson(configPath)).toMatchObject({ maxConcurrentAgents: 3, maxTotalProcesses: 20 });+ await expect(store.saveConfig({ ...failedConfig })).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });+ });++ it.each(['state', 'config'])('reads the disk outcome when a %s write succeeds before reporting failure', async kind => {+ const load = kind === 'state' ? store.loadState : store.loadConfig;+ const save = kind === 'state' ? store.saveState : store.saveConfig;+ const queue = kind === 'state' ? store.withStateLock : store.withConfigLock;+ const path = kind === 'state' ? statePath : configPath;+ const value = await load();+ value.exampleCommitted = true;+ spies.atomicWrite.mockImplementationOnce(async (...args) => {+ await realWrite(...args);+ throw writeFailure();+ });+ await expect(queue(() => save(value))).rejects.toMatchObject({ code: 'EIO' });+ await queue(async () => {+ const current = await load();+ expect(current.exampleCommitted).toBe(true);+ current.exampleNextWrite = true;+ await save(current);+ });+ expect(readJson(path)).toMatchObject({ exampleCommitted: true, exampleNextWrite: true });+ });++ it.each(['state', 'config'])('refuses to default unreadable %s after a failed write until repaired', async kind => {+ const load = kind === 'state' ? store.loadState : store.loadConfig;+ const save = kind === 'state' ? store.saveState : store.saveConfig;+ const path = kind === 'state' ? statePath : configPath;+ const value = await load();+ spies.atomicWrite.mockImplementationOnce(async () => {+ writeFileSync(path, 'unreadable');+ throw writeFailure();+ });+ await expect(save(value)).rejects.toMatchObject({ code: 'EIO' });+ await expect(load()).rejects.toMatchObject({ code: 'COS_PERSISTENCE_RECOVERY_REQUIRED' });+ await expect(save({})).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });+ expect(readFileSync(path, 'utf8')).toBe('unreadable');+ writeJson(path, { repaired: true });+ await save(await load());+ expect(readJson(path).repaired).toBe(true);+ });++ it('retries a rejected legacy config split before runtime writes can remove the legacy settings', async () => {+ rmSync(configPath);+ writeJson(statePath, { agents: {}, config: { maxConcurrentAgents: 9 } });+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());+ await expect(store.loadState()).rejects.toMatchObject({ code: 'EIO' });+ expect(readJson(statePath).config.maxConcurrentAgents).toBe(9);+ await saveHealthCheck();+ expect(readJson(configPath).maxConcurrentAgents).toBe(9);+ expect(readJson(statePath)).not.toHaveProperty('config');+ });++ it('shares a pending legacy split so a late loader cannot overwrite the next config update', async () => {+ rmSync(configPath);+ writeJson(statePath, { agents: {}, config: { maxConcurrentAgents: 9 } });+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());+ await expect(store.saveConfig({ maxConcurrentAgents: 9 })).rejects.toMatchObject({ code: 'EIO' });+ // Keep the interleaving deterministic: only the first write waits; all+ // other file work is synchronous over these temporary synthetic files.+ spies.ensureDirs.mockResolvedValue(undefined);+ spies.readJSONFileStrict.mockImplementation(async (path, fallback) => ({+ ok: true, value: existsSync(path) ? readJson(path) : fallback,+ }));+ let releaseWrite;+ let signalStarted;+ const gate = new Promise(resolve => { releaseWrite = resolve; });+ const started = new Promise(resolve => { signalStarted = resolve; });+ let first = true;+ spies.atomicWrite.mockImplementation(async (path, value) => {+ if (first) {+ first = false;+ signalStarted();+ await gate;+ }+ writeJson(path, value);+ });+ const initial = store.loadConfig();+ await started;+ const update = store.withConfigLock(async () => {+ await store.saveConfig({ ...(await store.loadConfig()), maxConcurrentAgents: 5 });+ });+ // Drain the synchronous I/O doubles' promise continuations before allowing+ // the original split to finish, reproducing a late legacy overwrite.+ await new Promise(resolve => setImmediate(resolve));+ releaseWrite();+ await Promise.all([initial, update]);+ expect(readJson(configPath).maxConcurrentAgents).toBe(5);+ expect((await store.loadState()).config.maxConcurrentAgents).toBe(5);+ });++ it('keeps the state queue until the rejected write settles while config can progress', async () => {+ let rejectWrite;+ let signalStarted;+ const started = new Promise(resolve => { signalStarted = resolve; });+ spies.atomicWrite.mockImplementationOnce(() => new Promise((resolve, reject) => {+ rejectWrite = reject;+ signalStarted();+ }));+ const failed = store.withStateLock(async () => {+ const state = await store.loadState();+ state.agents.example.metadata.claimBranches.push(binding);+ await store.saveState(state);+ });+ const failureAssertion = expect(failed).rejects.toMatchObject({ code: 'EIO' });+ await started;+ const next = saveHealthCheck();+ await store.withConfigLock(async () => {+ await store.saveConfig({ ...(await store.loadConfig()), maxConcurrentAgents: 5 });+ });+ expect(readJson(statePath)).not.toHaveProperty('lastHealthCheck');+ rejectWrite(writeFailure());+ await failureAssertion;+ await next;+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual([]);+ expect((await store.loadState()).config.maxConcurrentAgents).toBe(5);+ });+});diff --git a/server/services/cosState.js b/server/services/cosState.jsindex 9b7646d1cb..5a57e505af 100644--- a/server/services/cosState.js+++ b/server/services/cosState.js@@ -269,20 +269,48 @@ async function recoverFromUnreadableState(content) {// In-memory config cache. Every mutation goes through `withConfigLock` +// `saveConfig`, so the cache stays consistent.let configCache = null;+let configLoad = null;// Enumerable symbols survive a caller's shallow copy but never reach JSON.// Reject delayed writeback of a pre-restore read, even if queued afterwards.const RESTORE_GENERATION = Symbol('cosRestoreGeneration');+const WRITE_GENERATION = Symbol('cosWriteGeneration');+const stateRecovery = { generation: 0, required: false };+const configRecovery = { generation: 0, required: false };let restoreGeneration = 0;-function stampGeneration(value) {+function stampGeneration(value, recovery) {value[RESTORE_GENERATION] = restoreGeneration;+ value[WRITE_GENERATION] = recovery.generation;return value;}-function assertCurrentGeneration(value) {+function assertCurrentGeneration(value, recovery) {if (value[RESTORE_GENERATION] !== undefined && value[RESTORE_GENERATION] !== restoreGeneration) {throw new ServerError('CoS files were restored. Read the current state before retrying this change.', {status: 409, code: 'COS_RESTORE_STALE_WRITE',});}+ if (recovery.required || (value[WRITE_GENERATION] !== undefined && value[WRITE_GENERATION] !== recovery.generation)) {+ throw new ServerError('A CoS write failed. Read the current persisted state before retrying this change.', {+ status: 409, code: 'COS_PERSISTENCE_STALE_WRITE',+ });+ }+}++function invalidateFailedWrite(recovery) {+ recovery.generation += 1;+ recovery.required = true;+}++// After an uncertain write, never replace unreadable ownership/settings with+// defaults. Only a fresh trusted read may clear the recovery requirement.+async function readRecoveryFile(filePath) {+ const missing = Symbol('missing');+ const { ok, value } = await readJSONFileStrict(filePath, missing, { allowArray: false, logError: true });+ if (!ok || (value !== missing && !isPlainObject(value))) {+ throw new ServerError('CoS persistence recovery requires a readable state/config file. Repair the file before retrying.', {+ status: 503, code: 'COS_PERSISTENCE_RECOVERY_REQUIRED',+ });+ }+ return value === missing ? null : value;}/**@@ -300,7 +328,9 @@ function assertCurrentGeneration(value) {* never boots on bare defaults.*/async function readPersistedConfig() {- const { ok, value } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false, logError: true });+ const { ok, value } = configRecovery.required+ ? { ok: true, value: await readRecoveryFile(CONFIG_FILE) }+ : await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false, logError: true });if (ok && isPlainObject(value)) return { config: value, fromConfigFile: true };if (!ok) {console.log(`⚠️ Corrupted or empty config file at ${CONFIG_FILE}, falling back to legacy/default config`);@@ -309,9 +339,10 @@ async function readPersistedConfig() {}// Back-compat read. Deliberately does NOT quarantine or rewrite state.json —// loadState() owns that file's recovery.- if (!existsSync(STATE_FILE)) return { config: null, fromConfigFile: false };- const stateContent = await readFile(STATE_FILE, 'utf-8');- const state = parseStateFile(stateContent, 'CoS state (legacy config)');+ if (!configRecovery.required && !existsSync(STATE_FILE)) return { config: null, fromConfigFile: false };+ const state = configRecovery.required+ ? await readRecoveryFile(STATE_FILE)+ : parseStateFile(await readFile(STATE_FILE, 'utf-8'), 'CoS state (legacy config)');const legacy = state?.config;return { config: isPlainObject(legacy) ? legacy : null, fromConfigFile: false };}@@ -321,9 +352,25 @@ async function readPersistedConfig() {*/export async function loadConfig() {if (configCache) return configCache;+ // The legacy split writes during a read. Share its settlement so another+ // loader cannot overwrite a subsequent settings change with the legacy copy.+ if (configLoad) return configLoad;+ configLoad = readConfig();+ try {+ return await configLoad;+ } finally {+ configLoad = null;+ }+}++async function readConfig() {+ if (configCache) return configCache;+ const generation = configRecovery.generation;+ const restored = restoreGeneration;await ensureDirectories();const { config, fromConfigFile } = await readPersistedConfig();- configCache = stampGeneration(mergeStoredConfig(config));+ if (configCache || generation !== configRecovery.generation || restored !== restoreGeneration) return readConfig();+ const loaded = stampGeneration(mergeStoredConfig(config), configRecovery);// Recovering settings out of a legacy state.json is a READ; nothing has// written them to their own file yet. `saveState` strips `config` from// state.json on the very next runtime write, so leaving the recovery in@@ -332,8 +379,18 @@ export async function loadConfig() {// restored pre-split backup on a machine that already recorded it as applied).if (config && !fromConfigFile) {console.log(`📦 Recovered CoS config from a legacy state.json — writing ${CONFIG_FILE}`);- await atomicWrite(CONFIG_FILE, configCache);+ try {+ await atomicWrite(CONFIG_FILE, loaded);+ } catch (error) {+ configCache = null;+ invalidateFailedWrite(configRecovery);+ throw error;+ }}+ if (configCache || generation !== configRecovery.generation || restored !== restoreGeneration) return readConfig();+ configCache = loaded;+ configRecovery.required = false;+ if (stateCache) stateCache.config = configCache;return configCache;}@@ -343,12 +400,21 @@ export async function loadConfig() {* before the write does not go stale.*/export async function saveConfig(config) {- await ensureDirectories();- assertCurrentGeneration(config);- configCache = stampGeneration(config);- if (stateCache) stateCache.config = config;- await atomicWrite(CONFIG_FILE, config);- return config;+ assertCurrentGeneration(config, configRecovery);+ stampGeneration(config, configRecovery);+ try {+ await ensureDirectories();+ await atomicWrite(CONFIG_FILE, config);+ } catch (error) {+ // Callers may have mutated the shared object in place; a prior pointer is+ // not a rollback image. The write may also have reached disk before failing.+ configCache = null;+ invalidateFailedWrite(configRecovery);+ throw error;+ }+ configCache = stampGeneration(config, configRecovery);+ if (stateCache) stateCache.config = configCache;+ return configCache;}// In-memory state cache — avoids re-reading state.json from disk on every call.@@ -381,38 +447,37 @@ export async function getConfig() {}export async function loadState() {- if (stateCache) return stateCache;-- await ensureDirectories();-- // Config is its own file now, so it survives every state.json failure path.+ // Config recovery must also repair a warm runtime cache's config view.const config = await loadConfig();-- if (!existsSync(STATE_FILE)) {- stateCache = Object.assign(structuredClone(DEFAULT_STATE), { config });- return stampGeneration(stateCache);+ if (stateCache) {+ stateCache.config = config;+ return stateCache;}- const content = await readFile(STATE_FILE, 'utf-8');- const state = parseStateFile(content);-- if (!state) {- stateCache = Object.assign(await recoverFromUnreadableState(content), { config });- return stampGeneration(stateCache);+ const generation = stateRecovery.generation;+ const restored = restoreGeneration;+ await ensureDirectories();+ let state;+ if (stateRecovery.required) {+ state = await readRecoveryFile(STATE_FILE);+ } else if (existsSync(STATE_FILE)) {+ const content = await readFile(STATE_FILE, 'utf-8');+ state = parseStateFile(content) ?? await recoverFromUnreadableState(content);}+ // A concurrent read/write or restore may have superseded this disk read.+ if (stateCache || generation !== stateRecovery.generation || restored !== restoreGeneration) return loadState();- stateCache = {+ stateCache = state ? {...DEFAULT_STATE,...state,- // Always the config file's copy — a legacy `config` slice still sitting in- // state.json was already folded in by `readPersistedConfig()`, and must- // never shadow the (newer) config file on an install carrying both.+ // Legacy state.json settings must never shadow the independent config file.config,stats: { ...DEFAULT_STATE.stats, ...state.stats },persistentMind: normalizePersistentMindState(state.persistentMind),agents: state.agents ?? {}- };- return stampGeneration(stateCache);+ } : Object.assign(structuredClone(DEFAULT_STATE), { config });+ stateRecovery.required = false;+ return stampGeneration(stateCache, stateRecovery);}// Read the persisted state for safety checks, bypassing both the cache and@@ -468,14 +533,21 @@ export async function readAgentsStateForSafetyCheck() {* `updateConfig()`; `state.config` is a read-only view.*/export async function saveState(state) {- await ensureDirectories();- assertCurrentGeneration(state);- state.config = await loadConfig();- stateCache = stampGeneration(state);- // Config lives in its own file; never re-serialize it onto the hot path.- const persisted = { ...state };- delete persisted.config;- await atomicWrite(STATE_FILE, persisted);+ assertCurrentGeneration(state, stateRecovery);+ stampGeneration(state, stateRecovery);+ try {+ await ensureDirectories();+ state.config = await loadConfig();+ // Config lives in its own file; never re-serialize it onto the hot path.+ const persisted = { ...state };+ delete persisted.config;+ await atomicWrite(STATE_FILE, persisted);+ } catch (error) {+ stateCache = null;+ invalidateFailedWrite(stateRecovery);+ throw error;+ }+ stateCache = stampGeneration(state, stateRecovery);}/**