Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(cos): keep rejected state changes out of later saves (#12185)

Adam Eivyfb19cff8458b
diff --git a/server/services/cosState.failedWrite.test.js b/server/services/cosState.failedWrite.test.js
new file mode 100644
index 0000000000..afa2348b76
--- /dev/null
+++ b/server/services/cosState.failedWrite.test.js
@@ -0,0 +1,209 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';
+
+const { tempRoot, makeProxy, spies, cleanup } = mockPathsDataRoot({
+ prefix: 'cos-failed-write-', wrapExports: ['atomicWrite', 'ensureDirs', 'readJSONFileStrict'], makeSpy: vi.fn,
+});
+vi.mock('../lib/fileUtils.js', async importOriginal => makeProxy(await importOriginal()));
+
+const cosDir = join(tempRoot, 'cos');
+const statePath = join(cosDir, 'state.json');
+const configPath = join(cosDir, 'config.json');
+const binding = { branch: 'claim/issue-example', repository: 'github.com/example/project' };
+const writeJson = (path, value) => writeFileSync(path, JSON.stringify(value));
+const readJson = path => JSON.parse(readFileSync(path, 'utf8'));
+const writeFailure = () => Object.assign(new Error('Synthetic persistence failure'), { code: 'EIO' });
+let store;
+let realWrite;
+
+beforeEach(async () => {
+ vi.resetModules();
+ rmSync(cosDir, { force: true, recursive: true });
+ mkdirSync(cosDir, { recursive: true });
+ writeJson(configPath, { maxConcurrentAgents: 3 });
+ writeJson(statePath, { agents: { example: { id: 'example', status: 'running', metadata: { claimBranches: [] } } } });
+ store = await import('./cosState.js');
+ const actual = await vi.importActual('../lib/fileUtils.js');
+ realWrite = actual.atomicWrite;
+ for (const [name, spy] of Object.entries(spies)) spy.mockReset().mockImplementation(actual[name]);
+});
+afterAll(cleanup);
+
+async function saveHealthCheck() {
+ await store.withStateLock(async () => {
+ const state = await store.loadState();
+ state.lastHealthCheck = '2026-01-01T00:00:00.000Z';
+ await store.saveState(state);
+ });
+}
+
+describe('CoS failed persistence recovery', () => {
+ it.each(['bind', 'release'])('does not carry a rejected %s into the next queued write or a stale retry', async action => {
+ const durable = action === 'bind' ? [] : [binding];
+ writeJson(statePath, { agents: { example: { status: 'running', metadata: { claimBranches: durable } } } });
+ let rejectedState;
+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());
+ await expect(store.withStateLock(async () => {
+ const state = await store.loadState();
+ rejectedState = { ...state, agents: { ...state.agents, example: {
+ ...state.agents.example, metadata: { claimBranches: action === 'bind' ? [binding] : [] },
+ } } };
+ await store.saveState(rejectedState);
+ })).rejects.toMatchObject({ code: 'EIO' });
+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual(durable);
+ await saveHealthCheck();
+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual(durable);
+ expect(readJson(statePath).lastHealthCheck).toBe('2026-01-01T00:00:00.000Z');
+ await expect(store.saveState({ ...rejectedState })).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });
+ });
+
+ it('discards an existing agent lifecycle caller\'s in-place mutation after rejection', async () => {
+ const { updateAgent } = await import('./cosAgentLifecycle.js');
+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());
+ await expect(updateAgent('example', { metadata: { claimBranches: [binding] } })).rejects.toMatchObject({ code: 'EIO' });
+ await updateAgent('example', { status: 'paused' });
+ expect(readJson(statePath).agents.example).toMatchObject({ status: 'paused', metadata: { claimBranches: [] } });
+ });
+
+ it('reloads rejected in-place config changes and re-anchors a live runtime read independently', async () => {
+ const state = await store.loadState();
+ const failedConfig = state.config;
+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());
+ await expect(store.withConfigLock(async () => {
+ failedConfig.maxConcurrentAgents = 12;
+ await store.saveConfig(failedConfig);
+ })).rejects.toMatchObject({ code: 'EIO' });
+ // An already-started runtime transaction remains valid after a config error.
+ await store.withStateLock(async () => {
+ state.paused = true;
+ await store.saveState(state);
+ });
+ expect(state.config.maxConcurrentAgents).toBe(3);
+ expect(readJson(statePath)).toMatchObject({ paused: true });
+ expect(readJson(statePath)).not.toHaveProperty('config');
+ await store.withConfigLock(async () => {
+ const config = await store.loadConfig();
+ expect((await store.loadState()).config).toBe(config);
+ await store.saveConfig({ ...config, maxTotalProcesses: 20 });
+ });
+ expect(readJson(configPath)).toMatchObject({ maxConcurrentAgents: 3, maxTotalProcesses: 20 });
+ await expect(store.saveConfig({ ...failedConfig })).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });
+ });
+
+ it.each(['state', 'config'])('reads the disk outcome when a %s write succeeds before reporting failure', async kind => {
+ const load = kind === 'state' ? store.loadState : store.loadConfig;
+ const save = kind === 'state' ? store.saveState : store.saveConfig;
+ const queue = kind === 'state' ? store.withStateLock : store.withConfigLock;
+ const path = kind === 'state' ? statePath : configPath;
+ const value = await load();
+ value.exampleCommitted = true;
+ spies.atomicWrite.mockImplementationOnce(async (...args) => {
+ await realWrite(...args);
+ throw writeFailure();
+ });
+ await expect(queue(() => save(value))).rejects.toMatchObject({ code: 'EIO' });
+ await queue(async () => {
+ const current = await load();
+ expect(current.exampleCommitted).toBe(true);
+ current.exampleNextWrite = true;
+ await save(current);
+ });
+ expect(readJson(path)).toMatchObject({ exampleCommitted: true, exampleNextWrite: true });
+ });
+
+ it.each(['state', 'config'])('refuses to default unreadable %s after a failed write until repaired', async kind => {
+ const load = kind === 'state' ? store.loadState : store.loadConfig;
+ const save = kind === 'state' ? store.saveState : store.saveConfig;
+ const path = kind === 'state' ? statePath : configPath;
+ const value = await load();
+ spies.atomicWrite.mockImplementationOnce(async () => {
+ writeFileSync(path, 'unreadable');
+ throw writeFailure();
+ });
+ await expect(save(value)).rejects.toMatchObject({ code: 'EIO' });
+ await expect(load()).rejects.toMatchObject({ code: 'COS_PERSISTENCE_RECOVERY_REQUIRED' });
+ await expect(save({})).rejects.toMatchObject({ code: 'COS_PERSISTENCE_STALE_WRITE' });
+ expect(readFileSync(path, 'utf8')).toBe('unreadable');
+ writeJson(path, { repaired: true });
+ await save(await load());
+ expect(readJson(path).repaired).toBe(true);
+ });
+
+ it('retries a rejected legacy config split before runtime writes can remove the legacy settings', async () => {
+ rmSync(configPath);
+ writeJson(statePath, { agents: {}, config: { maxConcurrentAgents: 9 } });
+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());
+ await expect(store.loadState()).rejects.toMatchObject({ code: 'EIO' });
+ expect(readJson(statePath).config.maxConcurrentAgents).toBe(9);
+ await saveHealthCheck();
+ expect(readJson(configPath).maxConcurrentAgents).toBe(9);
+ expect(readJson(statePath)).not.toHaveProperty('config');
+ });
+
+ it('shares a pending legacy split so a late loader cannot overwrite the next config update', async () => {
+ rmSync(configPath);
+ writeJson(statePath, { agents: {}, config: { maxConcurrentAgents: 9 } });
+ spies.atomicWrite.mockRejectedValueOnce(writeFailure());
+ await expect(store.saveConfig({ maxConcurrentAgents: 9 })).rejects.toMatchObject({ code: 'EIO' });
+ // Keep the interleaving deterministic: only the first write waits; all
+ // other file work is synchronous over these temporary synthetic files.
+ spies.ensureDirs.mockResolvedValue(undefined);
+ spies.readJSONFileStrict.mockImplementation(async (path, fallback) => ({
+ ok: true, value: existsSync(path) ? readJson(path) : fallback,
+ }));
+ let releaseWrite;
+ let signalStarted;
+ const gate = new Promise(resolve => { releaseWrite = resolve; });
+ const started = new Promise(resolve => { signalStarted = resolve; });
+ let first = true;
+ spies.atomicWrite.mockImplementation(async (path, value) => {
+ if (first) {
+ first = false;
+ signalStarted();
+ await gate;
+ }
+ writeJson(path, value);
+ });
+ const initial = store.loadConfig();
+ await started;
+ const update = store.withConfigLock(async () => {
+ await store.saveConfig({ ...(await store.loadConfig()), maxConcurrentAgents: 5 });
+ });
+ // Drain the synchronous I/O doubles' promise continuations before allowing
+ // the original split to finish, reproducing a late legacy overwrite.
+ await new Promise(resolve => setImmediate(resolve));
+ releaseWrite();
+ await Promise.all([initial, update]);
+ expect(readJson(configPath).maxConcurrentAgents).toBe(5);
+ expect((await store.loadState()).config.maxConcurrentAgents).toBe(5);
+ });
+
+ it('keeps the state queue until the rejected write settles while config can progress', async () => {
+ let rejectWrite;
+ let signalStarted;
+ const started = new Promise(resolve => { signalStarted = resolve; });
+ spies.atomicWrite.mockImplementationOnce(() => new Promise((resolve, reject) => {
+ rejectWrite = reject;
+ signalStarted();
+ }));
+ const failed = store.withStateLock(async () => {
+ const state = await store.loadState();
+ state.agents.example.metadata.claimBranches.push(binding);
+ await store.saveState(state);
+ });
+ const failureAssertion = expect(failed).rejects.toMatchObject({ code: 'EIO' });
+ await started;
+ const next = saveHealthCheck();
+ await store.withConfigLock(async () => {
+ await store.saveConfig({ ...(await store.loadConfig()), maxConcurrentAgents: 5 });
+ });
+ expect(readJson(statePath)).not.toHaveProperty('lastHealthCheck');
+ rejectWrite(writeFailure());
+ await failureAssertion;
+ await next;
+ expect(readJson(statePath).agents.example.metadata.claimBranches).toEqual([]);
+ expect((await store.loadState()).config.maxConcurrentAgents).toBe(5);
+ });
+});
diff --git a/server/services/cosState.js b/server/services/cosState.js
index 9b7646d1cb..5a57e505af 100644
--- a/server/services/cosState.js
+++ b/server/services/cosState.js
@@ -269,20 +269,48 @@ async function recoverFromUnreadableState(content) {
// In-memory config cache. Every mutation goes through `withConfigLock` +
// `saveConfig`, so the cache stays consistent.
let configCache = null;
+let configLoad = null;
// Enumerable symbols survive a caller's shallow copy but never reach JSON.
// Reject delayed writeback of a pre-restore read, even if queued afterwards.
const RESTORE_GENERATION = Symbol('cosRestoreGeneration');
+const WRITE_GENERATION = Symbol('cosWriteGeneration');
+const stateRecovery = { generation: 0, required: false };
+const configRecovery = { generation: 0, required: false };
let restoreGeneration = 0;
-function stampGeneration(value) {
+function stampGeneration(value, recovery) {
value[RESTORE_GENERATION] = restoreGeneration;
+ value[WRITE_GENERATION] = recovery.generation;
return value;
}
-function assertCurrentGeneration(value) {
+function assertCurrentGeneration(value, recovery) {
if (value[RESTORE_GENERATION] !== undefined && value[RESTORE_GENERATION] !== restoreGeneration) {
throw new ServerError('CoS files were restored. Read the current state before retrying this change.', {
status: 409, code: 'COS_RESTORE_STALE_WRITE',
});
}
+ if (recovery.required || (value[WRITE_GENERATION] !== undefined && value[WRITE_GENERATION] !== recovery.generation)) {
+ throw new ServerError('A CoS write failed. Read the current persisted state before retrying this change.', {
+ status: 409, code: 'COS_PERSISTENCE_STALE_WRITE',
+ });
+ }
+}
+
+function invalidateFailedWrite(recovery) {
+ recovery.generation += 1;
+ recovery.required = true;
+}
+
+// After an uncertain write, never replace unreadable ownership/settings with
+// defaults. Only a fresh trusted read may clear the recovery requirement.
+async function readRecoveryFile(filePath) {
+ const missing = Symbol('missing');
+ const { ok, value } = await readJSONFileStrict(filePath, missing, { allowArray: false, logError: true });
+ if (!ok || (value !== missing && !isPlainObject(value))) {
+ throw new ServerError('CoS persistence recovery requires a readable state/config file. Repair the file before retrying.', {
+ status: 503, code: 'COS_PERSISTENCE_RECOVERY_REQUIRED',
+ });
+ }
+ return value === missing ? null : value;
}
/**
@@ -300,7 +328,9 @@ function assertCurrentGeneration(value) {
* never boots on bare defaults.
*/
async function readPersistedConfig() {
- const { ok, value } = await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false, logError: true });
+ const { ok, value } = configRecovery.required
+ ? { ok: true, value: await readRecoveryFile(CONFIG_FILE) }
+ : await readJSONFileStrict(CONFIG_FILE, null, { allowArray: false, logError: true });
if (ok && isPlainObject(value)) return { config: value, fromConfigFile: true };
if (!ok) {
console.log(`⚠️ Corrupted or empty config file at ${CONFIG_FILE}, falling back to legacy/default config`);
@@ -309,9 +339,10 @@ async function readPersistedConfig() {
}
// Back-compat read. Deliberately does NOT quarantine or rewrite state.json —
// loadState() owns that file's recovery.
- if (!existsSync(STATE_FILE)) return { config: null, fromConfigFile: false };
- const stateContent = await readFile(STATE_FILE, 'utf-8');
- const state = parseStateFile(stateContent, 'CoS state (legacy config)');
+ if (!configRecovery.required && !existsSync(STATE_FILE)) return { config: null, fromConfigFile: false };
+ const state = configRecovery.required
+ ? await readRecoveryFile(STATE_FILE)
+ : parseStateFile(await readFile(STATE_FILE, 'utf-8'), 'CoS state (legacy config)');
const legacy = state?.config;
return { config: isPlainObject(legacy) ? legacy : null, fromConfigFile: false };
}
@@ -321,9 +352,25 @@ async function readPersistedConfig() {
*/
export async function loadConfig() {
if (configCache) return configCache;
+ // The legacy split writes during a read. Share its settlement so another
+ // loader cannot overwrite a subsequent settings change with the legacy copy.
+ if (configLoad) return configLoad;
+ configLoad = readConfig();
+ try {
+ return await configLoad;
+ } finally {
+ configLoad = null;
+ }
+}
+
+async function readConfig() {
+ if (configCache) return configCache;
+ const generation = configRecovery.generation;
+ const restored = restoreGeneration;
await ensureDirectories();
const { config, fromConfigFile } = await readPersistedConfig();
- configCache = stampGeneration(mergeStoredConfig(config));
+ if (configCache || generation !== configRecovery.generation || restored !== restoreGeneration) return readConfig();
+ const loaded = stampGeneration(mergeStoredConfig(config), configRecovery);
// Recovering settings out of a legacy state.json is a READ; nothing has
// written them to their own file yet. `saveState` strips `config` from
// state.json on the very next runtime write, so leaving the recovery in
@@ -332,8 +379,18 @@ export async function loadConfig() {
// restored pre-split backup on a machine that already recorded it as applied).
if (config && !fromConfigFile) {
console.log(`📦 Recovered CoS config from a legacy state.json — writing ${CONFIG_FILE}`);
- await atomicWrite(CONFIG_FILE, configCache);
+ try {
+ await atomicWrite(CONFIG_FILE, loaded);
+ } catch (error) {
+ configCache = null;
+ invalidateFailedWrite(configRecovery);
+ throw error;
+ }
}
+ if (configCache || generation !== configRecovery.generation || restored !== restoreGeneration) return readConfig();
+ configCache = loaded;
+ configRecovery.required = false;
+ if (stateCache) stateCache.config = configCache;
return configCache;
}
@@ -343,12 +400,21 @@ export async function loadConfig() {
* before the write does not go stale.
*/
export async function saveConfig(config) {
- await ensureDirectories();
- assertCurrentGeneration(config);
- configCache = stampGeneration(config);
- if (stateCache) stateCache.config = config;
- await atomicWrite(CONFIG_FILE, config);
- return config;
+ assertCurrentGeneration(config, configRecovery);
+ stampGeneration(config, configRecovery);
+ try {
+ await ensureDirectories();
+ await atomicWrite(CONFIG_FILE, config);
+ } catch (error) {
+ // Callers may have mutated the shared object in place; a prior pointer is
+ // not a rollback image. The write may also have reached disk before failing.
+ configCache = null;
+ invalidateFailedWrite(configRecovery);
+ throw error;
+ }
+ configCache = stampGeneration(config, configRecovery);
+ if (stateCache) stateCache.config = configCache;
+ return configCache;
}
// In-memory state cache — avoids re-reading state.json from disk on every call.
@@ -381,38 +447,37 @@ export async function getConfig() {
}
export async function loadState() {
- if (stateCache) return stateCache;
-
- await ensureDirectories();
-
- // Config is its own file now, so it survives every state.json failure path.
+ // Config recovery must also repair a warm runtime cache's config view.
const config = await loadConfig();
-
- if (!existsSync(STATE_FILE)) {
- stateCache = Object.assign(structuredClone(DEFAULT_STATE), { config });
- return stampGeneration(stateCache);
+ if (stateCache) {
+ stateCache.config = config;
+ return stateCache;
}
- const content = await readFile(STATE_FILE, 'utf-8');
- const state = parseStateFile(content);
-
- if (!state) {
- stateCache = Object.assign(await recoverFromUnreadableState(content), { config });
- return stampGeneration(stateCache);
+ const generation = stateRecovery.generation;
+ const restored = restoreGeneration;
+ await ensureDirectories();
+ let state;
+ if (stateRecovery.required) {
+ state = await readRecoveryFile(STATE_FILE);
+ } else if (existsSync(STATE_FILE)) {
+ const content = await readFile(STATE_FILE, 'utf-8');
+ state = parseStateFile(content) ?? await recoverFromUnreadableState(content);
}
+ // A concurrent read/write or restore may have superseded this disk read.
+ if (stateCache || generation !== stateRecovery.generation || restored !== restoreGeneration) return loadState();
- stateCache = {
+ stateCache = state ? {
...DEFAULT_STATE,
...state,
- // Always the config file's copy — a legacy `config` slice still sitting in
- // state.json was already folded in by `readPersistedConfig()`, and must
- // never shadow the (newer) config file on an install carrying both.
+ // Legacy state.json settings must never shadow the independent config file.
config,
stats: { ...DEFAULT_STATE.stats, ...state.stats },
persistentMind: normalizePersistentMindState(state.persistentMind),
agents: state.agents ?? {}
- };
- return stampGeneration(stateCache);
+ } : Object.assign(structuredClone(DEFAULT_STATE), { config });
+ stateRecovery.required = false;
+ return stampGeneration(stateCache, stateRecovery);
}
// Read the persisted state for safety checks, bypassing both the cache and
@@ -468,14 +533,21 @@ export async function readAgentsStateForSafetyCheck() {
* `updateConfig()`; `state.config` is a read-only view.
*/
export async function saveState(state) {
- await ensureDirectories();
- assertCurrentGeneration(state);
- state.config = await loadConfig();
- stateCache = stampGeneration(state);
- // Config lives in its own file; never re-serialize it onto the hot path.
- const persisted = { ...state };
- delete persisted.config;
- await atomicWrite(STATE_FILE, persisted);
+ assertCurrentGeneration(state, stateRecovery);
+ stampGeneration(state, stateRecovery);
+ try {
+ await ensureDirectories();
+ state.config = await loadConfig();
+ // Config lives in its own file; never re-serialize it onto the hot path.
+ const persisted = { ...state };
+ delete persisted.config;
+ await atomicWrite(STATE_FILE, persisted);
+ } catch (error) {
+ stateCache = null;
+ invalidateFailedWrite(stateRecovery);
+ throw error;
+ }
+ stateCache = stampGeneration(state, stateRecovery);
}
/**