Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve restored task schedules during concurrent writes (#10699)
Adam Eivye2f95619bae1
diff --git a/docs/BACKUP.md b/docs/BACKUP.mdindex 7f7d324991..b6199c518f 100644--- a/docs/BACKUP.md+++ b/docs/BACKUP.md@@ -610,11 +610,27 @@ queue while retaining the existing restore diagnostics. Malformed restoredsettings still invalidate the cache rather than broadcasting empty defaults.Dry runs and unrelated selective restores do not acquire this settings boundary.-A full restore acquires boundaries in this order: settings, CoS configuration, CoS-runtime state, media model registry. The settings queue remains held through CoS reconciliation. A+A full restore acquires boundaries in this order: shared snapshot cut, task schedule,+settings, CoS configuration, CoS runtime state, media model registry. The settings queue remains held through CoS reconciliation. Arestore callback must never call a queued settings write API; cache reload readsdirectly and does not re-enter the queue.+### Restoring task schedules in a running server++Full live file restores, `cos` restores, and exact `cos/task-schedule.json`+restores join the schedule's existing write queue after the shared snapshot cut+and before settings/CoS ownership. Schedule mutations can consult CoS state, so+reversing that order could deadlock. Already admitted writes finish before+transfer; later writes read the restored schedule and preserve untouched task+settings, prompts, and templates.++The queue stays held through compatibility normalization and the schedule-change+notification, including after a partial transfer failure. Normalization reads+and saves directly inside that turn without re-entering the queue. Transfer+errors retain partial-overwrite diagnostics; reconciliation errors are surfaced+without discarding the transfer failure. Dry runs and unrelated scopes do not+acquire the schedule boundary.+### Restoring CoS files in a running serverA full live file restore or selective `cos` restore requires the CoS daemon anddiff --git a/server/services/backup.js b/server/services/backup.jsindex bf2986d75c..e18e24f191 100644--- a/server/services/backup.js+++ b/server/services/backup.js@@ -1627,17 +1627,25 @@ export async function restoreSnapshot(destPath, snapshotId, { dryRun = true, sub}return restoreWithMediaBoundary();};- // Fixed acquisition order: snapshot cut -> settings -> CoS config -> CoS+ // Fixed acquisition order: snapshot cut -> schedule -> settings -> CoS config -> CoS// runtime -> media registry. Drain publications BEFORE holding any domain// queue they may need to finish, and keep admission closed through transfer// and all cache reconciliation, including a partially failed transfer.const restoreWithSettingsBoundary = () => !dryRun && (!scope || scope === 'settings.json')? withLiveSettingsRestore(restoreWithCosBoundary): restoreWithCosBoundary();- if (dryRun) return restoreWithSettingsBoundary();+ // Schedule mutations can await CoS state, so drain them before CoS ownership.+ const restoreWithScheduleBoundary = async () => {+ if (!dryRun && (!scope || scope === 'cos' || scope === 'cos/task-schedule.json')) {+ const { withLiveTaskScheduleRestore } = await import('./taskScheduleStore.js');+ return withLiveTaskScheduleRestore(restoreWithSettingsBoundary);+ }+ return restoreWithSettingsBoundary();+ };+ if (dryRun) return restoreWithScheduleBoundary();const releaseSnapshotCut = await acquireBackupSnapshotCut();try {- return await restoreWithSettingsBoundary();+ return await restoreWithScheduleBoundary();} finally {releaseSnapshotCut();}diff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex fb739d1bd9..b5691c6381 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -2987,6 +2987,15 @@ import { withLiveCosRestore } from './cosState.js';vi.mock('../lib/mediaModels.js', () => ({ withLiveMediaModelsRestore: vi.fn(fn => fn()) }));import { withLiveMediaModelsRestore } from '../lib/mediaModels.js';+const scheduleStoreBoundary = vi.hoisted(() => ({ restore: null }));+vi.mock('./taskScheduleStore.js', async (importOriginal) => {+ const actual = await importOriginal();+ scheduleStoreBoundary.restore = actual.withLiveTaskScheduleRestore;+ return { ...actual, withLiveTaskScheduleRestore: vi.fn(fn => fn()) };+});+import { withLiveTaskScheduleRestore, updateSchedule } from './taskScheduleStore.js';+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';+describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () => {beforeEach(() => {spawn.mockReset();@@ -2995,6 +3004,7 @@ describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () =>withLiveCosRestore.mockClear();withLiveMediaModelsRestore.mockClear();withLiveSettingsRestore.mockClear();+ withLiveTaskScheduleRestore.mockReset().mockImplementation(fn => fn());});// Drive a mocked rsync to a clean exit so restoreSnapshot resolves.@@ -3121,6 +3131,64 @@ describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () =>});});+ describe('schedule restore ownership boundary', () => {+ it.each([+ [undefined, 0], ['cos', 0], ['cos/', 0], ['cos/task-schedule.json', 0],+ ['./cos//task-schedule.json/', 0], ['cos/task-schedule.json', 1]+ ])('drains schedule writes before CoS and preserves restored values for scope %s, exit %s', async (subdirFilter, exitCode) => {+ withLiveTaskScheduleRestore.mockImplementation(scheduleStoreBoundary.restore);+ const stateQueue = createFileWriteQueue();+ const consultedState = vi.fn();+ if (!subdirFilter || subdirFilter.replace(/\/$/, '') === 'cos') {+ withLiveCosRestore.mockImplementationOnce(fn => stateQueue(fn));+ }+ const admitted = Promise.withResolvers();+ const finishWrite = Promise.withResolvers();+ const writing = updateSchedule(async schedule => {+ admitted.resolve();+ await finishWrite.promise;+ await stateQueue(consultedState);+ schedule.tasks.security.enabled = true;+ return { changed: true };+ });+ await admitted.promise;+ const proc = fakeProc();+ spawn.mockReturnValue(proc);+ const pending = restoreSnapshot('/dest', 'snap-1', { dryRun: false, subdirFilter });+ await vi.waitFor(() => expect(withLiveTaskScheduleRestore).toHaveBeenCalledTimes(1));+ expect(withLiveCosRestore).not.toHaveBeenCalled();+ expect(spawn).not.toHaveBeenCalled();+ finishWrite.resolve();+ await flush();+ expect(consultedState).toHaveBeenCalledTimes(1);+ const restored = {+ version: 2, tasks: { security: { enabled: false, prompt: 'Restored custom prompt' } },+ templates: [{ id: 'restored-template', prompt: 'Example template' }], executions: {}+ };+ // Synthetic rsync boundary only: replace bytes in the isolated test root.+ writeFileSync(join(PATHS.cos, 'task-schedule.json'), JSON.stringify(restored));+ const later = vi.fn(schedule => {+ schedule.executions['task:security'] = { count: 7 };+ return { changed: true };+ });+ const waitingWrite = updateSchedule(later);+ expect(later).not.toHaveBeenCalled();+ proc.emit('close', exitCode);+ if (exitCode) await expect(pending).rejects.toThrow(/Some files may already have been overwritten/);+ else await pending;+ await Promise.all([writing, waitingWrite]);+ const persisted = JSON.parse(readFileSync(join(PATHS.cos, 'task-schedule.json'), 'utf8'));+ expect(persisted.tasks.security).toMatchObject({ enabled: false, prompt: 'Restored custom prompt' });+ expect(persisted.templates).toEqual(restored.templates);+ expect(persisted.executions['task:security']).toEqual({ count: 7 });+ });++ it.each([{ dryRun: true }, { dryRun: true, subdirFilter: 'cos/task-schedule.json' }, { dryRun: false, subdirFilter: 'cos/state.json' }, { dryRun: false, subdirFilter: 'images' }])('does not acquire schedule ownership for %j', async options => {+ await runRestore('/dest', 'snap-1', options);+ expect(withLiveTaskScheduleRestore).not.toHaveBeenCalled();+ });+ });+describe('CoS restore ownership boundary', () => {it.each([undefined, 'cos', 'cos/', 'cos/config.json', 'cos/state.json', 'cos/agents', 'cos/agents/', 'cos/agents/index.json'])('holds the boundary for affected live scope %s', async subdirFilter => {await runRestore('/dest', 'snap-1', { dryRun: false, subdirFilter });diff --git a/server/services/taskScheduleStore.js b/server/services/taskScheduleStore.jsindex 97d03d44a9..434c3a25a1 100644--- a/server/services/taskScheduleStore.js+++ b/server/services/taskScheduleStore.js@@ -314,3 +314,29 @@ async function saveScheduleNow(schedule) {export function saveSchedule(schedule) {return queueScheduleWrite(() => saveScheduleNow(schedule));}++/**+ * Own the schedule queue through an out-of-band restore and compatibility repair.+ * Acquire after the snapshot cut, but before settings/CoS locks: a schedule+ * mutation may consult CoS state. Never call queued schedule APIs in `transfer`.+ */+export function withLiveTaskScheduleRestore(transfer) {+ return queueScheduleWrite(async () => {+ const [result] = await Promise.allSettled([Promise.resolve().then(transfer)]);+ const [reload] = await Promise.allSettled([(async () => {+ const { schedule, needsSave } = await readSchedule();+ if (needsSave) await saveScheduleNow(schedule);+ else dashboardEvents.emit('cos:schedule:changed');+ })()]);+ if (reload.status === 'rejected') {+ const message = `Task schedule reconciliation failed: ${reload.reason.message}`;+ const error = new Error(result.status === 'rejected'+ ? `${result.reason.message}. ${message}` : message,+ { cause: result.status === 'rejected' ? result.reason : reload.reason });+ if (result.status === 'rejected' && result.reason.code) error.code = result.reason.code;+ throw error;+ }+ if (result.status === 'rejected') throw result.reason;+ return result.value;+ });+}diff --git a/server/services/taskScheduleStore.test.js b/server/services/taskScheduleStore.test.jsindex 0e1cd4e2a3..0ea2120a13 100644--- a/server/services/taskScheduleStore.test.js+++ b/server/services/taskScheduleStore.test.js@@ -65,7 +65,9 @@ vi.mock('../lib/fileUtils.js', async () => {};});-import { loadSchedule, updateSchedule } from './taskScheduleStore.js';+import { loadSchedule, updateSchedule, withLiveTaskScheduleRestore } from './taskScheduleStore.js';+import { dashboardEvents } from './dashboardEvents.js';+import { readJSONFile } from '../lib/fileUtils.js';import { updateTaskInterval } from './taskSchedule.js';import { recordTaskTypeFailure } from './taskScheduleBackoff.js';@@ -157,4 +159,50 @@ describe('taskScheduleStore', () => {expect(stages[2]).toMatchObject({ role: 'actions', providerId: 'codex-cli', model: 'gpt-5.6', executionProfile: 'public-review-gate' });expect(state.writes.at(-1).tasks['pr-reviewer'].taskMetadata.pipeline.stages).toHaveLength(3);});++ it('holds later mutations and compatibility repairs through partial restore reconciliation', async () => {+ const transferred = Promise.withResolvers();+ const finishTransfer = Promise.withResolvers();+ const restored = {+ version: 1,+ selfImprovement: { security: { enabled: false, prompt: 'Restored custom prompt' } },+ templates: [{ id: 'restored-template', prompt: 'Restored template' }]+ };+ const restore = withLiveTaskScheduleRestore(async () => {+ state.persisted = structuredClone(restored);+ transferred.resolve();+ await finishTransfer.promise;+ throw Object.assign(new Error('Partial transfer'), { code: 'RSYNC_FAILED' });+ });+ await transferred.promise;+ const repair = loadSchedule();+ const later = updateSchedule(schedule => {+ schedule.executions['task:security'] = { count: 1 };+ return { changed: true };+ });+ expect(state.writes).toHaveLength(0);+ const notification = vi.fn(() => expect(state.persisted.version).toBe(2));+ dashboardEvents.on('cos:schedule:changed', notification);+ finishTransfer.resolve();+ await expect(restore).rejects.toMatchObject({ code: 'RSYNC_FAILED' });+ await Promise.all([repair, later]);+ dashboardEvents.off('cos:schedule:changed', notification);+ expect(notification).toHaveBeenCalled();+ expect(state.persisted.tasks.security).toMatchObject({ enabled: false, prompt: 'Restored custom prompt' });+ expect(state.persisted.templates).toEqual(restored.templates);+ expect(state.persisted.executions['task:security'].count).toBe(1);+ });++ it('retains transfer diagnostics when reconciliation fails and releases the queue', async () => {+ const restore = withLiveTaskScheduleRestore(async () => {+ readJSONFile.mockRejectedValueOnce(new Error('Invalid restored JSON'));+ throw Object.assign(new Error('Some files may already have been overwritten'), { code: 'RSYNC_FAILED' });+ });+ await expect(restore).rejects.toMatchObject({+ code: 'RSYNC_FAILED',+ message: expect.stringMatching(/Some files may already have been overwritten.*Task schedule reconciliation failed: Invalid restored JSON/)+ });+ await expect(updateSchedule(() => ({ result: 'released', changed: false }))).resolves.toBe('released');+ });+});