Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve restored task schedules during concurrent writes (#10699)

Adam Eivye2f95619bae1
diff --git a/docs/BACKUP.md b/docs/BACKUP.md
index 7f7d324991..b6199c518f 100644
--- a/docs/BACKUP.md
+++ b/docs/BACKUP.md
@@ -610,11 +610,27 @@ queue while retaining the existing restore diagnostics. Malformed restored
settings still invalidate the cache rather than broadcasting empty defaults.
Dry runs and unrelated selective restores do not acquire this settings boundary.
-A full restore acquires boundaries in this order: settings, CoS configuration, CoS
-runtime state, media model registry. The settings queue remains held through CoS reconciliation. A
+A full restore acquires boundaries in this order: shared snapshot cut, task schedule,
+settings, CoS configuration, CoS runtime state, media model registry. The settings queue remains held through CoS reconciliation. A
restore callback must never call a queued settings write API; cache reload reads
directly and does not re-enter the queue.
+### Restoring task schedules in a running server
+
+Full live file restores, `cos` restores, and exact `cos/task-schedule.json`
+restores join the schedule's existing write queue after the shared snapshot cut
+and before settings/CoS ownership. Schedule mutations can consult CoS state, so
+reversing that order could deadlock. Already admitted writes finish before
+transfer; later writes read the restored schedule and preserve untouched task
+settings, prompts, and templates.
+
+The queue stays held through compatibility normalization and the schedule-change
+notification, including after a partial transfer failure. Normalization reads
+and saves directly inside that turn without re-entering the queue. Transfer
+errors retain partial-overwrite diagnostics; reconciliation errors are surfaced
+without discarding the transfer failure. Dry runs and unrelated scopes do not
+acquire the schedule boundary.
+
### Restoring CoS files in a running server
A full live file restore or selective `cos` restore requires the CoS daemon and
diff --git a/server/services/backup.js b/server/services/backup.js
index bf2986d75c..e18e24f191 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -1627,17 +1627,25 @@ export async function restoreSnapshot(destPath, snapshotId, { dryRun = true, sub
}
return restoreWithMediaBoundary();
};
- // Fixed acquisition order: snapshot cut -> settings -> CoS config -> CoS
+ // Fixed acquisition order: snapshot cut -> schedule -> settings -> CoS config -> CoS
// runtime -> media registry. Drain publications BEFORE holding any domain
// queue they may need to finish, and keep admission closed through transfer
// and all cache reconciliation, including a partially failed transfer.
const restoreWithSettingsBoundary = () => !dryRun && (!scope || scope === 'settings.json')
? withLiveSettingsRestore(restoreWithCosBoundary)
: restoreWithCosBoundary();
- if (dryRun) return restoreWithSettingsBoundary();
+ // Schedule mutations can await CoS state, so drain them before CoS ownership.
+ const restoreWithScheduleBoundary = async () => {
+ if (!dryRun && (!scope || scope === 'cos' || scope === 'cos/task-schedule.json')) {
+ const { withLiveTaskScheduleRestore } = await import('./taskScheduleStore.js');
+ return withLiveTaskScheduleRestore(restoreWithSettingsBoundary);
+ }
+ return restoreWithSettingsBoundary();
+ };
+ if (dryRun) return restoreWithScheduleBoundary();
const releaseSnapshotCut = await acquireBackupSnapshotCut();
try {
- return await restoreWithSettingsBoundary();
+ return await restoreWithScheduleBoundary();
} finally {
releaseSnapshotCut();
}
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index fb739d1bd9..b5691c6381 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -2987,6 +2987,15 @@ import { withLiveCosRestore } from './cosState.js';
vi.mock('../lib/mediaModels.js', () => ({ withLiveMediaModelsRestore: vi.fn(fn => fn()) }));
import { withLiveMediaModelsRestore } from '../lib/mediaModels.js';
+const scheduleStoreBoundary = vi.hoisted(() => ({ restore: null }));
+vi.mock('./taskScheduleStore.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ scheduleStoreBoundary.restore = actual.withLiveTaskScheduleRestore;
+ return { ...actual, withLiveTaskScheduleRestore: vi.fn(fn => fn()) };
+});
+import { withLiveTaskScheduleRestore, updateSchedule } from './taskScheduleStore.js';
+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
+
describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () => {
beforeEach(() => {
spawn.mockReset();
@@ -2995,6 +3004,7 @@ describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () =>
withLiveCosRestore.mockClear();
withLiveMediaModelsRestore.mockClear();
withLiveSettingsRestore.mockClear();
+ withLiveTaskScheduleRestore.mockReset().mockImplementation(fn => fn());
});
// Drive a mocked rsync to a clean exit so restoreSnapshot resolves.
@@ -3121,6 +3131,64 @@ describe('restoreSnapshot snapshotId, filter flags, and settings re-sync', () =>
});
});
+ describe('schedule restore ownership boundary', () => {
+ it.each([
+ [undefined, 0], ['cos', 0], ['cos/', 0], ['cos/task-schedule.json', 0],
+ ['./cos//task-schedule.json/', 0], ['cos/task-schedule.json', 1]
+ ])('drains schedule writes before CoS and preserves restored values for scope %s, exit %s', async (subdirFilter, exitCode) => {
+ withLiveTaskScheduleRestore.mockImplementation(scheduleStoreBoundary.restore);
+ const stateQueue = createFileWriteQueue();
+ const consultedState = vi.fn();
+ if (!subdirFilter || subdirFilter.replace(/\/$/, '') === 'cos') {
+ withLiveCosRestore.mockImplementationOnce(fn => stateQueue(fn));
+ }
+ const admitted = Promise.withResolvers();
+ const finishWrite = Promise.withResolvers();
+ const writing = updateSchedule(async schedule => {
+ admitted.resolve();
+ await finishWrite.promise;
+ await stateQueue(consultedState);
+ schedule.tasks.security.enabled = true;
+ return { changed: true };
+ });
+ await admitted.promise;
+ const proc = fakeProc();
+ spawn.mockReturnValue(proc);
+ const pending = restoreSnapshot('/dest', 'snap-1', { dryRun: false, subdirFilter });
+ await vi.waitFor(() => expect(withLiveTaskScheduleRestore).toHaveBeenCalledTimes(1));
+ expect(withLiveCosRestore).not.toHaveBeenCalled();
+ expect(spawn).not.toHaveBeenCalled();
+ finishWrite.resolve();
+ await flush();
+ expect(consultedState).toHaveBeenCalledTimes(1);
+ const restored = {
+ version: 2, tasks: { security: { enabled: false, prompt: 'Restored custom prompt' } },
+ templates: [{ id: 'restored-template', prompt: 'Example template' }], executions: {}
+ };
+ // Synthetic rsync boundary only: replace bytes in the isolated test root.
+ writeFileSync(join(PATHS.cos, 'task-schedule.json'), JSON.stringify(restored));
+ const later = vi.fn(schedule => {
+ schedule.executions['task:security'] = { count: 7 };
+ return { changed: true };
+ });
+ const waitingWrite = updateSchedule(later);
+ expect(later).not.toHaveBeenCalled();
+ proc.emit('close', exitCode);
+ if (exitCode) await expect(pending).rejects.toThrow(/Some files may already have been overwritten/);
+ else await pending;
+ await Promise.all([writing, waitingWrite]);
+ const persisted = JSON.parse(readFileSync(join(PATHS.cos, 'task-schedule.json'), 'utf8'));
+ expect(persisted.tasks.security).toMatchObject({ enabled: false, prompt: 'Restored custom prompt' });
+ expect(persisted.templates).toEqual(restored.templates);
+ expect(persisted.executions['task:security']).toEqual({ count: 7 });
+ });
+
+ it.each([{ dryRun: true }, { dryRun: true, subdirFilter: 'cos/task-schedule.json' }, { dryRun: false, subdirFilter: 'cos/state.json' }, { dryRun: false, subdirFilter: 'images' }])('does not acquire schedule ownership for %j', async options => {
+ await runRestore('/dest', 'snap-1', options);
+ expect(withLiveTaskScheduleRestore).not.toHaveBeenCalled();
+ });
+ });
+
describe('CoS restore ownership boundary', () => {
it.each([undefined, 'cos', 'cos/', 'cos/config.json', 'cos/state.json', 'cos/agents', 'cos/agents/', 'cos/agents/index.json'])('holds the boundary for affected live scope %s', async subdirFilter => {
await runRestore('/dest', 'snap-1', { dryRun: false, subdirFilter });
diff --git a/server/services/taskScheduleStore.js b/server/services/taskScheduleStore.js
index 97d03d44a9..434c3a25a1 100644
--- a/server/services/taskScheduleStore.js
+++ b/server/services/taskScheduleStore.js
@@ -314,3 +314,29 @@ async function saveScheduleNow(schedule) {
export function saveSchedule(schedule) {
return queueScheduleWrite(() => saveScheduleNow(schedule));
}
+
+/**
+ * Own the schedule queue through an out-of-band restore and compatibility repair.
+ * Acquire after the snapshot cut, but before settings/CoS locks: a schedule
+ * mutation may consult CoS state. Never call queued schedule APIs in `transfer`.
+ */
+export function withLiveTaskScheduleRestore(transfer) {
+ return queueScheduleWrite(async () => {
+ const [result] = await Promise.allSettled([Promise.resolve().then(transfer)]);
+ const [reload] = await Promise.allSettled([(async () => {
+ const { schedule, needsSave } = await readSchedule();
+ if (needsSave) await saveScheduleNow(schedule);
+ else dashboardEvents.emit('cos:schedule:changed');
+ })()]);
+ if (reload.status === 'rejected') {
+ const message = `Task schedule reconciliation failed: ${reload.reason.message}`;
+ const error = new Error(result.status === 'rejected'
+ ? `${result.reason.message}. ${message}` : message,
+ { cause: result.status === 'rejected' ? result.reason : reload.reason });
+ if (result.status === 'rejected' && result.reason.code) error.code = result.reason.code;
+ throw error;
+ }
+ if (result.status === 'rejected') throw result.reason;
+ return result.value;
+ });
+}
diff --git a/server/services/taskScheduleStore.test.js b/server/services/taskScheduleStore.test.js
index 0e1cd4e2a3..0ea2120a13 100644
--- a/server/services/taskScheduleStore.test.js
+++ b/server/services/taskScheduleStore.test.js
@@ -65,7 +65,9 @@ vi.mock('../lib/fileUtils.js', async () => {
};
});
-import { loadSchedule, updateSchedule } from './taskScheduleStore.js';
+import { loadSchedule, updateSchedule, withLiveTaskScheduleRestore } from './taskScheduleStore.js';
+import { dashboardEvents } from './dashboardEvents.js';
+import { readJSONFile } from '../lib/fileUtils.js';
import { updateTaskInterval } from './taskSchedule.js';
import { recordTaskTypeFailure } from './taskScheduleBackoff.js';
@@ -157,4 +159,50 @@ describe('taskScheduleStore', () => {
expect(stages[2]).toMatchObject({ role: 'actions', providerId: 'codex-cli', model: 'gpt-5.6', executionProfile: 'public-review-gate' });
expect(state.writes.at(-1).tasks['pr-reviewer'].taskMetadata.pipeline.stages).toHaveLength(3);
});
+
+ it('holds later mutations and compatibility repairs through partial restore reconciliation', async () => {
+ const transferred = Promise.withResolvers();
+ const finishTransfer = Promise.withResolvers();
+ const restored = {
+ version: 1,
+ selfImprovement: { security: { enabled: false, prompt: 'Restored custom prompt' } },
+ templates: [{ id: 'restored-template', prompt: 'Restored template' }]
+ };
+ const restore = withLiveTaskScheduleRestore(async () => {
+ state.persisted = structuredClone(restored);
+ transferred.resolve();
+ await finishTransfer.promise;
+ throw Object.assign(new Error('Partial transfer'), { code: 'RSYNC_FAILED' });
+ });
+ await transferred.promise;
+ const repair = loadSchedule();
+ const later = updateSchedule(schedule => {
+ schedule.executions['task:security'] = { count: 1 };
+ return { changed: true };
+ });
+ expect(state.writes).toHaveLength(0);
+ const notification = vi.fn(() => expect(state.persisted.version).toBe(2));
+ dashboardEvents.on('cos:schedule:changed', notification);
+ finishTransfer.resolve();
+ await expect(restore).rejects.toMatchObject({ code: 'RSYNC_FAILED' });
+ await Promise.all([repair, later]);
+ dashboardEvents.off('cos:schedule:changed', notification);
+ expect(notification).toHaveBeenCalled();
+ expect(state.persisted.tasks.security).toMatchObject({ enabled: false, prompt: 'Restored custom prompt' });
+ expect(state.persisted.templates).toEqual(restored.templates);
+ expect(state.persisted.executions['task:security'].count).toBe(1);
+ });
+
+ it('retains transfer diagnostics when reconciliation fails and releases the queue', async () => {
+ const restore = withLiveTaskScheduleRestore(async () => {
+ readJSONFile.mockRejectedValueOnce(new Error('Invalid restored JSON'));
+ throw Object.assign(new Error('Some files may already have been overwritten'), { code: 'RSYNC_FAILED' });
+ });
+ await expect(restore).rejects.toMatchObject({
+ code: 'RSYNC_FAILED',
+ message: expect.stringMatching(/Some files may already have been overwritten.*Task schedule reconciliation failed: Invalid restored JSON/)
+ });
+ await expect(updateSchedule(() => ({ result: 'released', changed: false }))).resolves.toBe('released');
+ });
+
});