Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve concurrent CoS queue mutations (#13300)

[._.]/ Adam Eivyb35e4d5e837a
diff --git a/server/services/_cosTaskRepositoryContract.js b/server/services/_cosTaskRepositoryContract.js
index c83c930520..2e8830d502 100644
--- a/server/services/_cosTaskRepositoryContract.js
+++ b/server/services/_cosTaskRepositoryContract.js
@@ -21,9 +21,9 @@ export function __runCosTaskRepositoryContract({ name, makeBackend }) {
let backend;
// Saves a whole list the way the store does: plan against what is loaded, then apply.
const save = async (source, next) => {
- const { rows } = await backend.loadWorkingSet(source);
+ const { rows, version } = await backend.loadWorkingSet(source);
const plan = planListSave({ previous: rows, next, source });
- await backend.apply(source, { upserts: plan.upserts, deletes: plan.deletes });
+ await backend.apply(source, { ...plan, expectedVersions: { [source]: version } });
return plan;
};
const ids = async (source) => (await backend.loadWorkingSet(source)).rows.map((row) => row.task.id);
@@ -61,6 +61,18 @@ export function __runCosTaskRepositoryContract({ name, makeBackend }) {
expect(rows.map((row) => row.task.description)).toEqual(['Edited']);
});
+ it('refuses stale or unversioned replacements, including after an explicit deletion', async () => {
+ await save('user', [task(1), task(2)]);
+ const snapshot = await backend.loadWorkingSet('user');
+ const plan = planListSave({ previous: snapshot.rows, next: [task(1, { description: 'Stale edit' }), task(3)], source: 'user' });
+ await backend.deleteById(id(1), 'user');
+ await expect(backend.apply('user', { ...plan, expectedVersions: { user: snapshot.version } }))
+ .rejects.toMatchObject({ code: 'COS_TASK_LIST_CONFLICT' });
+ await expect(backend.apply('user', plan)).rejects.toThrow('require the versions');
+ expect(await ids('user')).toEqual([id(2)]);
+ expect(await backend.readVersion('user')).toBe(snapshot.version + 1);
+ });
+
it('returns copies, never references into the backend', async () => {
await save('user', [task(1, { metadata: { note: 'kept' } })]);
const first = await backend.loadWorkingSet('user');
diff --git a/server/services/cosTaskRepository.db.test.js b/server/services/cosTaskRepository.db.test.js
index f6f428d97b..b226c606d4 100644
--- a/server/services/cosTaskRepository.db.test.js
+++ b/server/services/cosTaskRepository.db.test.js
@@ -3,7 +3,7 @@
* _cosTaskRepositoryContract.js. Runs via `npm run test:db` (→ `portos_test`)
* ONLY; skips cleanly when no database is reachable.
*/
-import { afterAll, describe, expect, it } from 'vitest';
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { checkHealth, close, ensureSchema, query, withTransaction } from '../lib/db.js';
import { requireDbOrSkip } from '../lib/dbTestGate.js';
import { __makePgCosTaskBackend } from './cosTaskRepository.js';
@@ -45,9 +45,9 @@ describe.skipIf(!runDb)('cos_tasks repository text PostgreSQL would refuse', ()
// The cut askPromote makes: 279 characters, then half of an emoji's surrogate pair.
const split = `${'a'.repeat(279)}${'😀'.slice(0, 1)}`;
const next = [task(1, split), task(2, 'before\0after')];
- const { rows } = await backend.loadWorkingSet('user');
+ const { rows, version } = await backend.loadWorkingSet('user');
const plan = planListSave({ previous: rows, next, source: 'user' });
- await backend.apply('user', { upserts: plan.upserts, deletes: plan.deletes });
+ await backend.apply('user', { ...plan, expectedVersions: { user: version } });
const saved = Object.fromEntries((await backend.loadWorkingSet('user')).rows.map((row) => [row.task.id, row.task]));
expect(saved['task-example-1'].description).toBe(`${'a'.repeat(279)}\uFFFD`);
@@ -56,3 +56,106 @@ describe.skipIf(!runDb)('cos_tasks repository text PostgreSQL would refuse', ()
expect(column.description).toBe('before\uFFFDafter');
});
});
+
+// Two independently loaded stores have separate caches. The lock double does
+// not serialize them, just as two service processes cannot share a JS lock.
+// All task persistence below still uses real transactions in portos_test.
+describe.skipIf(!runDb)('independent task writers', () => {
+ let helper;
+ let daemon;
+ let beforeApply;
+ let beforeLookup;
+ const events = [];
+ const notifications = [];
+ const backend = __makePgCosTaskBackend({ query, withTransaction });
+ const gate = () => Promise.withResolvers();
+ const holdNextWrite = ({ afterRead = false } = {}) => {
+ const entered = gate();
+ const release = gate();
+ const hook = async () => { entered.resolve(); await release.promise; };
+ if (afterRead) beforeLookup = hook; else beforeApply = hook;
+ return { entered: entered.promise, release: release.resolve };
+ };
+
+ beforeEach(async () => {
+ await query('DELETE FROM cos_tasks');
+ await query('DELETE FROM cos_task_lists');
+ beforeApply = null;
+ beforeLookup = null;
+ events.length = 0;
+ notifications.length = 0;
+ vi.doMock('./cosState.js', () => ({ withStateLock: fn => fn(), loadState: async () => ({ agents: {} }) }));
+ vi.doMock('./cosEvents.js', () => ({ cosEvents: { emit: (name, payload) => events.push({ name, ...payload }) } }));
+ vi.doMock('./notifications.js', () => ({ removeByMetadata: async (...args) => notifications.push(args) }));
+ vi.doMock('./codeReview.js', () => ({ runLocalCodeReview: vi.fn(), getCodeReviewDefaults: vi.fn() }));
+ vi.doMock('./cosTaskRepository.js', () => ({
+ readTaskListVersion: source => backend.readVersion(source),
+ loadTaskWorkingSet: source => backend.loadWorkingSet(source),
+ getStoredTask: async (id, source = null) => {
+ const hook = beforeLookup;
+ beforeLookup = null;
+ if (hook) await hook();
+ return backend.getTask(id, source);
+ },
+ deleteStoredTask: (id, source = null) => backend.deleteById(id, source),
+ getCompletedTaskIds: (source, ids) => backend.completedIds(source, ids),
+ countCompletedTasks: source => backend.countCompleted(source),
+ pageCompletedTasks: (source, options) => backend.pageCompleted(source, options),
+ applyTaskListSave: async (source, plan) => {
+ const hook = beforeApply;
+ beforeApply = null;
+ if (hook) await hook();
+ return backend.apply(source, plan);
+ },
+ }));
+ vi.resetModules();
+ helper = await import('./cosTaskStore.js');
+ vi.resetModules();
+ daemon = await import('./cosTaskStore.js');
+ expect(helper.addTask).not.toBe(daemon.addTask);
+ });
+
+ it('recomputes a stale addition, preserving concurrent additions and edits with one event per commit', async () => {
+ const a = await daemon.addTask({ description: 'A' });
+ const pause = holdNextWrite({ afterRead: true });
+ const adding = helper.addTask({ description: 'C' });
+ await pause.entered;
+ await daemon.updateTask(a.id, { description: 'Newer A' });
+ const b = await daemon.addTask({ description: 'B' });
+ pause.release();
+ const c = await adding;
+ expect((await helper.getUserTasks()).tasks.map(task => task.description)).toEqual(['Newer A', 'B', 'C']);
+ expect(events.filter(event => event.action === 'added' && event.task.id === c.id)).toHaveLength(1);
+ await helper.deleteTask(b.id);
+ expect((await daemon.getUserTasks()).tasks.map(task => task.id)).toEqual([a.id, c.id]);
+ });
+
+ it('rechecks a follow-up work identity in the other queue and emits no duplicate notification', async () => {
+ const metadata = { app: 'example', reviewLoopPRUrl: 'https://github.com/example/example/pull/42' };
+ const pause = holdNextWrite({ afterRead: true });
+ const adding = helper.addTask({ description: 'Scheduled follow-up', metadata }, 'internal');
+ await pause.entered;
+ const existing = await daemon.addTask({ description: 'Manual follow-up', metadata });
+ pause.release();
+ expect(await adding).toMatchObject({ id: existing.id, duplicate: true });
+ expect((await helper.getCosTasks()).tasks).toEqual([]);
+ expect(events.filter(event => event.action === 'added')).toHaveLength(1);
+ });
+
+ it('rejects a stale approval before emitting or removing its notification', async () => {
+ const task = await daemon.addTask({ description: 'Approval', approvalRequired: true }, 'internal');
+ const pause = holdNextWrite();
+ const approval = helper.approveTask(task.id);
+ const rejected = expect(approval).rejects.toMatchObject({ code: 'COS_TASK_LIST_CONFLICT' });
+ await pause.entered;
+ await daemon.updateTask(task.id, { description: 'Changed before approval' }, 'internal');
+ pause.release();
+ await rejected;
+ expect(events.filter(event => event.action === 'approved')).toEqual([]);
+ expect(notifications).toEqual([]);
+ expect((await daemon.getCosTasks()).tasks[0]).toMatchObject({ description: 'Changed before approval', approvalRequired: true });
+ await helper.approveTask(task.id);
+ expect(events.filter(event => event.action === 'approved')).toHaveLength(1);
+ expect(notifications).toEqual([['taskId', task.id]]);
+ });
+});
diff --git a/server/services/cosTaskRepository.js b/server/services/cosTaskRepository.js
index affe704dd4..573848c766 100644
--- a/server/services/cosTaskRepository.js
+++ b/server/services/cosTaskRepository.js
@@ -25,6 +25,26 @@ import { toPgJson } from '../lib/pgJson.js';
const SOURCES = ['user', 'internal'];
// Bounds a single statement's parameter payload (prompts can be MBs).
const UPSERT_BATCH = 100;
+const conflict = () => Object.assign(new Error('Task queue changed; reload and retry the operation'), {
+ code: 'COS_TASK_LIST_CONFLICT', status: 409,
+});
+
+function checkExpectedVersions(source, expectedVersions, versions) {
+ if (!Number.isSafeInteger(expectedVersions?.[source]) || expectedVersions[source] < 0
+ || Object.entries(expectedVersions).some(([key, value]) => !SOURCES.includes(key) || !Number.isSafeInteger(value) || value < 0)) {
+ throw new Error('Task saves require the versions of the snapshots they read');
+ }
+ if (Object.entries(expectedVersions).some(([key, value]) => versions[key] !== value)) throw conflict();
+}
+
+// Every writer locks the same version rows before touching tasks. The fixed
+// order also makes a save conditional on both queues safe for cross-queue dedup.
+async function lockVersions(client) {
+ await client.query(`INSERT INTO cos_task_lists (source, version) VALUES ('user', 0), ('internal', 0) ON CONFLICT DO NOTHING`);
+ const { rows } = await client.query('SELECT source, version FROM cos_task_lists ORDER BY source FOR UPDATE');
+ return Object.fromEntries(rows.map((row) => [row.source, Number(row.version)]));
+}
+
const tasksFile = () => join(PATHS.data, 'cos-tasks.json');
const byIdDesc = (a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0);
@@ -37,23 +57,27 @@ function makePgBackend(db) {
return rows[0] ? Number(rows[0].version) : 0;
},
async loadWorkingSet(source, { now = Date.now() } = {}) {
- const version = await this.readVersion(source);
+ // One MVCC snapshot binds the version to the rows, including an empty list.
const { rows } = await query(
`WITH newest AS (
SELECT task_id FROM cos_tasks WHERE source = $1 AND status = 'completed'
ORDER BY completed_at DESC NULLS LAST, task_id DESC LIMIT $3
- )
+ ), working AS (
SELECT task, position FROM cos_tasks
WHERE source = $1 AND (status <> 'completed' OR completed_at >= $2 OR task_id IN (SELECT task_id FROM newest))
ORDER BY CASE status WHEN 'pending' THEN 0 WHEN 'in_progress' THEN 1 WHEN 'challenged' THEN 2
WHEN 'blocked' THEN 3 ELSE 4 END,
- priority_value DESC, position, task_id`,
+ priority_value DESC, position, task_id
+ )
+ SELECT COALESCE((SELECT version FROM cos_task_lists WHERE source = $1), 0) AS version,
+ COALESCE(json_agg(working), '[]'::json) AS rows FROM working`,
[source, new Date(now - WORKING_SET_COMPLETED_WINDOW_MS).toISOString(), WORKING_SET_KEEP_COMPLETED],
);
- return { version, rows: rows.map((row) => ({ task: row.task, position: row.position })) };
+ return { version: Number(rows[0].version), rows: rows[0].rows };
},
- async apply(source, { upserts, deletes }) {
+ async apply(source, { upserts, deletes, expectedVersions }) {
return withTransaction(async (client) => {
+ checkExpectedVersions(source, expectedVersions, await lockVersions(client));
if (deletes.length > 0) await client.query('DELETE FROM cos_tasks WHERE source = $1 AND task_id = ANY($2::text[])', [source, deletes]);
for (let start = 0; start < upserts.length; start += UPSERT_BATCH) {
const columns = upserts.slice(start, start + UPSERT_BATCH).map(({ task, position }) => ({ task, position, ...rowColumns(task) }));
@@ -106,6 +130,7 @@ function makePgBackend(db) {
},
async deleteById(taskId, source) {
return withTransaction(async (client) => {
+ await lockVersions(client);
const { rows } = await client.query(
'DELETE FROM cos_tasks WHERE task_id = $1 AND ($2::text IS NULL OR source = $2) RETURNING source',
[taskId, source],
@@ -156,7 +181,8 @@ async function makeMemoryBackend({ persist }) {
async loadWorkingSet(source, { now = Date.now() } = {}) {
return { version: versions[source], rows: clone(selectWorkingSet([...lists[source].values()], { now })) };
},
- async apply(source, { upserts, deletes }) {
+ async apply(source, { upserts, deletes, expectedVersions }) {
+ checkExpectedVersions(source, expectedVersions, versions);
for (const id of deletes) lists[source].delete(id);
for (const { task, position } of upserts) lists[source].set(task.id, clone({ task, position }));
versions[source] += 1;
@@ -207,7 +233,7 @@ export async function readTaskListVersion(source) { return (await backend()).rea
/** `{ version, rows: [{ task, position }] }` in queue order for one queue's working set. */
export async function loadTaskWorkingSet(source) { return (await backend()).loadWorkingSet(source); }
-/** Persist a save plan (`planListSave`'s upserts/deletes) in one transaction; returns the new version. */
+/** Persist a plan with expectedVersions in one transaction; stale snapshots fail with COS_TASK_LIST_CONFLICT. */
export async function applyTaskListSave(source, plan) { return (await backend()).apply(source, plan); }
/** One task as `{ source, task, position }` whatever its status or age, user queue first. Omit `source` to search both. */
diff --git a/server/services/cosTaskRepository.test.js b/server/services/cosTaskRepository.test.js
index 8a6e85549b..67eb683c3b 100644
--- a/server/services/cosTaskRepository.test.js
+++ b/server/services/cosTaskRepository.test.js
@@ -8,7 +8,7 @@ describe('cosTaskRepository facade (test runner)', () => {
it('selects the in-memory backend and does not touch disk', async () => {
__resetCosTaskRepository();
const task = { id: 'task-example-0001', status: 'pending', priority: 'MEDIUM', priorityValue: 2, description: 'Example', approvalRequired: false, autoApproved: true, metadata: {} };
- await applyTaskListSave('user', { upserts: [{ task, position: 0 }], deletes: [] });
+ await applyTaskListSave('user', { upserts: [{ task, position: 0 }], deletes: [], expectedVersions: { user: 0 } });
expect((await loadTaskWorkingSet('user')).rows).toHaveLength(1);
expect(await readTaskListVersion('user')).toBe(1);
__resetCosTaskRepository();
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index ecebc3b176..0b6ba251c4 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -175,6 +175,9 @@ function isContentEdit(updates, existingMetadata = {}) {
// exactly one (nobody else wrote in between); otherwise they drop it.
const snapshotCache = new Map(); // source -> { version, snapshot }
const snapshotLoads = createSingleFlight();
+// A list may be mutated or transformed before saving. Keep its ORIGINAL read
+// snapshot, never a fresh read whose omissions could turn into accidental deletes.
+const listSnapshots = new WeakMap();
// Private snapshots never leave this module: list readers clone the array,
// while ID lookups clone only the matching task. Both share invalidation.
@@ -185,7 +188,7 @@ async function readTaskSnapshot(source) {
if (cached?.version === version) return cached.snapshot;
return snapshotLoads.run(`${source}:${version}`, async () => {
const loaded = await loadTaskWorkingSet(source);
- const snapshot = { rows: loaded.rows, tasks: loaded.rows.map((row) => row.task) };
+ const snapshot = { version: loaded.version, rows: loaded.rows, tasks: loaded.rows.map((row) => row.task) };
if (!snapshotCache.get(source) || snapshotCache.get(source).version <= loaded.version) {
snapshotCache.set(source, { version: loaded.version, snapshot });
}
@@ -194,7 +197,10 @@ async function readTaskSnapshot(source) {
}
async function readTaskFile(source) {
- return structuredClone((await readTaskSnapshot(source)).tasks);
+ const snapshot = await readTaskSnapshot(source);
+ const tasks = structuredClone(snapshot.tasks);
+ listSnapshots.set(tasks, snapshot);
+ return tasks;
}
async function findTaskInFile(source, taskId) {
@@ -215,19 +221,22 @@ async function findTaskInFile(source, taskId) {
* only the rows that changed, in one transaction. Every write path in this
* module goes through here. Resolves to whether anything was written.
*/
-async function writeTaskList(source, tasks) {
- const snapshot = await readTaskSnapshot(source);
+async function writeTaskList(source, tasks, original = tasks, dependencies = {}) {
+ const snapshot = listSnapshots.get(original);
+ if (!snapshot) throw new Error('Task saves require their original read snapshot');
const cached = snapshotCache.get(source);
const plan = planListSave({ previous: snapshot.rows, next: tasks, source });
if (plan.upserts.length === 0 && plan.deletes.length === 0) return false;
- const version = await applyTaskListSave(source, plan);
+ const version = await applyTaskListSave(source, {
+ ...plan, expectedVersions: { ...dependencies, [source]: snapshot.version },
+ });
const positions = new Map(snapshot.rows.map((row) => [row.task.id, row.position]));
for (const { task, position } of plan.upserts) positions.set(task.id, position);
if (cached?.snapshot === snapshot && version === cached.version + 1) {
// Same bound as a reload, so completions never accumulate in the cache.
const rows = selectWorkingSet(plan.saved.map((task) => ({ task, position: positions.get(task.id) })));
- snapshotCache.set(source, { version, snapshot: { rows, tasks: rows.map((row) => row.task) } });
+ snapshotCache.set(source, { version, snapshot: { version, rows, tasks: rows.map((row) => row.task) } });
} else {
snapshotCache.delete(source);
}
@@ -488,7 +497,20 @@ async function uniqueTaskId(source, id, tasks) {
* force-spawn the returned task itself; the change event still reaches socket
* consumers, but the normal scheduler must not race that dispatch.
*/
-export async function addTask(taskData, taskType = 'user', { raw = false, ignoreTaskId = null, now = Date.now(), suppressDequeue = false } = {}) {
+export async function addTask(taskData, taskType = 'user', options = {}) {
+ // Only addition is replayed: it has no effects before persistence. Re-read
+ // both queues and recompute dedup/IDs on conflict; never retry a stale plan.
+ // Other mutations surface 409 so decisions or side effects are not replayed.
+ for (let attempt = 0; ; attempt++) {
+ const result = await addTaskAttempt(taskData, taskType, options).catch(error => {
+ if (error.code !== 'COS_TASK_LIST_CONFLICT' || attempt >= 3) throw error;
+ return null;
+ });
+ if (result !== null) return result;
+ }
+}
+
+async function addTaskAttempt(taskData, taskType, { raw = false, ignoreTaskId = null, now = Date.now(), suppressDequeue = false }) {
validateRequest(cosTaskTypeSchema, taskType);
if (!raw && taskData.metadata) taskData = { ...taskData, metadata: newAuditMetadata(taskData.metadata) };
return withStateLock(async () => {
@@ -497,8 +519,10 @@ export async function addTask(taskData, taskType = 'user', { raw = false, ignore
// One state lock covers BOTH queues: manual and autonomous producers
// must acquire the same durable work identity even with different titles.
const newWorkIdentity = developmentWorkIdentity(taskData);
+ const dependencies = {};
if (newWorkIdentity) {
const other = await readTaskFile(taskType === 'user' ? 'internal' : 'user');
+ dependencies[taskType === 'user' ? 'internal' : 'user'] = listSnapshots.get(other).version;
const owner = [...tasks, ...other].find(task => task.id !== ignoreTaskId
&& DEVELOPMENT_ACTIVE_STATUSES.has(task.status) && sameDevelopmentWork(task, taskData)
// Targeted claims and the backlog drain coexist in either admission order.
@@ -617,7 +641,7 @@ export async function addTask(taskData, taskType = 'user', { raw = false, ignore
tasks.push(newTask);
}
- await writeTaskList(taskType, tasks);
+ await writeTaskList(taskType, tasks, tasks, dependencies);
// cos.js init listens for this event. For user tasks it fires
// tryImmediateSpawn so the task starts instantly if slots are available,
@@ -1010,7 +1034,7 @@ export async function mergePeerTasks(taskType, remoteTasks, { now = Date.now() }
// stored a terminal verdict before upgrading, or a crash before a prior consume, leaves
// the task terminal with the merge producing no change, so a changed-only consume would
// never catch up). Re-offering is cheap: the consumer is a durable no-op after the first.
- if (!await writeTaskList(taskType, merged)) return { changed: false, merged };
+ if (!await writeTaskList(taskType, merged, localTasks)) return { changed: false, merged };
cosEvents.emit('tasks:changed', { type: taskType, action: 'peer-merged' });
return { changed: true, count: merged.length, merged };
@@ -1276,7 +1300,7 @@ export async function reorderTasks(taskIds) {
reorderedTasks.push(task);
}
- await writeTaskList('user', reorderedTasks);
+ await writeTaskList('user', reorderedTasks, tasks);
cosEvents.emit('tasks:changed', { type: 'user', action: 'reordered' });
return { success: true, order: reorderedTasks.map(t => t.id) };
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index cde3b281af..26adea2ccd 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -129,9 +129,9 @@ const baseState = () => ({ agents: {} });
// Test-side access to what is stored, bypassing the store's cache and the
// failure/after-write hooks above (they wrap only the store's own saves).
const seed = async (source, tasks) => {
- const { rows } = await actualRepository.loadTaskWorkingSet(source);
+ const { rows, version } = await actualRepository.loadTaskWorkingSet(source);
const plan = planListSave({ previous: rows, next: tasks, source });
- await actualRepository.applyTaskListSave(source, plan);
+ await actualRepository.applyTaskListSave(source, { ...plan, expectedVersions: { [source]: version } });
__resetTaskCache();
};
const stored = async (source) => (await actualRepository.loadTaskWorkingSet(source)).rows.map((row) => row.task);