Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve concurrent CoS queue mutations (#13300)
[._.]/ Adam Eivyb35e4d5e837a
diff --git a/server/services/_cosTaskRepositoryContract.js b/server/services/_cosTaskRepositoryContract.jsindex c83c930520..2e8830d502 100644--- a/server/services/_cosTaskRepositoryContract.js+++ b/server/services/_cosTaskRepositoryContract.js@@ -21,9 +21,9 @@ export function __runCosTaskRepositoryContract({ name, makeBackend }) {let backend;// Saves a whole list the way the store does: plan against what is loaded, then apply.const save = async (source, next) => {- const { rows } = await backend.loadWorkingSet(source);+ const { rows, version } = await backend.loadWorkingSet(source);const plan = planListSave({ previous: rows, next, source });- await backend.apply(source, { upserts: plan.upserts, deletes: plan.deletes });+ await backend.apply(source, { ...plan, expectedVersions: { [source]: version } });return plan;};const ids = async (source) => (await backend.loadWorkingSet(source)).rows.map((row) => row.task.id);@@ -61,6 +61,18 @@ export function __runCosTaskRepositoryContract({ name, makeBackend }) {expect(rows.map((row) => row.task.description)).toEqual(['Edited']);});+ it('refuses stale or unversioned replacements, including after an explicit deletion', async () => {+ await save('user', [task(1), task(2)]);+ const snapshot = await backend.loadWorkingSet('user');+ const plan = planListSave({ previous: snapshot.rows, next: [task(1, { description: 'Stale edit' }), task(3)], source: 'user' });+ await backend.deleteById(id(1), 'user');+ await expect(backend.apply('user', { ...plan, expectedVersions: { user: snapshot.version } }))+ .rejects.toMatchObject({ code: 'COS_TASK_LIST_CONFLICT' });+ await expect(backend.apply('user', plan)).rejects.toThrow('require the versions');+ expect(await ids('user')).toEqual([id(2)]);+ expect(await backend.readVersion('user')).toBe(snapshot.version + 1);+ });+it('returns copies, never references into the backend', async () => {await save('user', [task(1, { metadata: { note: 'kept' } })]);const first = await backend.loadWorkingSet('user');diff --git a/server/services/cosTaskRepository.db.test.js b/server/services/cosTaskRepository.db.test.jsindex f6f428d97b..b226c606d4 100644--- a/server/services/cosTaskRepository.db.test.js+++ b/server/services/cosTaskRepository.db.test.js@@ -3,7 +3,7 @@* _cosTaskRepositoryContract.js. Runs via `npm run test:db` (→ `portos_test`)* ONLY; skips cleanly when no database is reachable.*/-import { afterAll, describe, expect, it } from 'vitest';+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';import { checkHealth, close, ensureSchema, query, withTransaction } from '../lib/db.js';import { requireDbOrSkip } from '../lib/dbTestGate.js';import { __makePgCosTaskBackend } from './cosTaskRepository.js';@@ -45,9 +45,9 @@ describe.skipIf(!runDb)('cos_tasks repository text PostgreSQL would refuse', ()// The cut askPromote makes: 279 characters, then half of an emoji's surrogate pair.const split = `${'a'.repeat(279)}${'😀'.slice(0, 1)}`;const next = [task(1, split), task(2, 'before\0after')];- const { rows } = await backend.loadWorkingSet('user');+ const { rows, version } = await backend.loadWorkingSet('user');const plan = planListSave({ previous: rows, next, source: 'user' });- await backend.apply('user', { upserts: plan.upserts, deletes: plan.deletes });+ await backend.apply('user', { ...plan, expectedVersions: { user: version } });const saved = Object.fromEntries((await backend.loadWorkingSet('user')).rows.map((row) => [row.task.id, row.task]));expect(saved['task-example-1'].description).toBe(`${'a'.repeat(279)}\uFFFD`);@@ -56,3 +56,106 @@ describe.skipIf(!runDb)('cos_tasks repository text PostgreSQL would refuse', ()expect(column.description).toBe('before\uFFFDafter');});});++// Two independently loaded stores have separate caches. The lock double does+// not serialize them, just as two service processes cannot share a JS lock.+// All task persistence below still uses real transactions in portos_test.+describe.skipIf(!runDb)('independent task writers', () => {+ let helper;+ let daemon;+ let beforeApply;+ let beforeLookup;+ const events = [];+ const notifications = [];+ const backend = __makePgCosTaskBackend({ query, withTransaction });+ const gate = () => Promise.withResolvers();+ const holdNextWrite = ({ afterRead = false } = {}) => {+ const entered = gate();+ const release = gate();+ const hook = async () => { entered.resolve(); await release.promise; };+ if (afterRead) beforeLookup = hook; else beforeApply = hook;+ return { entered: entered.promise, release: release.resolve };+ };++ beforeEach(async () => {+ await query('DELETE FROM cos_tasks');+ await query('DELETE FROM cos_task_lists');+ beforeApply = null;+ beforeLookup = null;+ events.length = 0;+ notifications.length = 0;+ vi.doMock('./cosState.js', () => ({ withStateLock: fn => fn(), loadState: async () => ({ agents: {} }) }));+ vi.doMock('./cosEvents.js', () => ({ cosEvents: { emit: (name, payload) => events.push({ name, ...payload }) } }));+ vi.doMock('./notifications.js', () => ({ removeByMetadata: async (...args) => notifications.push(args) }));+ vi.doMock('./codeReview.js', () => ({ runLocalCodeReview: vi.fn(), getCodeReviewDefaults: vi.fn() }));+ vi.doMock('./cosTaskRepository.js', () => ({+ readTaskListVersion: source => backend.readVersion(source),+ loadTaskWorkingSet: source => backend.loadWorkingSet(source),+ getStoredTask: async (id, source = null) => {+ const hook = beforeLookup;+ beforeLookup = null;+ if (hook) await hook();+ return backend.getTask(id, source);+ },+ deleteStoredTask: (id, source = null) => backend.deleteById(id, source),+ getCompletedTaskIds: (source, ids) => backend.completedIds(source, ids),+ countCompletedTasks: source => backend.countCompleted(source),+ pageCompletedTasks: (source, options) => backend.pageCompleted(source, options),+ applyTaskListSave: async (source, plan) => {+ const hook = beforeApply;+ beforeApply = null;+ if (hook) await hook();+ return backend.apply(source, plan);+ },+ }));+ vi.resetModules();+ helper = await import('./cosTaskStore.js');+ vi.resetModules();+ daemon = await import('./cosTaskStore.js');+ expect(helper.addTask).not.toBe(daemon.addTask);+ });++ it('recomputes a stale addition, preserving concurrent additions and edits with one event per commit', async () => {+ const a = await daemon.addTask({ description: 'A' });+ const pause = holdNextWrite({ afterRead: true });+ const adding = helper.addTask({ description: 'C' });+ await pause.entered;+ await daemon.updateTask(a.id, { description: 'Newer A' });+ const b = await daemon.addTask({ description: 'B' });+ pause.release();+ const c = await adding;+ expect((await helper.getUserTasks()).tasks.map(task => task.description)).toEqual(['Newer A', 'B', 'C']);+ expect(events.filter(event => event.action === 'added' && event.task.id === c.id)).toHaveLength(1);+ await helper.deleteTask(b.id);+ expect((await daemon.getUserTasks()).tasks.map(task => task.id)).toEqual([a.id, c.id]);+ });++ it('rechecks a follow-up work identity in the other queue and emits no duplicate notification', async () => {+ const metadata = { app: 'example', reviewLoopPRUrl: 'https://github.com/example/example/pull/42' };+ const pause = holdNextWrite({ afterRead: true });+ const adding = helper.addTask({ description: 'Scheduled follow-up', metadata }, 'internal');+ await pause.entered;+ const existing = await daemon.addTask({ description: 'Manual follow-up', metadata });+ pause.release();+ expect(await adding).toMatchObject({ id: existing.id, duplicate: true });+ expect((await helper.getCosTasks()).tasks).toEqual([]);+ expect(events.filter(event => event.action === 'added')).toHaveLength(1);+ });++ it('rejects a stale approval before emitting or removing its notification', async () => {+ const task = await daemon.addTask({ description: 'Approval', approvalRequired: true }, 'internal');+ const pause = holdNextWrite();+ const approval = helper.approveTask(task.id);+ const rejected = expect(approval).rejects.toMatchObject({ code: 'COS_TASK_LIST_CONFLICT' });+ await pause.entered;+ await daemon.updateTask(task.id, { description: 'Changed before approval' }, 'internal');+ pause.release();+ await rejected;+ expect(events.filter(event => event.action === 'approved')).toEqual([]);+ expect(notifications).toEqual([]);+ expect((await daemon.getCosTasks()).tasks[0]).toMatchObject({ description: 'Changed before approval', approvalRequired: true });+ await helper.approveTask(task.id);+ expect(events.filter(event => event.action === 'approved')).toHaveLength(1);+ expect(notifications).toEqual([['taskId', task.id]]);+ });+});diff --git a/server/services/cosTaskRepository.js b/server/services/cosTaskRepository.jsindex affe704dd4..573848c766 100644--- a/server/services/cosTaskRepository.js+++ b/server/services/cosTaskRepository.js@@ -25,6 +25,26 @@ import { toPgJson } from '../lib/pgJson.js';const SOURCES = ['user', 'internal'];// Bounds a single statement's parameter payload (prompts can be MBs).const UPSERT_BATCH = 100;+const conflict = () => Object.assign(new Error('Task queue changed; reload and retry the operation'), {+ code: 'COS_TASK_LIST_CONFLICT', status: 409,+});++function checkExpectedVersions(source, expectedVersions, versions) {+ if (!Number.isSafeInteger(expectedVersions?.[source]) || expectedVersions[source] < 0+ || Object.entries(expectedVersions).some(([key, value]) => !SOURCES.includes(key) || !Number.isSafeInteger(value) || value < 0)) {+ throw new Error('Task saves require the versions of the snapshots they read');+ }+ if (Object.entries(expectedVersions).some(([key, value]) => versions[key] !== value)) throw conflict();+}++// Every writer locks the same version rows before touching tasks. The fixed+// order also makes a save conditional on both queues safe for cross-queue dedup.+async function lockVersions(client) {+ await client.query(`INSERT INTO cos_task_lists (source, version) VALUES ('user', 0), ('internal', 0) ON CONFLICT DO NOTHING`);+ const { rows } = await client.query('SELECT source, version FROM cos_task_lists ORDER BY source FOR UPDATE');+ return Object.fromEntries(rows.map((row) => [row.source, Number(row.version)]));+}+const tasksFile = () => join(PATHS.data, 'cos-tasks.json');const byIdDesc = (a, b) => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0);@@ -37,23 +57,27 @@ function makePgBackend(db) {return rows[0] ? Number(rows[0].version) : 0;},async loadWorkingSet(source, { now = Date.now() } = {}) {- const version = await this.readVersion(source);+ // One MVCC snapshot binds the version to the rows, including an empty list.const { rows } = await query(`WITH newest AS (SELECT task_id FROM cos_tasks WHERE source = $1 AND status = 'completed'ORDER BY completed_at DESC NULLS LAST, task_id DESC LIMIT $3- )+ ), working AS (SELECT task, position FROM cos_tasksWHERE source = $1 AND (status <> 'completed' OR completed_at >= $2 OR task_id IN (SELECT task_id FROM newest))ORDER BY CASE status WHEN 'pending' THEN 0 WHEN 'in_progress' THEN 1 WHEN 'challenged' THEN 2WHEN 'blocked' THEN 3 ELSE 4 END,- priority_value DESC, position, task_id`,+ priority_value DESC, position, task_id+ )+ SELECT COALESCE((SELECT version FROM cos_task_lists WHERE source = $1), 0) AS version,+ COALESCE(json_agg(working), '[]'::json) AS rows FROM working`,[source, new Date(now - WORKING_SET_COMPLETED_WINDOW_MS).toISOString(), WORKING_SET_KEEP_COMPLETED],);- return { version, rows: rows.map((row) => ({ task: row.task, position: row.position })) };+ return { version: Number(rows[0].version), rows: rows[0].rows };},- async apply(source, { upserts, deletes }) {+ async apply(source, { upserts, deletes, expectedVersions }) {return withTransaction(async (client) => {+ checkExpectedVersions(source, expectedVersions, await lockVersions(client));if (deletes.length > 0) await client.query('DELETE FROM cos_tasks WHERE source = $1 AND task_id = ANY($2::text[])', [source, deletes]);for (let start = 0; start < upserts.length; start += UPSERT_BATCH) {const columns = upserts.slice(start, start + UPSERT_BATCH).map(({ task, position }) => ({ task, position, ...rowColumns(task) }));@@ -106,6 +130,7 @@ function makePgBackend(db) {},async deleteById(taskId, source) {return withTransaction(async (client) => {+ await lockVersions(client);const { rows } = await client.query('DELETE FROM cos_tasks WHERE task_id = $1 AND ($2::text IS NULL OR source = $2) RETURNING source',[taskId, source],@@ -156,7 +181,8 @@ async function makeMemoryBackend({ persist }) {async loadWorkingSet(source, { now = Date.now() } = {}) {return { version: versions[source], rows: clone(selectWorkingSet([...lists[source].values()], { now })) };},- async apply(source, { upserts, deletes }) {+ async apply(source, { upserts, deletes, expectedVersions }) {+ checkExpectedVersions(source, expectedVersions, versions);for (const id of deletes) lists[source].delete(id);for (const { task, position } of upserts) lists[source].set(task.id, clone({ task, position }));versions[source] += 1;@@ -207,7 +233,7 @@ export async function readTaskListVersion(source) { return (await backend()).rea/** `{ version, rows: [{ task, position }] }` in queue order for one queue's working set. */export async function loadTaskWorkingSet(source) { return (await backend()).loadWorkingSet(source); }-/** Persist a save plan (`planListSave`'s upserts/deletes) in one transaction; returns the new version. */+/** Persist a plan with expectedVersions in one transaction; stale snapshots fail with COS_TASK_LIST_CONFLICT. */export async function applyTaskListSave(source, plan) { return (await backend()).apply(source, plan); }/** One task as `{ source, task, position }` whatever its status or age, user queue first. Omit `source` to search both. */diff --git a/server/services/cosTaskRepository.test.js b/server/services/cosTaskRepository.test.jsindex 8a6e85549b..67eb683c3b 100644--- a/server/services/cosTaskRepository.test.js+++ b/server/services/cosTaskRepository.test.js@@ -8,7 +8,7 @@ describe('cosTaskRepository facade (test runner)', () => {it('selects the in-memory backend and does not touch disk', async () => {__resetCosTaskRepository();const task = { id: 'task-example-0001', status: 'pending', priority: 'MEDIUM', priorityValue: 2, description: 'Example', approvalRequired: false, autoApproved: true, metadata: {} };- await applyTaskListSave('user', { upserts: [{ task, position: 0 }], deletes: [] });+ await applyTaskListSave('user', { upserts: [{ task, position: 0 }], deletes: [], expectedVersions: { user: 0 } });expect((await loadTaskWorkingSet('user')).rows).toHaveLength(1);expect(await readTaskListVersion('user')).toBe(1);__resetCosTaskRepository();diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.jsindex ecebc3b176..0b6ba251c4 100644--- a/server/services/cosTaskStore.js+++ b/server/services/cosTaskStore.js@@ -175,6 +175,9 @@ function isContentEdit(updates, existingMetadata = {}) {// exactly one (nobody else wrote in between); otherwise they drop it.const snapshotCache = new Map(); // source -> { version, snapshot }const snapshotLoads = createSingleFlight();+// A list may be mutated or transformed before saving. Keep its ORIGINAL read+// snapshot, never a fresh read whose omissions could turn into accidental deletes.+const listSnapshots = new WeakMap();// Private snapshots never leave this module: list readers clone the array,// while ID lookups clone only the matching task. Both share invalidation.@@ -185,7 +188,7 @@ async function readTaskSnapshot(source) {if (cached?.version === version) return cached.snapshot;return snapshotLoads.run(`${source}:${version}`, async () => {const loaded = await loadTaskWorkingSet(source);- const snapshot = { rows: loaded.rows, tasks: loaded.rows.map((row) => row.task) };+ const snapshot = { version: loaded.version, rows: loaded.rows, tasks: loaded.rows.map((row) => row.task) };if (!snapshotCache.get(source) || snapshotCache.get(source).version <= loaded.version) {snapshotCache.set(source, { version: loaded.version, snapshot });}@@ -194,7 +197,10 @@ async function readTaskSnapshot(source) {}async function readTaskFile(source) {- return structuredClone((await readTaskSnapshot(source)).tasks);+ const snapshot = await readTaskSnapshot(source);+ const tasks = structuredClone(snapshot.tasks);+ listSnapshots.set(tasks, snapshot);+ return tasks;}async function findTaskInFile(source, taskId) {@@ -215,19 +221,22 @@ async function findTaskInFile(source, taskId) {* only the rows that changed, in one transaction. Every write path in this* module goes through here. Resolves to whether anything was written.*/-async function writeTaskList(source, tasks) {- const snapshot = await readTaskSnapshot(source);+async function writeTaskList(source, tasks, original = tasks, dependencies = {}) {+ const snapshot = listSnapshots.get(original);+ if (!snapshot) throw new Error('Task saves require their original read snapshot');const cached = snapshotCache.get(source);const plan = planListSave({ previous: snapshot.rows, next: tasks, source });if (plan.upserts.length === 0 && plan.deletes.length === 0) return false;- const version = await applyTaskListSave(source, plan);+ const version = await applyTaskListSave(source, {+ ...plan, expectedVersions: { ...dependencies, [source]: snapshot.version },+ });const positions = new Map(snapshot.rows.map((row) => [row.task.id, row.position]));for (const { task, position } of plan.upserts) positions.set(task.id, position);if (cached?.snapshot === snapshot && version === cached.version + 1) {// Same bound as a reload, so completions never accumulate in the cache.const rows = selectWorkingSet(plan.saved.map((task) => ({ task, position: positions.get(task.id) })));- snapshotCache.set(source, { version, snapshot: { rows, tasks: rows.map((row) => row.task) } });+ snapshotCache.set(source, { version, snapshot: { version, rows, tasks: rows.map((row) => row.task) } });} else {snapshotCache.delete(source);}@@ -488,7 +497,20 @@ async function uniqueTaskId(source, id, tasks) {* force-spawn the returned task itself; the change event still reaches socket* consumers, but the normal scheduler must not race that dispatch.*/-export async function addTask(taskData, taskType = 'user', { raw = false, ignoreTaskId = null, now = Date.now(), suppressDequeue = false } = {}) {+export async function addTask(taskData, taskType = 'user', options = {}) {+ // Only addition is replayed: it has no effects before persistence. Re-read+ // both queues and recompute dedup/IDs on conflict; never retry a stale plan.+ // Other mutations surface 409 so decisions or side effects are not replayed.+ for (let attempt = 0; ; attempt++) {+ const result = await addTaskAttempt(taskData, taskType, options).catch(error => {+ if (error.code !== 'COS_TASK_LIST_CONFLICT' || attempt >= 3) throw error;+ return null;+ });+ if (result !== null) return result;+ }+}++async function addTaskAttempt(taskData, taskType, { raw = false, ignoreTaskId = null, now = Date.now(), suppressDequeue = false }) {validateRequest(cosTaskTypeSchema, taskType);if (!raw && taskData.metadata) taskData = { ...taskData, metadata: newAuditMetadata(taskData.metadata) };return withStateLock(async () => {@@ -497,8 +519,10 @@ export async function addTask(taskData, taskType = 'user', { raw = false, ignore// One state lock covers BOTH queues: manual and autonomous producers// must acquire the same durable work identity even with different titles.const newWorkIdentity = developmentWorkIdentity(taskData);+ const dependencies = {};if (newWorkIdentity) {const other = await readTaskFile(taskType === 'user' ? 'internal' : 'user');+ dependencies[taskType === 'user' ? 'internal' : 'user'] = listSnapshots.get(other).version;const owner = [...tasks, ...other].find(task => task.id !== ignoreTaskId&& DEVELOPMENT_ACTIVE_STATUSES.has(task.status) && sameDevelopmentWork(task, taskData)// Targeted claims and the backlog drain coexist in either admission order.@@ -617,7 +641,7 @@ export async function addTask(taskData, taskType = 'user', { raw = false, ignoretasks.push(newTask);}- await writeTaskList(taskType, tasks);+ await writeTaskList(taskType, tasks, tasks, dependencies);// cos.js init listens for this event. For user tasks it fires// tryImmediateSpawn so the task starts instantly if slots are available,@@ -1010,7 +1034,7 @@ export async function mergePeerTasks(taskType, remoteTasks, { now = Date.now() }// stored a terminal verdict before upgrading, or a crash before a prior consume, leaves// the task terminal with the merge producing no change, so a changed-only consume would// never catch up). Re-offering is cheap: the consumer is a durable no-op after the first.- if (!await writeTaskList(taskType, merged)) return { changed: false, merged };+ if (!await writeTaskList(taskType, merged, localTasks)) return { changed: false, merged };cosEvents.emit('tasks:changed', { type: taskType, action: 'peer-merged' });return { changed: true, count: merged.length, merged };@@ -1276,7 +1300,7 @@ export async function reorderTasks(taskIds) {reorderedTasks.push(task);}- await writeTaskList('user', reorderedTasks);+ await writeTaskList('user', reorderedTasks, tasks);cosEvents.emit('tasks:changed', { type: 'user', action: 'reordered' });return { success: true, order: reorderedTasks.map(t => t.id) };diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.jsindex cde3b281af..26adea2ccd 100644--- a/server/services/cosTaskStore.test.js+++ b/server/services/cosTaskStore.test.js@@ -129,9 +129,9 @@ const baseState = () => ({ agents: {} });// Test-side access to what is stored, bypassing the store's cache and the// failure/after-write hooks above (they wrap only the store's own saves).const seed = async (source, tasks) => {- const { rows } = await actualRepository.loadTaskWorkingSet(source);+ const { rows, version } = await actualRepository.loadTaskWorkingSet(source);const plan = planListSave({ previous: rows, next: tasks, source });- await actualRepository.applyTaskListSave(source, plan);+ await actualRepository.applyTaskListSave(source, { ...plan, expectedVersions: { [source]: version } });__resetTaskCache();};const stored = async (source) => (await actualRepository.loadTaskWorkingSet(source)).rows.map((row) => row.task);