Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound queued claim ownership requests without late mutations (#12125)
Adam Eivybd7fd72e0122
diff --git a/server/routes/cosAgentRoutes.ownershipResponse.test.js b/server/routes/cosAgentRoutes.ownershipResponse.test.jsindex 02416f7397..f2e7e4e781 100644--- a/server/routes/cosAgentRoutes.ownershipResponse.test.js+++ b/server/routes/cosAgentRoutes.ownershipResponse.test.js@@ -5,13 +5,16 @@ import { execFile } from '../lib/childProcess.js';import { startLoopbackServer, closeLoopbackServer } from '../lib/testHelper.js';import { errorMiddleware } from '../lib/errorHandler.js';-const fixture = vi.hoisted(() => ({ state: null, apiBase: '', unreadable: false }));+const fixture = vi.hoisted(() => ({ state: null, apiBase: '', unreadable: false, stateGate: null, onStateQueued: null }));vi.mock('../services/cos.js', () => ({}));vi.mock('../services/agentOrchestrator.js', () => ({}));vi.mock('../services/apps.js', () => ({ getAppById: async () => ({ repoPath: '/synthetic/repo' }) }));vi.mock('../services/cosState.js', () => ({readAgentsStateForSafetyCheck: async () => ({ trusted: !fixture.unreadable, agents: fixture.unreadable ? null : fixture.state.agents }),- withStateLock: (work) => work(),+ withStateLock: async work => {+ if (fixture.stateGate) { fixture.onStateQueued(); await fixture.stateGate; }+ return work();+ },loadState: async () => structuredClone(fixture.state),saveState: async (state) => { fixture.state = structuredClone(state); },}));@@ -38,7 +41,7 @@ vi.mock('../lib/networkExposure.js', () => ({ localApiBaseUrl: () => fixture.apiimport routes from './cosAgentRoutes.js';import { formatInFlightForPrompt } from '../services/branchReconcile.js';-import { updateClaimOwnership } from '../services/cosClaimOwnership.js';+import { updateClaimOwnership, withClaimOwnershipLock, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS } from '../services/cosClaimOwnership.js';const execute = promisify(execFile);const coordinator = 'synthetic-coordinator';@@ -47,6 +50,7 @@ let server;beforeEach(async () => {fixture.unreadable = false;+ fixture.stateGate = fixture.onStateQueued = null;fixture.state = { agents: {[coordinator]: { id: coordinator, status: 'running', workspacePath: '/synthetic/repo', metadata: {} },} };@@ -66,6 +70,7 @@ beforeEach(async () => {afterEach(async () => {if (server) await closeLoopbackServer(server);server = null;+ vi.useRealTimers();vi.restoreAllMocks();});@@ -124,6 +129,27 @@ describe('generated ownership recheck raw HTTP response (#11348)', () => {expect(fixture.state).toEqual(before);});+ it('returns a bounded JSON refusal for a queued reservation and cannot reserve after the response', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ let resume;+ fixture.stateGate = new Promise(resolve => { resume = resolve; });+ const queued = new Promise(resolve => { fixture.onStateQueued = resolve; });+ const before = structuredClone(fixture.state);+ const response = fetch(`${fixture.apiBase}/api/cos/claim-ownership`, {+ method: 'POST', headers: { Authorization: 'Bearer synthetic-test-token', 'Content-Type': 'application/json' },+ body: JSON.stringify({ action: 'check', appId: 'synthetic-app', branch, agentId: coordinator }),+ });+ await queued;+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ const result = await response;+ expect(result.status).toBe(200);+ expect(await result.json()).toEqual({ admitted: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });+ resume();+ await withClaimOwnershipLock(() => {});+ expect(fixture.state).toEqual(before);+ });+it('keeps an HTTP authentication error separate from admission and leaves ownership unchanged', async () => {const before = structuredClone(fixture.state);const capture = await captureRecheck({ token: 'invalid-synthetic-token' });diff --git a/server/services/agentManagement.claimResume.test.js b/server/services/agentManagement.claimResume.test.jsindex dcfbc14d51..0b6a8fb779 100644--- a/server/services/agentManagement.claimResume.test.js+++ b/server/services/agentManagement.claimResume.test.js@@ -10,6 +10,7 @@ vi.mock('./cosState.js', () => ({withStateLock: createFileWriteQueue(),loadState: async () => f.state,saveState: async state => { f.state = state; f.disk = structuredClone(state); },+ readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: structuredClone(f.disk.agents) }),readMergeAdmissionStateForSafetyCheck: async () => ({ trusted: true, ...structuredClone(f.disk) }),}));vi.mock('./cosAgentLifecycle.js', () => ({diff --git a/server/services/cosClaimOwnership.js b/server/services/cosClaimOwnership.jsindex c5add5797a..925f763da6 100644--- a/server/services/cosClaimOwnership.js+++ b/server/services/cosClaimOwnership.js@@ -49,26 +49,90 @@ export async function readClaimOwnershipAgents() {return entries.map(([, agent]) => agent);}+// Match merge admission's pre-start budget, across BOTH queues. A deadline is+// cancellation only until the state callback starts; afterwards it diagnoses+// pending work while both locks stay owned through real settlement.+export const CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS = 15_000;+const retryAfterMs = 5000;++async function ownershipTransaction(operation, done, work) {+ const { withStateLock, loadState, saveState } = await import('./cosState.js');+ const queuedAt = performance.now();+ let started = false;+ let cancelled = false;+ let phase = 'ownershipQueue';+ let phaseStartedAt = queuedAt;+ const timings = { ownershipQueue: 0, stateQueue: 0, ownerRead: 0, ownershipCheck: 0, persistence: 0 };+ const elapsed = since => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.round(performance.now() - since)));+ const finishPhase = () => { timings[phase] = elapsed(phaseStartedAt); };+ const enter = name => { finishPhase(); phase = name; phaseStartedAt = performance.now(); };+ const report = status => {+ const log = status === 'failed' ? console.error : status === 'settled' ? console.log : console.warn;+ log(`🔒 Claim ownership operation=${operation} status=${status} phase=${phase} phaseMs=${elapsed(phaseStartedAt)} totalMs=${elapsed(queuedAt)} ownershipQueueMs=${timings.ownershipQueue} stateQueueMs=${timings.stateQueue} ownerReadMs=${timings.ownerRead} ownershipCheckMs=${timings.ownershipCheck} persistenceMs=${timings.persistence}`);+ };+ const refuse = reason => ({ [done]: false, reason });+ return new Promise((resolve, reject) => {+ const expire = () => {+ cancelled = true;+ finishPhase();+ report('queue-timeout');+ resolve({ ...refuse(phase === 'ownershipQueue' ? 'ownership-queue-timeout' : 'state-queue-timeout'), retryAfterMs });+ };+ const fenced = () => {+ if (cancelled) return true;+ // A busy event loop can resume the queue before its overdue timer fires.+ if (performance.now() - queuedAt >= CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS) { expire(); return true; }+ return false;+ };+ const timer = setTimeout(() => {+ if (!started) expire();+ else report('pending');+ }, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ const queued = withClaimOwnershipLock(() => {+ if (fenced()) return;+ enter('stateQueue');+ return withStateLock(async () => {+ if (fenced()) return;+ started = true;+ enter('ownerRead');+ const agents = await readClaimOwnershipAgents();+ if (!agents) return refuse('ownership-unreadable');+ const state = await loadState();+ if (JSON.stringify(Object.values(state.agents ?? {})) !== JSON.stringify(agents)) return refuse('ownership-stale');+ // Do not alter loadState's shared pre-image before persistence starts.+ const next = { ...state, agents: { ...state.agents } };+ enter('ownershipCheck');+ return work(next, async () => {+ enter('persistence');+ await saveState(next);+ });+ });+ });+ queued.then(result => {+ if (!cancelled) { finishPhase(); report('settled'); resolve(result); }+ }, error => {+ if (!cancelled) { finishPhase(); report('failed'); reject(error); }+ }).finally(() => clearTimeout(timer));+ });+}+/*** @param {{ agentId: string, action: 'bind'|'release', branch: string }} input* @returns {Promise<{ bound: true, branch: string }|{ released: true, branch: string }|{ bound?: false, released?: false, reason: string }>}*/export async function updateClaimOwnership({ agentId, action, branch }) {const done = action === 'bind' ? 'bound' : 'released';- const { withStateLock, loadState, saveState } = await import('./cosState.js');const { cosEvents } = await import('./cosEvents.js');- return withClaimOwnershipLock(() => withStateLock(async () => {- const state = await loadState();+ return ownershipTransaction(action === 'bind' ? 'bind' : 'release', done, async (state, save) => {const agent = state.agents?.[agentId];if (!agent || agent.id !== agentId) return { [done]: false, reason: 'owner-unknown' };const patch = action === 'bind' ? bindClaimBranch(agent, branch, Object.values(state.agents)) : releaseClaimBranch(agent, branch);if (patch.refused) return { [done]: false, reason: patch.refused };state.agents[agentId] = { ...agent, metadata: { ...agent.metadata, ...patch } };- await saveState(state);+ await save();cosEvents.emit('agent:updated', state.agents[agentId]);- console.log(`🔒 Claim ownership ${done}: ${branch} → ${agentId}`);return { [done]: true, branch };- }));+ });}/**@@ -119,25 +183,22 @@ export async function checkReconcileOwnership(input, deps = {}) {};if (!agentId) return admit();- const { withStateLock, loadState, saveState } = await import('./cosState.js');const { cosEvents } = await import('./cosEvents.js');// Decided inside the lock a bind takes, so no bind can land between// "admitted" and "reserved".- return withClaimOwnershipLock(() => withStateLock(async () => {+ return ownershipTransaction('reserve', 'admitted', async (state, save) => {const admission = await admit();if (!admission.admitted) return admission;- const state = await loadState();const agent = state.agents?.[agentId];if (!agent || agent.id !== agentId) return { admitted: false, reason: 'owner-unknown' };const patch = reserveReconcileBranch(agent, branch);if (!patch) return admission;if (patch.refused) return { admitted: false, reason: patch.refused };state.agents[agentId] = { ...agent, metadata: { ...agent.metadata, ...patch } };- await saveState(state);+ await save();cosEvents.emit('agent:updated', state.agents[agentId]);- console.log(`🔒 Reconcile reservation: ${branch} → ${agentId}`);return admission;- }));+ });}/** Read-only snapshot publication gate. Registered review work remains owneddiff --git a/server/services/cosClaimOwnership.test.js b/server/services/cosClaimOwnership.test.jsindex f703adce67..61421aafb1 100644--- a/server/services/cosClaimOwnership.test.js+++ b/server/services/cosClaimOwnership.test.js@@ -1,20 +1,36 @@-import { describe, expect, it, vi, beforeEach } from 'vitest';+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';-const store = { state: null };+const store = { state: null, stateGate: null, readGate: null, saveGate: null, unreadable: false, stale: false };+const deferred = () => {+ let resolve;+ let reject;+ const promise = new Promise((yes, no) => { resolve = yes; reject = no; });+ return { promise, resolve, reject };+};vi.mock('./cosState.js', () => ({- withStateLock: (work) => work(),- loadState: async () => structuredClone(store.state),- saveState: async (next) => { store.state = structuredClone(next); },+ withStateLock: vi.fn(async work => { if (store.stateGate) await store.stateGate; return work(); }),+ readAgentsStateForSafetyCheck: vi.fn(async () => {+ if (store.readGate) await store.readGate;+ return { trusted: !store.unreadable, agents: structuredClone(store.state.agents) };+ }),+ loadState: async () => store.stale ? { agents: {} } : store.state,+ saveState: vi.fn(async next => { if (store.saveGate) await store.saveGate; store.state = structuredClone(next); }),}));vi.mock('./cosEvents.js', () => ({ cosEvents: { emit: vi.fn() } }));-const { updateClaimOwnership, checkReconcileOwnership } = await import('./cosClaimOwnership.js');+const { updateClaimOwnership, checkReconcileOwnership, withClaimOwnershipLock, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS } = await import('./cosClaimOwnership.js');const { claimContinuationAdmission, claimBranchAdmission } = await import('../lib/claimContinuation.js');+const { withStateLock, readAgentsStateForSafetyCheck, saveState } = await import('./cosState.js');+const { cosEvents } = await import('./cosEvents.js');+const SOURCE = '/repos/app-x';const CHILD = '/repos/app-x/data/cos/worktrees/claim-issue-101';beforeEach(() => {+ store.stateGate = store.readGate = store.saveGate = null;+ store.unreadable = store.stale = false;+ vi.clearAllMocks();store.state = {agents: {'agent-epicrun': {@@ -25,8 +41,12 @@ beforeEach(() => {};});+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });+describe('updateClaimOwnership', () => {it('persists the bound child branch on the run record and settles its selection', async () => {+ expect(await updateClaimOwnership({ agentId: 'agent-epicrun', action: 'bind', branch: 'claim/issue-101' }))+ .toEqual({ bound: true, branch: 'claim/issue-101' });expect(await updateClaimOwnership({ agentId: 'agent-epicrun', action: 'bind', branch: 'claim/issue-101' })).toEqual({ bound: true, branch: 'claim/issue-101' });expect(store.state.agents['agent-epicrun'].metadata).toMatchObject({@@ -48,6 +68,123 @@ describe('updateClaimOwnership', () => {});});+describe('claim ownership queued transaction deadlines', () => {+ const update = action => updateClaimOwnership({ agentId: 'agent-epicrun', action, branch: 'claim/issue-101' });+ const tick = () => vi.advanceTimersByTimeAsync(0);+ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ });++ it.each(['bind', 'release'])('fences a timed-out %s at each queued boundary', async action => {+ store.state.agents['agent-epicrun'].metadata.claimBranches = action === 'release' ? ['claim/issue-101'] : [];+ const before = structuredClone(store.state);+ // A retirement can hold ownership while unrelated state writers hold state.+ // Neither resumed callback may mutate after the caller received a refusal.+ for (const queue of ['ownership', 'state']) {+ const gate = deferred();+ const held = queue === 'ownership' ? withClaimOwnershipLock(() => gate.promise) : null;+ if (queue === 'state') store.stateGate = gate.promise;+ const pending = update(action);+ await tick();+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ expect(await pending).toEqual({ [action === 'bind' ? 'bound' : 'released']: false,+ reason: `${queue}-queue-timeout`, retryAfterMs: 5000 });+ expect(readAgentsStateForSafetyCheck).not.toHaveBeenCalled();+ gate.resolve();+ if (held) await held;+ await withClaimOwnershipLock(() => {});+ expect(store.state).toEqual(before);+ expect(saveState).not.toHaveBeenCalled();+ expect(cosEvents.emit).not.toHaveBeenCalled();+ }+ store.stateGate = null;+ expect(await update(action)).toEqual({ [action === 'bind' ? 'bound' : 'released']: true, branch: 'claim/issue-101' });+ });++ it('fences overdue callbacks even when the timer has not run, sharing the budget across queues', async () => {+ const ownership = deferred();+ const state = deferred();+ const held = withClaimOwnershipLock(() => ownership.promise);+ store.stateGate = state.promise;+ const pending = update('bind');+ await tick();+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS - 1);+ ownership.resolve();+ await held;+ await tick();+ expect(withStateLock).toHaveBeenCalledOnce();+ // Advance only the monotonic clock; leave the scheduled timer unfired.+ vi.spyOn(performance, 'now').mockReturnValue(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ state.resolve();+ expect(await pending).toEqual({ bound: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });+ expect(readAgentsStateForSafetyCheck).not.toHaveBeenCalled();+ expect(saveState).not.toHaveBeenCalled();+ await withClaimOwnershipLock(() => {});+ });++ it('keeps a started save owned until durable settlement and reports redacted persistence evidence', async () => {+ const save = deferred();+ store.saveGate = save.promise;+ const before = structuredClone(store.state);+ let settled = false;+ const pending = update('bind').then(result => { settled = true; return result; });+ await tick();+ expect(saveState).toHaveBeenCalledOnce();+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ expect(settled).toBe(false);+ expect(store.state).toEqual(before);+ expect(cosEvents.emit).not.toHaveBeenCalled();+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=persistence'));+ let nextEntered = false;+ const following = withClaimOwnershipLock(() => { nextEntered = true; });+ await tick();+ expect(nextEntered).toBe(false);+ save.resolve();+ expect(await pending).toEqual({ bound: true, branch: 'claim/issue-101' });+ await following;+ expect(store.state.agents['agent-epicrun'].metadata.claimBranches).toEqual(['claim/issue-101']);+ expect(cosEvents.emit).toHaveBeenCalledOnce();+ const diagnostics = [...console.warn.mock.calls, ...console.log.mock.calls].flat().join(' ');+ expect(diagnostics).not.toMatch(/agent-epicrun|claim\/issue|repos/);+ expect(diagnostics).toMatch(/ownershipQueueMs=\d+ stateQueueMs=\d+ ownerReadMs=\d+ ownershipCheckMs=\d+ persistenceMs=\d+/);+ });++ it('diagnoses an in-flight trusted owner read and refuses unreadable or stale ownership without writes', async () => {+ const read = deferred();+ store.readGate = read.promise;+ store.unreadable = true;+ const pending = update('bind');+ await tick();+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=ownerRead'));+ read.resolve();+ expect(await pending).toEqual({ bound: false, reason: 'ownership-unreadable' });+ store.unreadable = false;+ store.stale = true;+ expect(await update('release')).toEqual({ released: false, reason: 'ownership-stale' });+ expect(saveState).not.toHaveBeenCalled();+ });++ it('propagates a started persistence failure and releases the queue only after rejection', async () => {+ const save = deferred();+ store.saveGate = save.promise;+ const before = structuredClone(store.state);+ const pending = update('bind');+ const failure = expect(pending).rejects.toThrow('synthetic write failure');+ await tick();+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);+ save.reject(new Error('synthetic write failure'));+ await failure;+ await withClaimOwnershipLock(() => {});+ expect(store.state).toEqual(before);+ expect(cosEvents.emit).not.toHaveBeenCalled();+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('status=failed phase=persistence'));+ });+});+// The scan that dispatched a reconcile worker is stale by the time it mutates:// ownership acquired in between must refuse at the worker's own recheck.describe('checkReconcileOwnership', () => {