Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: bound queued claim ownership requests without late mutations (#12125)

Adam Eivybd7fd72e0122
diff --git a/server/routes/cosAgentRoutes.ownershipResponse.test.js b/server/routes/cosAgentRoutes.ownershipResponse.test.js
index 02416f7397..f2e7e4e781 100644
--- a/server/routes/cosAgentRoutes.ownershipResponse.test.js
+++ b/server/routes/cosAgentRoutes.ownershipResponse.test.js
@@ -5,13 +5,16 @@ import { execFile } from '../lib/childProcess.js';
import { startLoopbackServer, closeLoopbackServer } from '../lib/testHelper.js';
import { errorMiddleware } from '../lib/errorHandler.js';
-const fixture = vi.hoisted(() => ({ state: null, apiBase: '', unreadable: false }));
+const fixture = vi.hoisted(() => ({ state: null, apiBase: '', unreadable: false, stateGate: null, onStateQueued: null }));
vi.mock('../services/cos.js', () => ({}));
vi.mock('../services/agentOrchestrator.js', () => ({}));
vi.mock('../services/apps.js', () => ({ getAppById: async () => ({ repoPath: '/synthetic/repo' }) }));
vi.mock('../services/cosState.js', () => ({
readAgentsStateForSafetyCheck: async () => ({ trusted: !fixture.unreadable, agents: fixture.unreadable ? null : fixture.state.agents }),
- withStateLock: (work) => work(),
+ withStateLock: async work => {
+ if (fixture.stateGate) { fixture.onStateQueued(); await fixture.stateGate; }
+ return work();
+ },
loadState: async () => structuredClone(fixture.state),
saveState: async (state) => { fixture.state = structuredClone(state); },
}));
@@ -38,7 +41,7 @@ vi.mock('../lib/networkExposure.js', () => ({ localApiBaseUrl: () => fixture.api
import routes from './cosAgentRoutes.js';
import { formatInFlightForPrompt } from '../services/branchReconcile.js';
-import { updateClaimOwnership } from '../services/cosClaimOwnership.js';
+import { updateClaimOwnership, withClaimOwnershipLock, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS } from '../services/cosClaimOwnership.js';
const execute = promisify(execFile);
const coordinator = 'synthetic-coordinator';
@@ -47,6 +50,7 @@ let server;
beforeEach(async () => {
fixture.unreadable = false;
+ fixture.stateGate = fixture.onStateQueued = null;
fixture.state = { agents: {
[coordinator]: { id: coordinator, status: 'running', workspacePath: '/synthetic/repo', metadata: {} },
} };
@@ -66,6 +70,7 @@ beforeEach(async () => {
afterEach(async () => {
if (server) await closeLoopbackServer(server);
server = null;
+ vi.useRealTimers();
vi.restoreAllMocks();
});
@@ -124,6 +129,27 @@ describe('generated ownership recheck raw HTTP response (#11348)', () => {
expect(fixture.state).toEqual(before);
});
+ it('returns a bounded JSON refusal for a queued reservation and cannot reserve after the response', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ let resume;
+ fixture.stateGate = new Promise(resolve => { resume = resolve; });
+ const queued = new Promise(resolve => { fixture.onStateQueued = resolve; });
+ const before = structuredClone(fixture.state);
+ const response = fetch(`${fixture.apiBase}/api/cos/claim-ownership`, {
+ method: 'POST', headers: { Authorization: 'Bearer synthetic-test-token', 'Content-Type': 'application/json' },
+ body: JSON.stringify({ action: 'check', appId: 'synthetic-app', branch, agentId: coordinator }),
+ });
+ await queued;
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ const result = await response;
+ expect(result.status).toBe(200);
+ expect(await result.json()).toEqual({ admitted: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });
+ resume();
+ await withClaimOwnershipLock(() => {});
+ expect(fixture.state).toEqual(before);
+ });
+
it('keeps an HTTP authentication error separate from admission and leaves ownership unchanged', async () => {
const before = structuredClone(fixture.state);
const capture = await captureRecheck({ token: 'invalid-synthetic-token' });
diff --git a/server/services/agentManagement.claimResume.test.js b/server/services/agentManagement.claimResume.test.js
index dcfbc14d51..0b6a8fb779 100644
--- a/server/services/agentManagement.claimResume.test.js
+++ b/server/services/agentManagement.claimResume.test.js
@@ -10,6 +10,7 @@ vi.mock('./cosState.js', () => ({
withStateLock: createFileWriteQueue(),
loadState: async () => f.state,
saveState: async state => { f.state = state; f.disk = structuredClone(state); },
+ readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: structuredClone(f.disk.agents) }),
readMergeAdmissionStateForSafetyCheck: async () => ({ trusted: true, ...structuredClone(f.disk) }),
}));
vi.mock('./cosAgentLifecycle.js', () => ({
diff --git a/server/services/cosClaimOwnership.js b/server/services/cosClaimOwnership.js
index c5add5797a..925f763da6 100644
--- a/server/services/cosClaimOwnership.js
+++ b/server/services/cosClaimOwnership.js
@@ -49,26 +49,90 @@ export async function readClaimOwnershipAgents() {
return entries.map(([, agent]) => agent);
}
+// Match merge admission's pre-start budget, across BOTH queues. A deadline is
+// cancellation only until the state callback starts; afterwards it diagnoses
+// pending work while both locks stay owned through real settlement.
+export const CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS = 15_000;
+const retryAfterMs = 5000;
+
+async function ownershipTransaction(operation, done, work) {
+ const { withStateLock, loadState, saveState } = await import('./cosState.js');
+ const queuedAt = performance.now();
+ let started = false;
+ let cancelled = false;
+ let phase = 'ownershipQueue';
+ let phaseStartedAt = queuedAt;
+ const timings = { ownershipQueue: 0, stateQueue: 0, ownerRead: 0, ownershipCheck: 0, persistence: 0 };
+ const elapsed = since => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.round(performance.now() - since)));
+ const finishPhase = () => { timings[phase] = elapsed(phaseStartedAt); };
+ const enter = name => { finishPhase(); phase = name; phaseStartedAt = performance.now(); };
+ const report = status => {
+ const log = status === 'failed' ? console.error : status === 'settled' ? console.log : console.warn;
+ log(`🔒 Claim ownership operation=${operation} status=${status} phase=${phase} phaseMs=${elapsed(phaseStartedAt)} totalMs=${elapsed(queuedAt)} ownershipQueueMs=${timings.ownershipQueue} stateQueueMs=${timings.stateQueue} ownerReadMs=${timings.ownerRead} ownershipCheckMs=${timings.ownershipCheck} persistenceMs=${timings.persistence}`);
+ };
+ const refuse = reason => ({ [done]: false, reason });
+ return new Promise((resolve, reject) => {
+ const expire = () => {
+ cancelled = true;
+ finishPhase();
+ report('queue-timeout');
+ resolve({ ...refuse(phase === 'ownershipQueue' ? 'ownership-queue-timeout' : 'state-queue-timeout'), retryAfterMs });
+ };
+ const fenced = () => {
+ if (cancelled) return true;
+ // A busy event loop can resume the queue before its overdue timer fires.
+ if (performance.now() - queuedAt >= CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS) { expire(); return true; }
+ return false;
+ };
+ const timer = setTimeout(() => {
+ if (!started) expire();
+ else report('pending');
+ }, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ const queued = withClaimOwnershipLock(() => {
+ if (fenced()) return;
+ enter('stateQueue');
+ return withStateLock(async () => {
+ if (fenced()) return;
+ started = true;
+ enter('ownerRead');
+ const agents = await readClaimOwnershipAgents();
+ if (!agents) return refuse('ownership-unreadable');
+ const state = await loadState();
+ if (JSON.stringify(Object.values(state.agents ?? {})) !== JSON.stringify(agents)) return refuse('ownership-stale');
+ // Do not alter loadState's shared pre-image before persistence starts.
+ const next = { ...state, agents: { ...state.agents } };
+ enter('ownershipCheck');
+ return work(next, async () => {
+ enter('persistence');
+ await saveState(next);
+ });
+ });
+ });
+ queued.then(result => {
+ if (!cancelled) { finishPhase(); report('settled'); resolve(result); }
+ }, error => {
+ if (!cancelled) { finishPhase(); report('failed'); reject(error); }
+ }).finally(() => clearTimeout(timer));
+ });
+}
+
/**
* @param {{ agentId: string, action: 'bind'|'release', branch: string }} input
* @returns {Promise<{ bound: true, branch: string }|{ released: true, branch: string }|{ bound?: false, released?: false, reason: string }>}
*/
export async function updateClaimOwnership({ agentId, action, branch }) {
const done = action === 'bind' ? 'bound' : 'released';
- const { withStateLock, loadState, saveState } = await import('./cosState.js');
const { cosEvents } = await import('./cosEvents.js');
- return withClaimOwnershipLock(() => withStateLock(async () => {
- const state = await loadState();
+ return ownershipTransaction(action === 'bind' ? 'bind' : 'release', done, async (state, save) => {
const agent = state.agents?.[agentId];
if (!agent || agent.id !== agentId) return { [done]: false, reason: 'owner-unknown' };
const patch = action === 'bind' ? bindClaimBranch(agent, branch, Object.values(state.agents)) : releaseClaimBranch(agent, branch);
if (patch.refused) return { [done]: false, reason: patch.refused };
state.agents[agentId] = { ...agent, metadata: { ...agent.metadata, ...patch } };
- await saveState(state);
+ await save();
cosEvents.emit('agent:updated', state.agents[agentId]);
- console.log(`🔒 Claim ownership ${done}: ${branch} → ${agentId}`);
return { [done]: true, branch };
- }));
+ });
}
/**
@@ -119,25 +183,22 @@ export async function checkReconcileOwnership(input, deps = {}) {
};
if (!agentId) return admit();
- const { withStateLock, loadState, saveState } = await import('./cosState.js');
const { cosEvents } = await import('./cosEvents.js');
// Decided inside the lock a bind takes, so no bind can land between
// "admitted" and "reserved".
- return withClaimOwnershipLock(() => withStateLock(async () => {
+ return ownershipTransaction('reserve', 'admitted', async (state, save) => {
const admission = await admit();
if (!admission.admitted) return admission;
- const state = await loadState();
const agent = state.agents?.[agentId];
if (!agent || agent.id !== agentId) return { admitted: false, reason: 'owner-unknown' };
const patch = reserveReconcileBranch(agent, branch);
if (!patch) return admission;
if (patch.refused) return { admitted: false, reason: patch.refused };
state.agents[agentId] = { ...agent, metadata: { ...agent.metadata, ...patch } };
- await saveState(state);
+ await save();
cosEvents.emit('agent:updated', state.agents[agentId]);
- console.log(`🔒 Reconcile reservation: ${branch} → ${agentId}`);
return admission;
- }));
+ });
}
/** Read-only snapshot publication gate. Registered review work remains owned
diff --git a/server/services/cosClaimOwnership.test.js b/server/services/cosClaimOwnership.test.js
index f703adce67..61421aafb1 100644
--- a/server/services/cosClaimOwnership.test.js
+++ b/server/services/cosClaimOwnership.test.js
@@ -1,20 +1,36 @@
-import { describe, expect, it, vi, beforeEach } from 'vitest';
+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
-const store = { state: null };
+const store = { state: null, stateGate: null, readGate: null, saveGate: null, unreadable: false, stale: false };
+const deferred = () => {
+ let resolve;
+ let reject;
+ const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
+ return { promise, resolve, reject };
+};
vi.mock('./cosState.js', () => ({
- withStateLock: (work) => work(),
- loadState: async () => structuredClone(store.state),
- saveState: async (next) => { store.state = structuredClone(next); },
+ withStateLock: vi.fn(async work => { if (store.stateGate) await store.stateGate; return work(); }),
+ readAgentsStateForSafetyCheck: vi.fn(async () => {
+ if (store.readGate) await store.readGate;
+ return { trusted: !store.unreadable, agents: structuredClone(store.state.agents) };
+ }),
+ loadState: async () => store.stale ? { agents: {} } : store.state,
+ saveState: vi.fn(async next => { if (store.saveGate) await store.saveGate; store.state = structuredClone(next); }),
}));
vi.mock('./cosEvents.js', () => ({ cosEvents: { emit: vi.fn() } }));
-const { updateClaimOwnership, checkReconcileOwnership } = await import('./cosClaimOwnership.js');
+const { updateClaimOwnership, checkReconcileOwnership, withClaimOwnershipLock, CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS } = await import('./cosClaimOwnership.js');
const { claimContinuationAdmission, claimBranchAdmission } = await import('../lib/claimContinuation.js');
+const { withStateLock, readAgentsStateForSafetyCheck, saveState } = await import('./cosState.js');
+const { cosEvents } = await import('./cosEvents.js');
+
const SOURCE = '/repos/app-x';
const CHILD = '/repos/app-x/data/cos/worktrees/claim-issue-101';
beforeEach(() => {
+ store.stateGate = store.readGate = store.saveGate = null;
+ store.unreadable = store.stale = false;
+ vi.clearAllMocks();
store.state = {
agents: {
'agent-epicrun': {
@@ -25,8 +41,12 @@ beforeEach(() => {
};
});
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+
describe('updateClaimOwnership', () => {
it('persists the bound child branch on the run record and settles its selection', async () => {
+ expect(await updateClaimOwnership({ agentId: 'agent-epicrun', action: 'bind', branch: 'claim/issue-101' }))
+ .toEqual({ bound: true, branch: 'claim/issue-101' });
expect(await updateClaimOwnership({ agentId: 'agent-epicrun', action: 'bind', branch: 'claim/issue-101' }))
.toEqual({ bound: true, branch: 'claim/issue-101' });
expect(store.state.agents['agent-epicrun'].metadata).toMatchObject({
@@ -48,6 +68,123 @@ describe('updateClaimOwnership', () => {
});
});
+describe('claim ownership queued transaction deadlines', () => {
+ const update = action => updateClaimOwnership({ agentId: 'agent-epicrun', action, branch: 'claim/issue-101' });
+ const tick = () => vi.advanceTimersByTimeAsync(0);
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ });
+
+ it.each(['bind', 'release'])('fences a timed-out %s at each queued boundary', async action => {
+ store.state.agents['agent-epicrun'].metadata.claimBranches = action === 'release' ? ['claim/issue-101'] : [];
+ const before = structuredClone(store.state);
+ // A retirement can hold ownership while unrelated state writers hold state.
+ // Neither resumed callback may mutate after the caller received a refusal.
+ for (const queue of ['ownership', 'state']) {
+ const gate = deferred();
+ const held = queue === 'ownership' ? withClaimOwnershipLock(() => gate.promise) : null;
+ if (queue === 'state') store.stateGate = gate.promise;
+ const pending = update(action);
+ await tick();
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ expect(await pending).toEqual({ [action === 'bind' ? 'bound' : 'released']: false,
+ reason: `${queue}-queue-timeout`, retryAfterMs: 5000 });
+ expect(readAgentsStateForSafetyCheck).not.toHaveBeenCalled();
+ gate.resolve();
+ if (held) await held;
+ await withClaimOwnershipLock(() => {});
+ expect(store.state).toEqual(before);
+ expect(saveState).not.toHaveBeenCalled();
+ expect(cosEvents.emit).not.toHaveBeenCalled();
+ }
+ store.stateGate = null;
+ expect(await update(action)).toEqual({ [action === 'bind' ? 'bound' : 'released']: true, branch: 'claim/issue-101' });
+ });
+
+ it('fences overdue callbacks even when the timer has not run, sharing the budget across queues', async () => {
+ const ownership = deferred();
+ const state = deferred();
+ const held = withClaimOwnershipLock(() => ownership.promise);
+ store.stateGate = state.promise;
+ const pending = update('bind');
+ await tick();
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS - 1);
+ ownership.resolve();
+ await held;
+ await tick();
+ expect(withStateLock).toHaveBeenCalledOnce();
+ // Advance only the monotonic clock; leave the scheduled timer unfired.
+ vi.spyOn(performance, 'now').mockReturnValue(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ state.resolve();
+ expect(await pending).toEqual({ bound: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });
+ expect(readAgentsStateForSafetyCheck).not.toHaveBeenCalled();
+ expect(saveState).not.toHaveBeenCalled();
+ await withClaimOwnershipLock(() => {});
+ });
+
+ it('keeps a started save owned until durable settlement and reports redacted persistence evidence', async () => {
+ const save = deferred();
+ store.saveGate = save.promise;
+ const before = structuredClone(store.state);
+ let settled = false;
+ const pending = update('bind').then(result => { settled = true; return result; });
+ await tick();
+ expect(saveState).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ expect(settled).toBe(false);
+ expect(store.state).toEqual(before);
+ expect(cosEvents.emit).not.toHaveBeenCalled();
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=persistence'));
+ let nextEntered = false;
+ const following = withClaimOwnershipLock(() => { nextEntered = true; });
+ await tick();
+ expect(nextEntered).toBe(false);
+ save.resolve();
+ expect(await pending).toEqual({ bound: true, branch: 'claim/issue-101' });
+ await following;
+ expect(store.state.agents['agent-epicrun'].metadata.claimBranches).toEqual(['claim/issue-101']);
+ expect(cosEvents.emit).toHaveBeenCalledOnce();
+ const diagnostics = [...console.warn.mock.calls, ...console.log.mock.calls].flat().join(' ');
+ expect(diagnostics).not.toMatch(/agent-epicrun|claim\/issue|repos/);
+ expect(diagnostics).toMatch(/ownershipQueueMs=\d+ stateQueueMs=\d+ ownerReadMs=\d+ ownershipCheckMs=\d+ persistenceMs=\d+/);
+ });
+
+ it('diagnoses an in-flight trusted owner read and refuses unreadable or stale ownership without writes', async () => {
+ const read = deferred();
+ store.readGate = read.promise;
+ store.unreadable = true;
+ const pending = update('bind');
+ await tick();
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=ownerRead'));
+ read.resolve();
+ expect(await pending).toEqual({ bound: false, reason: 'ownership-unreadable' });
+ store.unreadable = false;
+ store.stale = true;
+ expect(await update('release')).toEqual({ released: false, reason: 'ownership-stale' });
+ expect(saveState).not.toHaveBeenCalled();
+ });
+
+ it('propagates a started persistence failure and releases the queue only after rejection', async () => {
+ const save = deferred();
+ store.saveGate = save.promise;
+ const before = structuredClone(store.state);
+ const pending = update('bind');
+ const failure = expect(pending).rejects.toThrow('synthetic write failure');
+ await tick();
+ await vi.advanceTimersByTimeAsync(CLAIM_OWNERSHIP_QUEUE_TIMEOUT_MS);
+ save.reject(new Error('synthetic write failure'));
+ await failure;
+ await withClaimOwnershipLock(() => {});
+ expect(store.state).toEqual(before);
+ expect(cosEvents.emit).not.toHaveBeenCalled();
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('status=failed phase=persistence'));
+ });
+});
+
// The scan that dispatched a reconcile worker is stale by the time it mutates:
// ownership acquired in between must refuse at the worker's own recheck.
describe('checkReconcileOwnership', () => {