Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(cos): bound queued merge admission requests (#10790)

[._.]/ Adam Eivy253a74dcc7d1
diff --git a/docs/features/chief-of-staff.md b/docs/features/chief-of-staff.md
index a4fff0db94..3af28acd5b 100644
--- a/docs/features/chief-of-staff.md
+++ b/docs/features/chief-of-staff.md
@@ -58,7 +58,17 @@ holds the lease only around its own read → assess → pinned merge.
Leases extend the existing machine-local CoS runtime ownership in
`data/cos/state.json`, under its existing write queue; there is no new store,
-seed, migration, or federation payload. Missing lease fields are compatible
+seed, migration, or federation payload. Admission waits at most 15 seconds to
+start its state-queue callback; an expired waiter returns `admitted: false`,
+`reason: state-queue-timeout`, and `retryAfterMs: 5000`. Its fenced callback
+cannot later grant or release a lease. A refused check grants no merge authority;
+retry and require a fresh successful check. Once a transaction starts, it retains
+the queue until its actual result settles, including persistence failures.
+The deadline then logs the pending phase rather than reporting an incomplete
+write as a refusal or successful release. Diagnostics contain fixed operation
+and phase names plus numeric timings for queue wait, trusted-state read, cached
+state load, origin lookup, owner recovery, and persistence; no owner, token,
+repository, or path is logged. Missing lease fields are compatible
with older state. Unreadable, mismatched or stale ownership refuses admission.
An agent lease is recoverable from a matching finalized parent record
(including its archive), or by any participant once its 5-minute hold deadline
diff --git a/server/routes/cosAgentRoutes.admissionQueue.test.js b/server/routes/cosAgentRoutes.admissionQueue.test.js
new file mode 100644
index 0000000000..f26739a30d
--- /dev/null
+++ b/server/routes/cosAgentRoutes.admissionQueue.test.js
@@ -0,0 +1,148 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
+import { request } from '../lib/testHelper.js';
+import { errorMiddleware } from '../lib/errorHandler.js';
+
+const fixture = vi.hoisted(() => ({ state: null, disk: null, queue: null, enqueued: null, read: null, origin: null, save: null }));
+vi.mock('../services/cos.js', () => ({}));
+vi.mock('../services/agentOrchestrator.js', () => ({}));
+vi.mock('../services/cosState.js', () => ({
+ withStateLock: (fn) => { const result = fixture.queue(fn); fixture.enqueued?.(); return result; },
+ loadState: async () => fixture.state,
+ readMergeAdmissionStateForSafetyCheck: async () => {
+ await fixture.read?.();
+ return { trusted: true, ...structuredClone(fixture.disk) };
+ },
+ saveState: async (state) => {
+ await fixture.save?.();
+ fixture.disk = structuredClone(state);
+ fixture.state = state;
+ },
+}));
+vi.mock('../lib/gitRemote.js', () => ({ getOriginInfo: async () => {
+ await fixture.origin?.();
+ return { host: 'github.com', fullName: 'example/repo' };
+} }));
+import routes from './cosAgentRoutes.js';
+import { ADMISSION_QUEUE_TIMEOUT_MS, claimMergeAdmission } from '../services/cosMergeAdmission.js';
+
+const repository = 'github.com/example/repo';
+const agentId = 'synthetic-parent';
+const deferred = () => {
+ let resolve;
+ let reject;
+ const promise = new Promise((yes, no) => { resolve = yes; reject = no; });
+ return { promise, resolve, reject };
+};
+const input = (action, token) => ({ agentId, action, ...(token && { token }), ...(action === 'release' && { outcome: 'merged' }) });
+const call = (action, token) => claimMergeAdmission(input(action, token));
+const app = express();
+app.use(express.json());
+app.use('/api/cos', routes);
+app.use(errorMiddleware);
+const post = (action, token) => Promise.resolve(request(app).post('/api/cos/merge-admission').send(input(action, token)));
+
+beforeEach(() => {
+ fixture.queue = createFileWriteQueue();
+ fixture.enqueued = fixture.read = fixture.origin = fixture.save = null;
+ fixture.state = { agents: { [agentId]: { id: agentId, taskId: 'synthetic-task', status: 'running',
+ startedAt: '2026-01-01T00:00:00Z', metadata: { sourceWorkspace: '/repos/example', claimPicksOwnBranch: true } } }, mergeAdmissions: {} };
+ fixture.disk = structuredClone(fixture.state);
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });
+});
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+
+describe('merge admission state queue deadline', () => {
+ it.each(['acquire', 'check', 'release'])('returns a bounded HTTP refusal and fences a queued %s', async (action) => {
+ const holder = action === 'acquire' ? null : await call('acquire');
+ const before = structuredClone(fixture.disk);
+ const gate = deferred();
+ const blocked = fixture.queue(() => gate.promise);
+ const enqueued = deferred();
+ fixture.enqueued = enqueued.resolve;
+ const response = post(action, holder?.token);
+ await enqueued.promise;
+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);
+ const res = await response;
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({ admitted: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('phase=queue phaseMs=15000'));
+ gate.resolve();
+ await blocked;
+ await fixture.queue(() => {});
+ expect(fixture.disk).toEqual(before);
+ // No timed-out check is authority: only this new successful check is.
+ const fresh = holder ?? await call('acquire');
+ expect(await call('check', fresh.token)).toMatchObject({ admitted: true, token: fresh.token });
+ expect(await call('release', fresh.token)).toMatchObject({ released: true });
+ });
+
+ it('fences an expired callback even if the timer has not fired', async () => {
+ const gate = deferred();
+ fixture.queue(() => gate.promise);
+ const enqueued = deferred();
+ fixture.enqueued = enqueued.resolve;
+ const response = call('acquire');
+ await enqueued.promise;
+ vi.spyOn(performance, 'now').mockReturnValue(ADMISSION_QUEUE_TIMEOUT_MS);
+ gate.resolve();
+ expect(await response).toMatchObject({ admitted: false, reason: 'state-queue-timeout' });
+ expect(fixture.disk.mergeAdmissions).toEqual({});
+ });
+
+ it.each(['acquire', 'release', 'failed-release'])('retains queue ownership until started %s persistence actually settles', async (mode) => {
+ const holder = mode === 'acquire' ? null : await call('acquire');
+ const before = structuredClone(fixture.disk);
+ const entered = deferred();
+ const save = deferred();
+ fixture.save = () => { entered.resolve(); return save.promise; };
+ let settled = false;
+ const response = call(mode === 'acquire' ? 'acquire' : 'release', holder?.token);
+ const observed = response.then(result => { settled = true; return { result }; }, error => { settled = true; return { error }; });
+ await entered.promise;
+ const next = call('check', holder?.token);
+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);
+ expect(settled).toBe(false);
+ expect(fixture.disk).toEqual(before);
+ expect(await next).toMatchObject({ admitted: false, reason: 'state-queue-timeout' });
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=persistence'));
+ if (mode === 'failed-release') save.reject(new Error('synthetic persistence failure'));
+ else save.resolve();
+ const result = await observed;
+ fixture.save = null;
+ if (mode === 'failed-release') {
+ expect(result.error.message).toBe('synthetic persistence failure');
+ expect(fixture.disk).toEqual(before);
+ expect(await call('check', holder.token)).toMatchObject({ admitted: true });
+ } else if (mode === 'acquire') {
+ expect(result.result.admitted).toBe(true);
+ expect(await call('check', result.result.token)).toMatchObject({ admitted: true });
+ } else {
+ expect(result.result.released).toBe(true);
+ expect(fixture.disk.mergeAdmissions[repository]).toBeUndefined();
+ }
+ });
+
+ it.each([['read', 'trustedState'], ['origin', 'origin']])('diagnoses a stalled %s phase using only redacted names and numeric timings', async (hook, phase) => {
+ const entered = deferred();
+ const gate = deferred();
+ fixture[hook] = () => { entered.resolve(); return gate.promise; };
+ const response = call('acquire');
+ await entered.promise;
+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);
+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`status=pending phase=${phase}`));
+ gate.resolve();
+ const admission = await response;
+ const logs = [...console.log.mock.calls, ...console.warn.mock.calls].flat();
+ for (const line of logs) {
+ expect(line).toMatch(/^🚦 Merge admission operation=acquire status=(pending|settled) phase=(trustedState|origin|persistence) phaseMs=\d+ totalMs=\d+ queueMs=\d+ trustedStateMs=\d+ stateLoadMs=\d+ originMs=\d+ ownerRecoveryMs=\d+ persistenceMs=\d+$/);
+ expect(line).not.toContain(admission.token);
+ expect(line).not.toContain(agentId);
+ expect(line).not.toContain('/repos');
+ }
+ });
+});
diff --git a/server/services/cosMergeAdmission.js b/server/services/cosMergeAdmission.js
index 11af88c4b6..386634705f 100644
--- a/server/services/cosMergeAdmission.js
+++ b/server/services/cosMergeAdmission.js
@@ -9,6 +9,7 @@ import { isPlainObject } from '../lib/objects.js';
import { isTruthyMeta } from '../lib/metadataFlags.js';
const retryAfterMs = 5000;
+export const ADMISSION_QUEUE_TIMEOUT_MS = 15_000;
// An agent lease covers only verify-and-merge. Past this deadline it is
// reclaimable even from a running owner: that holder's pinned merge
// (--match-head-commit) cannot land an unverified head, so the worst case is
@@ -35,26 +36,70 @@ const validLease = (lease, repository) => isPlainObject(lease) && lease.reposito
: lease.kind === 'sweep' && typeof lease.serverOwner === 'string' && Number.isInteger(lease.pid) && lease.pid > 0);
const expired = (lease) => lease.kind === 'agent' && Date.now() - Date.parse(lease.acquiredAt) >= AGENT_LEASE_MAX_HOLD_MS;
-async function transaction(work) {
+async function transaction(work, operation) {
// Lazy: almost every task-generator suite reaches prWatcher, even when no
// pending merge exists. Admission needs runtime state only on the merge path.
const { withStateLock, readMergeAdmissionStateForSafetyCheck, loadState, saveState } = await import('./cosState.js');
- return withStateLock(async () => {
- const snapshot = await readMergeAdmissionStateForSafetyCheck().catch(() => null);
- if (!snapshot?.trusted || !isPlainObject(snapshot.agents) || !isPlainObject(snapshot.mergeAdmissions)) {
- return refuse('ownership-unreadable');
- }
- const state = await loadState();
- // A recovered/defaulted or externally replaced state is not proof of an
- // empty ownership set. Do not let the cache overwrite contradictory disk.
- if (JSON.stringify(state.agents) !== JSON.stringify(snapshot.agents)
- || JSON.stringify(state.mergeAdmissions ?? {}) !== JSON.stringify(snapshot.mergeAdmissions)) {
- return refuse('ownership-stale');
- }
- // Copy before mutation: a failed atomic write must not admit from a cache
- // that falsely remembers a successful grant/release.
- const next = { ...state, agents: { ...state.agents }, mergeAdmissions: { ...snapshot.mergeAdmissions } };
- return work(next, saveState);
+ const queuedAt = performance.now();
+ let started = false;
+ let cancelled = false;
+ let phase = 'queue';
+ let phaseStartedAt = queuedAt;
+ const timings = { queue: 0, trustedState: 0, stateLoad: 0, origin: 0, ownerRecovery: 0, persistence: 0 };
+ const elapsed = (since) => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.round(performance.now() - since)));
+ // Only fixed operation/phase names and bounded numeric timings reach logs.
+ const report = (status) => {
+ const log = status === 'failed' ? console.error : status === 'settled' ? console.log : console.warn;
+ log(`🚦 Merge admission operation=${operation} status=${status} phase=${phase} phaseMs=${elapsed(phaseStartedAt)} totalMs=${elapsed(queuedAt)} queueMs=${timings.queue} trustedStateMs=${timings.trustedState} stateLoadMs=${timings.stateLoad} originMs=${timings.origin} ownerRecoveryMs=${timings.ownerRecovery} persistenceMs=${timings.persistence}`);
+ };
+ const measure = async (name, fn) => {
+ phase = name;
+ phaseStartedAt = performance.now();
+ try { return await fn(); }
+ finally { timings[name] = elapsed(phaseStartedAt); }
+ };
+ return new Promise((resolve, reject) => {
+ const expire = () => {
+ cancelled = true;
+ timings.queue = elapsed(queuedAt);
+ report('queue-timeout');
+ resolve(refuse('state-queue-timeout'));
+ };
+ const timer = setTimeout(() => {
+ if (!started) expire();
+ else report('pending');
+ }, ADMISSION_QUEUE_TIMEOUT_MS);
+ const queued = withStateLock(async () => {
+ // Check the clock as well: a delayed timer must not let an expired
+ // callback mutate a lease when the state queue resumes first.
+ if (cancelled) return;
+ if (performance.now() - queuedAt >= ADMISSION_QUEUE_TIMEOUT_MS) { expire(); return; }
+ started = true;
+ timings.queue = elapsed(queuedAt);
+ const snapshot = await measure('trustedState', () => readMergeAdmissionStateForSafetyCheck().catch(() => null));
+ if (!snapshot?.trusted || !isPlainObject(snapshot.agents) || !isPlainObject(snapshot.mergeAdmissions)) {
+ return refuse('ownership-unreadable');
+ }
+ const state = await measure('stateLoad', loadState);
+ // A recovered/defaulted or externally replaced state is not proof of an
+ // empty ownership set. Do not let the cache overwrite contradictory disk.
+ if (JSON.stringify(state.agents) !== JSON.stringify(snapshot.agents)
+ || JSON.stringify(state.mergeAdmissions ?? {}) !== JSON.stringify(snapshot.mergeAdmissions)) {
+ return refuse('ownership-stale');
+ }
+ // Copy before mutation: a failed atomic write must not admit from a cache
+ // that falsely remembers a successful grant/release.
+ const next = { ...state, agents: { ...state.agents }, mergeAdmissions: { ...snapshot.mergeAdmissions } };
+ return work(next, (value) => measure('persistence', () => saveState(value)), measure);
+ });
+ // Once started, the queue owns the entire transaction through durable
+ // settlement. The deadline diagnoses a slow phase but cannot return a
+ // refusal while an unfenced acquire/release is still able to commit.
+ queued.then((result) => {
+ if (!cancelled) { report('settled'); resolve(result); }
+ }, (error) => {
+ if (!cancelled) { report('failed'); reject(error); }
+ }).finally(() => clearTimeout(timer));
});
}
@@ -76,7 +121,7 @@ async function inactive(lease, state) {
return sameAgent(lease, owner) && owner.status === 'completed' && Number.isFinite(Date.parse(owner.completedAt));
}
-async function admit(state, save, repository, owner) {
+async function admit(state, save, repository, owner, measure) {
const held = state.mergeAdmissions[repository];
if (Object.hasOwn(state.mergeAdmissions, repository)) {
if (!validLease(held, repository)) return refuse('lease-unreadable');
@@ -85,7 +130,7 @@ async function admit(state, save, repository, owner) {
&& held.taskId === owner.taskId && !expired(held)) {
return { admitted: true, token: held.token, repository };
}
- if (!await inactive(held, state)) return refuse('owner-active-or-unverified');
+ if (!await measure('ownerRecovery', () => inactive(held, state))) return refuse('owner-active-or-unverified');
}
const lease = { ...owner, repository, token: randomUUID(), acquiredAt: new Date().toISOString() };
state.mergeAdmissions[repository] = lease;
@@ -96,21 +141,21 @@ async function admit(state, save, repository, owner) {
/** The ID is the registered parent, never a worker-child ID or a cwd PID. */
export async function claimMergeAdmission({ agentId, action, token, outcome }) {
- return transaction(async (state, save) => {
+ return transaction(async (state, save, measure) => {
const agent = state.agents[agentId];
if (!agent || agent.id !== agentId || agent.status !== 'running' || !agent.startedAt || !agent.taskId
|| !(isTruthyMeta(agent.metadata?.claimPicksOwnBranch) || agent.metadata?.claimBranch)) {
return refuse('claim-owner-unverified');
}
if (!ownerPath(agent)) return refuse('repository-unreadable');
- const repository = repositoryKey(await getOriginInfo(ownerPath(agent)).catch(() => null));
+ const repository = repositoryKey(await measure('origin', () => getOriginInfo(ownerPath(agent)).catch(() => null)));
if (!repository) return refuse('repository-unreadable');
if (action === 'acquire') {
if (Object.entries(state.mergeAdmissions).some(([key, held]) =>
!validLease(held, key) || (held.agentId === agentId && (!sameAgent(held, agent) || key !== repository)))) {
return refuse('lease-owner-mismatch');
}
- return admit(state, save, repository, { kind: 'agent', ...binding(agent) });
+ return admit(state, save, repository, { kind: 'agent', ...binding(agent) }, measure);
}
const lease = state.mergeAdmissions[repository];
if (!validLease(lease, repository) || lease.kind !== 'agent' || !sameAgent(lease, agent) || lease.token !== token) {
@@ -126,16 +171,16 @@ export async function claimMergeAdmission({ agentId, action, token, outcome }) {
} };
await save(state);
return { admitted: false, released: true, outcome };
- });
+ }, ['acquire', 'check', 'release'].includes(action) ? action : 'unknown');
}
/** A sweep holds admission only around its final read/assessment/merge attempt. */
export async function withPendingMergeAdmission(origin, work) {
const repository = repositoryKey(origin);
if (!repository) return refuse('repository-unreadable');
- const admission = await transaction((state, save) => admit(state, save, repository, {
+ const admission = await transaction((state, save, measure) => admit(state, save, repository, {
kind: 'sweep', serverOwner, pid: process.pid,
- }));
+ }, measure), 'sweep-acquire');
if (!admission.admitted) return admission;
try {
return { admitted: true, result: await work() };
@@ -147,7 +192,7 @@ export async function withPendingMergeAdmission(origin, work) {
delete state.mergeAdmissions[repository];
await save(state);
return { released: true };
- });
+ }, 'sweep-release');
if (!released.released) console.error(`❌ Pending merge admission release withheld: ${released.reason}`);
} finally {
// The callback has stopped even if persistence failed. Keep the durable