Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(cos): bound queued merge admission requests (#10790)
[._.]/ Adam Eivy253a74dcc7d1
diff --git a/docs/features/chief-of-staff.md b/docs/features/chief-of-staff.mdindex a4fff0db94..3af28acd5b 100644--- a/docs/features/chief-of-staff.md+++ b/docs/features/chief-of-staff.md@@ -58,7 +58,17 @@ holds the lease only around its own read → assess → pinned merge.Leases extend the existing machine-local CoS runtime ownership in`data/cos/state.json`, under its existing write queue; there is no new store,-seed, migration, or federation payload. Missing lease fields are compatible+seed, migration, or federation payload. Admission waits at most 15 seconds to+start its state-queue callback; an expired waiter returns `admitted: false`,+`reason: state-queue-timeout`, and `retryAfterMs: 5000`. Its fenced callback+cannot later grant or release a lease. A refused check grants no merge authority;+retry and require a fresh successful check. Once a transaction starts, it retains+the queue until its actual result settles, including persistence failures.+The deadline then logs the pending phase rather than reporting an incomplete+write as a refusal or successful release. Diagnostics contain fixed operation+and phase names plus numeric timings for queue wait, trusted-state read, cached+state load, origin lookup, owner recovery, and persistence; no owner, token,+repository, or path is logged. Missing lease fields are compatiblewith older state. Unreadable, mismatched or stale ownership refuses admission.An agent lease is recoverable from a matching finalized parent record(including its archive), or by any participant once its 5-minute hold deadlinediff --git a/server/routes/cosAgentRoutes.admissionQueue.test.js b/server/routes/cosAgentRoutes.admissionQueue.test.jsnew file mode 100644index 0000000000..f26739a30d--- /dev/null+++ b/server/routes/cosAgentRoutes.admissionQueue.test.js@@ -0,0 +1,148 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';+import { request } from '../lib/testHelper.js';+import { errorMiddleware } from '../lib/errorHandler.js';++const fixture = vi.hoisted(() => ({ state: null, disk: null, queue: null, enqueued: null, read: null, origin: null, save: null }));+vi.mock('../services/cos.js', () => ({}));+vi.mock('../services/agentOrchestrator.js', () => ({}));+vi.mock('../services/cosState.js', () => ({+ withStateLock: (fn) => { const result = fixture.queue(fn); fixture.enqueued?.(); return result; },+ loadState: async () => fixture.state,+ readMergeAdmissionStateForSafetyCheck: async () => {+ await fixture.read?.();+ return { trusted: true, ...structuredClone(fixture.disk) };+ },+ saveState: async (state) => {+ await fixture.save?.();+ fixture.disk = structuredClone(state);+ fixture.state = state;+ },+}));+vi.mock('../lib/gitRemote.js', () => ({ getOriginInfo: async () => {+ await fixture.origin?.();+ return { host: 'github.com', fullName: 'example/repo' };+} }));+import routes from './cosAgentRoutes.js';+import { ADMISSION_QUEUE_TIMEOUT_MS, claimMergeAdmission } from '../services/cosMergeAdmission.js';++const repository = 'github.com/example/repo';+const agentId = 'synthetic-parent';+const deferred = () => {+ let resolve;+ let reject;+ const promise = new Promise((yes, no) => { resolve = yes; reject = no; });+ return { promise, resolve, reject };+};+const input = (action, token) => ({ agentId, action, ...(token && { token }), ...(action === 'release' && { outcome: 'merged' }) });+const call = (action, token) => claimMergeAdmission(input(action, token));+const app = express();+app.use(express.json());+app.use('/api/cos', routes);+app.use(errorMiddleware);+const post = (action, token) => Promise.resolve(request(app).post('/api/cos/merge-admission').send(input(action, token)));++beforeEach(() => {+ fixture.queue = createFileWriteQueue();+ fixture.enqueued = fixture.read = fixture.origin = fixture.save = null;+ fixture.state = { agents: { [agentId]: { id: agentId, taskId: 'synthetic-task', status: 'running',+ startedAt: '2026-01-01T00:00:00Z', metadata: { sourceWorkspace: '/repos/example', claimPicksOwnBranch: true } } }, mergeAdmissions: {} };+ fixture.disk = structuredClone(fixture.state);+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });+});+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });++describe('merge admission state queue deadline', () => {+ it.each(['acquire', 'check', 'release'])('returns a bounded HTTP refusal and fences a queued %s', async (action) => {+ const holder = action === 'acquire' ? null : await call('acquire');+ const before = structuredClone(fixture.disk);+ const gate = deferred();+ const blocked = fixture.queue(() => gate.promise);+ const enqueued = deferred();+ fixture.enqueued = enqueued.resolve;+ const response = post(action, holder?.token);+ await enqueued.promise;+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);+ const res = await response;+ expect(res.status).toBe(200);+ expect(res.body).toEqual({ admitted: false, reason: 'state-queue-timeout', retryAfterMs: 5000 });+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('phase=queue phaseMs=15000'));+ gate.resolve();+ await blocked;+ await fixture.queue(() => {});+ expect(fixture.disk).toEqual(before);+ // No timed-out check is authority: only this new successful check is.+ const fresh = holder ?? await call('acquire');+ expect(await call('check', fresh.token)).toMatchObject({ admitted: true, token: fresh.token });+ expect(await call('release', fresh.token)).toMatchObject({ released: true });+ });++ it('fences an expired callback even if the timer has not fired', async () => {+ const gate = deferred();+ fixture.queue(() => gate.promise);+ const enqueued = deferred();+ fixture.enqueued = enqueued.resolve;+ const response = call('acquire');+ await enqueued.promise;+ vi.spyOn(performance, 'now').mockReturnValue(ADMISSION_QUEUE_TIMEOUT_MS);+ gate.resolve();+ expect(await response).toMatchObject({ admitted: false, reason: 'state-queue-timeout' });+ expect(fixture.disk.mergeAdmissions).toEqual({});+ });++ it.each(['acquire', 'release', 'failed-release'])('retains queue ownership until started %s persistence actually settles', async (mode) => {+ const holder = mode === 'acquire' ? null : await call('acquire');+ const before = structuredClone(fixture.disk);+ const entered = deferred();+ const save = deferred();+ fixture.save = () => { entered.resolve(); return save.promise; };+ let settled = false;+ const response = call(mode === 'acquire' ? 'acquire' : 'release', holder?.token);+ const observed = response.then(result => { settled = true; return { result }; }, error => { settled = true; return { error }; });+ await entered.promise;+ const next = call('check', holder?.token);+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);+ expect(settled).toBe(false);+ expect(fixture.disk).toEqual(before);+ expect(await next).toMatchObject({ admitted: false, reason: 'state-queue-timeout' });+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('status=pending phase=persistence'));+ if (mode === 'failed-release') save.reject(new Error('synthetic persistence failure'));+ else save.resolve();+ const result = await observed;+ fixture.save = null;+ if (mode === 'failed-release') {+ expect(result.error.message).toBe('synthetic persistence failure');+ expect(fixture.disk).toEqual(before);+ expect(await call('check', holder.token)).toMatchObject({ admitted: true });+ } else if (mode === 'acquire') {+ expect(result.result.admitted).toBe(true);+ expect(await call('check', result.result.token)).toMatchObject({ admitted: true });+ } else {+ expect(result.result.released).toBe(true);+ expect(fixture.disk.mergeAdmissions[repository]).toBeUndefined();+ }+ });++ it.each([['read', 'trustedState'], ['origin', 'origin']])('diagnoses a stalled %s phase using only redacted names and numeric timings', async (hook, phase) => {+ const entered = deferred();+ const gate = deferred();+ fixture[hook] = () => { entered.resolve(); return gate.promise; };+ const response = call('acquire');+ await entered.promise;+ await vi.advanceTimersByTimeAsync(ADMISSION_QUEUE_TIMEOUT_MS);+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining(`status=pending phase=${phase}`));+ gate.resolve();+ const admission = await response;+ const logs = [...console.log.mock.calls, ...console.warn.mock.calls].flat();+ for (const line of logs) {+ expect(line).toMatch(/^🚦 Merge admission operation=acquire status=(pending|settled) phase=(trustedState|origin|persistence) phaseMs=\d+ totalMs=\d+ queueMs=\d+ trustedStateMs=\d+ stateLoadMs=\d+ originMs=\d+ ownerRecoveryMs=\d+ persistenceMs=\d+$/);+ expect(line).not.toContain(admission.token);+ expect(line).not.toContain(agentId);+ expect(line).not.toContain('/repos');+ }+ });+});diff --git a/server/services/cosMergeAdmission.js b/server/services/cosMergeAdmission.jsindex 11af88c4b6..386634705f 100644--- a/server/services/cosMergeAdmission.js+++ b/server/services/cosMergeAdmission.js@@ -9,6 +9,7 @@ import { isPlainObject } from '../lib/objects.js';import { isTruthyMeta } from '../lib/metadataFlags.js';const retryAfterMs = 5000;+export const ADMISSION_QUEUE_TIMEOUT_MS = 15_000;// An agent lease covers only verify-and-merge. Past this deadline it is// reclaimable even from a running owner: that holder's pinned merge// (--match-head-commit) cannot land an unverified head, so the worst case is@@ -35,26 +36,70 @@ const validLease = (lease, repository) => isPlainObject(lease) && lease.reposito: lease.kind === 'sweep' && typeof lease.serverOwner === 'string' && Number.isInteger(lease.pid) && lease.pid > 0);const expired = (lease) => lease.kind === 'agent' && Date.now() - Date.parse(lease.acquiredAt) >= AGENT_LEASE_MAX_HOLD_MS;-async function transaction(work) {+async function transaction(work, operation) {// Lazy: almost every task-generator suite reaches prWatcher, even when no// pending merge exists. Admission needs runtime state only on the merge path.const { withStateLock, readMergeAdmissionStateForSafetyCheck, loadState, saveState } = await import('./cosState.js');- return withStateLock(async () => {- const snapshot = await readMergeAdmissionStateForSafetyCheck().catch(() => null);- if (!snapshot?.trusted || !isPlainObject(snapshot.agents) || !isPlainObject(snapshot.mergeAdmissions)) {- return refuse('ownership-unreadable');- }- const state = await loadState();- // A recovered/defaulted or externally replaced state is not proof of an- // empty ownership set. Do not let the cache overwrite contradictory disk.- if (JSON.stringify(state.agents) !== JSON.stringify(snapshot.agents)- || JSON.stringify(state.mergeAdmissions ?? {}) !== JSON.stringify(snapshot.mergeAdmissions)) {- return refuse('ownership-stale');- }- // Copy before mutation: a failed atomic write must not admit from a cache- // that falsely remembers a successful grant/release.- const next = { ...state, agents: { ...state.agents }, mergeAdmissions: { ...snapshot.mergeAdmissions } };- return work(next, saveState);+ const queuedAt = performance.now();+ let started = false;+ let cancelled = false;+ let phase = 'queue';+ let phaseStartedAt = queuedAt;+ const timings = { queue: 0, trustedState: 0, stateLoad: 0, origin: 0, ownerRecovery: 0, persistence: 0 };+ const elapsed = (since) => Math.min(Number.MAX_SAFE_INTEGER, Math.max(0, Math.round(performance.now() - since)));+ // Only fixed operation/phase names and bounded numeric timings reach logs.+ const report = (status) => {+ const log = status === 'failed' ? console.error : status === 'settled' ? console.log : console.warn;+ log(`🚦 Merge admission operation=${operation} status=${status} phase=${phase} phaseMs=${elapsed(phaseStartedAt)} totalMs=${elapsed(queuedAt)} queueMs=${timings.queue} trustedStateMs=${timings.trustedState} stateLoadMs=${timings.stateLoad} originMs=${timings.origin} ownerRecoveryMs=${timings.ownerRecovery} persistenceMs=${timings.persistence}`);+ };+ const measure = async (name, fn) => {+ phase = name;+ phaseStartedAt = performance.now();+ try { return await fn(); }+ finally { timings[name] = elapsed(phaseStartedAt); }+ };+ return new Promise((resolve, reject) => {+ const expire = () => {+ cancelled = true;+ timings.queue = elapsed(queuedAt);+ report('queue-timeout');+ resolve(refuse('state-queue-timeout'));+ };+ const timer = setTimeout(() => {+ if (!started) expire();+ else report('pending');+ }, ADMISSION_QUEUE_TIMEOUT_MS);+ const queued = withStateLock(async () => {+ // Check the clock as well: a delayed timer must not let an expired+ // callback mutate a lease when the state queue resumes first.+ if (cancelled) return;+ if (performance.now() - queuedAt >= ADMISSION_QUEUE_TIMEOUT_MS) { expire(); return; }+ started = true;+ timings.queue = elapsed(queuedAt);+ const snapshot = await measure('trustedState', () => readMergeAdmissionStateForSafetyCheck().catch(() => null));+ if (!snapshot?.trusted || !isPlainObject(snapshot.agents) || !isPlainObject(snapshot.mergeAdmissions)) {+ return refuse('ownership-unreadable');+ }+ const state = await measure('stateLoad', loadState);+ // A recovered/defaulted or externally replaced state is not proof of an+ // empty ownership set. Do not let the cache overwrite contradictory disk.+ if (JSON.stringify(state.agents) !== JSON.stringify(snapshot.agents)+ || JSON.stringify(state.mergeAdmissions ?? {}) !== JSON.stringify(snapshot.mergeAdmissions)) {+ return refuse('ownership-stale');+ }+ // Copy before mutation: a failed atomic write must not admit from a cache+ // that falsely remembers a successful grant/release.+ const next = { ...state, agents: { ...state.agents }, mergeAdmissions: { ...snapshot.mergeAdmissions } };+ return work(next, (value) => measure('persistence', () => saveState(value)), measure);+ });+ // Once started, the queue owns the entire transaction through durable+ // settlement. The deadline diagnoses a slow phase but cannot return a+ // refusal while an unfenced acquire/release is still able to commit.+ queued.then((result) => {+ if (!cancelled) { report('settled'); resolve(result); }+ }, (error) => {+ if (!cancelled) { report('failed'); reject(error); }+ }).finally(() => clearTimeout(timer));});}@@ -76,7 +121,7 @@ async function inactive(lease, state) {return sameAgent(lease, owner) && owner.status === 'completed' && Number.isFinite(Date.parse(owner.completedAt));}-async function admit(state, save, repository, owner) {+async function admit(state, save, repository, owner, measure) {const held = state.mergeAdmissions[repository];if (Object.hasOwn(state.mergeAdmissions, repository)) {if (!validLease(held, repository)) return refuse('lease-unreadable');@@ -85,7 +130,7 @@ async function admit(state, save, repository, owner) {&& held.taskId === owner.taskId && !expired(held)) {return { admitted: true, token: held.token, repository };}- if (!await inactive(held, state)) return refuse('owner-active-or-unverified');+ if (!await measure('ownerRecovery', () => inactive(held, state))) return refuse('owner-active-or-unverified');}const lease = { ...owner, repository, token: randomUUID(), acquiredAt: new Date().toISOString() };state.mergeAdmissions[repository] = lease;@@ -96,21 +141,21 @@ async function admit(state, save, repository, owner) {/** The ID is the registered parent, never a worker-child ID or a cwd PID. */export async function claimMergeAdmission({ agentId, action, token, outcome }) {- return transaction(async (state, save) => {+ return transaction(async (state, save, measure) => {const agent = state.agents[agentId];if (!agent || agent.id !== agentId || agent.status !== 'running' || !agent.startedAt || !agent.taskId|| !(isTruthyMeta(agent.metadata?.claimPicksOwnBranch) || agent.metadata?.claimBranch)) {return refuse('claim-owner-unverified');}if (!ownerPath(agent)) return refuse('repository-unreadable');- const repository = repositoryKey(await getOriginInfo(ownerPath(agent)).catch(() => null));+ const repository = repositoryKey(await measure('origin', () => getOriginInfo(ownerPath(agent)).catch(() => null)));if (!repository) return refuse('repository-unreadable');if (action === 'acquire') {if (Object.entries(state.mergeAdmissions).some(([key, held]) =>!validLease(held, key) || (held.agentId === agentId && (!sameAgent(held, agent) || key !== repository)))) {return refuse('lease-owner-mismatch');}- return admit(state, save, repository, { kind: 'agent', ...binding(agent) });+ return admit(state, save, repository, { kind: 'agent', ...binding(agent) }, measure);}const lease = state.mergeAdmissions[repository];if (!validLease(lease, repository) || lease.kind !== 'agent' || !sameAgent(lease, agent) || lease.token !== token) {@@ -126,16 +171,16 @@ export async function claimMergeAdmission({ agentId, action, token, outcome }) {} };await save(state);return { admitted: false, released: true, outcome };- });+ }, ['acquire', 'check', 'release'].includes(action) ? action : 'unknown');}/** A sweep holds admission only around its final read/assessment/merge attempt. */export async function withPendingMergeAdmission(origin, work) {const repository = repositoryKey(origin);if (!repository) return refuse('repository-unreadable');- const admission = await transaction((state, save) => admit(state, save, repository, {+ const admission = await transaction((state, save, measure) => admit(state, save, repository, {kind: 'sweep', serverOwner, pid: process.pid,- }));+ }, measure), 'sweep-acquire');if (!admission.admitted) return admission;try {return { admitted: true, result: await work() };@@ -147,7 +192,7 @@ export async function withPendingMergeAdmission(origin, work) {delete state.mergeAdmissions[repository];await save(state);return { released: true };- });+ }, 'sweep-release');if (!released.released) console.error(`❌ Pending merge admission release withheld: ${released.reason}`);} finally {// The callback has stopped even if persistence failed. Keep the durable