Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: poll a pending merge handoff instead of waiting out the 30-minute sweep
[._.]/ Adam Eivy0201ad95356b
diff --git a/docs/features/chief-of-staff.md b/docs/features/chief-of-staff.mdindex f9f4f4403b..44a639c10d 100644--- a/docs/features/chief-of-staff.md+++ b/docs/features/chief-of-staff.md@@ -128,6 +128,21 @@ sweeps of one app (timer, task generator, receipt) join a single pass; a receiptthat lands during a pass which already read the queue gets one follow-up passafter it settles.+When that immediate pass leaves the receipt's entry queued (CI still running,+admission deferred, the forge briefly unreadable), the receipt arms a bounded+follow-up poll for the app so a head whose CI turns green a minute later does+not wait out the rest of the 30-minute timer. It re-runs the receipt sweep+every 60 seconds for the first five polls, then every 120 seconds+(`HANDOFF_FOLLOW_UP_*` in `server/services/prWatcher.js`), and stops once+every receipt it tracks has merged, escalated, timed out to a notification or+left the queue. An app has at most one poller: a receipt that lands while one+is active joins it with its own 45-minute cap, after which the 30-minute timer+sweep owns the entry exactly as before. Polls spend no tick budget, so+`MAX_PENDING_MERGE_TICKS` still measures wall-clock hours on the timer. Each+poll passes the timer's own gate — daemon running, not paused, CoS auto-run set+to `execute` — and is skipped otherwise; stopping the daemon cancels every+poller, and a receipt accepted while the daemon is stopped arms none.+Leases extend the existing machine-local CoS runtime ownership in`data/cos/state.json`, under its existing write queue; there is no new store,seed, migration, or federation payload. Admission waits at most 15 seconds todiff --git a/server/services/cos.js b/server/services/cos.jsindex 86d5de5465..d8cf1a44a2 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -560,15 +560,16 @@ async function bootDaemon(state) {// queued merges. Same autonomy gate as the evaluateTasks call site: merging// writes to a default branch, so it only runs when the user has CoS auto-run// set to `execute` and the daemon is not paused.- const { PENDING_MERGE_SWEEP_INTERVAL_MS } = await import('./prWatcher.js');+ // The handoff follow-up poll (prWatcher.followUpHandoffReceipt) shares the+ // gate and is cancelled with the daemon.+ const { PENDING_MERGE_SWEEP_INTERVAL_MS, setHandoffFollowUpGate } = await import('./prWatcher.js');+ setHandoffFollowUpGate(pendingMergeSweepBlocked);scheduleEvent({id: 'cos-pending-merge-sweep',type: 'interval',intervalMs: PENDING_MERGE_SWEEP_INTERVAL_MS,handler: async () => {- if (!isDaemonRunning()) return;- const s = await loadState();- if (s.paused || getDomainMode(s.config, 'cos') !== 'execute') return;+ if (await pendingMergeSweepBlocked()) return;const prWatcher = await import('./prWatcher.js');const sweep = await prWatcher.sweepPendingMergePrs().catch((err) => {console.warn(`⚠️ sweepPendingMergePrs failed: ${err?.message || err}`);@@ -614,6 +615,15 @@ async function bootDaemon(state) {}, POST_STARTUP_QUEUE_DELAY_MS);}+// Null when a pending-merge sweep may merge, else the skip reason.+async function pendingMergeSweepBlocked() {+ if (!isDaemonRunning()) return 'daemon-stopped';+ const s = await loadState();+ if (s.paused) return 'daemon paused';+ if (getDomainMode(s.config, 'cos') !== 'execute') return 'CoS auto-run is not set to execute';+ return null;+}+/*** Stop the CoS daemon*/@@ -642,6 +652,7 @@ async function teardownDaemon() {cancelEvent('cos-rehabilitation-check');cancelEvent('cos-improvement-check');cancelEvent('cos-pending-merge-sweep');+ (await import('./prWatcher.js')).cancelHandoffFollowUps();await unregisterJobSchedules();await withStateLock(async () => {diff --git a/server/services/cos.startStop.test.js b/server/services/cos.startStop.test.jsindex 2bccdc50a5..1de12667d2 100644--- a/server/services/cos.startStop.test.js+++ b/server/services/cos.startStop.test.js@@ -79,9 +79,14 @@ vi.mock('./persistentMindSupervisor.js', async (importOriginal) => ({shutdownPersistentMindSupervisor: vi.fn(async () => {}),registerPersistentMindTurnAdapter: vi.fn(async () => {}),}));+// The handoff follow-up poll lives in prWatcher; the daemon arms its gate and+// cancels it, so "armed" here is the registered gate.+const handoffFollowUp = vi.hoisted(() => ({ gate: null }));vi.mock('./prWatcher.js', () => ({PENDING_MERGE_SWEEP_INTERVAL_MS: 1800000,sweepPendingMergePrs: vi.fn(async () => null),+ setHandoffFollowUpGate: (gate) => { handoffFollowUp.gate = gate; },+ cancelHandoffFollowUps: () => { handoffFollowUp.gate = null; },}));const { start, stop } = await import('./cos.js');@@ -128,11 +133,26 @@ describe('CoS daemon start/stop lifecycle (#11860)', () => {await expect(stopping).resolves.toEqual({ success: true });expect(cosEventIds()).toEqual([]);+ expect(handoffFollowUp.gate).toBe(null);expect(isDaemonRunning()).toBe(false);expect(persisted.running).toBe(false);expect(statuses.at(-1)).toEqual({ running: false });});+ it('the handoff follow-up poll shares the pending-merge gate and is cancelled on stop', async () => {+ persisted = { ...persisted, paused: false, config: { ...persisted.config, domainAutonomy: { cos: 'execute' } } };+ const starting = start();+ await waitFor(() => typeof releaseRecovery === 'function');+ releaseRecovery();+ await starting;+ expect(await handoffFollowUp.gate()).toBe(null);+ persisted = { ...persisted, paused: true };+ expect(await handoffFollowUp.gate()).toBe('daemon paused');++ await stop();+ expect(handoffFollowUp.gate).toBe(null);+ });+it('a boot step that throws rolls the daemon back so the next start() succeeds', async () => {boot.cleanupOrphanedAgents = vi.fn(async () => { throw new Error('agent state unreadable'); });await expect(start()).rejects.toThrow('agent state unreadable');diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex 0bbcc100c7..daef8fde8a 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -2621,11 +2621,16 @@ describe('pending-merge sweep — own timer, not the evaluation cadence (#3630)'it('drives the interval off PENDING_MERGE_SWEEP_INTERVAL_MS, not a local literal', () => {expect(sweepBlock).toMatch(/intervalMs:\s*PENDING_MERGE_SWEEP_INTERVAL_MS/);- expect(COS_SRC).toMatch(/PENDING_MERGE_SWEEP_INTERVAL_MS\s*\}\s*=\s*await import\('\.\/prWatcher\.js'\)/);+ expect(COS_SRC).toMatch(/\{\s*PENDING_MERGE_SWEEP_INTERVAL_MS\b[^}]*\}\s*=\s*await import\('\.\/prWatcher\.js'\)/);});- it('gates the sweep on !paused AND cos auto-run === execute', () => {- expect(sweepBlock).toMatch(/s\.paused\s*\|\|\s*getDomainMode\(s\.config,\s*'cos'\)\s*!==\s*'execute'/);+ it('gates the sweep and the handoff follow-up poll on !paused AND cos auto-run === execute', () => {+ expect(sweepBlock).toMatch(/if \(await pendingMergeSweepBlocked\(\)\) return;/);+ expect(COS_SRC).toMatch(/setHandoffFollowUpGate\(pendingMergeSweepBlocked\)/);+ const gateFn = extractFnBody(COS_SRC, COS_SRC.indexOf('async function pendingMergeSweepBlocked'));+ expect(gateFn).toMatch(/!isDaemonRunning\(\)/);+ expect(gateFn).toMatch(/s\.paused\)/);+ expect(gateFn).toMatch(/getDomainMode\(s\.config,\s*'cos'\)\s*!==\s*'execute'/);});it('keeps the opportunistic drain in evaluateTasks so the call site cannot be deleted silently', () => {diff --git a/server/services/cosMergeHandoff.js b/server/services/cosMergeHandoff.jsindex 906cb0ccf9..31392282f3 100644--- a/server/services/cosMergeHandoff.js+++ b/server/services/cosMergeHandoff.js@@ -144,7 +144,7 @@ export async function handoffPendingMerge({ agentId, prUrl, headSha, worktreePatid: randomUUID(), appId: app.id, prUrl, prNumber: pr.number, headSha: head, repository,queuedAt: new Date().toISOString(),};- const { queuePendingMerge, sweepPendingMergeApp } = await import('./prWatcher.js');+ const { queuePendingMerge, followUpHandoffReceipt } = await import('./prWatcher.js');const queued = await queuePendingMerge(app.id, {prUrl, prNumber: pr.number, prBranch: view.headRefName, sourceAgentId: agentId,sourceTask: {@@ -167,9 +167,10 @@ export async function handoffPendingMerge({ agentId, prUrl, headSha, worktreePat});if (!queued) return refuse('queue-write-failed');console.log(`🤝 PR #${pr.number} handed to the pending-merge coordinator at ${head.slice(0, 12)} by ${agentId}`);- // Detached: the receipt must not wait on forge reads, and the 30-minute timer- // still owns the entry if this pass finds CI pending or fails.- sweepPendingMergeApp(app.id);+ // Detached: the receipt must not wait on forge reads. One immediate pass+ // merges a head whose CI already settled; a still-pending entry gets the+ // bounded follow-up poll, and the 30-minute timer owns it after that.+ followUpHandoffReceipt(app.id, receipt.id);return { queued: true, receipt };}diff --git a/server/services/cosMergeHandoff.test.js b/server/services/cosMergeHandoff.test.jsindex 8bc4fc152c..a7483c01b8 100644--- a/server/services/cosMergeHandoff.test.js+++ b/server/services/cosMergeHandoff.test.js@@ -1,4 +1,4 @@-import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';import { mkdtemp, rm, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join } from 'node:path';@@ -243,3 +243,131 @@ describe('pending-merge handoff', () => {expect(pending()).toEqual([]);});});++// A receipt whose CI is still running gets a bounded follow-up poll instead of+// waiting up to a whole 30-minute timer interval after CI turns green.+describe('handoff follow-up poll', () => {+ let gate;+ let prWatcher;+ // setImmediate stays real: it drains the poll's promise chain between ticks.+ const settle = async () => { for (let i = 0; i < 50; i++) await new Promise((r) => setImmediate(r)); };+ const forgeViews = () => execGhMock.mock.calls.filter(([args]) => args[0] === 'pr' && args[1] === 'view').length;+ const minutes = (n) => vi.advanceTimersByTimeAsync(n * 60 * 1000).then(settle);++ // The coordinator's lazy imports load from disk, which a setImmediate drain+ // does not wait for; warm them so a fresh worker settles like a warm one.+ beforeAll(() => Promise.all(['./prReviewerSecurity.js', '../lib/validation.js'].map((m) => import(m))));++ beforeEach(async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });+ prWatcher = await import('./prWatcher.js');+ gate = vi.fn(async () => null);+ prWatcher.setHandoffFollowUpGate(gate);+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ });++ afterEach(() => {+ prWatcher.cancelHandoffFollowUps();+ // Spies first: one captured under fake timers must not outlive them.+ vi.restoreAllMocks();+ vi.useRealTimers();+ });++ const handOffPending = async () => {+ const result = await handoff();+ await settle();+ return result;+ };++ it('polls a pending head, merges it once CI turns green, then stops', async () => {+ await handOffPending();+ expect(vi.getTimerCount()).toBe(1);+ expect(mergePrMock).not.toHaveBeenCalled();++ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(mergePrMock).not.toHaveBeenCalled();+ expect(vi.getTimerCount()).toBe(1);++ greenChecks();+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(mergePrMock).toHaveBeenCalledWith('/repos/source', 7, { forgeAccount: null, expectedHeadSha: HEAD });+ expect(pending()).toEqual([]);+ expect(vi.getTimerCount()).toBe(0);+ expect(console.log).toHaveBeenCalledWith(expect.stringContaining('every handed-off entry settled'));+ });++ it('never arms when the immediate pass already merged', async () => {+ greenChecks();+ await handOffPending();+ expect(mergePrMock).toHaveBeenCalled();+ expect(vi.getTimerCount()).toBe(0);+ });++ it('extends the active poller for a second receipt instead of arming another timer', async () => {+ const first = await handOffPending();+ const second = await handOffPending();+ expect(second.receipt.id).not.toBe(first.receipt.id);+ expect(vi.getTimerCount()).toBe(1);++ greenChecks();+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(mergePrMock).toHaveBeenCalledTimes(1);+ expect(vi.getTimerCount()).toBe(0);+ });++ it('backs off, spends no tick budget, and hands back to the timer sweep at the cap', async () => {+ await handOffPending();+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_FAST_POLLS);+ const fastReads = forgeViews();+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_BACKOFF_MS / 60000 / 2);+ expect(forgeViews()).toBe(fastReads);++ await minutes(prWatcher.HANDOFF_FOLLOW_UP_CAP_MS / 60000);+ expect(vi.getTimerCount()).toBe(0);+ expect(pending()).toEqual([expect.objectContaining({ ticks: 0 })]);+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('the timer sweep owns 1 entry'));+ const finalReads = forgeViews();+ await minutes(60);+ expect(forgeViews()).toBe(finalReads);+ });++ it('skips a poll the daemon gate blocks without reading the forge', async () => {+ await handOffPending();+ const reads = forgeViews();+ gate.mockResolvedValue('daemon paused');+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(forgeViews()).toBe(reads);+ expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('skipped: daemon paused'));+ expect(vi.getTimerCount()).toBe(1);+ });++ it('stops when the daemon stops', async () => {+ await handOffPending();+ const reads = forgeViews();+ prWatcher.cancelHandoffFollowUps();+ expect(vi.getTimerCount()).toBe(0);+ await minutes(10);+ expect(forgeViews()).toBe(reads);++ // A receipt while the daemon is stopped leaves the entry to the timer sweep.+ await handOffPending();+ expect(vi.getTimerCount()).toBe(0);+ });++ it('owns a throwing poll and keeps polling', async () => {+ await handOffPending();+ const read = f.apps.get.bind(f.apps);+ f.apps.get = () => { throw new Error('apps store unreadable'); };+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('apps store unreadable'));+ expect(vi.getTimerCount()).toBe(1);++ f.apps.get = read;+ greenChecks();+ await minutes(prWatcher.HANDOFF_FOLLOW_UP_INTERVAL_MS / 60000);+ expect(mergePrMock).toHaveBeenCalledTimes(1);+ expect(vi.getTimerCount()).toBe(0);+ });+});diff --git a/server/services/prWatcher.js b/server/services/prWatcher.jsindex a81291c906..716de3b0bc 100644--- a/server/services/prWatcher.js+++ b/server/services/prWatcher.js@@ -42,8 +42,20 @@ const PR_LIST_LIMIT = 200;// before a merge-only PR silently leaks forever. cos.js drives its interval off// PENDING_MERGE_SWEEP_INTERVAL_MS so `MAX_PENDING_MERGE_TICKS` keeps mapping to// wall-clock hours rather than to restarts (#3630).+//+// A handoff receipt whose CI is still running would otherwise wait up to one+// full interval after CI turns green, so the receipt arms a bounded per-app+// follow-up poll (`followUpHandoffReceipt`): every HANDOFF_FOLLOW_UP_INTERVAL_MS+// for the first HANDOFF_FOLLOW_UP_FAST_POLLS polls (a scoped CI run), then every+// HANDOFF_FOLLOW_UP_BACKOFF_MS, until its receipts settle or HANDOFF_FOLLOW_UP_CAP_MS+// passes after the latest one. Those polls spend no ticks, so the timer above+// stays the sole owner of the budget and of every entry after the cap.export const PENDING_MERGE_SWEEP_INTERVAL_MS = 30 * 60 * 1000;export const MAX_PENDING_MERGE_TICKS = 12;+export const HANDOFF_FOLLOW_UP_INTERVAL_MS = 60 * 1000;+export const HANDOFF_FOLLOW_UP_FAST_POLLS = 5;+export const HANDOFF_FOLLOW_UP_BACKOFF_MS = 2 * 60 * 1000;+export const HANDOFF_FOLLOW_UP_CAP_MS = 45 * 60 * 1000;const GREEN_CHECK_VERDICTS = new Set(['SUCCESS', 'NEUTRAL', 'SKIPPED']);// Every app lives in the same apps.json store, so this must be one global tail@@ -52,6 +64,10 @@ const GREEN_CHECK_VERDICTS = new Set(['SUCCESS', 'NEUTRAL', 'SKIPPED']);let pendingMergeWriteTail = Promise.resolve();const pendingMergeSweeps = new Map();const pendingMergeResweeps = new Map();+// appId -> { receipts: Map<receiptId, deadlineMs>, polls, timer }+const handoffFollowUps = new Map();+// Registered by the CoS daemon while it runs; null = daemon stopped.+let handoffFollowUpGate = null;// Cache the gh-authenticated login PER HOST, for the process lifetime. Host-keyed// for two reasons: (1) `gh api` does NOT infer the host from the working@@ -610,6 +626,118 @@ function receiptSweep(appId) {});}+/**+ * The receipt sweep for one handoff, then — if that receipt's entry is still+ * queued (CI pending, admission deferred, forge briefly unreadable) — the app's+ * bounded follow-up poll. Never rejects.+ */+export function followUpHandoffReceipt(appId, receiptId) {+ return sweepPendingMergeApp(appId)+ .then(() => armHandoffFollowUp(appId, receiptId))+ .catch((err) => {+ console.error(`❌ Handoff follow-up for app ${appId} could not be armed: ${err.message}`);+ });+}++/**+ * The CoS daemon registers the same gate its pending-merge timer uses: it+ * resolves null when a poll may merge, else a skip reason. A poll that finds no+ * gate (daemon stopped) ends its poller.+ */+export function setHandoffFollowUpGate(gate) {+ handoffFollowUpGate = typeof gate === 'function' ? gate : null;+}++/** Daemon stop: drop every follow-up poll and the gate. */+export function cancelHandoffFollowUps() {+ handoffFollowUpGate = null;+ for (const poller of handoffFollowUps.values()) clearTimeout(poller.timer);+ handoffFollowUps.clear();+}++async function queuedReceiptIds(appId) {+ const app = await getAppById(appId);+ return new Set(readPendingMergePrs(app).map((entry) => entry?.handoff?.receiptId).filter(Boolean));+}++async function armHandoffFollowUp(appId, receiptId) {+ if (!receiptId || !(await queuedReceiptIds(appId)).has(receiptId)) return;+ if (!handoffFollowUpGate) {+ console.warn(`⏸️ Handoff follow-up for app ${appId} not armed: CoS daemon is stopped; the timer sweep owns the entry`);+ return;+ }+ const deadline = Date.now() + HANDOFF_FOLLOW_UP_CAP_MS;+ const active = handoffFollowUps.get(appId);+ if (active) {+ active.receipts.set(receiptId, deadline);+ return;+ }+ const poller = { receipts: new Map([[receiptId, deadline]]), polls: 0, timer: null };+ handoffFollowUps.set(appId, poller);+ scheduleHandoffFollowUp(appId, poller);+ console.log(`⏱️ Handoff follow-up armed for app ${appId}: entry still pending, polling every ${HANDOFF_FOLLOW_UP_INTERVAL_MS / 1000}s`);+}++// A bounded wait on an external system: the forge cannot push CI completion to+// a private install, which exposes no public webhook endpoint.+function scheduleHandoffFollowUp(appId, poller) {+ const delay = poller.polls < HANDOFF_FOLLOW_UP_FAST_POLLS ? HANDOFF_FOLLOW_UP_INTERVAL_MS : HANDOFF_FOLLOW_UP_BACKOFF_MS;+ poller.timer = setTimeout(() => {+ pollHandoffFollowUp(appId, poller).catch((err) => {+ console.error(`❌ Handoff follow-up poll for app ${appId} failed: ${err.message}`);+ if (handoffFollowUps.get(appId) !== poller) return;+ // Still bounded when every poll throws: the latest receipt's cap ends it.+ if (Date.now() >= Math.max(...poller.receipts.values())) {+ handoffFollowUps.delete(appId);+ console.warn(`⏸️ Handoff follow-up for app ${appId} reached its cap; the timer sweep owns its entries`);+ } else {+ scheduleHandoffFollowUp(appId, poller);+ }+ });+ }, delay);+ poller.timer.unref?.();+}++async function pollHandoffFollowUp(appId, poller) {+ poller.polls += 1;+ const blocked = handoffFollowUpGate ? await handoffFollowUpGate() : 'daemon-stopped';+ if (handoffFollowUps.get(appId) !== poller) return;+ if (blocked === 'daemon-stopped') {+ handoffFollowUps.delete(appId);+ console.warn(`⏸️ Handoff follow-up for app ${appId} stopped: CoS daemon is stopped`);+ return;+ }+ if (blocked) {+ console.warn(`⏸️ Handoff follow-up poll for app ${appId} skipped: ${blocked}`);+ } else {+ const result = await sweepPendingMergeApp(appId);+ // null: receiptSweep already logged the thrown failure.+ if (result && !result.ok) {+ console.error(`❌ Handoff follow-up poll for app ${appId} failed: ${result.reason || 'sweep error'}`);+ }+ }+ const queued = await queuedReceiptIds(appId);+ // Each await is a window for a daemon stop to cancel this poller.+ if (handoffFollowUps.get(appId) !== poller) return;+ const now = Date.now();+ let expired = 0;+ for (const [receiptId, deadline] of poller.receipts) {+ if (!queued.has(receiptId)) poller.receipts.delete(receiptId);+ else if (now >= deadline) { poller.receipts.delete(receiptId); expired += 1; }+ }+ if (poller.receipts.size === 0) {+ handoffFollowUps.delete(appId);+ if (expired) {+ console.warn(`⏸️ Handoff follow-up for app ${appId} reached its ${HANDOFF_FOLLOW_UP_CAP_MS / 60000}-minute cap; the timer sweep owns ${expired} entr${expired === 1 ? 'y' : 'ies'}`);+ } else {+ console.log(`✅ Handoff follow-up for app ${appId} done: every handed-off entry settled`);+ }+ return;+ }+ scheduleHandoffFollowUp(appId, poller);+ console.log(`⏳ Handoff follow-up for app ${appId}: ${poller.receipts.size} handed-off entr${poller.receipts.size === 1 ? 'y' : 'ies'} still pending after poll ${poller.polls}`);+}+/*** Merge a patch into the app's persisted watcher state. Re-reads the app first* so the merge is against the freshest record.