Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
feat(core): capture stalled child operation phases [A7,C7]
Corey Olson203671b3cbac
diff --git a/src/cli-main.js b/src/cli-main.jsindex ce60596a..2698a048 100644--- a/src/cli-main.js+++ b/src/cli-main.js@@ -290,9 +290,7 @@ async function runMain(argv, streams) {(await import('./person.js')).requirePersonChannel(io.personChannel);io.caller('person');}- traceServeStartup('cli-bind-start');checkoutSession = await bindCheckoutSession(io, parsed.positionals, parsed.values);- traceServeStartup('cli-bind-ready');if (checkoutSession?.lease) CHECKOUT_LEASES.set(io, checkoutSession.lease);} catch (error) {if (!(error instanceof Refused)) throw error;@@ -304,29 +302,20 @@ async function runMain(argv, streams) {}}try {- traceServeStartup('cli-trunk-start');if (needsRepo) await rememberTrunk(io, git);- traceServeStartup('cli-trunk-ready');traceServeStartup('cli-command-preamble-start');// The relay and baseline work loads only where a relay token, relay state, a link or a pending result exists./** Load the work around the command when any of it could act here, or null. */const syncWork = async () => (sync && (await import('./local-state.js')).syncMayAct(io.cwd) ? import('./cli-sync.js') : null);await (await syncWork())?.syncBefore(io, main);traceServeStartup('cli-command-preamble-ready');- traceServeStartup('cli-command-dispatch');const code = await runCommand(argv, io);- traceServeStartup('cli-command-returned');if (code === 0) await checkoutSession?.finish?.();- traceServeStartup('cli-final-sync-start');if (code === 0) await (await syncWork())?.retrySync(io);- traceServeStartup('cli-final-sync-ready');const status = io.exitCode(code);if (status === 0 && needsRepo && !parsed?.values['dry-run']) (await import('./blocks.js')).clearCompletedBlocks(io);- traceServeStartup('cli-post-sync-ready');if (status !== 0) await io.prepareRefusal();- traceServeStartup('cli-flush-start');io.flush(status);- traceServeStartup('cli-flush-ready');return status;} finally {CHECKOUT_LEASES.delete(io);diff --git a/src/command-phase.js b/src/command-phase.jsnew file mode 100644index 00000000..e8194b35--- /dev/null+++ b/src/command-phase.js@@ -0,0 +1,74 @@+/** Opt-in credential-free command phases let private child watchdogs explain a stalled operation [A7,C7]. */+import {+ appendFileSync,+ readFileSync,+ statSync,+} from 'node:fs';+import { performance } from 'node:perf_hooks';++const PHASES = new Set(['waiting-relay', 'waiting-lock', 'local-export']);+const MAX_TRACE_BYTES = 1024 * 1024;+let sequence = 0;++/** An absent observer leaves production behavior unchanged and performs no file operation. */+function noPhase() {}++/** Begin one known operation; its id lets nested or concurrent operations finish independently. */+export function beginCommandPhase(phase) {+ const file = process.env.PULLBOARD_COMMAND_PHASE_FILE;+ if (!file || !PHASES.has(phase)) return noPhase;+ const id = ++sequence;+ let ended = false;+ /** Diagnostic failure cannot change the underlying command or its refusal. */+ function record(event) {+ try {+ appendFileSync(file, JSON.stringify({ pid: process.pid, id, phase, event,+ at: Date.now(), uptimeMs: performance.now() }) + '\n', { mode: 0o600 });+ } catch { /* An unavailable diagnostic file never changes command behavior. */ }+ }+ record('start');+ return () => { if (!ended) { ended = true; record('end'); } };+}++/** Report uncertainty explicitly; an absent phase is never fabricated as a zero-duration operation. */+function unavailable(pid, reason, timings = []) {+ return { pid, phase: 'unmeasured', elapsedMs: null, timings, unavailable: reason };+}++/** Read only bounded phase metadata for the actual child, excluding URLs, payloads, paths and credentials. */+export function commandPhaseSnapshot(file, pid) {+ if (!file) return unavailable(pid, 'no phase capture was configured');+ let events;+ try {+ if (statSync(file).size > MAX_TRACE_BYTES) return unavailable(pid, 'phase capture exceeded its byte bound');+ events = readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));+ } catch { return unavailable(pid, 'phase capture was not available or complete'); }+ const active = new Map();+ const timings = [];+ for (const event of events) {+ if (event.pid !== pid) continue;+ if (!PHASES.has(event.phase) || !Number.isSafeInteger(event.id) || event.id < 1+ || !Number.isFinite(event.at) || !Number.isFinite(event.uptimeMs)+ || !['start', 'end'].includes(event.event)) return unavailable(pid, 'phase capture contained invalid metadata');+ if (event.event === 'start') {+ if (active.has(event.id)) return unavailable(pid, 'phase capture repeated an operation');+ active.set(event.id, event);+ } else {+ const start = active.get(event.id);+ if (!start || start.phase !== event.phase || event.uptimeMs < start.uptimeMs) return unavailable(pid, 'phase capture had an unmatched completion');+ timings.push({ phase: event.phase, elapsedMs: event.uptimeMs - start.uptimeMs, complete: true });+ active.delete(event.id);+ }+ }+ const observedAt = Date.now();+ const pending = [...active.values()].sort((left, right) => left.id - right.id);+ if (!pending.length) return unavailable(pid, 'no instrumented operation was active at the stall', timings);+ for (const start of pending) {+ if (observedAt < start.at) return unavailable(pid, 'the wall clock moved before the active phase', timings);+ timings.push({ phase: start.phase, elapsedMs: observedAt - start.at, complete: false });+ }+ const latest = pending.at(-1);+ const activePhases = [...new Set(pending.map(start => start.phase))];+ return { pid, phase: activePhases.length === 1 ? latest.phase : 'concurrent',+ activePhases, elapsedMs: observedAt - latest.at, timings, observedAt };+}diff --git a/src/exchange.js b/src/exchange.jsindex 99742957..68a6dd2a 100644--- a/src/exchange.js+++ b/src/exchange.js@@ -1,3 +1,4 @@+import { beginCommandPhase } from './command-phase.js';import { LANDING_LOCAL_PREFIX } from './landing-local-lease.js';import { PROOF_COUNTER_PREFIX } from './proof-counters.js';import { rebuildProofCounters } from './proof-counters.js';@@ -59,19 +60,22 @@ function nativeRows(db, table) {* @returns {{ version: number, tables: Record<string, any[]> }}*/export function exportBoard(board) {- const db = board.db;- db.exec('BEGIN DEFERRED');+ const finishPhase = beginCommandPhase('local-export');try {- const tables = Object.fromEntries(tableNames(db).map((table) => [- table,- nativeRows(db, table),- ]));- db.exec('COMMIT');- return { version: VERSION, tables };- } catch (error) {- db.exec('ROLLBACK');- throw error;- }+ const db = board.db;+ db.exec('BEGIN DEFERRED');+ try {+ const tables = Object.fromEntries(tableNames(db).map((table) => [+ table,+ nativeRows(db, table),+ ]));+ db.exec('COMMIT');+ return { version: VERSION, tables };+ } catch (error) {+ db.exec('ROLLBACK');+ throw error;+ }+ } finally { finishPhase(); }}/** Hash semantic native state while excluding only per-agent inbox read cursors. */diff --git a/src/relay-lock.js b/src/relay-lock.jsindex d3ef8a7d..15129089 100644--- a/src/relay-lock.js+++ b/src/relay-lock.js@@ -1,3 +1,4 @@+import { beginCommandPhase } from './command-phase.js';/** The per-board link lock: one worktree sends at a time, and a sibling waits instead of failing (#919). */import { randomUUID } from 'node:crypto';import {@@ -97,37 +98,41 @@ function takeDirectory(lock) {* after 15 s failed for a reason the person cannot fix (#919); the wait is up to 2 minutes with a progress line.*/export async function locked(file, work, io) {- const lock = file + '.lock';- mkdirSync(dirname(file), { recursive: true });- const startedAt = Date.now();- const deadline = startedAt + (io?.relayLockWaitMs ?? LOCK_WAIT_MS);- const progressEvery = io?.relayLockProgressMs ?? LOCK_PROGRESS_MS;- let nextProgress = startedAt + progressEvery;- const lease = openLease(file);- const token = randomUUID();+ const finishWait = beginCommandPhase('waiting-lock');try {- while (!(takeLease(lease, token) && takeDirectory(lock))) {- if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', `another worktree was still sending this board after ${Math.round((Date.now() - startedAt) / 1000)} s; this attempt was not recorded; run the command again`);- if (io?.err && Date.now() >= nextProgress) {- nextProgress = Date.now() + progressEvery;- io.err(`pullboard: waiting for another worktree to finish sending this board (${Math.round((Date.now() - startedAt) / 1000)} s of ${Math.round((deadline - startedAt) / 1000)} s)`);- }- await new Promise((resolve) => setTimeout(resolve, 100));- }+ const lock = file + '.lock';+ mkdirSync(dirname(file), { recursive: true });+ const startedAt = Date.now();+ const deadline = startedAt + (io?.relayLockWaitMs ?? LOCK_WAIT_MS);+ const progressEvery = io?.relayLockProgressMs ?? LOCK_PROGRESS_MS;+ let nextProgress = startedAt + progressEvery;+ const lease = openLease(file);+ const token = randomUUID();try {- // Only the current lock holder can have a live link write; remove crashed legacy bearer fragments.- for (const name of readdirSync(dirname(file))) {- if (/^relay\.json\.[0-9a-f-]+\.tmp$/u.test(name)) rmSync(join(dirname(file), name), { force: true });+ while (!(takeLease(lease, token) && takeDirectory(lock))) {+ if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', `another worktree was still sending this board after ${Math.round((Date.now() - startedAt) / 1000)} s; this attempt was not recorded; run the command again`);+ if (io?.err && Date.now() >= nextProgress) {+ nextProgress = Date.now() + progressEvery;+ io.err(`pullboard: waiting for another worktree to finish sending this board (${Math.round((Date.now() - startedAt) / 1000)} s of ${Math.round((deadline - startedAt) / 1000)} s)`);+ }+ await new Promise((resolve) => setTimeout(resolve, 100));+ }+ finishWait();+ try {+ // Only the current lock holder can have a live link write; remove crashed legacy bearer fragments.+ for (const name of readdirSync(dirname(file))) {+ if (/^relay\.json\.[0-9a-f-]+\.tmp$/u.test(name)) rmSync(join(dirname(file), name), { force: true });+ }+ return await work();+ } finally {+ // Remove only this command's own directory lock; another's is never ours to free.+ let owner = null;+ try { owner = ownerOf(lock); } catch { /* Already gone, or never named: nothing of ours to keep. */ }+ if (owner === process.pid) rmSync(lock, { recursive: true, force: true });}- return await work();} finally {- // Remove only this command's own directory lock; another's is never ours to free.- let owner = null;- try { owner = ownerOf(lock); } catch { /* Already gone, or never named: nothing of ours to keep. */ }- if (owner === process.pid) rmSync(lock, { recursive: true, force: true });+ releaseLease(lease, token);+ lease.close();}- } finally {- releaseLease(lease, token);- lease.close();- }+ } finally { finishWait(); }}diff --git a/src/relay.js b/src/relay.jsindex 7ae99f97..fb38c466 100644--- a/src/relay.js+++ b/src/relay.js@@ -1,3 +1,4 @@+import { beginCommandPhase } from './command-phase.js';import { recordedMoveRefusal } from './recorded-move.js';import {recoverySummary,@@ -339,50 +340,53 @@ async function rawRequest(state, path, { method = 'GET', body, allowMissing = fa// Validate first: a refusal below repeats this address, so it must already be credential-free.const origin = relayOrigin(state.url);const startedAt = Date.now();- let requestSignal;- for (let attempt = 0; ; attempt += 1) {- try {- requestSignal ??= signal ? AbortSignal.any([signal, AbortSignal.timeout(io?.relayTimeoutMs ?? 10000)]) : AbortSignal.timeout(io?.relayTimeoutMs ?? 10000); // budget: original per-request transport ceiling, shared by a replacement GET.- response = await fetch(origin + path, {- method, redirect: 'error', signal: requestSignal,- headers: { 'x-pullboard-engine': String(ENGINE_VERSION), ...(token ? { authorization: 'Bearer ' + token } : {}), ...(body === undefined ? {} : { 'content-type': 'application/json' }), ...(initialBaseline ? { 'if-none-match': '*' } : {}) },- ...(body === undefined ? {} : { body: JSON.stringify(body) }),- });- break;- } catch (error) {+ const finishPhase = beginCommandPhase('waiting-relay');+ try {+ let requestSignal;+ for (let attempt = 0; ; attempt += 1) {+ try {+ requestSignal ??= signal ? AbortSignal.any([signal, AbortSignal.timeout(io?.relayTimeoutMs ?? 10000)]) : AbortSignal.timeout(io?.relayTimeoutMs ?? 10000); // budget: original per-request transport ceiling, shared by a replacement GET.+ response = await fetch(origin + path, {+ method, redirect: 'error', signal: requestSignal,+ headers: { 'x-pullboard-engine': String(ENGINE_VERSION), ...(token ? { authorization: 'Bearer ' + token } : {}), ...(body === undefined ? {} : { 'content-type': 'application/json' }), ...(initialBaseline ? { 'if-none-match': '*' } : {}) },+ ...(body === undefined ? {} : { body: JSON.stringify(body) }),+ });+ break;+ } catch (error) {+ if (signal?.aborted) throw new Refused('OAUTH_CANCELLED', 'GitHub device sign-in was cancelled; no board was linked');+ // A stale cached connection reports ECONNRESET or EPIPE across supported Node runtimes. Only a read may try once more.+ if (method === 'GET' && attempt === 0 && ['ECONNRESET', 'EPIPE'].includes(transportFailure(error)) && requestSignal && !requestSignal.aborted) continue;+ const refusal = new Refused('RELAY_UNAVAILABLE', `the relay ${state.url} did not answer (${transportFailure(error)} after ${Date.now() - startedAt} ms on ${method} ${path.split('?')[0]}); read the local board offline, then retry this move when the relay is reachable`);+ refusal.transportFailure = transportFailure(error);+ throw refusal;+ }+ }+ let document;+ try { document = await response.json(); }+ catch {if (signal?.aborted) throw new Refused('OAUTH_CANCELLED', 'GitHub device sign-in was cancelled; no board was linked');- // A stale cached connection reports ECONNRESET or EPIPE across supported Node runtimes. Only a read may try once more.- if (method === 'GET' && attempt === 0 && ['ECONNRESET', 'EPIPE'].includes(transportFailure(error)) && requestSignal && !requestSignal.aborted) continue;- const refusal = new Refused('RELAY_UNAVAILABLE', `the relay ${state.url} did not answer (${transportFailure(error)} after ${Date.now() - startedAt} ms on ${method} ${path.split('?')[0]}); read the local board offline, then retry this move when the relay is reachable`);- refusal.transportFailure = transportFailure(error);- throw refusal;+ const error = new Refused('RELAY_RESPONSE', 'the relay returned an invalid response; retry with the configured relay address');+ error.httpStatus = response.status;+ throw error;}- }- let document;- try { document = await response.json(); }- catch {- if (signal?.aborted) throw new Refused('OAUTH_CANCELLED', 'GitHub device sign-in was cancelled; no board was linked');- const error = new Refused('RELAY_RESPONSE', 'the relay returned an invalid response; retry with the configured relay address');- error.httpStatus = response.status;- throw error;- }- relayNotices(document, io);- // Only off can forget a link after a supported API reports that the board is gone.- if (allowMissing && method === 'DELETE' && response.status === 404 && document.version === 1) return { alreadyDeleted: true };- if (!response.ok) {- const error = new Refused(document.error?.code || 'RELAY_UNAVAILABLE', (document.error?.code === 'AUTH_REQUIRED' ? ((state.agent || token?.startsWith('pa_')) ? 'this agent token is expired or revoked; use a new token issued by the person for this agent, then retry with PULLBOARD_RELAY_TOKEN' : 'the relay sign-in expired or was revoked; run pullboard relay on to sign in again without changing the board key') : (document.error?.message ? `the relay ${state.url}: ${document.error.message}` : null)) || 'the relay refused this send; retry this command explicitly when reachable',- document.error?.code === 'AUTH_REQUIRED' ? undefined : document.error?.next);- error.httpStatus = response.status;- error.relayVersion = document.version;- error.relayCode = document.error?.code;- error.relayMessage = document.error?.message;- if (/^[0-9a-f]{64}$/u.test(document.error?.page ?? '')) error.page = document.error.page;- const storage = relayStorage(document.error?.storage);- if (storage) error.storage = storage;- throw error;- }- if (document.version !== 1) throw new Refused('RELAY_VERSION', 'the relay API version is unsupported; upgrade Pullboard before syncing');- return document;+ relayNotices(document, io);+ // Only off can forget a link after a supported API reports that the board is gone.+ if (allowMissing && method === 'DELETE' && response.status === 404 && document.version === 1) return { alreadyDeleted: true };+ if (!response.ok) {+ const error = new Refused(document.error?.code || 'RELAY_UNAVAILABLE', (document.error?.code === 'AUTH_REQUIRED' ? ((state.agent || token?.startsWith('pa_')) ? 'this agent token is expired or revoked; use a new token issued by the person for this agent, then retry with PULLBOARD_RELAY_TOKEN' : 'the relay sign-in expired or was revoked; run pullboard relay on to sign in again without changing the board key') : (document.error?.message ? `the relay ${state.url}: ${document.error.message}` : null)) || 'the relay refused this send; retry this command explicitly when reachable',+ document.error?.code === 'AUTH_REQUIRED' ? undefined : document.error?.next);+ error.httpStatus = response.status;+ error.relayVersion = document.version;+ error.relayCode = document.error?.code;+ error.relayMessage = document.error?.message;+ if (/^[0-9a-f]{64}$/u.test(document.error?.page ?? '')) error.page = document.error.page;+ const storage = relayStorage(document.error?.storage);+ if (storage) error.storage = storage;+ throw error;+ }+ if (document.version !== 1) throw new Refused('RELAY_VERSION', 'the relay API version is unsupported; upgrade Pullboard before syncing');+ return document;+ } finally { finishPhase(); }}/** Read durable local event rows; their ids are the crash-safe outbox after the saved cursor. */diff --git a/test/relay-client-fixture.js b/test/relay-client-fixture.jsindex c4756c61..7852b8d5 100644--- a/test/relay-client-fixture.js+++ b/test/relay-client-fixture.js@@ -1,3 +1,4 @@+import { commandPhaseSnapshot } from '../src/command-phase.js';import { AGENT_SHELL_MARKERS, SSH_SHELL_MARKERS } from '../src/person.js';/** Private real CLI, auth and relay fixture; credentials never enter assertion messages [H1,H7,H18]. */import assert from 'node:assert/strict';@@ -107,7 +108,8 @@ function childResult(root, env, argv, observedSnapshotUploads, initialSnapshotUp/** Retain the exact command and work-derived bound without exposing credentials. */const reportFailure = (code, signal, detail = '') => reportFixtureChildFailure({command: process.execPath, args: argv, status: code, signal,- elapsedMs: performance.now() - startedAt, stderr, env, detail,+ elapsedMs: performance.now() - startedAt, stderr, env, detail: exceededDeadline+ ? detail + '\nstall phase capture: ' + JSON.stringify(commandPhaseSnapshot(env.PULLBOARD_COMMAND_PHASE_FILE, child.pid)) : detail,});child.once('error', error => {if (settled) return;@@ -142,7 +144,7 @@ export { childResult as relayFixtureChild };* fixture once used, a loaded machine kept the test process busy enough that browser replies queued past their* deadlines (#955, #979). A case that needs a faster poll passes pollMs by name and says why.*/-export async function relayClientFixture(t, { cliChildren, coordinatorModel, accountStorageBytes, pollMs = 200, traceCliPhases = false } = {}) {+export async function relayClientFixture(t, { cliChildren, coordinatorModel, accountStorageBytes, pollMs = 200 } = {}) {const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-relay-cleanup-')));const root = join(scratch, 'repo');const home = join(scratch, 'home');@@ -546,6 +548,7 @@ globalThis.fetch = function tracedFetch(input, init) {}/** Invoke the production CLI in this private repository. */let cliChildIndex = 0;+ let phaseSequence = 0;/** Verify and consume the next optional test-flow child descriptor. */function plannedSnapshotUploads(args) {if (!cliChildren) return undefined;@@ -563,8 +566,7 @@ globalThis.fetch = function tracedFetch(input, init) {/** Observe only this child's received work; retries consume the same finite six-unit allowance. */const observedWork = () => ({ legacyUploads: snapshotWriteCount - writesBefore,...Object.fromEntries(Object.entries(pageWork).map(([name, count]) => [name, count - pagesBefore[name]])) });- const phaseFile = (traceCliPhases || args[0] === 'export') ? join(scratch, `export-phase-${cliChildIndex}-${Date.now()}.jsonl`) : null;- const phaseEnv = phaseFile ? { ...env, PULLBOARD_STARTUP_TRACE_ENTRY: CLI, PULLBOARD_STARTUP_TRACE_FILE: phaseFile } : env;+ const phaseEnv = { ...env, PULLBOARD_COMMAND_PHASE_FILE: join(scratch, `command-phase-${++phaseSequence}.jsonl`) };const childEnv = traceSnapshotWrites ? { ...phaseEnv, PULLBOARD_TEST_RELAY_TRACE_FILE: traceFile,PULLBOARD_TEST_RELAY_TRACE_ORIGIN: origin,NODE_OPTIONS: [env.NODE_OPTIONS, '--import=' + pathToFileURL(tracePreload).href].filter(Boolean).join(' ') } : phaseEnv;@@ -586,13 +588,6 @@ globalThis.fetch = function tracedFetch(input, init) {throw new Error(`${message}; safe snapshot transport evidence: ${evidence}`, { cause: error });}throw error;- } finally {- if (phaseFile) {- try {- t.diagnostic('private CLI phase timing: ' + JSON.stringify({ child: cliChildIndex, command: args[0],- phases: existsSync(phaseFile) ? readFileSync(phaseFile, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [] }));- } catch (error) { t.diagnostic('private CLI phase timing unavailable: ' + error.code); }- }}if (traceSnapshotWrites) result.snapshotTrace = childTrace();if (cliChildren) {diff --git a/test/relay-person-requests.test.js b/test/relay-person-requests.test.jsindex 9712ef46..20e1a238 100644--- a/test/relay-person-requests.test.js+++ b/test/relay-person-requests.test.js@@ -277,7 +277,7 @@ test('real paired Chrome approval remains pending until coordinator applies andskip: !findChromeExecutable() && 'Chrome is not installed',timeout: approvalFlowTimeoutMs(APPROVAL_FLOW_CLI_CHILDREN),}, async t => {- const box = await relayClientFixture(t, { cliChildren: APPROVAL_FLOW_CLI_CHILDREN, traceCliPhases: true });+ const box = await relayClientFixture(t, { cliChildren: APPROVAL_FLOW_CLI_CHILDREN });const title = 'PERSON_REQUEST_APPROVAL_FIXTURE';const id = 'G98';const row = addPendingSpecRow(box, id);