Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: a hand-off is verified only by this brain's own autopilot

garrytan581aa2bb1de3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 334d5614a6..724b4c08f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,8 @@ The upgrade restarted autopilot only through an installed launchd or systemd ser
| Autopilot started by your own cron entry or wrapper | `restart_refused`, fix `kill <pid>` | still refuses, naming `gbrain autopilot --install` or `gbrain config set self_upgrade.autopilot_launcher external` (declare it only when the launcher starts autopilot again within 5 minutes); declared, it is handed over like the cron line |
| doctor `self_upgrade_health` | silent until an upgrade failed | runs the same pre-swap plan read-only: warns while `gbrain upgrade` would refuse, naming each process and its step (autopilot outside gbrain's service, a process from another install), and names a hand-off or a second Bun global install the upgrade replaces itself; a harness MCP server does not warn (`--stop-mcp` stops it), and a held or failed auto-upgrade is reported first |
+On a host whose autopilot runs from gbrain's cron line, the old binary still refuses with `restart_refused` on the way to this release, because it checks the restart before it installs. Run `gbrain upgrade --no-restart`, then `gbrain upgrade --restart-only`, so the new build does the hand-off.
+
**The daily stable check no longer sticks on a rolled-back release.** A release that was stable before (its `stable-v<version>` record tag exists) or that `stable.json` lists in `known_bad` is skipped and logged, so the newest release gets its soak instead of a refused promotion on every scheduled run (#8792).
**A release can be marked not promotable when it soaked clean but is known bad from outside evidence.** Dispatch `promote-stable.yml` for the current stable with `not_promotable`, `not_promotable_reason` and `not_promotable_fixed_in`. The stable record gains a `not_promotable` list, and no tag moves. The daily check never soaks or promotes a release on it, on the important path and the weekly floor alike, and a manual promotion of one is refused. Clients ignore the list, so unlike `withdraw` it moves no install to another release. `not_promotable_remove` takes a release off the list. A reason over 500 characters is refused, and so is a record larger than clients accept; nothing is pushed in either case.
diff --git a/src/commands/upgrade-restart.ts b/src/commands/upgrade-restart.ts
index a4b389fd53..9fe0c03a89 100644
--- a/src/commands/upgrade-restart.ts
+++ b/src/commands/upgrade-restart.ts
@@ -332,7 +332,7 @@ export async function runHostRestart(opts: HostRestartOptions & { dryRun: boolea
if (!opts.json) for (const r of results) console.log(` ${r.ok ? 'ok' : 'FAILED'}: ${r.detail}`);
const verified = await verifyRestart(plan, startedMs, warmBefore, {
collect: () => collectHostProcesses(sources), probeWarmSocket: () => probeCliSocket(),
- sleep: ms => new Promise(r => setTimeout(r, ms)), now: () => Date.now(),
+ sleep: ms => new Promise(r => setTimeout(r, ms)), now: () => Date.now(), autopilotLockPid: readAutopilotLockPid,
}, { timeoutMs: plan.steps.length ? Math.max(VERIFY_TIMEOUT_MS, ...plan.steps.map(step => step.kind === 'handoff' ? step.launcher.wait_ms : 0)) : 0, pollMs: VERIFY_POLL_MS });
const doc = { status: verified.ok ? 'ok' : 'unverified', ...planDoc, results, verified: { processes: verified.processes, warm_socket: verified.warm_socket, problems: verified.problems, unverified: verified.unverified } };
if (!verified.ok) {
diff --git a/src/core/upgrade-restart.ts b/src/core/upgrade-restart.ts
index aba7d1540d..61784868f5 100644
--- a/src/core/upgrade-restart.ts
+++ b/src/core/upgrade-restart.ts
@@ -432,7 +432,7 @@ export function mismatchedBuilds(plan: RestartPlan, processes: readonly HostProc
/** What still stands between the host and one build after a restart; empty when it is done. */
export function restartProblems(plan: RestartPlan, processes: readonly HostProcess[], restartStartedMs: number,
- warm: { before: WarmSocketState; after: WarmSocketState }): string[] {
+ warm: { before: WarmSocketState; after: WarmSocketState }, lockPid: number | null = null): string[] {
const problems: string[] = [];
const live = new Map(processes.map(p => [p.pid, p]));
const restarted = restartedPids(plan);
@@ -450,9 +450,15 @@ export function restartProblems(plan: RestartPlan, processes: readonly HostProce
}
for (const step of plan.steps) {
if (step.kind === 'stop' || step.kind === 'replace_copy') continue;
+ if (step.kind === 'handoff') {
+ if (!fresh.some(p => p.role === 'autopilot' && p.pid === lockPid && p.started_ms !== null && p.started_ms >= restartStartedMs)) {
+ problems.push(`autopilot did not come back after it was stopped for ${launcherName(step.launcher)} to start again (this brain's autopilot lock names ${lockPid === null ? 'no autopilot' : `pid ${lockPid}, which is not an autopilot started after the restart`}).`);
+ }
+ continue;
+ }
const role = step.kind === 'supervisor' ? 'supervisor' : step.role;
if (!fresh.some(p => p.role === role)) {
- problems.push(`${role} did not come back after ${step.kind === 'service' ? `\`${step.argv.join(' ')}\`` : step.kind === 'handoff' ? `it was stopped for ${launcherName(step.launcher)} to start again` : 'the supervisor restart'}.`);
+ problems.push(`${role} did not come back after ${step.kind === 'service' ? `\`${step.argv.join(' ')}\`` : 'the supervisor restart'}.`);
} else if (step.kind === 'supervisor' && step.workers.length && !fresh.some(p => p.role === 'worker' && live.get(p.ppid ?? -1)?.role === 'supervisor' && !restarted.has(p.ppid!))) {
problems.push(`the jobs worker did not come back under the restarted supervisor (\`${step.start.join(' ')}\`).`);
}
@@ -474,6 +480,8 @@ export interface VerifyDeps {
probeWarmSocket: () => Promise<WarmSocketState>;
sleep: (ms: number) => Promise<void>;
now: () => number;
+ /** This brain's autopilot lock pid, re-read on every poll: a hand-off is done only when an autopilot started after the restart holds it. */
+ autopilotLockPid?: () => number | null;
}
export interface VerifyResult { ok: boolean; processes: HostProcess[]; warm_socket: WarmSocketState; problems: string[]; unverified: string[] }
@@ -484,7 +492,9 @@ export async function verifyRestart(plan: RestartPlan, restartStartedMs: number,
for (;;) {
const processes = await deps.collect();
const warm = await deps.probeWarmSocket();
- const problems = restartProblems(plan, processes, restartStartedMs, { before: warmBefore, after: warm });
+ let lockPid: number | null = null;
+ try { lockPid = deps.autopilotLockPid?.() ?? null; } catch { lockPid = null; }
+ const problems = restartProblems(plan, processes, restartStartedMs, { before: warmBefore, after: warm }, lockPid);
if (!problems.length || deps.now() >= deadline) return { ok: !problems.length, processes, warm_socket: warm, problems, unverified: unverifiedBuilds(plan, processes) };
await deps.sleep(opts.pollMs);
}
diff --git a/test/upgrade-restart-external-autopilot-8776.test.ts b/test/upgrade-restart-external-autopilot-8776.test.ts
index 7590a5b6e5..a67022f723 100644
--- a/test/upgrade-restart-external-autopilot-8776.test.ts
+++ b/test/upgrade-restart-external-autopilot-8776.test.ts
@@ -76,8 +76,8 @@ describe('planRestart: an autopilot started outside gbrain\'s service', () => {
await withEnv({ GBRAIN_SELF_UPGRADE_MODE: 'auto' }, () => {
const check = checkSelfUpgradeHealth(planRestart([autopilot(30)], ctx({ expected: PENDING_BUILD, autopilotLauncher: { ...DECLARED, pid: null } })));
expect(check.status).toBe('warn');
- expect(check.fix?.ask_user).not.toBe(true);
- expect(check.fix?.argv).toEqual(['gbrain', 'upgrade', '--restart-only', '--dry-run', '--json']);
+ expect(check.fix).not.toMatchObject({ ask_user: true });
+ expect(check.fix).toMatchObject({ argv: ['gbrain', 'upgrade', '--restart-only', '--dry-run', '--json'] });
});
});
@@ -105,19 +105,48 @@ describe('runRestartSteps and verifyRestart: the hand-off', () => {
test('done once the launcher starts autopilot on the build; until then it names the launcher', async () => {
const back = autopilot(130, NEW, { started_ms: 9_000 });
- expect(restartProblems(plan, [back], 5_000, { before: { state: 'absent' }, after: { state: 'absent' } })).toEqual([]);
- expect(restartProblems(plan, [], 5_000, { before: { state: 'absent' }, after: { state: 'absent' } }))
- .toEqual(['autopilot did not come back after it was stopped for this brain\'s gbrain cron line to start again.']);
+ expect(restartProblems(plan, [back], 5_000, { before: { state: 'absent' }, after: { state: 'absent' } }, 130)).toEqual([]);
+ expect(restartProblems(plan, [], 5_000, { before: { state: 'absent' }, after: { state: 'absent' } }, null))
+ .toEqual(['autopilot did not come back after it was stopped for this brain\'s gbrain cron line to start again (this brain\'s autopilot lock names no autopilot).']);
let clock = 0;
let round = 0;
const verified = await verifyRestart(plan, 5_000, { state: 'absent' }, {
collect: async () => (round++ < 3 ? [] : [back]), probeWarmSocket: async () => ({ state: 'absent' }), sleep: async ms => { clock += ms; }, now: () => clock,
+ autopilotLockPid: () => (round > 3 ? 130 : null),
}, { timeoutMs: CRON.wait_ms, pollMs: 2_000 });
expect(verified.ok).toBe(true);
expect(clock).toBe(6_000);
});
});
+describe('verifyRestart: the hand-off is done only when this brain\'s autopilot lock names a fresh autopilot', () => {
+ const plan = planRestart([autopilot()], ctx({ autopilotLauncher: CRON }));
+ const absent = { state: 'absent' } as const;
+ const verify = (processes: HostProcess[], lockPid: () => number | null) => verifyRestart(plan, 5_000, absent,
+ { collect: async () => processes, probeWarmSocket: async () => absent, sleep: async () => {}, now: () => 0, autopilotLockPid: lockPid }, { timeoutMs: 0, pollMs: 2_000 });
+
+ test('two brains: only brain B\'s new autopilot alive after this brain\'s was handed off does not verify', async () => {
+ const brainB = autopilot(130, NEW, { started_ms: 9_000, args: ['autopilot', '--repo', '/srv/brain-b'] });
+ const result = await verify([brainB], () => 30);
+ expect(result.ok).toBe(false);
+ expect(result.problems).toEqual(['autopilot did not come back after it was stopped for this brain\'s gbrain cron line to start again (this brain\'s autopilot lock names pid 30, which is not an autopilot started after the restart).']);
+ });
+
+ test('a lock missing or unreadable at verify time reports that autopilot did not come back, naming the lock, and never verifies', async () => {
+ const fresh = autopilot(130, NEW, { started_ms: 9_000 });
+ for (const lockPid of [() => null, () => { throw new Error('EACCES'); }]) {
+ const result = await verify([fresh], lockPid);
+ expect(result.ok).toBe(false);
+ expect(result.problems).toEqual(['autopilot did not come back after it was stopped for this brain\'s gbrain cron line to start again (this brain\'s autopilot lock names no autopilot).']);
+ }
+ });
+
+ test('single brain: the launcher\'s new autopilot holding this brain\'s lock verifies', async () => {
+ const result = await verify([autopilot(130, NEW, { started_ms: 9_000 })], () => 130);
+ expect(result).toMatchObject({ ok: true, problems: [] });
+ });
+});
+
describe('planRestart: an autopilot known only from its heartbeat row', () => {
test('is not handed off, since the upgrade cannot confirm the process to stop, and names kill <pid>', () => {
const plan = planRestart([autopilot(30, OLD, { started_ms: null, args: undefined, ancestors: undefined })], ctx({ expected: PENDING_BUILD, autopilotLauncher: CRON }));