Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(recovery): draw the banner with magmux's overlay, not a pane that wedges Claude Code

Jack Rudenkoa261140ead16
diff --git a/packages/cli/src/handlers/composed-handler-recovery.test.ts b/packages/cli/src/handlers/composed-handler-recovery.test.ts
index f412275b..c25df069 100644
--- a/packages/cli/src/handlers/composed-handler-recovery.test.ts
+++ b/packages/cli/src/handlers/composed-handler-recovery.test.ts
@@ -24,20 +24,13 @@ import { probeLink } from "../providers/probe-live.js";
import type { ProviderTransport } from "../providers/transport/types.js";
import { resetRecoveryClock, setRecoveryClock } from "../recovery/clock.js";
import { closeAllEpisodes, episodeCount } from "../recovery/coordinator.js";
-import { UI_SUPPRESS_AFTER_BYE_MS, __handleCommandForTests } from "../recovery/magmux-ui.js";
-import {
- recoveryGiveUpActive,
- resetRecoveryFlagOverrides,
- resetRecoveryGiveUp,
- setRecoveryFlagOverrides,
-} from "../recovery/settings.js";
+import { resetRecoveryFlagOverrides, setRecoveryFlagOverrides } from "../recovery/settings.js";
import {
capturedRecoveryLines,
startLogCapture,
stopLogCapture,
} from "../recovery/test-helpers/capture-log.js";
import { FakeClock } from "../recovery/test-helpers/fake-clock.js";
-import { RECOVERY_PROTOCOL_VERSION } from "../recovery/types.js";
import { ComposedHandler } from "./composed-handler.js";
import { classifyConnectionError, markOwnTimeout } from "./shared/connection-error.js";
@@ -61,9 +54,6 @@ afterEach(() => {
closeAllEpisodes();
resetRecoveryClock();
resetRecoveryFlagOverrides();
- // PROCESS state, shared with every sibling test file in this Bun process. A
- // give-up left set would skip the ladder in tests that never pressed a key.
- resetRecoveryGiveUp();
});
let seq = 0;
@@ -491,69 +481,3 @@ describe("a transport's own request timeout is NOT a connection failure", () =>
}
});
});
-
-// ───────────────────────────────────────────────────────────────────────────
-// `[q] give up`
-// ───────────────────────────────────────────────────────────────────────────
-
-describe("[q] give up stops the HOLD, not only the banner", () => {
- /**
- * `bye` suppressed the pane for 60 s and gave up on the episodes alive at
- * that instant — and nothing else. The next request against the same dead
- * target opened a NEW episode, found the pane suppressed, got no pane and no
- * lease, and then held its socket for the full ~4.5-minute deadline with the
- * reason legible NOWHERE before answering 400.
- *
- * Claude Code issues concurrent requests during an outage — the main loop,
- * the title model, subagents — so a request arriving inside the suppression
- * window is the EXPECTED case. Before this feature existed those requests
- * failed in milliseconds.
- */
- test("a request arriving after the key does not hang — it fails fast, as it used to", async () => {
- // The real command, through the real handler the socket server calls.
- __handleCommandForTests({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "bye",
- reason: "user_quit",
- });
-
- const handler = new ComposedHandler(
- deadTransport(deadPort),
- "dead-model",
- "dead-model",
- 8080,
- {}
- );
- const req = new Request("http://127.0.0.1:8080/v1/messages", { method: "POST" });
- const { c, captured } = contextFor(req);
-
- const started = performance.now();
- await handler.handle(c, PAYLOAD);
- const elapsed = performance.now() - started;
-
- expect(elapsed).toBeLessThan(1_000);
- expect(captured.status).toBe(400);
- expect(captured.body?.error?.type).toBe("connection_error");
- // No episode: the hold is what the user asked to stop.
- expect(episodeCount()).toBe(0);
- expect((await capturedRecoveryLines()).some((l) => l.includes("skipped (gave-up)"))).toBe(true);
- });
-
- test("and the suppression expires with the banner's, not before or after it", async () => {
- const clock = new FakeClock(0);
- setRecoveryClock(clock);
- __handleCommandForTests({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "bye",
- reason: "user_quit",
- });
- expect(recoveryGiveUpActive()).toBe(true);
-
- // One number, read by the hold and by the surface. A minute later the
- // ladder is available again — the key is a pause, not an off switch.
- await clock.advance(UI_SUPPRESS_AFTER_BYE_MS - 1);
- expect(recoveryGiveUpActive()).toBe(true);
- await clock.advance(2);
- expect(recoveryGiveUpActive()).toBe(false);
- });
-});
diff --git a/packages/cli/src/handlers/composed-handler.ts b/packages/cli/src/handlers/composed-handler.ts
index c846570d..a6a7fd3f 100644
--- a/packages/cli/src/handlers/composed-handler.ts
+++ b/packages/cli/src/handlers/composed-handler.ts
@@ -52,7 +52,7 @@ import {
noteTargetReachable,
uiLeaseValid,
} from "../recovery/coordinator.js";
-import { recoveryGiveUpActive, resolveRecoveryEnabled } from "../recovery/settings.js";
+import { resolveRecoveryEnabled } from "../recovery/settings.js";
import {
type OpenAIImageBlock,
type VisionProxyAuthHeaders,
@@ -677,12 +677,6 @@ export class ComposedHandler implements ModelHandler {
// The CI / scripted-`-p` master switch. Off restores today's behaviour
// everywhere, byte for byte.
if (!resolveRecoveryEnabled()) return "recovery-disabled";
- // The user pressed `[q] give up`. It has to be read HERE and not only in
- // the UI manager: suppressing the banner alone left the NEXT request —
- // and during an outage Claude Code always has one — holding its socket for
- // the full deadline with the reason legible nowhere. The key says stop, so
- // the hold stops too, for the same window the surface does.
- if (recoveryGiveUpActive()) return "gave-up";
// Attempt 1 already spent the budget. A guard, not a hope.
if (recoveryClock().now() + MIN_ATTEMPT_SLOT_MS > deadlineAt) return "no-budget";
return null;
diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts
index d702af4b..603a588e 100644
--- a/packages/cli/src/index.ts
+++ b/packages/cli/src/index.ts
@@ -52,12 +52,7 @@ function classifyStartupKind(): string {
"quota",
"usage",
]);
- if (
- (first && management.has(first)) ||
- argv.includes("--mcp") ||
- first === "serve" ||
- first === "recovery-pane"
- ) {
+ if ((first && management.has(first)) || argv.includes("--mcp") || first === "serve") {
return "other";
}
return "run";
@@ -65,12 +60,7 @@ function classifyStartupKind(): string {
process.on("exit", () => {
const argv = process.argv.slice(2);
const first = argv.find((a) => !a.startsWith("-"));
- // `recovery-pane` joins the long-running exclusions for the same reason
- // `serve` and `--mcp` are there: it lives for as long as the outage does, so
- // an at-exit total is its lifetime rather than anyone's startup, and it would
- // pollute the metrics with a number that measures a network failure.
- const longRunningServer =
- argv.includes("--mcp") || first === "serve" || first === "recovery-pane";
+ const longRunningServer = argv.includes("--mcp") || first === "serve";
if (longRunningServer) return;
finalizeStartupTrace(classifyStartupKind(), { quiet: true });
});
@@ -328,11 +318,6 @@ const isStatsCommand = firstPositional === "stats";
const isConfigCommand = firstPositional === "config";
// Serve subcommand: claudish serve --port <n> --models <path> (Claude Desktop redirect gateway)
const isServeCommand = firstPositional === "serve";
-// Recovery pane: claudish recovery-pane --socket <path>. Launched BY claudish
-// into a magmux pane, never by a user — but it must be routed here all the
-// same, because an unrouted subcommand does not error: it falls through to the
-// default path and becomes a catalog search for the literal string.
-const isRecoveryPaneCommand = firstPositional === "recovery-pane";
// Providers subcommand: claudish providers --json (credential presence, no key material)
const isProvidersCommand = firstPositional === "providers";
// Keychain subcommand: claudish keychain status|list|import|set|rm|enable|disable
@@ -388,17 +373,6 @@ if (isMcpMode) {
process.exit(1);
})
);
-} else if (isRecoveryPaneCommand) {
- // The network-recovery banner. Its own process, its own PTY, its own stdin —
- // which is the entire reason the recovery UI is a pane: keys typed into it
- // reach THIS process rather than the Claude Code session beside it.
- const paneArgIndex = args.indexOf("recovery-pane");
- import("./recovery/pane-app.js").then((m) =>
- m.recoveryPaneCommand(args.slice(paneArgIndex + 1)).catch((e) => {
- console.error(`[claudish recovery-pane] ${e instanceof Error ? e.message : String(e)}`);
- process.exit(1);
- })
- );
} else if (isBehaviorCommand) {
// Layer 4 introspection: claudish behavior rules|corpus. Read-only unless
// --write is passed to `corpus`.
diff --git a/packages/cli/src/launcher/magmux-wrapper.test.ts b/packages/cli/src/launcher/magmux-wrapper.test.ts
index 65527496..ac81858e 100644
--- a/packages/cli/src/launcher/magmux-wrapper.test.ts
+++ b/packages/cli/src/launcher/magmux-wrapper.test.ts
@@ -257,9 +257,6 @@ describe("the script's directory is unguessable and cannot be pre-created", () =
* another uid could pre-create (or symlink) that path and have it adopted
* without a word, then read the session's secrets or swap the script between
* the write and the `exec`. CWE-377 / CWE-59.
- *
- * The rest of this change already knew the rule: `socket-server.ts` uses
- * `randomBytes(12)` for its directory and `O_EXCL` for the pane lock.
*/
test("the path does not contain the pid, and two plans never collide", () => {
const root = scratch();
diff --git a/packages/cli/src/launcher/magmux-wrapper.ts b/packages/cli/src/launcher/magmux-wrapper.ts
index 0c4b5a5b..fcd140d1 100644
--- a/packages/cli/src/launcher/magmux-wrapper.ts
+++ b/packages/cli/src/launcher/magmux-wrapper.ts
@@ -237,9 +237,7 @@ export function planMagmuxWrap(input: MagmuxWrapInput): MagmuxWrapPlan | null {
//
// `mkdtempSync` gives an unguessable name AND fails rather than adopting, and
// `wx` is `O_CREAT | O_EXCL`, so the file cannot pre-exist as a symlink to
- // somewhere else. The same rule the rest of this change already follows:
- // `socket-server.ts` uses `randomBytes(12)` for its directory and `O_EXCL`
- // for the pane lock.
+ // somewhere else.
const tmpRoot = input.tmpRoot ?? "/tmp";
mkdirSync(tmpRoot, { recursive: true });
const scriptDir = mkdtempSync(join(tmpRoot, "claudish-launch-"));
@@ -312,8 +310,8 @@ export function planMagmuxWrap(input: MagmuxWrapInput): MagmuxWrapPlan | null {
// A file holding the session's `ANTHROPIC_API_KEY` must not outlive the
// process on a SIGTERM or a crash-out — `watch()`'s `proc.exit` hook only
// covers the ordinary path, and claudish is not always the one that dies
- // last. Same discipline as `socket-server.ts`'s socket unlink; the listener
- // removes itself in `cleanup` so a long-lived host cannot accumulate them.
+ // last. The listener removes itself in `cleanup` so a long-lived host
+ // cannot accumulate them.
process.on("exit", cleanup);
return {
diff --git a/packages/cli/src/recovery/coordinator.test.ts b/packages/cli/src/recovery/coordinator.test.ts
index 48b5228d..1c46239c 100644
--- a/packages/cli/src/recovery/coordinator.test.ts
+++ b/packages/cli/src/recovery/coordinator.test.ts
@@ -33,7 +33,6 @@ import {
giveUp,
joinEpisode,
noteTargetReachable,
- tryNow,
uiLeaseValid,
} from "./coordinator.js";
import type { WaitOutcome } from "./coordinator.js";
@@ -576,22 +575,7 @@ describe("a successful attempt", () => {
});
});
-describe("the wake paths that later phases drive", () => {
- test("tryNow collapses the wait and advances the round", async () => {
- useFakeClock();
- const h = joinEpisode(seed());
- const parked = h.waitForNextAttempt(never(), 60_000);
- await drain();
- expect(clock.pending()).toBe(1);
-
- tryNow(h.episodeId);
- expect((await parked).kind).toBe("attempt");
- // Woken at t=0, not at t=60 000.
- expect(clock.now()).toBe(0);
- expect(clock.pending()).toBe(0);
- h.leave();
- });
-
+describe("ending an episode from outside the ladder", () => {
test("giveUp wakes every waiter with gave_up and latches", async () => {
useFakeClock();
const s = seed();
diff --git a/packages/cli/src/recovery/coordinator.ts b/packages/cli/src/recovery/coordinator.ts
index 5d7b3d70..77fddf22 100644
--- a/packages/cli/src/recovery/coordinator.ts
+++ b/packages/cli/src/recovery/coordinator.ts
@@ -31,11 +31,7 @@ import { randomUUID } from "node:crypto";
import { type ConnectionErrorKind, isLoopback } from "../handlers/shared/connection-error.js";
import { log, logRecovery } from "../logger.js";
import { recoveryClock } from "./clock.js";
-import {
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryEpisodeFrame,
- type RecoveryFrameState,
-} from "./types.js";
+import type { RecoveryEpisodeFrame, RecoveryFrameState } from "./types.js";
export type RecoveryState = "attempting" | "waiting" | "recovered" | "handoff" | "abandoned";
@@ -164,16 +160,16 @@ export function describeEpisode(episodeId: string): Readonly<Record<string, unkn
* What the UI manager (`magmux-ui.ts`) plugs into the coordinator.
*
* A REGISTRATION SEAM RATHER THAN AN IMPORT, and not for testability. The UI
- * manager reads episode state (to build a frame) and drives the ladder (`[r]`,
- * `[q]`), so it must import this module; a direct import back the other way is
- * a cycle. More importantly it keeps the dependency HONEST in the other
- * direction: nothing is installed in `-p`, in `serve`, in the MCP server or in
- * the test suite, so those paths cannot open a pane, cannot hold a lease, and
- * behave exactly as they did before this phase — by construction rather than by
- * a flag someone has to remember to check.
+ * manager reads episode state to build the banner, so it must import this
+ * module; a direct import back the other way is a cycle. More importantly it
+ * keeps the dependency HONEST in the other direction: nothing is installed in
+ * `-p`, in `serve`, in the MCP server or in the test suite, so those paths
+ * cannot draw a banner, cannot hold a lease, and behave exactly as they did
+ * before this feature — by construction rather than by a flag someone has to
+ * remember to check.
*/
export interface RecoveryUiHooks {
- /** A new episode exists. The manager decides whether to ask for a pane. */
+ /** A new episode exists. The manager starts drawing the banner. */
onEpisodeOpened(episodeId: string): void;
/** An episode reached a terminal state. */
onEpisodeClosed(episodeId: string, outcome: RecoveryOutcome): void;
@@ -210,7 +206,7 @@ export function uiLeaseValid(episodeId: string): boolean {
}
}
-/** Record that a pane was ASKED for. Asking and being told no is still asking. */
+/** Record that a banner was ASKED for. Asking and being told no is still asking. */
export function markPaneRequested(episodeId: string): void {
const ep = byId.get(episodeId);
if (!ep || ep.paneRequestedAtPerf !== null) return;
@@ -227,7 +223,7 @@ function liveEpisodes(): Episode[] {
}
/**
- * The episode a renderer should paint, as a wire frame — or null.
+ * The episode the banner should draw, as a frame — or null.
*
* Concurrent episodes are ORDINARY: the main loop's provider and the small
* title model's provider can be unreachable at the same moment. Which one wins
@@ -237,9 +233,9 @@ function liveEpisodes(): Episode[] {
* reason is legible" demands.
*
* THE UNIT CONVERSION HAPPENS HERE AND NOWHERE ELSE. Intervals inside this
- * module are process ms; everything on the wire is epoch ms, because the pane
- * is a different process with a different `performance.now()` origin and would
- * render a process-ms instant as an arbitrary number with no error anywhere.
+ * module are process ms; every instant in a frame is epoch ms, so the banner's
+ * countdown is computed against `Date.now()` in one unit, with no second
+ * conversion site to drift.
*/
export function renderableEpisodeFrame(): RecoveryEpisodeFrame | null {
const live = liveEpisodes();
@@ -251,7 +247,6 @@ export function renderableEpisodeFrame(): RecoveryEpisodeFrame | null {
const nextAttemptAtMs =
ep.nextAttemptAtPerf === null ? null : epochNow + (ep.nextAttemptAtPerf - perfNow);
return {
- v: RECOVERY_PROTOCOL_VERSION,
type: "episode",
episodeId: ep.episodeId,
state: ep.state as RecoveryFrameState,
@@ -273,23 +268,6 @@ export function renderableEpisodeFrame(): RecoveryEpisodeFrame | null {
};
}
-/** How many attempts this episode has made. For the manual-retry log line. */
-export function attemptsSoFar(episodeId: string): number {
- return byId.get(episodeId)?.attempts ?? 0;
-}
-
-/** Is this episode still live? Used by the UI manager's lease check. */
-export function episodeIsLive(episodeId: string): boolean {
- const ep = byId.get(episodeId);
- if (!ep) return false;
- return ep.state === "attempting" || ep.state === "waiting" || ep.state === "handoff";
-}
-
-/** `[q] give up` — the user said stop, so every live episode stops. */
-export function giveUpAll(): void {
- for (const ep of liveEpisodes()) giveUp(ep.episodeId);
-}
-
export interface EpisodeHandle {
readonly episodeId: string;
readonly key: string;
@@ -687,28 +665,15 @@ export function noteTargetReachable(providerName: string, endpoint: string): voi
}
/**
- * Collapse the wait for an episode and attempt now. The pane's `[r] try now`
- * key routes here in a later phase; nothing calls it in this one, and it is
- * exported now so the ladder never grows a second wake path later.
- */
-export function tryNow(episodeId: string): void {
- const ep = byId.get(episodeId);
- if (!ep || ep.attemptTimer === null) return;
- clearAttemptTimer(ep);
- ep.state = "attempting";
- ep.round++;
- for (const w of [...ep.waiters]) w.wake?.({ kind: "attempt" });
-}
-
-/**
- * `[q] give up` for ONE episode: every waiter answers today's inline error and
- * the episode is over.
+ * End ONE episode now: every waiter answers today's inline error.
*
* `gaveUp` latches before the close so a waiter that has not parked yet — one
- * between attempts at the instant the key was pressed — still reads the answer
- * rather than parking on a timer that no longer exists. The user said stop, so
- * stop, and say why inline: this is the missing half of an affordance the
- * superseded design labelled "give up" and gave no effect at all.
+ * between attempts at that instant — still reads the answer rather than
+ * parking on a timer that no longer exists.
+ *
+ * No key reaches this since the banner became magmux's overlay, which cannot
+ * read keystrokes. It stays because it is the one clean way to end an episode
+ * with waiters on it, and the tests end episodes through it.
*/
export function giveUp(episodeId: string): void {
const ep = byId.get(episodeId);
diff --git a/packages/cli/src/recovery/lease.test.ts b/packages/cli/src/recovery/lease.test.ts
deleted file mode 100644
index 59b203b0..00000000
--- a/packages/cli/src/recovery/lease.test.ts
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * The UI lease — the round-2 CRITICAL, and the reason this file exists at all.
- *
- * The lease is what will decide, one phase from now, whether an exhausted
- * episode may answer a RETRYABLE status instead of an inline error. The rule it
- * encodes is: *a retryable status is permissible exactly while claudish still
- * has a surface on which the reason is legible.*
- *
- * The superseded design renewed that lease from FRAME RECEIPTS, and frames only
- * ticked while the ladder was `waiting`. A connect against an unreachable host
- * takes 20–75 s (measured on this machine: `192.0.2.1` = 75 005 ms), during
- * which the proxy has nothing new to say — so the lease died ten seconds into
- * every slow attempt, and the exhaustion arm, which reads it immediately after
- * an attempt returns, saw it false for the ENTIRE failure class the feature was
- * built for, with a live painted banner on screen.
- *
- * And every test passed, because every fault in the suite was a ~1 ms loopback
- * refusal. These are the tests that can fail on it.
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { EventEmitter } from "node:events";
-import { resetRecoveryClock, setRecoveryClock } from "./clock.js";
-import {
- __handleCommandForTests,
- __resetUiStateForTests,
- __setPaneOpenForTests,
- describeLease,
- uiLeaseValid,
-} from "./magmux-ui.js";
-import { runRecoveryPane } from "./pane-app.js";
-import {
- type RecoverySocketServer,
- createRecoverySocketDir,
- recoverySocketPathIn,
- startRecoverySocketServer,
-} from "./socket-server.js";
-import { FakeClock } from "./test-helpers/fake-clock.js";
-import {
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryEpisodeFrame,
- UI_HEARTBEAT_MS,
- UI_LEASE_MS,
-} from "./types.js";
-
-const EPISODE: RecoveryEpisodeFrame = {
- v: RECOVERY_PROTOCOL_VERSION,
- type: "episode",
- episodeId: "ep-slow",
- // `attempting`, not `waiting`: this is the state the proxy sits in for the
- // whole of a 45-second connect, and the state in which the old design went
- // silent.
- state: "attempting",
- tier: 1,
- providerDisplayName: "Openai-codex",
- host: "chatgpt.com",
- endpoint: "https://chatgpt.com/backend-api/codex/responses",
- kind: "unreachable",
- code: "ConnectionClosed",
- loopback: false,
- reason: "Cannot reach Openai-codex at https://chatgpt.com/backend-api/codex/responses.",
- attempts: 4,
- clientRetries: 0,
- startedAtMs: 1_700_000_000_000,
- nextAttemptAtMs: null,
- lastOutcome: "ConnectionClosed after 45001ms",
- waiters: 1,
- otherEpisodes: 0,
-};
-
-let clock: FakeClock;
-
-beforeEach(() => {
- clock = new FakeClock(0);
- setRecoveryClock(clock);
- __resetUiStateForTests();
-});
-
-afterEach(() => {
- __resetUiStateForTests();
- resetRecoveryClock();
-});
-
-const ack = (episodeId: string) =>
- __handleCommandForTests({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "ack",
- episodeId,
- paintedAt: Date.now(),
- });
-
-describe("the UI lease (unit, fake clock)", () => {
- test("SURVIVES an attempt far longer than UI_LEASE_MS while the renderer heartbeats", async () => {
- // THE round-2 CRITICAL, expressed as an assertion. 45 seconds of attempt —
- // 4.5 leases — with NOT ONE FRAME emitted, because the proxy has nothing to
- // say while a connect hangs. The renderer's own heartbeat is the only thing
- // renewing anything.
- __setPaneOpenForTests(true);
- ack("ep-slow");
-
- for (let elapsed = 0; elapsed < 45_000; elapsed += UI_HEARTBEAT_MS) {
- await clock.advance(UI_HEARTBEAT_MS);
- ack("ep-slow"); // the renderer, on ITS timer, not on a frame
- expect(uiLeaseValid("ep-slow")).toBe(true);
- }
-
- expect(clock.now()).toBeGreaterThanOrEqual(45_000);
- expect(uiLeaseValid("ep-slow")).toBe(true);
- });
-
- test("expires UI_LEASE_MS after the last heartbeat — a frozen renderer holds nothing", async () => {
- __setPaneOpenForTests(true);
- ack("ep-slow");
- await clock.advance(UI_LEASE_MS);
- expect(uiLeaseValid("ep-slow")).toBe(true); // exactly at the boundary
- await clock.advance(1);
- expect(uiLeaseValid("ep-slow")).toBe(false);
- });
-
- test("a forged client cannot grant itself a lease: paneOpen is required", () => {
- // The socket has no same-uid boundary and cannot have one. What is defended
- // is the CONSEQUENCE: the one forbidden state (a retryable status with no
- // banner) additionally requires that WE opened a pane and hold the reply,
- // which is a fact no client can cause.
- __setPaneOpenForTests(false);
- ack("ep-slow");
- expect(uiLeaseValid("ep-slow")).toBe(false);
- expect(describeLease("ep-slow").lastAckAgoMs).toBe(0); // it DID heartbeat…
- expect(describeLease("ep-slow").paneOpen).toBe(false); // …and it still gets nothing
- });
-
- test("the lease is per-EPISODE: heartbeating one does not lease another", () => {
- __setPaneOpenForTests(true);
- ack("ep-slow");
- expect(uiLeaseValid("ep-slow")).toBe(true);
- expect(uiLeaseValid("ep-other")).toBe(false);
- });
-
- test("switching the painted episode lets the old lease lapse with no extra protocol", async () => {
- __setPaneOpenForTests(true);
- ack("ep-slow");
- for (let i = 0; i < 12; i++) {
- await clock.advance(UI_HEARTBEAT_MS);
- ack("ep-other"); // the renderer moved on
- }
- expect(uiLeaseValid("ep-other")).toBe(true);
- expect(uiLeaseValid("ep-slow")).toBe(false);
- });
-
- test("`bye` revokes every lease, and no later heartbeat can resurrect one", () => {
- // `[q] give up` is the affordance the superseded design labelled
- // "give up" and gave no server-side effect at all — so a user could close
- // the banner while the machine kept retrying and eventually answered a
- // retryable status with nothing on screen. That is the design's OWN stated
- // worst case, reached through a key that says "give up".
- __setPaneOpenForTests(true);
- ack("ep-slow");
- ack("ep-other");
- expect(uiLeaseValid("ep-slow")).toBe(true);
-
- __handleCommandForTests({ v: RECOVERY_PROTOCOL_VERSION, type: "bye", reason: "user_quit" });
-
- expect(uiLeaseValid("ep-slow")).toBe(false);
- expect(uiLeaseValid("ep-other")).toBe(false);
- // The pane is genuinely gone, not merely un-leased…
- expect(describeLease("ep-slow").paneOpen).toBe(false);
- // …so a client that keeps heartbeating into the closed socket gets nothing.
- // Without this, `bye` would be undone by the very next tick of a renderer
- // that had not noticed it was closed.
- ack("ep-slow");
- expect(uiLeaseValid("ep-slow")).toBe(false);
- });
-});
-
-// ─── The same property, end to end, over a real socket ───────────────────────
-
-class FakeStdin extends EventEmitter {
- rawMode = false;
- setRawMode(on: boolean) {
- this.rawMode = on;
- return this;
- }
- resume() {
- return this;
- }
- pause() {
- return this;
- }
-}
-
-class FakeStdout extends EventEmitter {
- columns = 100;
- chunks: string[] = [];
- write(s: string) {
- this.chunks.push(s);
- return true;
- }
-}
-
-describe("the UI lease (end to end, real socket, real renderer)", () => {
- test("the REAL pane heartbeats through total frame silence, and the lease holds", async () => {
- // One frame in the whole run — the replay — and then nothing, which is what
- // a hung connect looks like on the wire. If the renderer ever goes back to
- // acking per received frame, the ack count collapses to 1 and the lease is
- // dead by fake-t=10s. That is the defect this test exists to fail on.
- const path = recoverySocketPathIn(createRecoverySocketDir());
- let framesSent = 0;
- let acks = 0;
-
- let server: RecoverySocketServer | null = null;
- const stdin = new FakeStdin();
- const stdout = new FakeStdout();
-
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => {
- framesSent++;
- return EPISODE;
- },
- onCommand: (cmd) => {
- if (cmd.type === "ack") acks++;
- __handleCommandForTests(cmd);
- },
- tickMs: 0, // the proxy says NOTHING after the replay
- });
-
- __setPaneOpenForTests(true);
- const paneDone = runRecoveryPane({
- socketPath: path,
- stdin: stdin as unknown as NodeJS.ReadStream,
- stdout: stdout as unknown as NodeJS.WriteStream,
- });
-
- // ~4 real heartbeat intervals, with the LEASE clock advancing 5 s each time
- // — so the episode is 20 s old in lease terms while the socket has carried
- // exactly one frame.
- for (let i = 0; i < 4; i++) {
- await new Promise((r) => setTimeout(r, UI_HEARTBEAT_MS + 120));
- await clock.advance(5_000);
- expect(uiLeaseValid("ep-slow")).toBe(true);
- }
-
- expect(framesSent).toBe(1);
- expect(acks).toBeGreaterThanOrEqual(3);
- expect(clock.now()).toBeGreaterThan(UI_LEASE_MS);
- expect(uiLeaseValid("ep-slow")).toBe(true);
-
- stdin.emit("data", Buffer.from("q"));
- await paneDone;
- await server.close();
- }, 15_000);
-});
diff --git a/packages/cli/src/recovery/magmux-ui.ts b/packages/cli/src/recovery/magmux-ui.ts
index f78c8ad3..ea9bb542 100644
--- a/packages/cli/src/recovery/magmux-ui.ts
+++ b/packages/cli/src/recovery/magmux-ui.ts
@@ -1,167 +1,162 @@
/**
- * The recovery UI manager — the single owner of the socket, the pane, the
- * `paneOpen` fact, the cross-process lock and the lease.
+ * The recovery banner — drawn by magmux itself, over the pane claudish's Claude
+ * Code already occupies.
*
- * THE POST-`bye` SUPPRESSION IS NOT OWNED HERE, and that is a correction. It
- * used to be `state.suppressUntilPerf`, read only by `ensureRecoveryUi`, so
- * `[q] give up` stopped the BANNER while the next request against the same
- * dead target still held its socket for the full derived deadline with the
- * reason legible nowhere. The fact now lives in `recovery/settings.ts` where
- * `shouldSkipTier1` can read it too: one number, suppressing the hold and the
- * surface together.
+ * WHY magmux's `overlay`, AND NOT A PANE OF OURS. The first version opened a
+ * pane with `open_pane` and closed it with `close_pane`. Opening a stacked pane
+ * shrinks Claude Code's pane; closing it grows the pane back, and Claude Code
+ * 2.1.272 does not survive the grow: six rows of its bottom chrome collapse
+ * onto one line and the input box is destroyed, and nothing done from outside —
+ * Escape, a keystroke, Ctrl-L, SIGWINCH, a shrink-then-grow cycle — repairs it
+ * (`ai-docs/reports/network-recovery-pane-close-wedges-host-tui-20260915.md`).
+ * magmux resizes correctly; the reflow itself is the trigger. `overlay` draws a
+ * styled box OVER a pane and changes no layout, so there is no reflow to
+ * survive. It also retired the pane process, the NDJSON socket we served it
+ * from, that socket's wire protocol and the cross-process lock that let only
+ * one claudish in a grid own the single pane: magmux is the renderer now, and
+ * every claudish draws on its own pane.
*
- * OWNERSHIP IS PROCESS-SCOPED; ONLY EPISODES ARE EPISODE-SCOPED. There is ONE
- * pane and it multiplexes every episode, so "a pane exists" is a property of
- * this module and never of an episode. An earlier design hung `paneOpened` on
- * each episode, which meant the second concurrent episode could never be leased
- * — only the first one ever received an `open_pane` reply — and that closing
- * one episode tore down a socket a different episode's renderer was reading.
- * `releaseEpisodeUi` therefore emits one episode's `closed` frame and NOTHING
- * else; the socket and the pane come down only when no episode is live at all.
+ * THE LEASE, RESTATED FOR A RENDERER WE DO NOT OWN. A retryable 503 is allowed
+ * only while the reason is legible on screen, so:
*
- * THE LEASE IS THE WHOLE POINT OF THIS FILE. It is not a latch, not a config
- * read and not "a client connected":
+ * uiLeaseValid(id) ⇔ we hold a magmux control connection and know our pane
+ * ∧ magmux ACKNOWLEDGED an overlay write painting THIS
+ * episode within UI_LEASE_MS
*
- * uiLeaseValid(id) ⇔ paneOpen (WE opened it, and hold the reply)
- * ∧ some client heartbeated `ack` naming THIS episode
- * within UI_LEASE_MS
+ * magmux replies only to a message that carries an `id`, and a reply means it
+ * accepted the text for a pane it is drawing. If magmux dies, the writes stop
+ * being acknowledged, the lease lapses on its own, and the proxy answers the
+ * inline 400 at exhaustion. No path has to remember to revoke anything.
*
- * Both clauses are load-bearing. `paneOpen` is what a forged same-uid client
- * cannot manufacture, so it cannot talk the proxy into the one forbidden state
- * (a retryable status with no banner on screen). The heartbeat window is what
- * makes the lease self-clearing on EOF, on a killed pane, on a frozen renderer
- * and on `[q]`, with no cleanup path anyone has to remember.
+ * THE BANNER IS RE-ASSERTED EVERY TICK, NEVER WRITTEN ONCE. magmux writes the
+ * same overlay from its own Claude Code state tracker — `CtrlError` paints
+ * "✗ …" over whatever is there (`magmux/mux/mux.go`) — so a banner written once
+ * can be replaced mid-outage. The countdown needs one write per second anyway,
+ * and the same write restores ours within a second of being overwritten.
*
- * AND THE HEARTBEAT IS THE RENDERER'S, NOT OURS. The pane sends `ack` on its
- * own 1 Hz timer for as long as it is painting an episode, whether or not a
- * frame arrived. Renewing from frame receipts would tie the renderer's apparent
- * liveness to OUR emission cadence — and we have nothing to say for the 20–75 s
- * an unreachable connect takes (measured: `192.0.2.1` = 75 005 ms). The lease
- * would then read false at exhaustion for precisely the failure class this
- * feature exists for, with a live painted banner on screen, and every
- * loopback-only test would still pass.
- *
- * NOTHING HERE CHANGES A STATUS IN THIS PHASE. The lease is computed and
- * logged; the 503-vs-400 flip is the next one. Shipping the flip before the
- * banner exists recreates the buried-reason bug the feature is meant to kill.
+ * NO KEYS. An overlay is drawn by magmux; it is not a process and cannot read
+ * `[r] try now` or `[q] give up`. Esc in Claude Code still aborts the held
+ * request — a client disconnect ends its waiter — and the ladder retries on its
+ * own. Driving the status line and forwarding keys are requested of magmux in
+ * its `ai-docs/feature-request-status-line.md`.
*/
-import { closeSync, existsSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
+import { execFile } from "node:child_process";
+import { existsSync } from "node:fs";
import { type Socket, connect as netConnect } from "node:net";
-import { basename, join } from "node:path";
+import { promisify } from "node:util";
import { log } from "../logger.js";
import { recoveryClock } from "./clock.js";
import {
type RecoveryOutcome,
type RecoveryUiHooks,
- attemptsSoFar,
- episodeIsLive,
- giveUpAll,
markPaneRequested,
registerRecoveryUi,
renderableEpisodeFrame,
- tryNow,
} from "./coordinator.js";
-import { noteRecoveryGaveUp, recoveryGiveUpActive, resetRecoveryGiveUp } from "./settings.js";
-import {
- type RecoverySocketServer,
- cleanupRecoverySocket,
- createRecoverySocketDir,
- recoverySocketPathIn,
- startRecoverySocketServer,
-} from "./socket-server.js";
-import {
- PANE_LINGER_MS,
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryCommand,
- UI_LEASE_MS,
-} from "./types.js";
-
-/** After `[q]`, do not re-open a pane for this long. */
-export const UI_SUPPRESS_AFTER_BYE_MS = 60_000;
+import { FRAME_TICK_MS, type RecoveryEpisodeFrame, UI_LEASE_MS } from "./types.js";
-// ─── The control endpoint ────────────────────────────────────────────────────
+const execFileAsync = promisify(execFile);
-let explicitControlSock: string | null = null;
+/** How long the "recovered" banner stays up before the overlay is cleared. */
+export const OUTCOME_LINGER_MS = 4_000;
/**
- * The magmux control socket, handed in by the launch wrapper.
- *
- * AN EXPLICIT INPUT, NOT AN ENV LOOKUP, and that is the only thing that makes
- * this work at all: claudish is magmux's PARENT, and `MAGMUX_SOCK` is exported
- * DOWNWARD, so the parent can never learn a pid-derived path. The wrapper
- * generates `--id claudish-<pid>`, which fixes the path before magmux starts,
- * and hands it here.
+ * Wrap width for the reason sentence. magmux clamps the box to the pane and
+ * CLIPS a line that is too wide rather than wrapping it, so the text has to
+ * arrive already wrapped.
*/
-export function setMagmuxControlSocket(sock: string | null): void {
- explicitControlSock = sock;
-}
+const OVERLAY_WRAP_COLS = 60;
+
+/** A control request that gets no reply in this long is treated as refused. */
+const CONTROL_TIMEOUT_MS = 3_000;
+
+// ─── The control endpoint ────────────────────────────────────────────────────
+
+let explicitControlSock: string | null = null;
/**
- * Where can this process send `open_pane` right now?
+ * Where to send `overlay`, and whether claudish launched this magmux itself.
*
- * Order: the path the wrapper handed us, then an ambient `MAGMUX_SOCK` (the
- * already-inside-a-pane case, e.g. `team --grid --mode interactive`), then null.
+ * The launch wrapper hands its socket in EXPLICITLY: claudish is magmux's
+ * parent and `MAGMUX_SOCK` is exported downward only, so the parent can never
+ * discover a pid-derived path. The wrapper fixes the path with `--id` before
+ * magmux starts. Failing that, an ambient `MAGMUX_SOCK` means this claudish is
+ * running INSIDE someone else's magmux — `team --grid`, or a user's own pane.
*
- * IT ANSWERS A LAUNCH-ORDER QUESTION, NOT A USER PREFERENCE, and only this
- * module may read it. Null means "this process cannot open a pane" — true in
- * `-p`, in `serve`, in the MCP server, in every unwrapped launch and in the
- * whole test suite. It must never gate whether a RETRY happens: keying the
- * retry ladder on it made recovery skip universally in every phase before the
- * wrapper existed, twice, and both times the same rule decided the tests.
+ * It answers a launch-order question and gates only the banner. It must never
+ * gate whether a retry happens.
*/
-export function resolveMagmuxControl(): { sock: string } | null {
- if (explicitControlSock && existsSync(explicitControlSock)) return { sock: explicitControlSock };
+function resolveControlEndpoint(): { sock: string; wrapped: boolean } | null {
+ if (explicitControlSock && existsSync(explicitControlSock)) {
+ return { sock: explicitControlSock, wrapped: true };
+ }
const ambient = process.env.MAGMUX_SOCK;
- if (ambient && existsSync(ambient)) return { sock: ambient };
+ if (ambient && existsSync(ambient)) return { sock: ambient, wrapped: false };
return null;
}
-// ─── Manager state ───────────────────────────────────────────────────────────
+// ─── State ───────────────────────────────────────────────────────────────────
-interface ManagerState {
- server: RecoverySocketServer | null;
- socketPath: string | null;
+interface UiState {
control: Socket | null;
- /** True only once magmux has REPLIED to our `open_pane`. */
- paneOpen: boolean;
- /** magmux's own index for our pane. Read from the reply — never assumed. */
- paneIndex: number | null;
- /** episodeId → PROCESS ms of the last heartbeat naming it. */
+ connecting: Promise<boolean> | null;
+ /** The magmux pane Claude Code runs in. Null until found. */
+ targetPane: number | null;
+ /** episodeId → PROCESS ms of the last ACKNOWLEDGED overlay write painting it. */
leases: Map<string, number>;
- /** Guards the async open against re-entry. */
- opening: boolean;
- lockPath: string | null;
+ /** The last frame magmux acknowledged, kept to draw the outcome after close. */
+ lastFrame: RecoveryEpisodeFrame | null;
+ tickTimer: unknown | null;
lingerTimer: unknown | null;
+ /** Guards against a slow magmux stacking one paint on another. */
+ painting: boolean;
+ /** Whether the last paint was acknowledged, so a failure logs once, not per tick. */
+ lastPaintOk: boolean | null;
+ /** An unreachable magmux is logged once, not once per tick. */
+ unavailableLogged: boolean;
+ /**
+ * Ambient magmux only: our pane was looked for and is not there. Pane
+ * topology does not change under a running claudish, so the ancestry walk is
+ * not repeated every tick.
+ */
+ paneNotFound: boolean;
nextControlId: number;
}
-const state: ManagerState = {
- server: null,
- socketPath: null,
+const state: UiState = {
control: null,
- paneOpen: false,
- paneIndex: null,
+ connecting: null,
+ targetPane: null,
leases: new Map(),
- opening: false,
- lockPath: null,
+ lastFrame: null,
+ tickTimer: null,
lingerTimer: null,
+ painting: false,
+ lastPaintOk: null,
+ unavailableLogged: false,
+ paneNotFound: false,
nextControlId: 1,
};
+function noteUnavailable(message: string): void {
+ if (state.unavailableLogged) return;
+ state.unavailableLogged = true;
+ log(message);
+}
+
let installed = false;
// ─── The lease ───────────────────────────────────────────────────────────────
/**
- * Is a renderer painting this episode right now?
+ * Is this episode's reason on screen right now?
*
- * Read LIVE at the moment a status is chosen. Never cached: by the time a
- * caller destructures a boolean off a result object, the pane it describes may
- * have been killed.
+ * Read LIVE when a status is chosen, never cached: a boolean destructured off a
+ * result object can describe a magmux that has since exited.
*/
export function uiLeaseValid(episodeId: string): boolean {
- if (!state.paneOpen) return false;
+ if (!state.control || state.targetPane === null) return false;
const last = state.leases.get(episodeId);
if (last === undefined) return false;
return recoveryClock().now() - last <= UI_LEASE_MS;
@@ -169,13 +164,15 @@ export function uiLeaseValid(episodeId: string): boolean {
/** Diagnostics for the log and for tests. Never used to decide anything. */
export function describeLease(episodeId: string): {
- paneOpen: boolean;
+ connected: boolean;
+ targetPane: number | null;
lastAckAgoMs: number | null;
valid: boolean;
} {
const last = state.leases.get(episodeId);
return {
- paneOpen: state.paneOpen,
+ connected: state.control !== null,
+ targetPane: state.targetPane,
lastAckAgoMs: last === undefined ? null : Math.round(recoveryClock().now() - last),
valid: uiLeaseValid(episodeId),
};
@@ -199,7 +196,7 @@ async function connectControl(sock: string): Promise<Socket | null> {
function request(
socket: Socket,
msg: Record<string, unknown>,
- timeoutMs = 3_000
+ timeoutMs = CONTROL_TIMEOUT_MS
): Promise<Record<string, unknown> | null> {
const id = state.nextControlId++;
return new Promise((resolve) => {
@@ -239,342 +236,383 @@ function request(
});
}
-// ─── The cross-process single-pane lock ──────────────────────────────────────
+/** Forget the connection. Every lease goes with it, at once. */
+function dropConnection(): void {
+ state.control = null;
+ state.targetPane = null;
+ state.leases.clear();
+}
/**
- * ONE recovery pane per magmux, not N.
+ * This process and its ancestors, nearest first.
*
- * `team --grid --mode interactive` launches one claudish per pane and magmux
- * exports the same `MAGMUX_SOCK` to all of them, so one outage hits N processes
- * at once and a merely process-level singleton would open N panes into one
- * grid. The losers still run the ladder — they retry and they recover — they
- * just never open a pane, hold no lease, and answer the inline error at
- * exhaustion. That is honest: one banner names one slot's provider, and every
- * slot still recovers.
+ * In an ambient magmux the pane's process is whatever magmux started — usually
+ * a login shell or the `claudish` launcher — and this proxy runs somewhere
+ * beneath it. Asynchronous on purpose: it runs inside the proxy while requests
+ * are being served.
*/
-function acquirePaneLock(controlSock: string): string | null {
- const lockPath = join("/tmp", `claudish-recovery-${basename(controlSock)}.lock`);
- for (let attempt = 0; attempt < 2; attempt++) {
+async function ancestorPids(maxDepth = 6): Promise<Set<number>> {
+ const out = new Set<number>([process.pid]);
+ let pid = process.ppid;
+ for (let i = 0; i < maxDepth && pid > 1; i++) {
+ out.add(pid);
try {
- const fd = openSync(lockPath, "wx", 0o600);
- writeSync(fd, String(process.pid));
- closeSync(fd);
- return lockPath;
+ const { stdout } = await execFileAsync("ps", ["-o", "ppid=", "-p", String(pid)], {
+ timeout: 1_000,
+ });
+ const next = Number(stdout.trim());
+ if (!Number.isFinite(next) || next <= 1) break;
+ pid = next;
} catch {
- // Held — by a live peer, or by debris from one that was SIGKILLed.
+ break;
+ }
+ }
+ return out;
+}
+
+/**
+ * Which pane is ours, in a magmux claudish did not launch.
+ *
+ * magmux exports `MAGMUX_SOCK` and `MAGMUX_THEME` to its panes and nothing that
+ * names the pane, so it is found by process ancestry against `list`. Null when
+ * nothing matches: the banner is then unavailable, and recovery still retries
+ * and still answers the inline error at exhaustion.
+ */
+async function findOwnPane(socket: Socket): Promise<number | null> {
+ const reply = await request(socket, { type: "list" });
+ const result = reply?.result as { panes?: unknown } | unknown[] | undefined;
+ const panes = Array.isArray(result)
+ ? result
+ : Array.isArray((result as { panes?: unknown })?.panes)
+ ? ((result as { panes: unknown[] }).panes as unknown[])
+ : [];
+ const ancestors = await ancestorPids();
+ for (const p of panes) {
+ const pane = (p as { pane?: unknown }).pane;
+ const pid = (p as { pid?: unknown }).pid;
+ if (typeof pane === "number" && typeof pid === "number" && ancestors.has(pid)) return pane;
+ }
+ return null;
+}
+
+/** Connect once and find our pane. Concurrent callers share one attempt. */
+function ensureConnected(): Promise<boolean> {
+ if (state.control && state.targetPane !== null) return Promise.resolve(true);
+ if (state.paneNotFound) return Promise.resolve(false);
+ if (state.connecting) return state.connecting;
+ state.connecting = (async () => {
+ const endpoint = resolveControlEndpoint();
+ if (!endpoint) {
+ noteUnavailable(
+ "[Recovery] no magmux control endpoint — no banner; recovery continues without one"
+ );
+ return false;
+ }
+ const socket = await connectControl(endpoint.sock);
+ if (!socket) {
+ noteUnavailable(`[Recovery] could not reach magmux at ${endpoint.sock} — no banner`);
+ return false;
+ }
+ // Bound to THIS socket: a late `close` from a replaced connection must not
+ // tear down its successor and every lease with it.
+ const onGone = () => {
+ if (state.control === socket) dropConnection();
+ };
+ socket.on("error", onGone);
+ socket.on("close", onGone);
+ state.control = socket;
+ // A magmux claudish launched has exactly one `-e` pane, and magmux numbers
+ // `-e` panes 0..N-1 in argument order, so Claude Code is pane 0.
+ const pane = endpoint.wrapped ? 0 : await findOwnPane(socket);
+ if (pane === null) {
+ state.paneNotFound = true;
+ log("[Recovery] this claudish's pane was not found in magmux — no banner");
try {
- const owner = Number(readFileSync(lockPath, "utf-8").trim());
- if (Number.isFinite(owner) && owner > 0) {
- try {
- process.kill(owner, 0);
- return null; // a live peer owns the banner
- } catch {
- unlinkSync(lockPath); // the owner is gone; the lock is debris
- continue;
- }
- }
- unlinkSync(lockPath);
+ socket.end();
} catch {
- return null;
+ /* already gone */
}
+ dropConnection();
+ return false;
}
- }
- return null;
+ state.targetPane = pane;
+ state.unavailableLogged = false;
+ log(
+ `[Recovery] banner will draw on magmux pane ${pane} ` +
+ `(${endpoint.wrapped ? "magmux launched by claudish" : "ambient magmux"})`
+ );
+ return true;
+ })().finally(() => {
+ state.connecting = null;
+ });
+ return state.connecting;
}
-function releasePaneLock(): void {
- if (!state.lockPath) return;
- try {
- if (existsSync(state.lockPath)) {
- const owner = Number(readFileSync(state.lockPath, "utf-8").trim());
- if (owner === process.pid) unlinkSync(state.lockPath);
+// ─── The banner text ─────────────────────────────────────────────────────────
+
+/** `9s`, `1m 26s`, `2h 5m`. */
+export function formatElapsed(ms: number): string {
+ const s = Math.max(0, Math.round(ms / 1000));
+ if (s < 60) return `${s}s`;
+ const m = Math.floor(s / 60);
+ if (m < 60) return `${m}m ${String(s % 60).padStart(2, "0")}s`;
+ return `${Math.floor(m / 60)}h ${m % 60}m`;
+}
+
+/** Word-wrap one sentence to `cols`, never splitting a word. */
+function wrap(text: string, cols: number): string[] {
+ const out: string[] = [];
+ let line = "";
+ for (const word of text.split(/\s+/).filter(Boolean)) {
+ if (line && line.length + 1 + word.length > cols) {
+ out.push(line);
+ line = word;
+ } else {
+ line = line ? `${line} ${word}` : word;
}
- } catch {
- /* best effort */
}
- state.lockPath = null;
+ if (line) out.push(line);
+ return out;
}
-// ─── The pane command line ───────────────────────────────────────────────────
+const KIND_WORD: Record<RecoveryEpisodeFrame["kind"], string> = {
+ refused: "refused",
+ dns: "not found",
+ unreachable: "unreachable",
+};
/**
- * How to run `claudish recovery-pane` from wherever this build lives.
+ * The outage banner.
*
- * Built from `process.argv` rather than from the name `claudish`, because the
- * process that must start is THIS build: a global install mid-update, a `bun
- * run src/index.ts` in a worktree and an npm-installed bundle are three
- * different files, and a pane from a different build is the version-skew case
- * the protocol has to print a notice for. `claudish` on PATH stays as the
- * fallback for a packaging shape neither argv entry describes.
+ * THE FIRST LINE MUST STAND ALONE. On a pane too small for the box, magmux
+ * falls back to a one-line pill that shows only the first line.
*/
-export function recoveryPaneCommandLine(socketPath: string): string {
- const script = process.argv[1];
- const q = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
- if (script && (script.endsWith(".ts") || script.endsWith(".js") || script.endsWith(".cjs"))) {
- return `${q(process.execPath)} ${q(script)} recovery-pane --socket ${q(socketPath)}`;
+export function bannerText(frame: RecoveryEpisodeFrame, nowMs: number): string {
+ const lines: string[] = [`✗ NETWORK · ${frame.providerDisplayName} ${KIND_WORD[frame.kind]}`];
+ lines.push(...wrap(frame.reason, OVERLAY_WRAP_COLS));
+ lines.push(`${frame.host} · ${frame.code ?? frame.kind}${frame.loopback ? " · local" : ""}`);
+ const elapsed = formatElapsed(nowMs - frame.startedAtMs);
+ if (frame.state === "handoff") {
+ lines.push(`handed back to Claude Code · waiting for its retry · ${elapsed} in recovery`);
+ } else if (frame.nextAttemptAtMs === null) {
+ lines.push(`connecting to ${frame.host}… · attempt ${frame.attempts} · ${elapsed} in recovery`);
+ } else {
+ const secs = Math.max(0, Math.ceil((frame.nextAttemptAtMs - nowMs) / 1000));
+ lines.push(`next attempt in ${secs}s · attempt ${frame.attempts} · ${elapsed} in recovery`);
}
- return `claudish recovery-pane --socket ${q(socketPath)}`;
+ const held = `${frame.waiters} request${frame.waiters === 1 ? "" : "s"} held`;
+ const more =
+ frame.otherEpisodes > 0
+ ? ` · +${frame.otherEpisodes} more outage${frame.otherEpisodes === 1 ? "" : "s"}`
+ : "";
+ lines.push(`${held}${more}`);
+ lines.push("Esc in Claude Code stops the turn");
+ return lines.join("\n");
}
-// ─── Opening and closing the pane ────────────────────────────────────────────
+/** The banner shown for a moment after the connection comes back. */
+export function recoveredText(frame: RecoveryEpisodeFrame, nowMs: number): string {
+ return [
+ `✓ NETWORK · ${frame.providerDisplayName} recovered`,
+ `${frame.attempts} attempt${frame.attempts === 1 ? "" : "s"} over ${formatElapsed(nowMs - frame.startedAtMs)}`,
+ ].join("\n");
+}
-function onCommand(cmd: RecoveryCommand): void {
- switch (cmd.type) {
- case "hello":
- log(`[Recovery] pane connected (pid ${cmd.pid}, protocol ${cmd.protocol})`);
- break;
- case "ack": {
- // THE LEASE RENEWAL. Only here, and only from a renderer that painted.
- const first = !state.leases.has(cmd.episodeId);
- state.leases.set(cmd.episodeId, recoveryClock().now());
+// ─── Painting ────────────────────────────────────────────────────────────────
+
+/** Paint one outage frame. Only an acknowledged write renews a lease. */
+async function paint(frame: RecoveryEpisodeFrame): Promise<void> {
+ if (state.painting) return;
+ state.painting = true;
+ try {
+ if (!(await ensureConnected())) return;
+ const socket = state.control;
+ const pane = state.targetPane;
+ if (!socket || pane === null) return;
+ const reply = await request(socket, {
+ type: "overlay",
+ pane,
+ text: bannerText(frame, Date.now()),
+ style: "error",
+ });
+ const ok = reply?.ok === true;
+ if (ok) {
+ const first = !state.leases.has(frame.episodeId);
+ state.leases.set(frame.episodeId, recoveryClock().now());
+ state.lastFrame = frame;
if (first) {
log(
- `[Recovery] pane is painting episode ${cmd.episodeId} — lease granted ` +
- `(paneOpen=${state.paneOpen})`
+ `[Recovery] banner painted for episode ${frame.episodeId} on magmux pane ${pane} ` +
+ "— lease granted"
);
}
- break;
+ // Decoration. Its reply grants nothing, so nothing waits on it.
+ void request(socket, { type: "tint", pane, color: "red" });
+ } else if (state.lastPaintOk !== false) {
+ log(`[Recovery] magmux did not acknowledge the banner: ${JSON.stringify(reply)}`);
}
- case "retry_now":
- if (episodeIsLive(cmd.episodeId)) {
- const n = attemptsSoFar(cmd.episodeId) + 1;
- log(`[Recovery] attempt ${n} (manual) for episode ${cmd.episodeId}`);
- tryNow(cmd.episodeId);
- }
- break;
- case "bye":
- log(`[Recovery] pane said bye (${cmd.reason}) — giving up on every live episode`);
- // ONE fact, two readers. `giveUpAll()` ends the episodes alive at this
- // instant; this ends the ones that do not exist yet. Without it the next
- // request against the same dead target opened a new episode, found the
- // pane suppressed, and held its socket for the full deadline with the
- // reason legible nowhere — the forbidden state, reached from the key
- // whose whole purpose is to avoid it. The hold and the surface are
- // suppressed together, for the same window, from the same number.
- noteRecoveryGaveUp(recoveryClock().now() + UI_SUPPRESS_AFTER_BYE_MS);
- giveUpAll();
- // The pane the user just closed must not reappear two seconds later, and
- // closing it is ALSO what revokes every lease: `teardown()` drops
- // `paneOpen`, and without that clause no heartbeat can grant anything.
- // One revocation path, not two — a second one is a second thing to
- // forget, and the superseded design's bug was exactly a revocation path
- // that existed on paper and cleared nothing.
- void closePane();
- break;
+ state.lastPaintOk = ok;
+ } finally {
+ state.painting = false;
}
}
-async function openPane(): Promise<void> {
- const control = resolveMagmuxControl();
- if (!control) {
- log("[Recovery] no magmux control endpoint — no pane, recovery continues without a banner");
- return;
- }
- const lock = acquirePaneLock(control.sock);
- if (!lock) {
- log("[Recovery] another claudish already owns the recovery pane for this magmux");
- return;
- }
- state.lockPath = lock;
-
- const dir = createRecoverySocketDir();
- const socketPath = recoverySocketPathIn(dir);
- state.socketPath = socketPath;
- state.server = await startRecoverySocketServer({
- socketPath,
- snapshot: () => renderableEpisodeFrame(),
- onCommand,
- onDisconnect: () => {
- // EOF revokes nothing by itself — the lease expires on its own within
- // UI_LEASE_MS, and having exactly one expiry path means there is no
- // second one to forget. The retry loop is not the pane's business.
- log("[Recovery] pane disconnected");
- },
- });
+/** Replace the overlay outright. Grants no lease. */
+async function writeOverlay(text: string, style: string, tint: string): Promise<void> {
+ const socket = state.control;
+ const pane = state.targetPane;
+ if (!socket || pane === null) return;
+ await request(socket, { type: "overlay", pane, text, style });
+ await request(socket, { type: "tint", pane, color: tint });
+}
- const socket = await connectControl(control.sock);
- if (!socket) {
- log(`[Recovery] could not reach magmux at ${control.sock}`);
- await teardown();
- return;
- }
- state.control = socket;
- socket.on("error", () => {
- state.paneOpen = false;
- });
- socket.on("close", () => {
- state.paneOpen = false;
- });
+function clearOverlay(): Promise<void> {
+ return writeOverlay("", "", "reset");
+}
- const reply = await request(socket, {
- type: "open_pane",
- cmd: recoveryPaneCommandLine(socketPath),
- cwd: process.cwd(),
- split: "vertical",
- });
- if (!reply || reply.ok !== true) {
- log(`[Recovery] magmux refused open_pane: ${JSON.stringify(reply)}`);
- await teardown();
- return;
+function stopTick(): void {
+ if (state.tickTimer !== null) {
+ recoveryClock().clearTimeout(state.tickTimer);
+ state.tickTimer = null;
}
- const result = reply.result as { pane?: number } | undefined;
- // Read the index back. magmux's own control panel occupies an index, so the
- // first pane an agent opens is NOT necessarily 1 — measured: 2.
- state.paneIndex = typeof result?.pane === "number" ? result.pane : null;
- state.paneOpen = true;
- log(`[Recovery] recovery pane opened (magmux pane ${state.paneIndex}) at ${socketPath}`);
}
-async function closePane(): Promise<void> {
- if (state.control && state.paneIndex !== null) {
- await request(state.control, { type: "close_pane", pane: state.paneIndex, force: true }, 1_000);
+function cancelLinger(): void {
+ if (state.lingerTimer !== null) {
+ recoveryClock().clearTimeout(state.lingerTimer);
+ state.lingerTimer = null;
}
- await teardown();
}
-async function teardown(): Promise<void> {
- state.paneOpen = false;
- state.paneIndex = null;
- state.leases.clear();
- if (state.control) {
- try {
- state.control.end();
- } catch {
- /* already gone */
- }
- state.control = null;
- }
- if (state.server) {
- await state.server.close();
- state.server = null;
- } else if (state.socketPath) {
- cleanupRecoverySocket(state.socketPath);
- }
- state.socketPath = null;
- releasePaneLock();
+/**
+ * Paint now, then once per `FRAME_TICK_MS` for as long as any outage is live.
+ *
+ * The tick is independent of the ladder on purpose. A connect against an
+ * unreachable host takes up to 75 s with nothing new to say, and the lease has
+ * to stay fresh across it — renewing only on ladder events would let it lapse
+ * at exhaustion, for exactly the failure class this feature exists for.
+ */
+function startTick(): void {
+ if (state.tickTimer !== null) return;
+ const clock = recoveryClock();
+ const loop = () => {
+ state.tickTimer = null;
+ const frame = renderableEpisodeFrame();
+ if (!frame) return;
+ void paint(frame);
+ const t = clock.setTimeout(loop, FRAME_TICK_MS);
+ clock.unref?.(t);
+ state.tickTimer = t;
+ };
+ loop();
}
// ─── The hooks ───────────────────────────────────────────────────────────────
-/**
- * Ask for a surface for this episode. Idempotent, and never awaited by the
- * retry loop — a pane that takes 200 ms to appear must not delay the first
- * retry by 200 ms.
- */
-export function ensureRecoveryUi(episodeId: string): void {
- // Asking and being told no is still asking: the timestamp is what a later
- // phase's grace rule measures from, and it must be set even when the answer
- // is an immediate no.
+function onEpisodeOpened(episodeId: string): void {
+ // Asking and being told no is still asking: the coordinator's grace rule
+ // measures from this, so it is recorded even when there is no magmux.
markPaneRequested(episodeId);
- if (state.lingerTimer !== null) {
- recoveryClock().clearTimeout(state.lingerTimer);
- state.lingerTimer = null;
- }
- if (state.paneOpen || state.opening) return;
- if (recoveryGiveUpActive()) {
- log("[Recovery] pane suppressed — the user pressed [q] less than a minute ago");
- return;
- }
- state.opening = true;
- void openPane()
- .catch((err) => {
- log(`[Recovery] pane could not be opened: ${String(err)}`);
- })
- .finally(() => {
- state.opening = false;
- });
+ cancelLinger();
+ startTick();
}
/**
- * One episode ended. Tell the renderer, and NOTHING else — a concurrent
- * episode may still be painting through the same socket and the same pane.
+ * One episode ended. The coordinator has already removed it from the live set,
+ * so `renderableEpisodeFrame()` here answers "is another outage still live?".
*/
-export function releaseEpisodeUi(episodeId: string, outcome: RecoveryOutcome): void {
+function onEpisodeClosed(episodeId: string, outcome: RecoveryOutcome): void {
state.leases.delete(episodeId);
- state.server?.broadcast({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "closed",
- episodeId,
- outcome,
- });
- if (renderableEpisodeFrame() !== null) return; // something else is still live
- // Nothing left to paint. Linger, so the pane does not flap open and shut on
- // every client-retry cycle, and so the last thing that happened stays
- // readable for someone who looked away.
- if (state.lingerTimer !== null) recoveryClock().clearTimeout(state.lingerTimer);
+ if (renderableEpisodeFrame() !== null) return; // the tick keeps drawing the other one
+ stopTick();
+ const last = state.lastFrame?.episodeId === episodeId ? state.lastFrame : null;
+ state.lastFrame = null;
+ if (outcome !== "recovered" || !last) {
+ // Esc in Claude Code, shutdown, an expired hand-off: nothing is being held
+ // any more, so there is nothing to explain.
+ void clearOverlay();
+ return;
+ }
+ void writeOverlay(recoveredText(last, Date.now()), "success", "green");
+ cancelLinger();
const clock = recoveryClock();
const timer = clock.setTimeout(() => {
state.lingerTimer = null;
- if (renderableEpisodeFrame() !== null) return;
- void closePane();
- }, PANE_LINGER_MS);
+ if (renderableEpisodeFrame() !== null) return; // a new outage owns the overlay
+ void clearOverlay();
+ }, OUTCOME_LINGER_MS);
clock.unref?.(timer);
state.lingerTimer = timer;
}
const hooks: RecoveryUiHooks = {
- onEpisodeOpened: (episodeId) => ensureRecoveryUi(episodeId),
- onEpisodeClosed: (episodeId, outcome) => releaseEpisodeUi(episodeId, outcome),
+ onEpisodeOpened,
+ onEpisodeClosed,
leaseValid: (episodeId) => uiLeaseValid(episodeId),
};
/**
- * Install the recovery UI into the coordinator.
+ * Install the banner into the coordinator.
*
- * Called from exactly one place — the magmux launch wrapper, once it knows the
- * control socket. Every other entry point leaves it uninstalled, so `-p`,
- * `--stdin`, `serve`, the MCP server and the test suite cannot open a pane and
- * cannot hold a lease, without any of them having to opt out.
+ * Called only from `claude-runner.ts`, for an interactive launch that has a
+ * magmux to draw on. Every other entry point — `-p`, `--stdin`, `serve`, the
+ * MCP server, the test suite — leaves it uninstalled, so none of them can hold
+ * a lease, without any of them having to opt out.
*/
export function installRecoveryUi(controlSock: string | null): void {
- setMagmuxControlSocket(controlSock);
+ explicitControlSock = controlSock;
if (installed) return;
installed = true;
registerRecoveryUi(hooks);
- process.on("exit", () => {
- releasePaneLock();
- if (state.socketPath) cleanupRecoverySocket(state.socketPath);
- });
}
-/** Tear everything down. Proxy shutdown, and tests. */
+/** Take the banner down and uninstall. Claude Code's exit, proxy shutdown, tests. */
export async function shutdownRecoveryUi(): Promise<void> {
- if (state.lingerTimer !== null) {
- recoveryClock().clearTimeout(state.lingerTimer);
- state.lingerTimer = null;
+ stopTick();
+ cancelLinger();
+ if (state.control && state.targetPane !== null) {
+ // Short: at Claude Code's exit magmux is usually exiting too.
+ const socket = state.control;
+ const pane = state.targetPane;
+ await request(socket, { type: "overlay", pane, text: "", style: "" }, 500);
+ await request(socket, { type: "tint", pane, color: "reset" }, 500);
}
- await closePane();
+ if (state.control) {
+ try {
+ state.control.end();
+ } catch {
+ /* already gone */
+ }
+ }
+ dropConnection();
+ state.lastFrame = null;
+ state.lastPaintOk = null;
+ state.unavailableLogged = false;
+ state.paneNotFound = false;
registerRecoveryUi(null);
installed = false;
- resetRecoveryGiveUp();
explicitControlSock = null;
}
-/** Tests only: the manager is process state shared with every sibling test. */
-export function __setPaneOpenForTests(open: boolean, socketPath?: string): void {
- state.paneOpen = open;
- if (socketPath !== undefined) state.socketPath = socketPath;
-}
-
-/** Tests only: feed a command as though a pane had sent it. */
-export function __handleCommandForTests(cmd: RecoveryCommand): void {
- onCommand(cmd);
-}
-
-/** Tests only: forget every lease and every suppression. */
+/** Tests only: the banner is process state shared with every sibling test. */
export function __resetUiStateForTests(): void {
- state.leases.clear();
- state.paneOpen = false;
- state.paneIndex = null;
- resetRecoveryGiveUp();
- state.opening = false;
- state.socketPath = null;
- state.server = null;
- state.control = null;
- if (state.lingerTimer !== null) {
- recoveryClock().clearTimeout(state.lingerTimer);
- state.lingerTimer = null;
+ stopTick();
+ cancelLinger();
+ if (state.control) {
+ try {
+ state.control.destroy();
+ } catch {
+ /* already gone */
+ }
}
- releasePaneLock();
+ dropConnection();
+ state.connecting = null;
+ state.lastFrame = null;
+ state.painting = false;
+ state.lastPaintOk = null;
+ state.unavailableLogged = false;
+ state.paneNotFound = false;
+ registerRecoveryUi(null);
+ installed = false;
explicitControlSock = null;
}
diff --git a/packages/cli/src/recovery/pane-app.test.ts b/packages/cli/src/recovery/pane-app.test.ts
deleted file mode 100644
index 7b92cc52..00000000
--- a/packages/cli/src/recovery/pane-app.test.ts
+++ /dev/null
@@ -1,342 +0,0 @@
-/**
- * The recovery banner.
- *
- * `renderPane` is pure, which is why it is the thing asserted on: the live
- * evidence (a capture of the real pane in a real magmux) and these tests then
- * look at the SAME function rather than at two implementations that drift.
- */
-
-import { afterEach, describe, expect, test } from "bun:test";
-import { EventEmitter } from "node:events";
-import { resetThemeModeForTests, setThemeMode } from "../theme/theme-mode.js";
-import {
- type PaneViewState,
- formatDuration,
- parsePaneArgs,
- renderPane,
- runRecoveryPane,
- stripAnsi,
- wrapText,
-} from "./pane-app.js";
-import {
- type RecoverySocketServer,
- createRecoverySocketDir,
- recoverySocketPathIn,
- startRecoverySocketServer,
-} from "./socket-server.js";
-import {
- PROXY_SILENT_MS,
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryCommand,
- type RecoveryEpisodeFrame,
-} from "./types.js";
-
-const T0 = 1_700_000_000_000;
-
-const EPISODE: RecoveryEpisodeFrame = {
- v: RECOVERY_PROTOCOL_VERSION,
- type: "episode",
- episodeId: "ep-1",
- state: "waiting",
- tier: 1,
- providerDisplayName: "Openai-codex",
- host: "chatgpt.com",
- endpoint: "https://chatgpt.com/backend-api/codex/responses",
- kind: "unreachable",
- code: "ConnectionClosed",
- loopback: false,
- reason:
- "Cannot reach Openai-codex at https://chatgpt.com/backend-api/codex/responses. Check your network connection.",
- attempts: 5,
- clientRetries: 0,
- startedAtMs: T0,
- nextAttemptAtMs: T0 + 132_000,
- lastOutcome: "ConnectionClosed after 11ms",
- waiters: 2,
- otherEpisodes: 0,
-};
-
-const view = (over: Partial<PaneViewState> = {}): PaneViewState => ({
- frame: EPISODE,
- closed: null,
- nowMs: T0 + 120_000,
- lastFrameAtMs: T0 + 120_000,
- connected: true,
- protocolMismatch: false,
- notice: null,
- ...over,
-});
-
-/** Escapes are the medium, not the message — strip them to read the banner. */
-/** The banner's text, without the paint. Shared with the renderer on purpose:
- * a second escape-stripper in the test would be a second thing to get wrong. */
-const plain = stripAnsi;
-
-afterEach(() => {
- // `setThemeMode` flips a PROCESS-GLOBAL palette and Bun runs sibling test
- // files in one process. Never leave it flipped.
- resetThemeModeForTests();
-});
-
-describe("renderPane", () => {
- test("names the provider, the host and the reason — C-3's three facts", () => {
- const out = plain(renderPane(view(), 100));
- expect(out).toContain("Openai-codex");
- expect(out).toContain("chatgpt.com");
- // The reason is `buildConnectionErrorMessage`'s OWN sentence, not a
- // paraphrase: if the banner and the inline error ever disagree, the user is
- // being told two different stories about one fault.
- expect(out).toContain("Cannot reach Openai-codex at");
- // The sentence is wrapped to the pane, so read it as one flow.
- expect(out.replace(/\s+/g, " ")).toContain("Check your network connection.");
- expect(out).toContain("ConnectionClosed");
- });
-
- test("offers both keys and does not promise focus it will not steal", () => {
- const out = plain(renderPane(view(), 100));
- expect(out).toContain("[r]");
- expect(out).toContain("[q]");
- // magmux forwards clicks to focus a pane. Stealing focus would swallow
- // keystrokes the user was queueing into Claude Code, which accepts messages
- // while a turn is in flight — a worse failure than one extra click.
- expect(out).toContain("click here");
- });
-
- test("the countdown DECREASES between two renders — C-4, as a unit", () => {
- const early = plain(renderPane(view({ nowMs: T0 + 70_000 }), 100));
- const later = plain(renderPane(view({ nowMs: T0 + 73_000 }), 100));
- expect(early).toContain("next attempt in 1m 02s");
- expect(later).toContain("next attempt in 59s");
- expect(early).not.toBe(later);
- });
-
- test("a countdown that has gone past reads 0s, never a negative", () => {
- expect(plain(renderPane(view({ nowMs: T0 + 200_000 }), 100))).toContain("next attempt in 0s");
- });
-
- test("`attempting` shows what we are doing, not a frozen countdown", () => {
- // A 45-second connect with a countdown stuck at the same number reads as a
- // hang. This line is why the proxy ticks in every live state.
- const out = plain(
- renderPane(view({ frame: { ...EPISODE, state: "attempting", nextAttemptAtMs: null } }), 100)
- );
- expect(out).toContain("connecting to chatgpt.com");
- expect(out).not.toContain("next attempt in");
- });
-
- test("`handoff` says the client is expected back, which is the truth", () => {
- const out = plain(
- renderPane(
- view({
- frame: { ...EPISODE, state: "handoff", nextAttemptAtMs: null, tier: 2, clientRetries: 2 },
- }),
- 100
- )
- );
- expect(out).toContain("waiting for Claude Code to retry");
- expect(out).toContain("client retry 2");
- });
-
- test("counters: attempts, elapsed, last outcome, held requests, other episodes", () => {
- const out = plain(renderPane(view({ frame: { ...EPISODE, otherEpisodes: 1 } }), 100));
- expect(out).toContain("attempt 5");
- expect(out).toContain("2m 00s in recovery");
- expect(out).toContain("last: ConnectionClosed after 11ms");
- expect(out).toContain("2 requests held");
- expect(out).toContain("+1 more");
- });
-
- test("a loopback endpoint is marked local", () => {
- const out = plain(renderPane(view({ frame: { ...EPISODE, loopback: true } }), 100));
- expect(out).toContain("local");
- });
-
- test("the dead-proxy notice appears only after real silence", () => {
- // C-18 asserts this notice is ABSENT for a whole slow-connect run, so it
- // must key on silence longer than the proxy's own tick and nothing else.
- const quiet = view({ lastFrameAtMs: T0 + 120_000 - (PROXY_SILENT_MS - 500) });
- expect(plain(renderPane(quiet, 100))).not.toContain("has not reported");
- const dead = view({ lastFrameAtMs: T0 + 120_000 - (PROXY_SILENT_MS + 500) });
- expect(plain(renderPane(dead, 100))).toContain("has not reported");
- });
-
- test("a closed episode says how it ended and that the pane will go by itself", () => {
- const ok = plain(
- renderPane(
- view({ closed: { v: 1, type: "closed", episodeId: "ep-1", outcome: "recovered" } }),
- 100
- )
- );
- expect(ok).toContain("recovered");
- expect(ok).toContain("this pane closes on its own");
- const bad = plain(
- renderPane(
- view({ closed: { v: 1, type: "closed", episodeId: "ep-1", outcome: "gave_up" } }),
- 100
- )
- );
- expect(bad).toContain("gave up");
- });
-
- test("version skew prints a notice instead of rendering fields it cannot read", () => {
- const out = plain(renderPane(view({ protocolMismatch: true }), 100));
- expect(out).toContain("version skew");
- expect(out).not.toContain("Openai-codex");
- });
-
- test("before the first frame it says what it is waiting for", () => {
- expect(plain(renderPane(view({ frame: null }), 100))).toContain("waiting for the proxy");
- expect(plain(renderPane(view({ frame: null, connected: false }), 100))).toContain(
- "connecting to the claudish proxy"
- );
- });
-
- test("every line fits a 40-column pane — a split pane is HALF the terminal", () => {
- // Measured on magmux 0.11.0: `split:"vertical"` took pane 0 from 40 rows to
- // 20 and gave the new pane 19. A user with a narrow window gets a narrow
- // pane, and a banner that runs off it is a banner that cannot be read.
- const narrow = renderPane(view(), 40)
- .split("\n")
- .map((l) => plain(l));
- for (const line of narrow) expect(line.length).toBeLessThanOrEqual(40);
- // …and the keys survive the squeeze, because they are the part a user
- // cannot work out for themselves.
- const joined = narrow.join(" ");
- expect(joined).toContain("[r]");
- expect(joined).toContain("[q]");
- });
-
- // ── THE THEMING INVARIANT ────────────────────────────────────────────────
- test("the palette is read at RENDER time, not at module load", () => {
- // This module was imported at the top of this file, long before the line
- // below runs — which is exactly the production ordering: command modules
- // are imported before theme detection completes. A module-level
- // `const C = cliAnsi()` would snapshot the dark palette here and never
- // change again. That bug class has been found six times in this codebase.
- setThemeMode("dark");
- const dark = renderPane(view(), 100);
- setThemeMode("light");
- const light = renderPane(view(), 100);
- expect(light).not.toBe(dark);
- expect(light).toContain("\x1b[38;2;"); // truecolor: the LIGHT palette
- expect(dark).not.toContain("\x1b[38;2;"); // classic 16-colour: dark/unknown
- });
-
- test("the red banner fill is the SAME in both themes — it is theme-independent", () => {
- // The one element that must be legible whatever happens does not depend on
- // detection succeeding: a self-contained mid-dark fill with bright-white
- // ink, the pairing `team-grid.ts` established.
- const FILL = "\x1b[1;97;48;2;160;50;70m";
- setThemeMode("dark");
- expect(renderPane(view(), 100)).toContain(FILL);
- setThemeMode("light");
- expect(renderPane(view(), 100)).toContain(FILL);
- });
-});
-
-describe("formatDuration / wrapText / parsePaneArgs", () => {
- test("durations", () => {
- expect(formatDuration(0)).toBe("0s");
- expect(formatDuration(-5_000)).toBe("0s");
- expect(formatDuration(47_000)).toBe("47s");
- expect(formatDuration(132_000)).toBe("2m 12s");
- });
-
- test("wrapping never drops or splits a word", () => {
- const words = "alpha beta gamma delta epsilon".split(" ");
- const lines = wrapText(words.join(" "), 12);
- expect(lines.join(" ").split(/\s+/)).toEqual(words);
- for (const l of lines) expect(l.length).toBeLessThanOrEqual(12);
- });
-
- test("--socket parses in both spellings, and is required", () => {
- expect(parsePaneArgs(["--socket", "/tmp/x/r.sock"]).socketPath).toBe("/tmp/x/r.sock");
- expect(parsePaneArgs(["--socket=/tmp/y/r.sock"]).socketPath).toBe("/tmp/y/r.sock");
- expect(parsePaneArgs([]).socketPath).toBeNull();
- });
-});
-
-// ─── Keys, over a real socket ────────────────────────────────────────────────
-
-class FakeStdin extends EventEmitter {
- setRawMode() {
- return this;
- }
- resume() {
- return this;
- }
- pause() {
- return this;
- }
-}
-class FakeStdout extends EventEmitter {
- columns = 100;
- chunks: string[] = [];
- write(s: string) {
- this.chunks.push(s);
- return true;
- }
-}
-
-describe("the pane's two keys", () => {
- test("`r`, `R` and Enter each ask for an attempt now; `q` gives up and exits", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- const received: RecoveryCommand[] = [];
- let server: RecoverySocketServer | null = null;
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => EPISODE,
- onCommand: (cmd) => received.push(cmd),
- tickMs: 0,
- });
-
- const stdin = new FakeStdin();
- const stdout = new FakeStdout();
- const done = runRecoveryPane({
- socketPath: path,
- stdin: stdin as unknown as NodeJS.ReadStream,
- stdout: stdout as unknown as NodeJS.WriteStream,
- });
- await new Promise((r) => setTimeout(r, 150));
-
- for (const key of ["r", "R", "\r"]) stdin.emit("data", Buffer.from(key));
- await new Promise((r) => setTimeout(r, 120));
- expect(received.filter((c) => c.type === "retry_now")).toHaveLength(3);
- expect(
- received
- .filter((c) => c.type === "retry_now")
- .every((c) => "episodeId" in c && c.episodeId === "ep-1")
- ).toBe(true);
-
- stdin.emit("data", Buffer.from("q"));
- expect(await done).toBe(0);
- await new Promise((r) => setTimeout(r, 80));
- expect(received.filter((c) => c.type === "bye")).toHaveLength(1);
- await server.close();
- }, 15_000);
-
- test("the banner is painted from the REPLAYED frame, with no tick at all", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- let server: RecoverySocketServer | null = null;
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => EPISODE,
- onCommand: () => {},
- tickMs: 0,
- });
- const stdin = new FakeStdin();
- const stdout = new FakeStdout();
- const done = runRecoveryPane({
- socketPath: path,
- stdin: stdin as unknown as NodeJS.ReadStream,
- stdout: stdout as unknown as NodeJS.WriteStream,
- });
- await new Promise((r) => setTimeout(r, 150));
- const painted = plain(stdout.chunks.join(""));
- expect(painted).toContain("Openai-codex");
- expect(painted).toContain("chatgpt.com");
- stdin.emit("data", Buffer.from("q"));
- await done;
- await server.close();
- }, 15_000);
-});
diff --git a/packages/cli/src/recovery/pane-app.ts b/packages/cli/src/recovery/pane-app.ts
deleted file mode 100644
index ee1a4a9b..00000000
--- a/packages/cli/src/recovery/pane-app.ts
+++ /dev/null
@@ -1,521 +0,0 @@
-/**
- * `claudish recovery-pane` — the five-line banner that earns claudish the right
- * to wait.
- *
- * PLAIN ANSI WITH RAW STDIN, NOT OpenTUI. This process exists to paint five
- * lines and read two keys. claudish already boots OpenTUI three times (config
- * TUI, probe TUI, resume picker) and a fourth boot for a banner is
- * disproportionate — it would pay a React reconciler and a renderer handshake
- * on the critical path of a user who is, by construction, already having a bad
- * minute. `team-grid.ts`'s self-contained banner is the established precedent
- * in this repo for exactly this shape.
- *
- * THEMING. `cliAnsi()` is called INSIDE `renderPane`, never at module load:
- * command modules are imported before theme detection runs, so a module-level
- * `const C = cliAnsi()` snapshots the pre-detection (dark) palette forever.
- * That bug class has been found six times in this codebase. The banner FILL is
- * additionally theme-independent by construction — a self-contained mid-dark
- * red with bright-white ink, the same pairing `team-grid.ts` uses — so the one
- * element that must be legible no matter what does not depend on detection
- * succeeding at all. (Measured: magmux DOES answer OSC 11 inside a pane, so
- * detection works; the banner does not rely on it.)
- *
- * WHY A PANE AND NOT AN OVERLAY. Keys typed here reach THIS process's own PTY,
- * not Claude Code's. That is the whole reason the recovery UI is a pane: it
- * dissolves the "how does a keystroke reach the proxy" question instead of
- * working around it. Focus is deliberately NOT stolen — Claude Code accepts
- * messages while a turn is in flight, and capturing those keystrokes into our
- * renderer would be a worse failure than one extra click.
- */
-
-import { cliAnsi } from "../theme/ansi.js";
-import { type RecoverySocketClient, connectRecoverySocket } from "./socket-client.js";
-import {
- PANE_LINGER_MS,
- PROXY_SILENT_MS,
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryClosedFrame,
- type RecoveryEpisodeFrame,
- UI_HEARTBEAT_MS,
-} from "./types.js";
-
-export interface PaneViewState {
- /** The episode being painted, or null before the first frame. */
- frame: RecoveryEpisodeFrame | null;
- /** Set once the episode ends; the pane lingers on this rather than vanishing. */
- closed: RecoveryClosedFrame | null;
- /** EPOCH ms — the same clock `nextAttemptAtMs` and `startedAtMs` are in. */
- nowMs: number;
- /** EPOCH ms of the last frame of any kind, or null. */
- lastFrameAtMs: number | null;
- connected: boolean;
- protocolMismatch: boolean;
- /** A transient line, e.g. after `[r]`. */
- notice: string | null;
-}
-
-function pad(text: string, width: number): string {
- return text.length >= width ? text.slice(0, width) : text + " ".repeat(width - text.length);
-}
-
-/**
- * Truncate to a VISIBLE width, stepping over the escapes.
- *
- * `String.length` counts an SGR escape as eight or nine characters and would
- * cut a line in the middle of one, leaving the rest of the pane painted in
- * whatever colour the fragment happened to start. A split pane is HALF the
- * terminal — measured: `split:"vertical"` took pane 0 from 40 rows to 20 — so
- * "the pane is always wide" is not an assumption this can make.
- */
-/** The SGR sequence starting at `i` (`ESC [ … m`), or null. */
-function matchSgr(text: string, i: number): string | null {
- if (text[i] !== "\x1b" || text[i + 1] !== "[") return null;
- let j = i + 2;
- while (j < text.length) {
- const c = text.charCodeAt(j);
- const isParam = (c >= 0x30 && c <= 0x39) || c === 0x3b; // 0-9 ;
- if (!isParam) break;
- j++;
- }
- return text[j] === "m" ? text.slice(i, j + 1) : null;
-}
-
-/**
- * Drop every SGR sequence, leaving the text.
- *
- * Exported for the tests and for the evidence harness: assertions are about
- * what a person reads, and a capture full of escapes cannot be read by either.
- */
-export function stripAnsi(text: string): string {
- let out = "";
- for (let i = 0; i < text.length; i++) {
- const m = matchSgr(text, i);
- if (m) {
- i += m.length - 1;
- continue;
- }
- out += text[i];
- }
- return out;
-}
-
-export function clipAnsi(text: string, width: number): string {
- let out = "";
- let visible = 0;
- for (let i = 0; i < text.length; i++) {
- const ch = text[i] as string;
- if (ch === "\x1b") {
- // ESC is the subject matter here, so the scan is written by hand rather
- // than as a regex: a regex literal containing the control character trips
- // `noControlCharactersInRegex`, and suppressing a lint rule to parse the
- // one thing this function exists to parse is worse than four lines of
- // explicit scanning.
- const m = matchSgr(text, i);
- if (m) {
- out += m;
- i += m.length - 1;
- continue;
- }
- }
- if (visible >= width) return `${out}\x1b[0m`;
- out += ch;
- visible++;
- }
- return out;
-}
-
-/** `2m 12s`, `47s`, `0s`. Never negative — a countdown that went past reads `0s`. */
-export function formatDuration(ms: number): string {
- const total = Math.max(0, Math.round(ms / 1000));
- if (total < 60) return `${total}s`;
- const m = Math.floor(total / 60);
- const s = total % 60;
- return `${m}m ${String(s).padStart(2, "0")}s`;
-}
-
-/** Greedy wrap. The reason sentence is the one thing that must stay readable. */
-export function wrapText(text: string, width: number): string[] {
- if (width <= 0) return [text];
- const words = text.split(/\s+/).filter(Boolean);
- const lines: string[] = [];
- let line = "";
- for (const w of words) {
- if (!line) {
- line = w;
- } else if (line.length + 1 + w.length <= width) {
- line += ` ${w}`;
- } else {
- lines.push(line);
- line = w;
- }
- }
- if (line) lines.push(line);
- return lines.length > 0 ? lines : [""];
-}
-
-/**
- * Build the banner. PURE — no clock, no terminal, no escape-code state.
- *
- * Pure because it is the only part of this process worth asserting on in a
- * unit test, and because the live evidence (a capture of the real pane) and the
- * test then look at the same function rather than at two implementations that
- * could drift.
- */
-export function renderPane(state: PaneViewState, width: number): string {
- // RENDER TIME. Never at module load.
- const C = cliAnsi();
- const w = Math.max(28, Math.min(width || 80, 200));
- const RED_FILL = "\x1b[1;97;48;2;160;50;70m";
- const GREEN_FILL = "\x1b[1;97;48;2;30;120;70m";
- const lines: string[] = [];
-
- // Every return goes through `finish`, so there is ONE place the pane's width
- // is enforced and no later branch can forget it.
- const finish = (ls: string[]) => ls.map((l) => clipAnsi(l, w)).join("\n");
-
- if (state.protocolMismatch) {
- return finish([
- `${RED_FILL}${pad(" ██ CLAUDISH RECOVERY · version skew", w)}\x1b[0m`,
- `${C.GRAY} This pane speaks protocol v${RECOVERY_PROTOCOL_VERSION}; the proxy does not.${C.RESET}`,
- `${C.GRAY} Restart the session so both halves are the same build.${C.RESET}`,
- "",
- ]);
- }
-
- const ep = state.frame;
- const closed = state.closed;
-
- if (closed) {
- const ok = closed.outcome === "recovered";
- const fill = ok ? GREEN_FILL : RED_FILL;
- const title = ok
- ? ` ██ NETWORK · ${ep?.providerDisplayName ?? "provider"} recovered`
- : ` ██ NETWORK · recovery ended (${closed.outcome.replace(/_/g, " ")})`;
- lines.push(`${fill}${pad(title, w)}\x1b[0m`);
- if (ep) {
- lines.push(
- `${C.GRAY} ${pad(`${ep.attempts} attempts over ${formatDuration(state.nowMs - ep.startedAtMs)}`, w - 2)}${C.RESET}`
- );
- }
- lines.push(`${C.GRAY} this pane closes on its own${C.RESET}`);
- return finish(lines);
- }
-
- if (!ep) {
- lines.push(`${RED_FILL}${pad(" ██ NETWORK · claudish is retrying", w)}\x1b[0m`);
- lines.push(
- `${C.GRAY} ${state.connected ? "waiting for the proxy to describe the failure…" : "connecting to the claudish proxy…"}${C.RESET}`
- );
- return finish(lines);
- }
-
- // 1 — the red line. Provider and the failure kind, in the fill that reads on
- // any page.
- const kindWord =
- ep.kind === "dns" ? "DNS failure" : ep.kind === "refused" ? "refused" : "unreachable";
- lines.push(`${RED_FILL}${pad(` ██ NETWORK · ${ep.providerDisplayName} ${kindWord}`, w)}\x1b[0m`);
-
- // 2 — the reason, in `buildConnectionErrorMessage`'s OWN words. The same
- // sentence the inline 400 would have carried; if the two ever diverge the
- // user is being told two different stories about one fault.
- for (const l of wrapText(ep.reason, w - 4)) lines.push(`${C.STRONG} ${l}${C.RESET}`);
-
- // 3 — host and code, as discrete tokens. The sentence above contains the host
- // too, but embedded in prose; this line is what a capture can be grepped for.
- lines.push(
- `${C.GRAY} host ${C.RESET}${C.CYAN}${ep.host}${C.RESET}${C.GRAY} · ${ep.code ?? "no code"}${
- ep.loopback ? " · local" : ""
- }${C.RESET}`
- );
-
- // 4 — the live line. A countdown while waiting; what we are doing instead
- // while an attempt is actually in flight, because a frozen countdown during a
- // 45-second connect reads as a hang.
- const elapsed = formatDuration(state.nowMs - ep.startedAtMs);
- const when =
- ep.state === "waiting" && ep.nextAttemptAtMs !== null
- ? `${C.YELLOW}next attempt in ${formatDuration(ep.nextAttemptAtMs - state.nowMs)}${C.RESET}`
- : ep.state === "handoff"
- ? `${C.YELLOW}waiting for Claude Code to retry${C.RESET}`
- : `${C.YELLOW}connecting to ${ep.host}…${C.RESET}`;
- const tierNote = ep.tier === 2 ? ` · client retry ${ep.clientRetries}` : "";
- lines.push(
- ` ${when}${C.GRAY} · attempt ${ep.attempts} · ${elapsed} in recovery${tierNote}${C.RESET}`
- );
-
- // 5 — last outcome and the other-episode count.
- const others = ep.otherEpisodes > 0 ? ` · +${ep.otherEpisodes} more` : "";
- const waiters = ep.waiters === 1 ? "1 request held" : `${ep.waiters} requests held`;
- lines.push(`${C.GRAY} last: ${ep.lastOutcome} · ${waiters}${others}${C.RESET}`);
-
- // 6 — the keys. The click instruction is not decoration: magmux forwards
- // mouse clicks to focus a pane, and focus is not stolen — so "click, then
- // press" is the actual interaction and the banner has to say so.
- //
- // A `split` pane is HALF the terminal (measured: 40 rows became 20 and 19),
- // and a user with a narrow window gets a narrow pane. Below 64 columns the
- // full sentence wraps or is cut, so the hint sheds the part a user can work
- // out — where to click — and keeps the part they cannot: which keys exist.
- //
- // `[r]` IS HIDDEN IN `handoff`, because in that state it does nothing:
- // `tryNow` returns early when no attempt timer is armed, and a handed-off
- // episode has no waiters and therefore no timer. Advertising a key that
- // prints "retrying now…" over an episode nobody is retrying is the same
- // class of defect as a give-up that does not give up — a control that
- // appears to act and does not.
- const keys =
- ep.state === "handoff"
- ? `${C.BOLD}[q]${C.RESET} give up`
- : `${C.BOLD}[r]${C.RESET} try now${C.GRAY} · ${C.RESET}${C.BOLD}[q]${C.RESET} give up`;
- lines.push(
- w < 64
- ? `${C.GRAY} ${C.RESET}${keys}`
- : `${C.GRAY} click here or Ctrl-G Tab, then ${C.RESET}${keys}`
- );
-
- if (state.notice) lines.push(`${C.GREEN} ${state.notice}${C.RESET}`);
-
- // The dead-proxy notice. Frame silence is the heuristic, which is exactly why
- // the proxy ticks in EVERY live state — a 45-second connect used to look
- // identical to a dead proxy.
- if (
- state.lastFrameAtMs !== null &&
- state.nowMs - state.lastFrameAtMs > PROXY_SILENT_MS &&
- state.connected
- ) {
- lines.push(
- `${C.RED} the claudish proxy has not reported in ${formatDuration(state.nowMs - state.lastFrameAtMs)}${C.RESET}`
- );
- }
- if (!state.connected) {
- lines.push(`${C.RED} the claudish proxy closed this connection${C.RESET}`);
- }
-
- return finish(lines);
-}
-
-export interface RecoveryPaneOptions {
- socketPath: string;
- stdin?: NodeJS.ReadStream;
- stdout?: NodeJS.WriteStream;
- /** EPOCH ms. Injectable so a test can drive the countdown deterministically. */
- now?: () => number;
- /** Stop after this long with no live episode. Tests pass something small. */
- lingerMs?: number;
-}
-
-/** Parse `--socket <path>` out of a recovery-pane argv slice. */
-export function parsePaneArgs(argv: string[]): { socketPath: string | null } {
- for (let i = 0; i < argv.length; i++) {
- const a = argv[i];
- if (a === "--socket" && argv[i + 1]) return { socketPath: argv[i + 1] as string };
- if (a?.startsWith("--socket=")) return { socketPath: a.slice("--socket=".length) };
- }
- return { socketPath: null };
-}
-
-/**
- * Run the pane until the user quits, the episode ends and the linger expires,
- * or the proxy goes away. Resolves with the process exit code.
- */
-export async function runRecoveryPane(opts: RecoveryPaneOptions): Promise<number> {
- const stdin = opts.stdin ?? process.stdin;
- const stdout = opts.stdout ?? process.stdout;
- const now = opts.now ?? (() => Date.now());
- const lingerMs = opts.lingerMs ?? PANE_LINGER_MS;
-
- const state: PaneViewState = {
- frame: null,
- closed: null,
- nowMs: now(),
- lastFrameAtMs: null,
- connected: true,
- protocolMismatch: false,
- notice: null,
- };
-
- let noticeUntilMs = 0;
- let closedAtMs: number | null = null;
- /** The episode this renderer is heartbeating. Exactly one at a time. */
- let paintedEpisodeId: string | null = null;
- let client: RecoverySocketClient | null = null;
-
- const paint = () => {
- state.nowMs = now();
- if (state.notice && state.nowMs > noticeUntilMs) state.notice = null;
- const width = stdout.columns ?? 80;
- const body = renderPane(state, width);
- // Home, repaint each line clearing to EOL, then clear the rest. No
- // alternate screen: the pane's content stays in magmux's scrollback, which
- // is where a user looks after the fact to find out what happened.
- const painted = body.split("\n").join("\x1b[K\n");
- stdout.write(`\x1b[H${painted}\x1b[K\x1b[J`);
-
- // THE LEASE. Sent after painting, and on this timer rather than on frame
- // arrival — the proxy has nothing to say for the 20–75 s an unreachable
- // connect takes, and a lease renewed by frames would expire in exactly the
- // failure class this feature exists for.
- if (paintedEpisodeId && client?.connected) {
- client.send({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "ack",
- episodeId: paintedEpisodeId,
- paintedAt: state.nowMs,
- });
- }
- };
-
- stdout.write("\x1b[?25l"); // hide the cursor; there is nothing to type into
- const restore = () => {
- try {
- stdout.write("\x1b[?25h\n");
- stdin.setRawMode?.(false);
- stdin.pause();
- } catch {
- /* the terminal went away first */
- }
- };
-
- client = await connectRecoverySocket({
- socketPath: opts.socketPath,
- onFrame: (frame) => {
- state.lastFrameAtMs = now();
- if (frame.type === "closed") {
- if (frame.outcome === "protocol_mismatch") {
- state.protocolMismatch = true;
- paint();
- return;
- }
- if (!state.frame || frame.episodeId === state.frame.episodeId) {
- state.closed = frame;
- closedAtMs = now();
- // Stop heartbeating: this episode is over, and a lease that outlived
- // its episode is the latch this design replaced.
- paintedEpisodeId = null;
- }
- paint();
- return;
- }
- // A new episode supersedes the old one. The renderer heartbeats exactly
- // one episode; the previous one's lease lapses on its own within
- // UI_LEASE_MS with no extra protocol.
- state.frame = frame;
- state.closed = null;
- closedAtMs = null;
- paintedEpisodeId = frame.episodeId;
- paint();
- },
- onClose: () => {
- state.connected = false;
- paint();
- },
- });
-
- if (!client) {
- state.connected = false;
- paint();
- restore();
- return 1;
- }
-
- let done: (code: number) => void = () => {};
- const finished = new Promise<number>((resolve) => {
- done = resolve;
- });
-
- // Raw stdin: the pane's two keys must not need a newline, and the terminal
- // must not echo them into the banner.
- try {
- stdin.setRawMode?.(true);
- } catch {
- /* not a TTY — the banner still renders, the keys simply do nothing */
- }
- // Outside the try: a stdin that cannot go raw can still deliver bytes, and
- // resume() is also what keeps this process's event loop alive.
- stdin.resume();
- stdin.on("data", (buf: Buffer) => {
- const s = buf.toString("utf-8");
- for (const ch of s) {
- if (ch === "\x03") {
- // Ctrl-C in this pane means the same as [q]: the user asked to stop.
- client?.send({ v: RECOVERY_PROTOCOL_VERSION, type: "bye", reason: "user_quit" });
- restore();
- done(0);
- return;
- }
- if (ch === "q" || ch === "Q") {
- client?.send({ v: RECOVERY_PROTOCOL_VERSION, type: "bye", reason: "user_quit" });
- restore();
- done(0);
- return;
- }
- if (ch === "r" || ch === "R" || ch === "\r" || ch === "\n") {
- // Not in `handoff`: there is no armed timer to collapse there, so the
- // key would only print a reassurance that is not true. The banner hides
- // it in that state too — one rule, both places.
- if (state.frame && state.frame.state !== "handoff" && client?.connected) {
- client.send({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "retry_now",
- episodeId: state.frame.episodeId,
- });
- state.notice = "retrying now…";
- noticeUntilMs = now() + 3_000;
- paint();
- }
- }
- }
- });
-
- // ONE timer drives both the countdown and the heartbeat, at UI_HEARTBEAT_MS.
- // They are the same act — "I am still painting this episode" — and splitting
- // them into two timers would let a frozen renderer keep its lease.
- const ticker = setInterval(() => {
- paint();
- if (closedAtMs !== null && now() - closedAtMs > lingerMs) {
- restore();
- done(0);
- }
- if (!state.connected && client && !client.connected) {
- // The proxy is gone. Linger briefly so the last state is readable, then
- // leave rather than sitting on a dead socket forever.
- if (closedAtMs === null) closedAtMs = now();
- }
- }, UI_HEARTBEAT_MS);
- ticker.unref?.();
-
- paint();
- const code = await finished;
- clearInterval(ticker);
- client.close();
- return code;
-}
-
-/** The `claudish recovery-pane` entry point. */
-export async function recoveryPaneCommand(argv: string[]): Promise<void> {
- const { socketPath } = parsePaneArgs(argv);
- if (!socketPath) {
- console.error("Usage: claudish recovery-pane --socket <path>");
- process.exit(2);
- }
- // The pane is its own PROCESS, reached from `index.ts` before `runCli()`, so
- // nothing has published a theme mode for it. `cliAnsi()` is read at paint
- // time — which is right, and which the module-load snapshot trap makes
- // necessary — but it cannot use a mode that was never detected, so the pane
- // stayed on the unknown/classic palette even where OSC 11, `COLORFGBG` or an
- // explicit `CLAUDISH_THEME=light` would have chosen the light one.
- //
- // It runs HERE rather than inside `runRecoveryPane` for the same reason the
- // TUI runs it before `createCliRenderer`: the OSC query puts stdin in raw
- // mode and reads it, so it must finish before the pane's own key handler
- // owns the stream. Bounded at 150 ms, a no-op without a TTY on both ends, and
- // never fatal — a banner with the wrong palette still beats no banner.
- try {
- const { detectAndSetThemeMode } = await import("../theme/theme-mode.js");
- await detectAndSetThemeMode();
- } catch {
- /* the classic palette is a working fallback */
- }
- const code = await runRecoveryPane({ socketPath });
- process.exit(code);
-}
diff --git a/packages/cli/src/recovery/settings.ts b/packages/cli/src/recovery/settings.ts
index 1073d5a2..a0ad257d 100644
--- a/packages/cli/src/recovery/settings.ts
+++ b/packages/cli/src/recovery/settings.ts
@@ -13,7 +13,6 @@
import { ENV } from "../config.js";
import { log } from "../logger.js";
import { readRecoveryEnabled, readRecoveryUi } from "../profile-config.js";
-import { recoveryClock } from "./clock.js";
/** Flag layer. Set once by `cli.ts`'s arg loop; undefined means "not given". */
interface RecoveryFlagOverrides {
@@ -124,52 +123,6 @@ export function recoverySurfaceAllowed(): boolean {
return resolveRecoveryEnabled() && resolveRecoveryUi();
}
-// ─── "The user said stop" ────────────────────────────────────────────────────
-
-/**
- * PROCESS ms until which `[q] give up` suppresses recovery. 0 = not suppressed.
- *
- * ── WHY THIS IS A PROCESS FACT AND NOT THE PANE'S OWN STATE ─────────────────
- *
- * `[q]` used to do two things, both inside the UI manager: suppress re-opening
- * the pane for a minute, and `giveUpAll()` — which iterates the episodes alive
- * AT THAT INSTANT. Nothing recorded that the user had asked recovery to stop,
- * so the next request against the same dead target opened a NEW episode, found
- * the pane suppressed, got no pane and no lease, and then held its socket for
- * the full derived deadline (~4.5 minutes at the default) before answering 400
- * — with the reason legible NOWHERE, which is the state this design calls
- * "strictly worse than the bug this feature exists to remove", reached from the
- * one affordance whose entire purpose is to end it. Before recovery existed
- * those requests failed in milliseconds.
- *
- * It is not an edge case: Claude Code issues concurrent requests during an
- * outage (main loop, title model, subagents), so a new request inside the
- * suppression window is the EXPECTED case. A control that appears to stop
- * something and does not is worse than no control.
- *
- * So the give-up suppresses the HOLD and the SURFACE together, for the same
- * window, from ONE fact — read by `shouldSkipTier1` (the hold) and by
- * `ensureRecoveryUi` (the surface). Two facts would be two things to keep in
- * step, and the asymmetry between them is exactly what created the forbidden
- * state.
- */
-let giveUpUntilPerf = 0;
-
-/** `[q] give up`: suppress recovery until `untilPerf` (PROCESS ms). */
-export function noteRecoveryGaveUp(untilPerf: number): void {
- giveUpUntilPerf = Math.max(giveUpUntilPerf, untilPerf);
-}
-
-/** Is the user's give-up still in force? Reads the clock LIVE — never cached. */
-export function recoveryGiveUpActive(): boolean {
- return giveUpUntilPerf > 0 && recoveryClock().now() < giveUpUntilPerf;
-}
-
-/** Pane teardown, `serve` rebuilds, and tests. */
-export function resetRecoveryGiveUp(): void {
- giveUpUntilPerf = 0;
-}
-
// ─── The deadline ────────────────────────────────────────────────────────────
/**
diff --git a/packages/cli/src/recovery/socket-client.ts b/packages/cli/src/recovery/socket-client.ts
deleted file mode 100644
index 14e8683d..00000000
--- a/packages/cli/src/recovery/socket-client.ts
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * The pane half of the recovery link.
- *
- * Shaped after `team-grid.ts`'s `subscribeToMagmux` — the same NDJSON framing,
- * the same tolerance of malformed lines, and above all the same CONNECT-RETRY
- * LOOP, because the socket appears asynchronously relative to the process that
- * dials it. There the race is magmux creating its socket after spawn; here it
- * is the pane starting ~100 ms after `open_pane` while the proxy is still
- * binding, or — in the normal case — long after. Either way a single
- * `connect()` that fails is indistinguishable from a proxy that is not coming,
- * and the difference is the whole user experience.
- */
-
-import { existsSync } from "node:fs";
-import { type Socket, connect as netConnect } from "node:net";
-import {
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryCommand,
- type RecoveryFrame,
- encodeLine,
- parseFrame,
-} from "./types.js";
-
-export interface RecoverySocketClientOptions {
- socketPath: string;
- onFrame: (frame: RecoveryFrame) => void;
- /** The proxy hung up, or was never there. */
- onClose?: () => void;
- /** Total connect attempts before giving up. 40 × 50 ms ≈ 2 s, as in team-grid. */
- attempts?: number;
- retryDelayMs?: number;
-}
-
-export interface RecoverySocketClient {
- send(cmd: RecoveryCommand): void;
- close(): void;
- readonly connected: boolean;
-}
-
-/**
- * Dial the recovery socket, retrying while it does not yet exist.
- *
- * Resolves null when every attempt failed — the caller decides what to print.
- * It never throws: a pane that crashes on a missing socket replaces a banner
- * with a stack trace in the middle of the user's screen.
- */
-export async function connectRecoverySocket(
- opts: RecoverySocketClientOptions
-): Promise<RecoverySocketClient | null> {
- const attempts = opts.attempts ?? 40;
- const delay = opts.retryDelayMs ?? 50;
- let socket: Socket | null = null;
-
- for (let attempt = 0; attempt < attempts; attempt++) {
- if (existsSync(opts.socketPath)) {
- try {
- socket = await new Promise<Socket>((resolve, reject) => {
- const s = netConnect(opts.socketPath);
- s.once("connect", () => resolve(s));
- s.once("error", reject);
- });
- break;
- } catch {
- /* not ready yet — the server binds after mkdir, so the file can exist
- for a few microseconds before anything is listening on it */
- }
- }
- await new Promise((r) => setTimeout(r, delay));
- }
-
- if (!socket) return null;
-
- const live = socket;
- let open = true;
- let buf = "";
-
- live.on("data", (chunk: Buffer) => {
- buf += chunk.toString("utf-8");
- if (buf.length > 256 * 1024) buf = buf.slice(-4096);
- let nl = buf.indexOf("\n");
- while (nl >= 0) {
- const line = buf.slice(0, nl).trim();
- buf = buf.slice(nl + 1);
- nl = buf.indexOf("\n");
- if (!line) continue;
- const frame = parseFrame(line);
- if (frame) opts.onFrame(frame);
- }
- });
-
- const gone = () => {
- if (!open) return;
- open = false;
- opts.onClose?.();
- };
- live.on("error", gone);
- live.on("close", gone);
- live.on("end", gone);
-
- const client: RecoverySocketClient = {
- get connected() {
- return open;
- },
- send(cmd: RecoveryCommand) {
- if (!open || live.destroyed) return;
- try {
- live.write(encodeLine(cmd));
- } catch {
- /* the proxy went away; `gone` will fire */
- }
- },
- close() {
- open = false;
- try {
- live.end();
- } catch {
- /* already gone */
- }
- },
- };
-
- client.send({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "hello",
- protocol: RECOVERY_PROTOCOL_VERSION,
- pid: process.pid,
- });
-
- return client;
-}
diff --git a/packages/cli/src/recovery/socket-server.test.ts b/packages/cli/src/recovery/socket-server.test.ts
deleted file mode 100644
index fa7d5aaf..00000000
--- a/packages/cli/src/recovery/socket-server.test.ts
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * The recovery socket, over a REAL unix socket with REAL clients.
- *
- * No `mock.module()` anywhere: this repo's standing rule is that mocking shared
- * infrastructure bleeds across sibling test files in Bun's module registry, and
- * `node:net` is about as shared as infrastructure gets. The socket is cheap
- * enough to use for real.
- */
-
-import { afterEach, describe, expect, test } from "bun:test";
-import { existsSync, statSync } from "node:fs";
-import { connect } from "node:net";
-import { dirname } from "node:path";
-import {
- type RecoverySocketServer,
- createRecoverySocketDir,
- recoverySocketPathIn,
- startRecoverySocketServer,
-} from "./socket-server.js";
-import {
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryCommand,
- type RecoveryEpisodeFrame,
- type RecoveryFrame,
- encodeLine,
- parseFrame,
-} from "./types.js";
-
-const EPISODE: RecoveryEpisodeFrame = {
- v: RECOVERY_PROTOCOL_VERSION,
- type: "episode",
- episodeId: "ep-replay",
- state: "attempting",
- tier: 1,
- providerDisplayName: "Openai-codex",
- host: "chatgpt.com",
- endpoint: "https://chatgpt.com/backend-api/codex/responses",
- kind: "unreachable",
- code: "ConnectionClosed",
- loopback: false,
- reason: "Cannot reach Openai-codex at https://chatgpt.com/backend-api/codex/responses.",
- attempts: 3,
- clientRetries: 0,
- startedAtMs: 1_700_000_000_000,
- nextAttemptAtMs: null,
- lastOutcome: "ConnectionClosed after 11ms",
- waiters: 1,
- otherEpisodes: 0,
-};
-
-/** A raw NDJSON client, so the test exercises the same bytes the pane does. */
-function rawClient(path: string): Promise<{
- frames: RecoveryFrame[];
- send: (line: string) => void;
- end: () => void;
-}> {
- return new Promise((resolve, reject) => {
- const s = connect(path);
- const frames: RecoveryFrame[] = [];
- let buf = "";
- s.on("data", (chunk: Buffer) => {
- buf += chunk.toString();
- let nl = buf.indexOf("\n");
- while (nl >= 0) {
- const line = buf.slice(0, nl);
- buf = buf.slice(nl + 1);
- nl = buf.indexOf("\n");
- const f = parseFrame(line);
- if (f) frames.push(f);
- }
- });
- s.once("connect", () =>
- resolve({
- frames,
- send: (line: string) => s.write(line),
- end: () => s.destroy(),
- })
- );
- s.once("error", reject);
- });
-}
-
-const settle = (ms = 60) => new Promise((r) => setTimeout(r, ms));
-
-let server: RecoverySocketServer | null = null;
-
-afterEach(async () => {
- await server?.close();
- server = null;
-});
-
-describe("recovery socket server", () => {
- test("replays the current episode to a client that connects LATE", async () => {
- // The pane starts ~100ms after `open_pane`, which is always after the
- // episode began. Without a replay its first paint is blank, and that blank
- // is the first thing a user ever sees of this feature.
- const path = recoverySocketPathIn(createRecoverySocketDir());
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => EPISODE,
- onCommand: () => {},
- tickMs: 0,
- });
-
- const client = await rawClient(path);
- await settle();
- expect(client.frames).toHaveLength(1);
- expect(client.frames[0]).toEqual(EPISODE);
- client.end();
- });
-
- test("ticks in EVERY live state, including `attempting` with no state change", async () => {
- // The state below never changes and no broadcast is ever called. Frames
- // must arrive anyway: a 45-second connect is 45 seconds of silence
- // otherwise, which looks identical to a dead proxy to the pane's own
- // liveness heuristic.
- const path = recoverySocketPathIn(createRecoverySocketDir());
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => ({ ...EPISODE, state: "attempting", nextAttemptAtMs: null }),
- onCommand: () => {},
- tickMs: 25,
- });
- const client = await rawClient(path);
- await settle(180);
- // 1 replay + several ticks.
- expect(client.frames.length).toBeGreaterThanOrEqual(4);
- expect(client.frames.every((f) => f.type === "episode" && f.state === "attempting")).toBe(true);
- client.end();
- });
-
- test("a null snapshot suppresses the replay and the tick rather than sending junk", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => null,
- onCommand: () => {},
- tickMs: 25,
- });
- const client = await rawClient(path);
- await settle(120);
- expect(client.frames).toHaveLength(0);
- client.end();
- });
-
- test("routes ack, retry_now and bye; ignores malformed and skewed lines", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- const seen: RecoveryCommand[] = [];
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => null,
- onCommand: (cmd) => seen.push(cmd),
- tickMs: 0,
- });
- const client = await rawClient(path);
- client.send(
- encodeLine({ v: 1, type: "ack", episodeId: "ep-replay", paintedAt: 1 }) +
- encodeLine({ v: 1, type: "retry_now", episodeId: "ep-replay" }) +
- "{bad json}\n" +
- `${JSON.stringify({ v: 9, type: "ack", episodeId: "ep-replay" })}\n` +
- encodeLine({ v: 1, type: "bye", reason: "user_quit" })
- );
- await settle();
- expect(seen.map((c) => c.type)).toEqual(["ack", "retry_now", "bye"]);
- client.end();
- });
-
- test("a client on the WRONG protocol is told so and dropped", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- const seen: RecoveryCommand[] = [];
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => null,
- onCommand: (cmd) => seen.push(cmd),
- tickMs: 0,
- });
- const client = await rawClient(path);
- client.send(`${JSON.stringify({ v: 9, type: "hello", protocol: 9, pid: 1 })}\n`);
- await settle();
- expect(client.frames).toHaveLength(1);
- expect(client.frames[0]).toMatchObject({ type: "closed", outcome: "protocol_mismatch" });
- // And the command is NOT delivered — a build we cannot talk to must not be
- // able to drive the ladder.
- expect(seen).toHaveLength(0);
- });
-
- test("the directory is 0700, the socket 0600, and both are gone after close", async () => {
- const dir = createRecoverySocketDir();
- const path = recoverySocketPathIn(dir);
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => null,
- onCommand: () => {},
- tickMs: 0,
- });
- expect(statSync(dirname(path)).mode & 0o777).toBe(0o700);
- expect(statSync(path).mode & 0o777).toBe(0o600);
- await server.close();
- server = null;
- expect(existsSync(path)).toBe(false);
- expect(existsSync(dir)).toBe(false);
- });
-
- test("two directories from two calls never collide", async () => {
- const a = createRecoverySocketDir();
- const b = createRecoverySocketDir();
- expect(a).not.toBe(b);
- // 24 hex characters of randomness — not the proxy port, which is printed at
- // startup and would make the path guessable by something not looking for us.
- expect(a).toMatch(/^\/tmp\/claudish-recovery-[0-9a-f]{24}$/);
- });
-
- test("disconnects are reported, and broadcasting afterwards does not throw", async () => {
- const path = recoverySocketPathIn(createRecoverySocketDir());
- let gone = 0;
- server = await startRecoverySocketServer({
- socketPath: path,
- snapshot: () => null,
- onCommand: () => {},
- onDisconnect: () => {
- gone++;
- },
- tickMs: 0,
- });
- const client = await rawClient(path);
- await settle();
- expect(server.clientCount()).toBe(1);
- client.end();
- await settle();
- expect(gone).toBe(1);
- expect(server.clientCount()).toBe(0);
- server.broadcast(EPISODE); // must be a no-op, not a throw
- });
-});
diff --git a/packages/cli/src/recovery/socket-server.ts b/packages/cli/src/recovery/socket-server.ts
deleted file mode 100644
index 91374b1b..00000000
--- a/packages/cli/src/recovery/socket-server.ts
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * The proxy half of the recovery link: an NDJSON server on a unix socket.
- *
- * WHY A SOCKET AND NOT THE MAGMUX CHANNEL. magmux's `send` writes KEYSTROKES
- * into a pane, and the receiving program reads them as user input. Using it as
- * a data bus would make the renderer parse one stream for both proxy state and
- * the user's `[r]`/`[q]` — two protocols on one fd, with this repo's own
- * paste-detection trap waiting at the bottom of it.
- *
- * WHY NOT A FILE. No poller exists to read one (`docs/usage/magmux.md`'s 500 ms
- * status-bar poller was removed in `168c814` and the docs were never
- * corrected), building one buys 500 ms granularity and whole-file rewrite
- * races, and decisively there is no upstream path for `[r] try now`.
- *
- * WHY `/tmp` AND NOT `~/.claudish`. macOS `sun_path` is 104 bytes, `$TMPDIR` on
- * macOS is `/var/folders/<28 chars>/T/`, and `~` is longer still for many
- * users. `/tmp` is also magmux's own convention for its control socket.
- *
- * WHAT THIS DEFENDS AND WHAT IT DOES NOT. The directory is 0700 with an
- * unguessable name and the socket is 0600, which excludes other uids. It is NOT
- * a boundary against a SAME-UID process and no unix socket can be: that process
- * can read our argv, read the pane's argv, or attach a debugger. Saying
- * otherwise would be theatre. What is defended is the consequence — a forged
- * client cannot manufacture the one forbidden state (a retryable status with no
- * banner), because the lease additionally requires that the proxy ITSELF opened
- * a pane, which is a fact no client can cause.
- */
-
-import { randomBytes } from "node:crypto";
-import { chmodSync, existsSync, mkdirSync, rmSync, unlinkSync } from "node:fs";
-import { type Server, type Socket, createServer } from "node:net";
-import { dirname, join } from "node:path";
-import { log } from "../logger.js";
-import {
- FRAME_TICK_MS,
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryCommand,
- type RecoveryEpisodeFrame,
- type RecoveryFrame,
- encodeLine,
- parseCommand,
-} from "./types.js";
-
-export interface RecoverySocketClientHandle {
- readonly id: number;
- /** The pid the client claimed in its `hello`, once it has sent one. */
- pid: number | null;
- send(frame: RecoveryFrame): void;
- close(): void;
-}
-
-export interface RecoverySocketServerOptions {
- socketPath: string;
- /**
- * The frame a NEWLY CONNECTED client must see immediately, and the frame the
- * tick broadcasts. Null means there is nothing live to paint.
- *
- * REPLAY-ON-CONNECT is what makes a late pane correct rather than blank. The
- * pane starts roughly 100 ms after `open_pane` and the episode began before
- * that; without a replay its first paint would be empty until the next tick,
- * which is the first thing a user sees of this feature.
- */
- snapshot: () => RecoveryEpisodeFrame | null;
- onCommand: (cmd: RecoveryCommand, client: RecoverySocketClientHandle) => void;
- onDisconnect?: (client: RecoverySocketClientHandle) => void;
- /**
- * Broadcast cadence. A tick fires in EVERY live state, not only while
- * `waiting`: the banner has to say "connecting…" during a 45 s attempt, and
- * the pane's dead-proxy heuristic is frame silence, so a quiet `attempting`
- * phase would make a healthy proxy look dead to its own renderer.
- */
- tickMs?: number;
-}
-
-export interface RecoverySocketServer {
- readonly path: string;
- broadcast(frame: RecoveryFrame): void;
- clientCount(): number;
- close(): Promise<void>;
-}
-
-/**
- * A private directory for one process's recovery socket.
- *
- * The name carries 96 bits of randomness rather than the proxy port, which is
- * printed at startup and would make the path guessable — and while a same-uid
- * process can find it anyway, an unguessable name costs one line and removes
- * the only version of this that is trivially findable by something that is NOT
- * looking for us.
- */
-export function createRecoverySocketDir(): string {
- const dir = join("/tmp", `claudish-recovery-${randomBytes(12).toString("hex")}`);
- mkdirSync(dir, { mode: 0o700, recursive: true });
- return dir;
-}
-
-/** The socket path inside a directory from `createRecoverySocketDir()`. */
-export function recoverySocketPathIn(dir: string): string {
- return join(dir, "r.sock");
-}
-
-/** Remove the socket and, if we made it, its private directory. Never throws. */
-export function cleanupRecoverySocket(socketPath: string): void {
- try {
- if (existsSync(socketPath)) unlinkSync(socketPath);
- } catch {
- /* already gone */
- }
- try {
- const dir = dirname(socketPath);
- if (dir.startsWith("/tmp/claudish-recovery-")) rmSync(dir, { recursive: true, force: true });
- } catch {
- /* already gone */
- }
-}
-
-export async function startRecoverySocketServer(
- opts: RecoverySocketServerOptions
-): Promise<RecoverySocketServer> {
- const { socketPath } = opts;
- const tickMs = opts.tickMs ?? FRAME_TICK_MS;
- const clients = new Map<number, { socket: Socket; handle: RecoverySocketClientHandle }>();
- let nextId = 1;
- let closed = false;
-
- // A stale socket file from a SIGKILLed predecessor makes `listen` fail with
- // EADDRINUSE even though nothing is listening. The directory name is unique
- // per process, so anything already at this path is debris by construction.
- try {
- if (existsSync(socketPath)) unlinkSync(socketPath);
- } catch {
- /* best effort */
- }
-
- const server: Server = createServer((socket: Socket) => {
- if (closed) {
- socket.destroy();
- return;
- }
- const id = nextId++;
- const handle: RecoverySocketClientHandle = {
- id,
- pid: null,
- send(frame: RecoveryFrame) {
- if (socket.destroyed) return;
- try {
- socket.write(encodeLine(frame));
- } catch {
- /* the pane went away mid-write; the close handler cleans up */
- }
- },
- close() {
- try {
- socket.end();
- } catch {
- /* already gone */
- }
- },
- };
- clients.set(id, { socket, handle });
-
- let buf = "";
- socket.on("data", (chunk: Buffer) => {
- buf += chunk.toString("utf-8");
- // Bound the buffer. A client that never sends a newline must not be able
- // to grow the proxy's heap without limit.
- if (buf.length > 64 * 1024) buf = buf.slice(-1024);
- let nl = buf.indexOf("\n");
- while (nl >= 0) {
- const line = buf.slice(0, nl).trim();
- buf = buf.slice(nl + 1);
- nl = buf.indexOf("\n");
- if (!line) continue;
- const cmd = parseCommand(line);
- if (!cmd) continue;
- if (cmd.type === "hello") {
- handle.pid = typeof cmd.pid === "number" ? cmd.pid : null;
- if (cmd.protocol !== RECOVERY_PROTOCOL_VERSION) {
- // Tell it, then drop it. A renderer that cannot read our frames must
- // print one line about version skew rather than paint garbage.
- handle.send({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "closed",
- episodeId: "",
- outcome: "protocol_mismatch",
- });
- handle.close();
- return;
- }
- }
- opts.onCommand(cmd, handle);
- }
- });
-
- const gone = () => {
- if (!clients.delete(id)) return;
- opts.onDisconnect?.(handle);
- };
- socket.on("error", gone);
- socket.on("close", gone);
- socket.on("end", gone);
-
- // REPLAY. Before anything else this client could possibly ask for.
- const current = opts.snapshot();
- if (current) handle.send(current);
- });
-
- await new Promise<void>((resolve, reject) => {
- server.once("error", reject);
- server.listen(socketPath, () => {
- server.removeListener("error", reject);
- resolve();
- });
- });
-
- try {
- chmodSync(socketPath, 0o600);
- } catch {
- /* the 0700 directory is the real gate; a chmod failure does not open it */
- }
-
- let ticker: ReturnType<typeof setInterval> | null = null;
- if (tickMs > 0) {
- ticker = setInterval(() => {
- if (closed || clients.size === 0) return;
- const frame = opts.snapshot();
- if (!frame) return;
- for (const { handle } of clients.values()) handle.send(frame);
- }, tickMs);
- // The proxy must still be able to exit. A recovery ticker that holds the
- // event loop open would turn every recovered outage into a hung process.
- ticker.unref?.();
- }
-
- const onExit = () => {
- cleanupRecoverySocket(socketPath);
- };
- process.on("exit", onExit);
-
- log(`[Recovery] socket listening at ${socketPath}`);
-
- return {
- path: socketPath,
- broadcast(frame: RecoveryFrame) {
- if (closed) return;
- for (const { handle } of clients.values()) handle.send(frame);
- },
- clientCount: () => clients.size,
- async close() {
- if (closed) return;
- closed = true;
- if (ticker) clearInterval(ticker);
- for (const { socket } of clients.values()) {
- try {
- socket.destroy();
- } catch {
- /* already gone */
- }
- }
- clients.clear();
- await new Promise<void>((resolve) => server.close(() => resolve()));
- process.removeListener("exit", onExit);
- cleanupRecoverySocket(socketPath);
- },
- };
-}
diff --git a/packages/cli/src/recovery/types.test.ts b/packages/cli/src/recovery/types.test.ts
deleted file mode 100644
index 0b47013e..00000000
--- a/packages/cli/src/recovery/types.test.ts
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * The wire contract.
- *
- * Worth its own file because both processes depend on it and NEITHER can
- * detect a breakage in the other: a proxy that emits a frame the pane silently
- * drops shows a blank banner, and a pane that sends a command the proxy silently
- * drops holds no lease. Both failures are silent by construction, so the
- * parsers' refusals are what has to be pinned.
- */
-
-import { describe, expect, test } from "bun:test";
-import {
- RECOVERY_PROTOCOL_VERSION,
- type RecoveryEpisodeFrame,
- UI_HEARTBEAT_MS,
- UI_LEASE_MS,
- encodeLine,
- parseCommand,
- parseFrame,
-} from "./types.js";
-
-const frame = (over: Partial<RecoveryEpisodeFrame> = {}): RecoveryEpisodeFrame => ({
- v: RECOVERY_PROTOCOL_VERSION,
- type: "episode",
- episodeId: "ep-1",
- state: "waiting",
- tier: 1,
- providerDisplayName: "Openai-codex",
- host: "chatgpt.com",
- endpoint: "https://chatgpt.com/backend-api/codex/responses",
- kind: "unreachable",
- code: "ConnectionClosed",
- loopback: false,
- reason: "Cannot reach Openai-codex at https://chatgpt.com/backend-api/codex/responses.",
- attempts: 5,
- clientRetries: 0,
- startedAtMs: 1_789_234_567_890,
- nextAttemptAtMs: 1_789_234_627_890,
- lastOutcome: "ConnectionClosed after 11ms",
- waiters: 2,
- otherEpisodes: 0,
- ...over,
-});
-
-describe("recovery wire contract", () => {
- test("an episode frame round-trips through one NDJSON line", () => {
- const line = encodeLine(frame());
- expect(line.endsWith("\n")).toBe(true);
- expect(line.indexOf("\n")).toBe(line.length - 1); // exactly one, at the end
- expect(parseFrame(line)).toEqual(frame());
- });
-
- test("a frame from a DIFFERENT protocol version is refused, not guessed at", () => {
- const skewed = encodeLine({ ...frame(), v: 2 });
- expect(parseFrame(skewed)).toBeNull();
- });
-
- test("malformed JSON and unknown types parse to null rather than throwing", () => {
- expect(parseFrame("{not json")).toBeNull();
- expect(parseFrame("null")).toBeNull();
- expect(parseFrame(JSON.stringify({ v: 1, type: "wat", episodeId: "x" }))).toBeNull();
- expect(parseFrame(JSON.stringify({ v: 1, type: "episode" }))).toBeNull(); // no episodeId
- });
-
- test("the SKEW NOTICE parses at the wrong version — it is the one frame that must", () => {
- // The mirror image of `hello` below, and for the same reason. The proxy
- // answers a mismatched pane with a `closed` / `protocol_mismatch` frame
- // carrying the PROXY'S version, so a blanket version gate dropped the very
- // message addressed to the build that cannot read the others:
- // `PaneViewState.protocolMismatch` was unreachable in a real skew and the
- // pane printed "the claudish proxy closed this connection" instead — a safe
- // failure with the wrong diagnosis, on the one day it mattered.
- const notice = JSON.stringify({
- v: 99,
- type: "closed",
- episodeId: "",
- outcome: "protocol_mismatch",
- });
- const parsed = parseFrame(notice);
- expect(parsed).not.toBeNull();
- expect(parsed?.type).toBe("closed");
- expect((parsed as { outcome?: string }).outcome).toBe("protocol_mismatch");
-
- // And ONLY that outcome. A skewed `closed` frame of any other kind is still
- // refused — the exemption is for the notice, not for the type.
- expect(
- parseFrame(JSON.stringify({ v: 99, type: "closed", episodeId: "e", outcome: "recovered" }))
- ).toBeNull();
- });
-
- test("`hello` parses even at the WRONG version — the mismatch must be answerable", () => {
- // The server cannot tell a skewed client it is skewed if the one message it
- // needs to answer is the one it drops.
- const cmd = parseCommand(JSON.stringify({ v: 99, type: "hello", protocol: 99, pid: 7 }));
- expect(cmd).not.toBeNull();
- expect(cmd?.type).toBe("hello");
- });
-
- test("every OTHER command is version-gated", () => {
- expect(parseCommand(JSON.stringify({ v: 2, type: "ack", episodeId: "ep-1" }))).toBeNull();
- expect(parseCommand(JSON.stringify({ v: 2, type: "retry_now", episodeId: "ep-1" }))).toBeNull();
- expect(parseCommand(JSON.stringify({ v: 2, type: "bye", reason: "user_quit" }))).toBeNull();
- });
-
- test("ack, retry_now and bye parse at the right version", () => {
- expect(
- parseCommand(JSON.stringify({ v: 1, type: "ack", episodeId: "ep-1", paintedAt: 5 }))
- ).toEqual({ v: 1, type: "ack", episodeId: "ep-1", paintedAt: 5 });
- expect(parseCommand(JSON.stringify({ v: 1, type: "retry_now", episodeId: "ep-1" }))).toEqual({
- v: 1,
- type: "retry_now",
- episodeId: "ep-1",
- });
- expect(parseCommand(JSON.stringify({ v: 1, type: "bye", reason: "user_quit" }))).toEqual({
- v: 1,
- type: "bye",
- reason: "user_quit",
- });
- });
-
- test("the heartbeat leaves an order of magnitude of headroom under the lease", () => {
- // Nine consecutive dropped heartbeats must be survivable. If these two ever
- // come within 2x of each other, one slow paint revokes a live banner's
- // lease and the exhaustion arm answers as though nothing were on screen.
- expect(UI_LEASE_MS / UI_HEARTBEAT_MS).toBeGreaterThanOrEqual(10);
- });
-});
diff --git a/packages/cli/src/recovery/types.ts b/packages/cli/src/recovery/types.ts
index c19ecbcd..335a7081 100644
--- a/packages/cli/src/recovery/types.ts
+++ b/packages/cli/src/recovery/types.ts
@@ -1,50 +1,30 @@
/**
- * The recovery wire contract — the ONE module both processes import.
+ * The shape of one outage, as the banner draws it.
*
- * It has no imports of its own, deliberately. The proxy links it into a Hono
- * request path that has already loaded the whole adapter stack; the pane links
- * it into a process whose entire job is to paint five lines and whose startup
- * cost is on the critical path of a user already having a bad minute. Anything
- * this file imported would be paid for twice, and the second time for nothing.
+ * No imports, deliberately: the coordinator builds this and `magmux-ui.ts`
+ * reads it, and neither should drag the other's dependencies along.
*
- * FRAMES ARE FAT ON PURPOSE. Every `episode` frame carries the whole banner —
- * provider, host, endpoint, reason sentence, counters, the next-attempt instant
- * — rather than a delta or an id to look up. The pane has nothing to call back
- * to: there is no request/response direction from renderer to proxy beyond the
- * three commands below, and adding one would mean the banner could be blank
- * while a round trip was in flight. A frame is ~400 bytes at 1 Hz.
+ * FRAMES ARE FAT ON PURPOSE. One frame carries the whole banner — provider,
+ * host, endpoint, reason sentence, counters, the next-attempt instant — so the
+ * banner can be drawn from a single value with nothing to look up.
*
- * EVERY TIMESTAMP ON THE WIRE IS EPOCH MS (`Date.now()`). The two processes
- * have different `performance.now()` origins, so a process-ms value would
- * render as an arbitrary number in the pane with no error anywhere. The rule
- * that makes this checkable: a field crossing this boundary is named `…AtMs`
- * and is epoch; a field used in an interval comparison is named `…AtPerf` and
- * never leaves its process.
+ * TIMESTAMPS HERE ARE EPOCH MS (`Date.now()`), named `…AtMs`. Everything the
+ * coordinator measures intervals with is process ms (`performance.now()`) and is
+ * converted exactly once, in `renderableEpisodeFrame()`. A process-ms instant
+ * drawn as a countdown is an arbitrary number with no error anywhere.
*/
-/**
- * Bumped when a frame or command changes shape incompatibly.
- *
- * Both processes are the same binary in practice, but not always: a
- * `team --grid` launches N claudish processes that may be different builds
- * against one magmux, and a global install can be mid-update. A mismatched
- * client is told so and prints one line, instead of rendering a banner with
- * fields it does not understand.
- */
-export const RECOVERY_PROTOCOL_VERSION = 1;
-
-/** The live states. `recovered`/`abandoned` arrive as a `closed` frame instead. */
+/** The live states. A recovered or abandoned episode is no longer drawn. */
export type RecoveryFrameState = "attempting" | "waiting" | "handoff";
export interface RecoveryEpisodeFrame {
- v: number;
type: "episode";
episodeId: string;
state: RecoveryFrameState;
/** DISPLAY ONLY — the highest tier any waiter in this episode has reached. */
tier: 1 | 2;
providerDisplayName: string;
- /** `chatgpt.com` — the host alone, for the banner's first line. */
+ /** `chatgpt.com` — the host alone, for the banner. */
host: string;
/** The endpoint that actually failed; the AUTH host on an auth-path failure. */
endpoint: string;
@@ -65,189 +45,18 @@ export interface RecoveryEpisodeFrame {
otherEpisodes: number;
}
-export type RecoveryClosedOutcome =
- | "recovered"
- | "handoff"
- | "client_gone"
- | "gave_up"
- | "grace_expired"
- | "shutdown"
- | "protocol_mismatch";
-
-export interface RecoveryClosedFrame {
- v: number;
- type: "closed";
- episodeId: string;
- outcome: RecoveryClosedOutcome;
-}
-
-export type RecoveryFrame = RecoveryEpisodeFrame | RecoveryClosedFrame;
-
-/** Version negotiation only. It grants NOTHING — see `ack`. */
-export interface RecoveryHelloCommand {
- v: number;
- type: "hello";
- protocol: number;
- pid: number;
-}
-
-/**
- * The lease heartbeat, and the single most consequential message in the
- * protocol.
- *
- * It is sent AFTER PAINTING, and then once per heartbeat interval for as long
- * as the renderer is still painting THIS `episodeId` — whether or not a frame
- * arrived in the meantime. It is not a frame receipt. Renewing the lease from
- * frame arrivals ties the renderer's apparent liveness to the PROXY's emission
- * cadence, and a connect against an unreachable host takes 20–75 s (measured:
- * `192.0.2.1` = 75 005 ms) during which the proxy has nothing new to say. The
- * lease would then be invalid at exhaustion for exactly the failure class this
- * feature exists for, with a live, painted banner on screen.
- */
-export interface RecoveryAckCommand {
- v: number;
- type: "ack";
- episodeId: string;
- /** EPOCH ms of the paint this ack attests to. */
- paintedAt: number;
-}
-
-/** `[r]` — collapse the current wait. Ignored unless it names the live episode. */
-export interface RecoveryRetryNowCommand {
- v: number;
- type: "retry_now";
- episodeId: string;
-}
-
-/** `[q]` — the user said stop. Every live episode gives up and answers inline. */
-export interface RecoveryByeCommand {
- v: number;
- type: "bye";
- reason: string;
-}
-
-export type RecoveryCommand =
- | RecoveryHelloCommand
- | RecoveryAckCommand
- | RecoveryRetryNowCommand
- | RecoveryByeCommand;
-
-/** One NDJSON line, newline included. */
-export function encodeLine(value: RecoveryFrame | RecoveryCommand): string {
- return `${JSON.stringify(value)}\n`;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
- return typeof value === "object" && value !== null;
-}
-
-/**
- * Parse one NDJSON line into a frame, or null.
- *
- * Null for malformed JSON, for an unknown `type`, and for a `v` this build does
- * not speak — a reader that guesses at an unknown shape is how a version skew
- * becomes a rendering bug instead of a one-line notice.
- *
- * ── THE ONE EXEMPTION, AND WHY IT IS NOT AN INCONSISTENCY ───────────────────
- *
- * The skew NOTICE itself is exempt, exactly as `parseCommand` exempts `hello`
- * for the mirror-image reason. The proxy answers a mismatched pane with a
- * `closed` / `protocol_mismatch` frame carrying the PROXY'S version — so a
- * blanket version gate dropped the very message addressed to the build that
- * cannot read the others, `PaneViewState.protocolMismatch` was unreachable in
- * any real skew, and the pane printed "the claudish proxy closed this
- * connection" instead. Mixed-build grids and sessions started during an update
- * failed safely with the WRONG diagnosis: the skew notice did not appear on the
- * one day it was needed.
- *
- * Exempting it is sound where exempting a data frame would not be. It carries
- * no payload to misread — a type, an empty `episodeId` and a terminal outcome —
- * and it is the LAST thing on the connection, so nothing can follow it that a
- * mis-parse could corrupt.
- */
-export function parseFrame(line: string): RecoveryFrame | null {
- let parsed: unknown;
- try {
- parsed = JSON.parse(line);
- } catch {
- return null;
- }
- if (!isRecord(parsed)) return null;
- if (parsed.type === "closed" && parsed.outcome === "protocol_mismatch") {
- return {
- v: RECOVERY_PROTOCOL_VERSION,
- type: "closed",
- episodeId: typeof parsed.episodeId === "string" ? parsed.episodeId : "",
- outcome: "protocol_mismatch",
- } as unknown as RecoveryClosedFrame;
- }
- if (parsed.v !== RECOVERY_PROTOCOL_VERSION) return null;
- if (parsed.type === "episode" && typeof parsed.episodeId === "string") {
- return parsed as unknown as RecoveryEpisodeFrame;
- }
- if (parsed.type === "closed" && typeof parsed.episodeId === "string") {
- return parsed as unknown as RecoveryClosedFrame;
- }
- return null;
-}
-
/**
- * Parse one NDJSON line into a command.
- *
- * A `hello` with the WRONG protocol still parses — the server has to be able to
- * tell a mismatched client so, and it cannot do that if the message it needs to
- * answer is the one it drops. Every other command is version-gated.
- */
-export function parseCommand(line: string): RecoveryCommand | null {
- let parsed: unknown;
- try {
- parsed = JSON.parse(line);
- } catch {
- return null;
- }
- if (!isRecord(parsed)) return null;
- if (parsed.type === "hello" && typeof parsed.protocol === "number") {
- return parsed as unknown as RecoveryHelloCommand;
- }
- if (parsed.v !== RECOVERY_PROTOCOL_VERSION) return null;
- if (parsed.type === "ack" && typeof parsed.episodeId === "string") {
- return parsed as unknown as RecoveryAckCommand;
- }
- if (parsed.type === "retry_now" && typeof parsed.episodeId === "string") {
- return parsed as unknown as RecoveryRetryNowCommand;
- }
- if (parsed.type === "bye") {
- return { v: RECOVERY_PROTOCOL_VERSION, type: "bye", reason: String(parsed.reason ?? "") };
- }
- return null;
-}
-
-// ─── Timings shared by both processes ────────────────────────────────────────
-
-/**
- * A lease lives this long after the last heartbeat naming its episode.
+ * A lease lives this long after magmux last acknowledged a banner write for its
+ * episode.
*
* NEVER compare this against a per-attempt connect cap. They measure unrelated
- * things — one the renderer's liveness, the other a network ceiling — and the
- * whole fix for the round-2 CRITICAL is that they are independent.
+ * things — one the renderer's liveness, the other a network ceiling — and they
+ * must stay independent.
*/
export const UI_LEASE_MS = 10_000;
-/** The pane heartbeats this often while painting. 10× headroom under the lease. */
-export const UI_HEARTBEAT_MS = 1_000;
-
-/** The proxy emits an `episode` frame this often in EVERY live state. */
-export const FRAME_TICK_MS = 1_000;
-
/**
- * No frame for this long and the pane says so.
- *
- * Strictly greater than `FRAME_TICK_MS` by enough to survive a scheduler hiccup
- * and a slow paint: the notice means "the proxy is gone", and saying that while
- * a healthy proxy is mid-connect is the symptom the every-live-state tick
- * exists to prevent.
+ * The banner is redrawn this often while any outage is live. It is both the
+ * countdown's resolution and the lease's renewal, and 10× inside `UI_LEASE_MS`.
*/
-export const PROXY_SILENT_MS = 6_000;
-
-/** Keep the pane up this long after the last episode ends, so it cannot flap. */
-export const PANE_LINGER_MS = 30_000;
+export const FRAME_TICK_MS = 1_000;