π OpenHands: AI-Driven Development
fix: fail the launch when the automation backend exits during startup (#18311)
Engel Nyst33fcb2e1b394
diff --git a/.agents/skills/local-stack-runtime/references/guide.md b/.agents/skills/local-stack-runtime/references/guide.mdindex d9371c779d..2c753d8e0b 100644--- a/.agents/skills/local-stack-runtime/references/guide.md+++ b/.agents/skills/local-stack-runtime/references/guide.md@@ -103,6 +103,7 @@ you are running inside of β NOT the automation backend.- `npm run dev` runs the full local stack via `uvx` (agent-server + automation backend + Vite dev server + ingress proxy) with no Docker dependency. `npm run dev:static` does the same but serves a production build of the frontend instead of the Vite dev server.- `scripts/dev-with-automation.mjs` runs the full stack: agent-server, automation backend (both via uvx), frontend server, and ingress proxy. It defaults to Vite when run directly, supports `--static` for an existing build, and supports `--dynamic` so wrappers that default static can opt back into Vite. Uses a standalone ingress proxy (`scripts/ingress.mjs`) to route traffic:- Keep `SIGINT`, `SIGTERM`, and `SIGHUP` wired through the coordinated shutdown handler. Services run in detached process groups on POSIX, so cleanup must use `signalProcessTree()` rather than signaling only the direct child; regression coverage lives in `__tests__/scripts/dev-with-automation.test.ts`.+ - `main()` prints the banner without waiting for the automation backend, then watches `/api/automation/health` in the background. If automation exits before it is healthy (a failed database migration, #18299), the launcher stops every service it started and calls `onAutomationStartupFailure`: the CLI exits 1, and Electron replaces the splash or main window with its startup-failure screen. A slow start only logs a timeout. The Docker entrypoint still tolerates a dead automation backend.- `/api/automation/*` β automation backend (:18001)- `/api/*`, `/sockets`, etc. β agent server (:18000)- `/*` (default) β frontend server (:3001), either Vite or static depending on launcher modediff --git a/__tests__/scripts/dev-with-automation.test.ts b/__tests__/scripts/dev-with-automation.test.tsindex d51044ef5b..8f89284e68 100644--- a/__tests__/scripts/dev-with-automation.test.ts+++ b/__tests__/scripts/dev-with-automation.test.ts@@ -8,7 +8,7 @@import net from "node:net";import { spawn } from "node:child_process";import { once } from "node:events";-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";import { homedir, tmpdir } from "node:os";import path from "node:path";import { setTimeout as delay } from "node:timers/promises";@@ -1035,6 +1035,197 @@ describe("dev-with-automation CLI", () => {15_000,);+ // Runs the real `main()` with `--backend-only` and lets the caller decide+ // what the automation backend does. `main()` starts it as+ // `uv run --project <checkout> uvicorn ... --host 127.0.0.1 --port <port>`;+ // the fake `uv` hands those arguments to `automationSource` (a Node ESM+ // script). The agent-server is an in-process stub that answers every request,+ // so the launcher reaches the automation wait without delay.+ async function launchBackendOnly(automationSource: string) {+ const moduleUrl = pathToFileURL(+ path.join(repoRoot, "scripts", "dev-with-automation.mjs"),+ ).href;+ const dir = mkdtempSync(path.join(tmpdir(), "automation-startup-"));+ const bin = path.join(dir, "bin");+ const checkout = path.join(dir, "automation");+ mkdirSync(bin);+ mkdirSync(checkout);+ writeFileSync(path.join(checkout, "pyproject.toml"), "");+ const automationScript = path.join(dir, "automation.mjs");+ writeFileSync(automationScript, automationSource);+ // `uvx` only has to exist; the agent-server is stubbed below.+ writeFileSync(path.join(bin, "uvx"), "#!/bin/sh\nexit 0\n", {+ mode: 0o755,+ });+ writeFileSync(+ path.join(bin, "uv"),+ `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(automationScript)} "$@"\n`,+ { mode: 0o755 },+ );+ const ports: number[] = [];+ for (let i = 0; i < 3; i += 1) {+ const server = net.createServer();+ await new Promise<void>((resolve) =>+ server.listen(0, "127.0.0.1", resolve),+ );+ ports.push((server.address() as net.AddressInfo).port);+ await new Promise((resolve) => server.close(resolve));+ }+ const [ingressPort, agentServerPort, automationPort] = ports;+ const launcherScript = path.join(dir, "launch.mjs");+ writeFileSync(+ launcherScript,+ [+ 'import http from "node:http";',+ `import { main } from ${JSON.stringify(moduleUrl)};`,+ "const startAgentServer = (config) =>",+ ' http.createServer((_req, res) => res.end("{}")).listen(config.agentServerPort, "127.0.0.1");',+ "main({ startAgentServer, agentServerReadyTimeoutMs: 10_000 })",+ ' .then(() => console.log("LAUNCHED"))',+ ' .catch((error) => { console.log("FAILED:", error.message); process.exit(1); });',+ ].join("\n"),+ );++ const launcher = spawn(+ process.execPath,+ [launcherScript, "--backend-only", "--port", String(ingressPort)],+ {+ cwd: repoRoot,+ env: {+ ...process.env,+ PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}`,+ HOME: dir,+ OH_CANVAS_SAFE_STATE_DIR: path.join(dir, "state"),+ OH_CANVAS_SAFE_BACKEND_PORT: String(agentServerPort),+ OH_CANVAS_SAFE_AUTOMATION_PORT: String(automationPort),+ OH_AUTOMATION_LOCAL_PATH: checkout,+ },+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ let output = "";+ launcher.stdout.on("data", (chunk) => {+ output += chunk.toString();+ });+ launcher.stderr.on("data", (chunk) => {+ output += chunk.toString();+ });+ const exited = once(launcher, "exit").then(([code]) => code as number);++ return {+ ingressPort,+ automationPort,+ output: () => output,+ exitCode: (timeoutMs: number) =>+ Promise.race([exited, delay(timeoutMs).then(() => "timed out")]),+ async waitForOutput(text: string, timeoutMs: number) {+ const deadline = Date.now() + timeoutMs;+ while (!output.includes(text) && Date.now() < deadline) {+ if (launcher.exitCode !== null) break;+ await delay(50);+ }+ return output.includes(text);+ },+ async stop() {+ // SIGTERM lets the launcher's shutdown handler stop its detached+ // services; that is also the cleanup when a regression leaves it up.+ if (launcher.exitCode === null) {+ launcher.kill("SIGTERM");+ await Promise.race([exited, delay(6_000)]);+ if (launcher.exitCode === null) launcher.kill("SIGKILL");+ }+ rmSync(dir, { recursive: true, force: true });+ },+ };+ }++ const isListening = (port: number) =>+ new Promise<boolean>((resolve) => {+ const socket = net.connect(port, "127.0.0.1");+ socket.once("connect", () => {+ socket.destroy();+ resolve(true);+ });+ socket.once("error", () => resolve(false));+ });++ it.skipIf(process.platform === "win32")(+ "stops the stack and exits 1 when the automation backend exits during startup",+ async () => {+ // Exits the way automation does when a database migration fails.+ const stack = await launchBackendOnly(+ 'console.error("Failed to apply SQLite migrations");\nprocess.exit(3);\n',+ );++ try {+ const exitCode = await stack.exitCode(25_000);++ const output = stack.output();+ expect(exitCode, output).toBe(1);+ // The stack is announced without waiting for automation, then+ // stopped once automation exits. "Ingress:" is the banner's first+ // entry, printed in every launch mode.+ const banner = output.indexOf("Ingress:");+ const failure = output.indexOf(+ "The automation backend exited during startup (code 3).",+ );+ expect(banner, output).toBeGreaterThan(-1);+ expect(failure, output).toBeGreaterThan(banner);+ expect(await isListening(stack.ingressPort)).toBe(false);+ } finally {+ await stack.stop();+ }+ },+ 30_000,+ );++ it.skipIf(process.platform === "win32")(+ "announces the stack without waiting for the automation backend",+ async () => {+ // Listening but not ready yet: the first two health probes get 503, so+ // "Ready" is only logged after the launcher has polled and waited.+ const stack = await launchBackendOnly(+ [+ 'import http from "node:http";',+ 'const port = Number(process.argv[process.argv.indexOf("--port") + 1]);',+ "let probes = 0;",+ "http",+ " .createServer((req, res) => {",+ ' const healthy = req.url === "/api/automation/health" && ++probes > 2;',+ " res.writeHead(healthy ? 200 : 503).end(healthy ? '{\"status\":\"ok\"}' : '');",+ " })",+ ' .listen(port, "127.0.0.1");',+ "// Never outlive the test, even if the launcher is killed.",+ "setTimeout(() => process.exit(0), 30_000);",+ ].join("\n"),+ );++ try {+ expect(+ await stack.waitForOutput("LAUNCHED", 20_000),+ stack.output(),+ ).toBe(true);++ const readyLine = `Ready at http://127.0.0.1:${stack.automationPort}/api/automation/health`;+ expect(+ await stack.waitForOutput(readyLine, 20_000),+ stack.output(),+ ).toBe(true);++ const output = stack.output();+ const ready = output.indexOf(readyLine);+ expect(output.indexOf("Ingress:"), output).toBeGreaterThan(-1);+ expect(output.indexOf("Ingress:"), output).toBeLessThan(ready);+ // A healthy start keeps the stack up.+ expect(await stack.exitCode(1_000)).toBe("timed out");+ expect(await isListening(stack.ingressPort)).toBe(true);+ } finally {+ await stack.stop();+ }+ },+ 30_000,+ );+it("shows help with --help flag", async () => {const child = spawn(process.execPath,diff --git a/electron/main.mjs b/electron/main.mjsindex b632dee9a7..d4f115a0cf 100644--- a/electron/main.mjs+++ b/electron/main.mjs@@ -341,6 +341,7 @@ function setLoadingWindowExpanded(expanded) {}function createLoadingWindow() {+ bootLogReady = false;loadingWin = new BrowserWindow({width: LOADING_WIN_WIDTH,// Tall enough to fit the streaming status line + the "first launch can@@ -690,6 +691,9 @@ async function startStack() {skipNpmCheck: true,agentServerReadyTimeoutMs: 10 * 60_000,onServiceLog: handleServiceLog,+ // main() announces the stack without waiting for the automation backend;+ // if it then exits during startup, the services are already stopped.+ onAutomationStartupFailure: failStartup,});// main() returns { config, agentServerReady } β treat a timeout as a fatal@@ -705,6 +709,44 @@ async function startStack() {}}+let startupFailed = false;++/**+ * Show why startup failed on the splash, keeping the full startup log+ * readable and copyable. The automation backend can fail after the main+ * window has opened, so the splash is reopened in its place when needed.+ */+function failStartup(err) {+ if (startupFailed) return;+ startupFailed = true;+ const summary =+ err.message ++ " Ensure ports 8000, 18000, and 18001 are free, then try again.";+ // Record the failure in the terminal and the startup-log buffer so it+ // shows (and copies) as the final console line.+ console.error("[desktop] Startup failed:", err);+ appendBootLog("desktop", summary, "error");+ if (mainWin && !mainWin.isDestroyed()) {+ // Open the splash before closing the main window so window-all-closed+ // does not quit the app.+ if (!loadingWin || loadingWin.isDestroyed()) createLoadingWindow();+ mainWin.destroy();+ }+ // Keep the splash open in its failure state so the full startup log can+ // be read and copied; the app quits via the splash's Quit button (or+ // Cmd+Q / closing the window).+ if (showStartupFailure(summary)) return;+ // Loading window already gone β fall back to the old dialog-and-quit.+ const errorTail = recentServiceErrors.length+ ? `\n\nRecent service errors:\n${recentServiceErrors.join("\n")}`+ : "";+ dialog.showErrorBox(+ "OpenHands Agent Canvas failed to start",+ summary + errorTail,+ );+ app.quit();+}+// ββ App lifecycle βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββapp.whenReady().then(async () => {@@ -764,29 +806,11 @@ app.whenReady().then(async () => {setBootPhase("Connecting to agent serverβ¦");await waitForAgentServer("http://localhost:8000/server_info", 60_000);+ if (startupFailed) return;setBootPhase("Ready.");createMainWindow();} catch (err) {- const summary =- err.message +- " Ensure ports 8000, 18000, and 18001 are free, then try again.";- // Record the failure in the terminal and the startup-log buffer so it- // shows (and copies) as the final console line.- console.error("[desktop] Startup failed:", err);- appendBootLog("desktop", summary, "error");- // Keep the splash open in its failure state so the full startup log can- // be read and copied; the app quits via the splash's Quit button (or- // Cmd+Q / closing the window).- if (showStartupFailure(summary)) return;- // Loading window already gone β fall back to the old dialog-and-quit.- const errorTail = recentServiceErrors.length- ? `\n\nRecent service errors:\n${recentServiceErrors.join("\n")}`- : "";- dialog.showErrorBox(- "OpenHands Agent Canvas failed to start",- summary + errorTail,- );- app.quit();+ failStartup(err);}});diff --git a/scripts/dev-with-automation.mjs b/scripts/dev-with-automation.mjsindex be842e6dd6..96dfb10298 100644--- a/scripts/dev-with-automation.mjs+++ b/scripts/dev-with-automation.mjs@@ -726,11 +726,14 @@ function spawnService(name, command, args, options = {}) {return proc;}-async function waitForService(name, url, timeoutMs = 30000) {+async function waitForService(name, url, timeoutMs = 30000, proc = null) {const start = Date.now();let lastError = null;while (Date.now() - start < timeoutMs) {+ // A service that already exited will never answer; let the caller report+ // the exit instead of polling until the timeout.+ if (proc && !isProcessRunning(proc)) return false;try {const res = await fetch(url, { signal: AbortSignal.timeout(5000) });if (res.ok) {@@ -1000,7 +1003,7 @@ function startAutomationBackend(config) {const automationCmd = buildAutomationCommand(process.env);logService("automation", `Using ${automationCmd.source}`, c.dim);- spawnService(+ return spawnService("automation",automationCmd.command,[@@ -1097,6 +1100,24 @@ function startAutomationBackend(config) {let shuttingDown = false;+// Services run in their own process groups, so they outlive the launcher+// unless it stops them: SIGTERM each tree, then SIGKILL what is still running.+async function stopServices() {+ for (const [name, proc] of processes) {+ logService(name, "Stopping...", c.dim);+ signalProcessTree(proc, "SIGTERM");+ }++ await delay(3000);++ for (const [name, proc] of processes) {+ if (isProcessRunning(proc)) {+ logService(name, "Force stopping...", c.dim);+ signalProcessTree(proc, "SIGKILL");+ }+ }+}+function shutdown() {if (shuttingDown) return;shuttingDown = true;@@ -1105,21 +1126,39 @@ function shutdown() {console.log(`${c.yellow}Shutting down...${c.reset}`);fileLog("info", "Shutting down...");- for (const [name, proc] of processes) {- logService(name, "Stopping...", c.dim);- signalProcessTree(proc, "SIGTERM");- }-- setTimeout(() => {- for (const [name, proc] of processes) {- if (isProcessRunning(proc)) {- logService(name, "Force stopping...", c.dim);- signalProcessTree(proc, "SIGKILL");- }- }+ stopServices().then(() => {shutdownHooks.run();process.exit(0);- }, 3000);+ });+}++/**+ * Watch the automation backend until it answers, and stop the stack if it+ * exits first. It exits during startup when, for example, a database+ * migration fails (#18299); leaving the stack up made Automate silently+ * unavailable. The launcher announces the stack without waiting for this, so+ * a healthy start is not delayed; a failure stops every service and then calls+ * `onFailure`. A slow start (first-run download) only logs a timeout.+ */+async function watchAutomationStartup(config, proc, timeoutMs, onFailure) {+ const ready = await waitForService(+ "automation",+ `http://127.0.0.1:${config.autoBackendPort}${AUTOMATION_ROUTE_PREFIX}/health`,+ timeoutMs,+ proc,+ );+ if (ready || isProcessRunning(proc) || shuttingDown) return;++ const exit =+ proc.exitCode !== null+ ? `code ${proc.exitCode}`+ : `signal ${proc.signalCode}`;+ const message =+ `The automation backend exited during startup (${exit}). ` ++ "Its [automation] log lines show why.";+ logError(message);+ await stopServices();+ onFailure(new Error(message));}process.on("SIGINT", shutdown);@@ -1508,6 +1547,11 @@ async function main(options = {}) {// download / install progress to the user. `level` is one of// "stdout" | "stderr" | "info" | "warn" | "error".onServiceLog,+ // Called with an Error when the automation backend exits before it+ // becomes healthy, after every service has been stopped. That can happen+ // after `main()` has returned. Defaults to exiting the process with+ // code 1; embedded launchers (Electron) pass their own handler.+ onAutomationStartupFailure = () => process.exit(1),} = options;// Install the listener early so log lines emitted before the first@@ -1659,9 +1703,9 @@ async function main(options = {}) {}// 3. Start automation backend- if (config.launchAutomation) {- startAutomationBackend(config);- }+ const automationProc = config.launchAutomation+ ? startAutomationBackend(config)+ : null;// 4. Start frontend server (Vite dev server OR static server)if (config.launchFrontend) {@@ -1683,6 +1727,17 @@ async function main(options = {}) {printBanner(config);+ // 7. Watch the automation backend in the background: if it exits before it+ // is healthy, stop the stack and report the failure.+ if (automationProc) {+ void watchAutomationStartup(+ config,+ automationProc,+ agentServerReadyTimeoutMs,+ onAutomationStartupFailure,+ );+ }+// Return the resolved config + readiness signal so embedded launchers can// (a) build URLs from the actual allocated ports and (b) decide whether to// show an error to the user when the agent-server never came up.