Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: fail the launch when the automation backend exits during startup (#18311)

Engel Nyst33fcb2e1b394
diff --git a/.agents/skills/local-stack-runtime/references/guide.md b/.agents/skills/local-stack-runtime/references/guide.md
index d9371c779d..2c753d8e0b 100644
--- a/.agents/skills/local-stack-runtime/references/guide.md
+++ b/.agents/skills/local-stack-runtime/references/guide.md
@@ -103,6 +103,7 @@ you are running inside of β€” NOT the automation backend.
- `npm run dev` runs the full local stack via `uvx` (agent-server + automation backend + Vite dev server + ingress proxy) with no Docker dependency. `npm run dev:static` does the same but serves a production build of the frontend instead of the Vite dev server.
- `scripts/dev-with-automation.mjs` runs the full stack: agent-server, automation backend (both via uvx), frontend server, and ingress proxy. It defaults to Vite when run directly, supports `--static` for an existing build, and supports `--dynamic` so wrappers that default static can opt back into Vite. Uses a standalone ingress proxy (`scripts/ingress.mjs`) to route traffic:
- Keep `SIGINT`, `SIGTERM`, and `SIGHUP` wired through the coordinated shutdown handler. Services run in detached process groups on POSIX, so cleanup must use `signalProcessTree()` rather than signaling only the direct child; regression coverage lives in `__tests__/scripts/dev-with-automation.test.ts`.
+ - `main()` prints the banner without waiting for the automation backend, then watches `/api/automation/health` in the background. If automation exits before it is healthy (a failed database migration, #18299), the launcher stops every service it started and calls `onAutomationStartupFailure`: the CLI exits 1, and Electron replaces the splash or main window with its startup-failure screen. A slow start only logs a timeout. The Docker entrypoint still tolerates a dead automation backend.
- `/api/automation/*` β†’ automation backend (:18001)
- `/api/*`, `/sockets`, etc. β†’ agent server (:18000)
- `/*` (default) β†’ frontend server (:3001), either Vite or static depending on launcher mode
diff --git a/__tests__/scripts/dev-with-automation.test.ts b/__tests__/scripts/dev-with-automation.test.ts
index d51044ef5b..8f89284e68 100644
--- a/__tests__/scripts/dev-with-automation.test.ts
+++ b/__tests__/scripts/dev-with-automation.test.ts
@@ -8,7 +8,7 @@
import net from "node:net";
import { spawn } from "node:child_process";
import { once } from "node:events";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
@@ -1035,6 +1035,197 @@ describe("dev-with-automation CLI", () => {
15_000,
);
+ // Runs the real `main()` with `--backend-only` and lets the caller decide
+ // what the automation backend does. `main()` starts it as
+ // `uv run --project <checkout> uvicorn ... --host 127.0.0.1 --port <port>`;
+ // the fake `uv` hands those arguments to `automationSource` (a Node ESM
+ // script). The agent-server is an in-process stub that answers every request,
+ // so the launcher reaches the automation wait without delay.
+ async function launchBackendOnly(automationSource: string) {
+ const moduleUrl = pathToFileURL(
+ path.join(repoRoot, "scripts", "dev-with-automation.mjs"),
+ ).href;
+ const dir = mkdtempSync(path.join(tmpdir(), "automation-startup-"));
+ const bin = path.join(dir, "bin");
+ const checkout = path.join(dir, "automation");
+ mkdirSync(bin);
+ mkdirSync(checkout);
+ writeFileSync(path.join(checkout, "pyproject.toml"), "");
+ const automationScript = path.join(dir, "automation.mjs");
+ writeFileSync(automationScript, automationSource);
+ // `uvx` only has to exist; the agent-server is stubbed below.
+ writeFileSync(path.join(bin, "uvx"), "#!/bin/sh\nexit 0\n", {
+ mode: 0o755,
+ });
+ writeFileSync(
+ path.join(bin, "uv"),
+ `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(automationScript)} "$@"\n`,
+ { mode: 0o755 },
+ );
+ const ports: number[] = [];
+ for (let i = 0; i < 3; i += 1) {
+ const server = net.createServer();
+ await new Promise<void>((resolve) =>
+ server.listen(0, "127.0.0.1", resolve),
+ );
+ ports.push((server.address() as net.AddressInfo).port);
+ await new Promise((resolve) => server.close(resolve));
+ }
+ const [ingressPort, agentServerPort, automationPort] = ports;
+ const launcherScript = path.join(dir, "launch.mjs");
+ writeFileSync(
+ launcherScript,
+ [
+ 'import http from "node:http";',
+ `import { main } from ${JSON.stringify(moduleUrl)};`,
+ "const startAgentServer = (config) =>",
+ ' http.createServer((_req, res) => res.end("{}")).listen(config.agentServerPort, "127.0.0.1");',
+ "main({ startAgentServer, agentServerReadyTimeoutMs: 10_000 })",
+ ' .then(() => console.log("LAUNCHED"))',
+ ' .catch((error) => { console.log("FAILED:", error.message); process.exit(1); });',
+ ].join("\n"),
+ );
+
+ const launcher = spawn(
+ process.execPath,
+ [launcherScript, "--backend-only", "--port", String(ingressPort)],
+ {
+ cwd: repoRoot,
+ env: {
+ ...process.env,
+ PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}`,
+ HOME: dir,
+ OH_CANVAS_SAFE_STATE_DIR: path.join(dir, "state"),
+ OH_CANVAS_SAFE_BACKEND_PORT: String(agentServerPort),
+ OH_CANVAS_SAFE_AUTOMATION_PORT: String(automationPort),
+ OH_AUTOMATION_LOCAL_PATH: checkout,
+ },
+ stdio: ["ignore", "pipe", "pipe"],
+ },
+ );
+ let output = "";
+ launcher.stdout.on("data", (chunk) => {
+ output += chunk.toString();
+ });
+ launcher.stderr.on("data", (chunk) => {
+ output += chunk.toString();
+ });
+ const exited = once(launcher, "exit").then(([code]) => code as number);
+
+ return {
+ ingressPort,
+ automationPort,
+ output: () => output,
+ exitCode: (timeoutMs: number) =>
+ Promise.race([exited, delay(timeoutMs).then(() => "timed out")]),
+ async waitForOutput(text: string, timeoutMs: number) {
+ const deadline = Date.now() + timeoutMs;
+ while (!output.includes(text) && Date.now() < deadline) {
+ if (launcher.exitCode !== null) break;
+ await delay(50);
+ }
+ return output.includes(text);
+ },
+ async stop() {
+ // SIGTERM lets the launcher's shutdown handler stop its detached
+ // services; that is also the cleanup when a regression leaves it up.
+ if (launcher.exitCode === null) {
+ launcher.kill("SIGTERM");
+ await Promise.race([exited, delay(6_000)]);
+ if (launcher.exitCode === null) launcher.kill("SIGKILL");
+ }
+ rmSync(dir, { recursive: true, force: true });
+ },
+ };
+ }
+
+ const isListening = (port: number) =>
+ new Promise<boolean>((resolve) => {
+ const socket = net.connect(port, "127.0.0.1");
+ socket.once("connect", () => {
+ socket.destroy();
+ resolve(true);
+ });
+ socket.once("error", () => resolve(false));
+ });
+
+ it.skipIf(process.platform === "win32")(
+ "stops the stack and exits 1 when the automation backend exits during startup",
+ async () => {
+ // Exits the way automation does when a database migration fails.
+ const stack = await launchBackendOnly(
+ 'console.error("Failed to apply SQLite migrations");\nprocess.exit(3);\n',
+ );
+
+ try {
+ const exitCode = await stack.exitCode(25_000);
+
+ const output = stack.output();
+ expect(exitCode, output).toBe(1);
+ // The stack is announced without waiting for automation, then
+ // stopped once automation exits. "Ingress:" is the banner's first
+ // entry, printed in every launch mode.
+ const banner = output.indexOf("Ingress:");
+ const failure = output.indexOf(
+ "The automation backend exited during startup (code 3).",
+ );
+ expect(banner, output).toBeGreaterThan(-1);
+ expect(failure, output).toBeGreaterThan(banner);
+ expect(await isListening(stack.ingressPort)).toBe(false);
+ } finally {
+ await stack.stop();
+ }
+ },
+ 30_000,
+ );
+
+ it.skipIf(process.platform === "win32")(
+ "announces the stack without waiting for the automation backend",
+ async () => {
+ // Listening but not ready yet: the first two health probes get 503, so
+ // "Ready" is only logged after the launcher has polled and waited.
+ const stack = await launchBackendOnly(
+ [
+ 'import http from "node:http";',
+ 'const port = Number(process.argv[process.argv.indexOf("--port") + 1]);',
+ "let probes = 0;",
+ "http",
+ " .createServer((req, res) => {",
+ ' const healthy = req.url === "/api/automation/health" && ++probes > 2;',
+ " res.writeHead(healthy ? 200 : 503).end(healthy ? '{\"status\":\"ok\"}' : '');",
+ " })",
+ ' .listen(port, "127.0.0.1");',
+ "// Never outlive the test, even if the launcher is killed.",
+ "setTimeout(() => process.exit(0), 30_000);",
+ ].join("\n"),
+ );
+
+ try {
+ expect(
+ await stack.waitForOutput("LAUNCHED", 20_000),
+ stack.output(),
+ ).toBe(true);
+
+ const readyLine = `Ready at http://127.0.0.1:${stack.automationPort}/api/automation/health`;
+ expect(
+ await stack.waitForOutput(readyLine, 20_000),
+ stack.output(),
+ ).toBe(true);
+
+ const output = stack.output();
+ const ready = output.indexOf(readyLine);
+ expect(output.indexOf("Ingress:"), output).toBeGreaterThan(-1);
+ expect(output.indexOf("Ingress:"), output).toBeLessThan(ready);
+ // A healthy start keeps the stack up.
+ expect(await stack.exitCode(1_000)).toBe("timed out");
+ expect(await isListening(stack.ingressPort)).toBe(true);
+ } finally {
+ await stack.stop();
+ }
+ },
+ 30_000,
+ );
+
it("shows help with --help flag", async () => {
const child = spawn(
process.execPath,
diff --git a/electron/main.mjs b/electron/main.mjs
index b632dee9a7..d4f115a0cf 100644
--- a/electron/main.mjs
+++ b/electron/main.mjs
@@ -341,6 +341,7 @@ function setLoadingWindowExpanded(expanded) {
}
function createLoadingWindow() {
+ bootLogReady = false;
loadingWin = new BrowserWindow({
width: LOADING_WIN_WIDTH,
// Tall enough to fit the streaming status line + the "first launch can
@@ -690,6 +691,9 @@ async function startStack() {
skipNpmCheck: true,
agentServerReadyTimeoutMs: 10 * 60_000,
onServiceLog: handleServiceLog,
+ // main() announces the stack without waiting for the automation backend;
+ // if it then exits during startup, the services are already stopped.
+ onAutomationStartupFailure: failStartup,
});
// main() returns { config, agentServerReady } β€” treat a timeout as a fatal
@@ -705,6 +709,44 @@ async function startStack() {
}
}
+let startupFailed = false;
+
+/**
+ * Show why startup failed on the splash, keeping the full startup log
+ * readable and copyable. The automation backend can fail after the main
+ * window has opened, so the splash is reopened in its place when needed.
+ */
+function failStartup(err) {
+ if (startupFailed) return;
+ startupFailed = true;
+ const summary =
+ err.message +
+ " Ensure ports 8000, 18000, and 18001 are free, then try again.";
+ // Record the failure in the terminal and the startup-log buffer so it
+ // shows (and copies) as the final console line.
+ console.error("[desktop] Startup failed:", err);
+ appendBootLog("desktop", summary, "error");
+ if (mainWin && !mainWin.isDestroyed()) {
+ // Open the splash before closing the main window so window-all-closed
+ // does not quit the app.
+ if (!loadingWin || loadingWin.isDestroyed()) createLoadingWindow();
+ mainWin.destroy();
+ }
+ // Keep the splash open in its failure state so the full startup log can
+ // be read and copied; the app quits via the splash's Quit button (or
+ // Cmd+Q / closing the window).
+ if (showStartupFailure(summary)) return;
+ // Loading window already gone β€” fall back to the old dialog-and-quit.
+ const errorTail = recentServiceErrors.length
+ ? `\n\nRecent service errors:\n${recentServiceErrors.join("\n")}`
+ : "";
+ dialog.showErrorBox(
+ "OpenHands Agent Canvas failed to start",
+ summary + errorTail,
+ );
+ app.quit();
+}
+
// ── App lifecycle ─────────────────────────────────────────────────────────────
app.whenReady().then(async () => {
@@ -764,29 +806,11 @@ app.whenReady().then(async () => {
setBootPhase("Connecting to agent server…");
await waitForAgentServer("http://localhost:8000/server_info", 60_000);
+ if (startupFailed) return;
setBootPhase("Ready.");
createMainWindow();
} catch (err) {
- const summary =
- err.message +
- " Ensure ports 8000, 18000, and 18001 are free, then try again.";
- // Record the failure in the terminal and the startup-log buffer so it
- // shows (and copies) as the final console line.
- console.error("[desktop] Startup failed:", err);
- appendBootLog("desktop", summary, "error");
- // Keep the splash open in its failure state so the full startup log can
- // be read and copied; the app quits via the splash's Quit button (or
- // Cmd+Q / closing the window).
- if (showStartupFailure(summary)) return;
- // Loading window already gone β€” fall back to the old dialog-and-quit.
- const errorTail = recentServiceErrors.length
- ? `\n\nRecent service errors:\n${recentServiceErrors.join("\n")}`
- : "";
- dialog.showErrorBox(
- "OpenHands Agent Canvas failed to start",
- summary + errorTail,
- );
- app.quit();
+ failStartup(err);
}
});
diff --git a/scripts/dev-with-automation.mjs b/scripts/dev-with-automation.mjs
index be842e6dd6..96dfb10298 100644
--- a/scripts/dev-with-automation.mjs
+++ b/scripts/dev-with-automation.mjs
@@ -726,11 +726,14 @@ function spawnService(name, command, args, options = {}) {
return proc;
}
-async function waitForService(name, url, timeoutMs = 30000) {
+async function waitForService(name, url, timeoutMs = 30000, proc = null) {
const start = Date.now();
let lastError = null;
while (Date.now() - start < timeoutMs) {
+ // A service that already exited will never answer; let the caller report
+ // the exit instead of polling until the timeout.
+ if (proc && !isProcessRunning(proc)) return false;
try {
const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
if (res.ok) {
@@ -1000,7 +1003,7 @@ function startAutomationBackend(config) {
const automationCmd = buildAutomationCommand(process.env);
logService("automation", `Using ${automationCmd.source}`, c.dim);
- spawnService(
+ return spawnService(
"automation",
automationCmd.command,
[
@@ -1097,6 +1100,24 @@ function startAutomationBackend(config) {
let shuttingDown = false;
+// Services run in their own process groups, so they outlive the launcher
+// unless it stops them: SIGTERM each tree, then SIGKILL what is still running.
+async function stopServices() {
+ for (const [name, proc] of processes) {
+ logService(name, "Stopping...", c.dim);
+ signalProcessTree(proc, "SIGTERM");
+ }
+
+ await delay(3000);
+
+ for (const [name, proc] of processes) {
+ if (isProcessRunning(proc)) {
+ logService(name, "Force stopping...", c.dim);
+ signalProcessTree(proc, "SIGKILL");
+ }
+ }
+}
+
function shutdown() {
if (shuttingDown) return;
shuttingDown = true;
@@ -1105,21 +1126,39 @@ function shutdown() {
console.log(`${c.yellow}Shutting down...${c.reset}`);
fileLog("info", "Shutting down...");
- for (const [name, proc] of processes) {
- logService(name, "Stopping...", c.dim);
- signalProcessTree(proc, "SIGTERM");
- }
-
- setTimeout(() => {
- for (const [name, proc] of processes) {
- if (isProcessRunning(proc)) {
- logService(name, "Force stopping...", c.dim);
- signalProcessTree(proc, "SIGKILL");
- }
- }
+ stopServices().then(() => {
shutdownHooks.run();
process.exit(0);
- }, 3000);
+ });
+}
+
+/**
+ * Watch the automation backend until it answers, and stop the stack if it
+ * exits first. It exits during startup when, for example, a database
+ * migration fails (#18299); leaving the stack up made Automate silently
+ * unavailable. The launcher announces the stack without waiting for this, so
+ * a healthy start is not delayed; a failure stops every service and then calls
+ * `onFailure`. A slow start (first-run download) only logs a timeout.
+ */
+async function watchAutomationStartup(config, proc, timeoutMs, onFailure) {
+ const ready = await waitForService(
+ "automation",
+ `http://127.0.0.1:${config.autoBackendPort}${AUTOMATION_ROUTE_PREFIX}/health`,
+ timeoutMs,
+ proc,
+ );
+ if (ready || isProcessRunning(proc) || shuttingDown) return;
+
+ const exit =
+ proc.exitCode !== null
+ ? `code ${proc.exitCode}`
+ : `signal ${proc.signalCode}`;
+ const message =
+ `The automation backend exited during startup (${exit}). ` +
+ "Its [automation] log lines show why.";
+ logError(message);
+ await stopServices();
+ onFailure(new Error(message));
}
process.on("SIGINT", shutdown);
@@ -1508,6 +1547,11 @@ async function main(options = {}) {
// download / install progress to the user. `level` is one of
// "stdout" | "stderr" | "info" | "warn" | "error".
onServiceLog,
+ // Called with an Error when the automation backend exits before it
+ // becomes healthy, after every service has been stopped. That can happen
+ // after `main()` has returned. Defaults to exiting the process with
+ // code 1; embedded launchers (Electron) pass their own handler.
+ onAutomationStartupFailure = () => process.exit(1),
} = options;
// Install the listener early so log lines emitted before the first
@@ -1659,9 +1703,9 @@ async function main(options = {}) {
}
// 3. Start automation backend
- if (config.launchAutomation) {
- startAutomationBackend(config);
- }
+ const automationProc = config.launchAutomation
+ ? startAutomationBackend(config)
+ : null;
// 4. Start frontend server (Vite dev server OR static server)
if (config.launchFrontend) {
@@ -1683,6 +1727,17 @@ async function main(options = {}) {
printBanner(config);
+ // 7. Watch the automation backend in the background: if it exits before it
+ // is healthy, stop the stack and report the failure.
+ if (automationProc) {
+ void watchAutomationStartup(
+ config,
+ automationProc,
+ agentServerReadyTimeoutMs,
+ onAutomationStartupFailure,
+ );
+ }
+
// Return the resolved config + readiness signal so embedded launchers can
// (a) build URLs from the actual allocated ports and (b) decide whether to
// show an error to the user when the agent-server never came up.