Multiplayer agent harness for work.
Report app start failures, restarts and outages on every deploy provider (#2226)
Regan Bellca4642fbcc51
diff --git a/docs/error-reporting.md b/docs/error-reporting.mdindex 55e989036..d73c9c4d8 100644--- a/docs/error-reporting.md+++ b/docs/error-reporting.md@@ -24,4 +24,4 @@ Browser Sentry ignores plain network `TypeError` failures (`Failed to fetch`, `N## Published apps-On the AWS and Porter app hosts, published-app failures become operator error records in the app owner's scope, so they reach the admin errors view and Sentry: `deploy:app_start_failed` (publish failed, including an app that only answered 5xx on `/` for the readiness window), `deploy:app_restarted` (the supervisor restarted the app since the last visit-time check), and `deploy:app_down` (the app and its supervisor are gone). Each record names the deployment and carries up to 4 KB of the app's recent output, with the app's env values and known secret patterns masked. Fly relies on its machine TCP check, so its readiness does not reject 5xx and its failures surface only as the publish error; Docker uses its restart policy and does not report.+Published-app failures become operator error records in the app owner's scope, so they reach the admin errors view and Sentry: `deploy:app_start_failed` (publish failed, including an app that only answered 5xx on `/` for the readiness window), `deploy:app_restarted` (the supervisor restarted the app since the last visit-time check), and `deploy:app_down` (the app and its supervisor are gone). Each record names the deployment and carries up to 4 KB of the app's recent output, with the app's env values and known secret patterns masked. Every app host (AWS, Porter, Fly, Docker) runs the app under the same supervisor and probes, so the app image must provide `sh` and `curl`.diff --git a/src/deploy/aws-deploy-provider.ts b/src/deploy/aws-deploy-provider.tsindex 3502fe1ba..dbc12c829 100644--- a/src/deploy/aws-deploy-provider.ts+++ b/src/deploy/aws-deploy-provider.ts@@ -11,10 +11,12 @@ import { createKeyedQueue, sleep } from "../util/async.ts";import { shq } from "../util/shell.ts";import { swallow } from "../util/errors.ts";import {+ APP_PID_PATH,appGone,+ appLaunch,+ appLogs,appReporter,listeningOn,- supervised,waitAppReady,type AppErrorSink,} from "./shared-deploy-provider.ts";@@ -36,8 +38,6 @@ const DATA_CREDS_RETRY_MS = 60_000;const failureBackoffStamp = (): number => Date.now() - DATA_CREDS_REFRESH_MS + DATA_CREDS_RETRY_MS;const MAX_DATA_TAR_BYTES = 180 * 1024 * 1024;const READY_PATH = `${APP_DIR}/.qm-ready`;-const PID_PATH = "/tmp/qm-app.pid";-const LOG_PATH = "/tmp/qm-app.log";const APP_READY_WINDOW_SEC = 60;const APP_PORT_RELEASE_TICKS = 40;const APP_START_EXEC_TIMEOUT_SEC = 60;@@ -440,9 +440,9 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPconst inner = `cd ${shq(APP_DIR)}; ${version.entrypoint}`;const script = [listeningOn(appPort),- `old=$(cat ${shq(PID_PATH)} 2>/dev/null || true)`,+ `old=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true)`,`if [ -n "$old" ]; then kill -TERM "-$old" 2>/dev/null || true; sleep 0.5; kill -KILL "-$old" 2>/dev/null || true; fi`,- `rm -f ${shq(PID_PATH)}`,+ `rm -f ${shq(APP_PID_PATH)}`,`for i in $(seq 1 ${APP_PORT_RELEASE_TICKS}); do listening || break; sleep 0.25; done`,`listening && exit ${EXIT_APP_PORT_HELD}`,...(s3 ? [`mkdir -p ${shq(DATA_DIR)}`] : []),@@ -450,7 +450,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployP`export PORT=${appPort}`,...(s3 ? [`export DATA_DIR=${shq(DATA_DIR)}`] : []),...(envExports ? [envExports] : []),- `setsid sh -c ${shq(supervised(inner, appPort, PID_PATH))} > ${shq(LOG_PATH)} 2>&1 & echo $! > ${shq(PID_PATH)}`,+ appLaunch(inner, appPort),].join("; ");const r = await execRaw(id, endpoint, script, APP_START_EXEC_TIMEOUT_SEC);if (r.code === EXIT_APP_PORT_HELD) {@@ -462,8 +462,6 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPawait waitAppReady((script, timeoutSec) => execRaw(id, endpoint, script, timeoutSec), {appPort,windowSec: APP_READY_WINDOW_SEC,- pidPath: PID_PATH,- logPath: LOG_PATH,report: appReporter(opts.onError, d, version.env),});}@@ -561,7 +559,7 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPconst exec = (script: string, timeoutSec: number) =>execRaw(stored.microvmId, stored.endpoint, script, timeoutSec);const report = appReporter(opts.onError, d, version.env);- if (await appGone(exec, { appPort, pidPath: PID_PATH, logPath: LOG_PATH, report })) return null;+ if (await appGone(exec, { appPort, report })) return null;} catch (e) {if (!(e instanceof AwsApiError && (e.status >= 500 || e.status === 429))) throw e;}@@ -658,14 +656,10 @@ export function createAwsDeployProvider(opts: AwsDeployProviderOptions): DeployPconst stored = await store.get(d.id);if (!stored) return null;await ensureRunning(stored.microvmId, stored.endpoint);- const lines = Math.max(1, Math.min(2000, Math.floor(opts.tailLines)));- const r = await execRaw(- stored.microvmId,- stored.endpoint,- `tail -n ${lines} ${shq(LOG_PATH)} 2>/dev/null || true`,- 30,+ return appLogs(+ (script, timeoutSec) => execRaw(stored.microvmId, stored.endpoint, script, timeoutSec),+ opts.tailLines,);- return r.stdout;},async destroy(d): Promise<void> {diff --git a/src/deploy/docker-deploy-provider.ts b/src/deploy/docker-deploy-provider.tsindex 0db1576d3..96f5a4bc0 100644--- a/src/deploy/docker-deploy-provider.ts+++ b/src/deploy/docker-deploy-provider.ts@@ -2,8 +2,20 @@ import type { Deployment, DeploymentVersion } from "./deploy-store.ts";import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";import { spawnDockerExec, type DockerExec } from "../sandbox/docker-exec.ts";import { errMessage } from "../util/errors.ts";+import {+ appBoot,+ appGone,+ appLogs,+ appReporter,+ recentAppChecks,+ waitAppReady,+ type AppErrorSink,+ type Exec,+} from "./shared-deploy-provider.ts";const APP_PORT = 8080;+const APP_READY_WINDOW_SEC = 60;+const RUN_TIMEOUT_MS = 300_000;const LEGACY_NETWORK = "agent-deploynet";const DAEMON_PROBE_TIMEOUT_MS = 10_000;@@ -12,6 +24,7 @@ export interface DockerDeployProviderOptions {docker?: string;basePort?: number;dockerExec?: DockerExec;+ onError?: AppErrorSink;}export interface DockerDaemonProbeOptions {@@ -34,7 +47,7 @@ export async function dockerDaemonFailure(opts: DockerDaemonProbeOptions = {}):export function createDockerDeployProvider(opts: DockerDeployProviderOptions = {}): DeployProvider {const docker = opts.docker ?? "docker";- const image = opts.image ?? "node:24-alpine";+ const image = opts.image ?? "node:24";let nextPort = opts.basePort ?? 9200;const ports = new Map<string, number>();const freed: number[] = [];@@ -56,6 +69,11 @@ export function createDockerDeployProvider(opts: DockerDeployProviderOptions = {const dexec = opts.dockerExec ?? spawnDockerExec(docker);const name = (d: Deployment) => `agent-deploy-${d.id.slice(0, 12)}`;+ const execIn =+ (d: Deployment): Exec =>+ (script, timeoutSec) =>+ dexec(["exec", name(d), "sh", "-c", script], timeoutSec * 1000);+ const checked = recentAppChecks();const network = (d: Deployment) => `${name(d)}-net`;const ensureNetwork = async (net: string): Promise<string> => {if ((await dexec(["network", "inspect", net])).code !== 0) {@@ -108,50 +126,57 @@ export function createDockerDeployProvider(opts: DockerDeployProviderOptions = {await dexec(["rm", "-f", name(d)]);const hostPort = allocPort(name(d));const envArgs = Object.entries(version.env ?? {}).flatMap(([k, v]) => ["-e", `${k}=${v}`]);- const r = await dexec([- "run",- "-d",- "--restart",- "unless-stopped",- "--name",- name(d),- "--network",- net,- "--memory",- "512m",- "--cpus",- "1",- "--pids-limit",- "256",- "-p",- `127.0.0.1:${hostPort}:${APP_PORT}`,- "-v",- `${version.snapshotDir}:/app:ro`,- "-w",- "/app",- "-e",- `PORT=${APP_PORT}`,- ...envArgs,- image,- "sh",- "-c",- version.entrypoint,- ]);- if (r.code !== 0) {+ const r = await dexec(+ [+ "run",+ "-d",+ "--restart",+ "unless-stopped",+ "--name",+ name(d),+ "--network",+ net,+ "--memory",+ "512m",+ "--cpus",+ "1",+ "--pids-limit",+ "256",+ "-p",+ `127.0.0.1:${hostPort}:${APP_PORT}`,+ "-v",+ `${version.snapshotDir}:/app:ro`,+ "-w",+ "/app",+ "-e",+ `PORT=${APP_PORT}`,+ ...envArgs,+ image,+ "sh",+ "-c",+ appBoot(`cd /app; ${version.entrypoint}`, APP_PORT),+ ],+ RUN_TIMEOUT_MS,+ );+ try {+ if (r.code !== 0) throw new Error(`deploy run failed: ${r.stderr.trim()}`);+ await waitAppReady(execIn(d), {+ appPort: APP_PORT,+ windowSec: APP_READY_WINDOW_SEC,+ report: appReporter(opts.onError, d, version.env),+ });+ } catch (e) {await dexec(["rm", "-f", name(d)]);await dexec(["network", "rm", net]);freePort(name(d));- throw new Error(`deploy run failed: ${r.stderr.trim()}`);+ throw e;}return { host: "127.0.0.1", port: hostPort };},async logs(d: Deployment, opts: { tailLines: number }): Promise<string | null> {if (!(await migrateTarget(name(d)))) return null;- const lines = Math.max(1, Math.min(2000, Math.floor(opts.tailLines)));- const r = await dexec(["logs", "--tail", String(lines), name(d)]);- if (r.code !== 0) return null;- return `${r.stdout}${r.stderr}`;+ return appLogs(execIn(d), opts.tailLines);},async destroy(d: Deployment): Promise<void> {@@ -160,8 +185,13 @@ export function createDockerDeployProvider(opts: DockerDeployProviderOptions = {freePort(name(d));},- async resolveEndpoint(d): Promise<DeployEndpoint | null> {- return (await migrateTarget(name(d))) ? d.endpoint : null;+ async resolveEndpoint(d, version): Promise<DeployEndpoint | null> {+ if (checked.has(d.id)) return d.endpoint;+ if (!(await migrateTarget(name(d)))) return null;+ const report = appReporter(opts.onError, d, version.env);+ if (await appGone(execIn(d), { appPort: APP_PORT, report })) return null;+ checked.set(d.id, true);+ return d.endpoint;},};}diff --git a/src/deploy/fly-deploy-provider.ts b/src/deploy/fly-deploy-provider.tsindex 5899d2de7..b1f67fd7c 100644--- a/src/deploy/fly-deploy-provider.ts+++ b/src/deploy/fly-deploy-provider.ts@@ -8,6 +8,16 @@ import { sleep } from "../util/async.ts";import { swallow, swallowAs } from "../util/errors.ts";import { shq } from "../util/shell.ts";import type { DurableMap } from "../persistence/durable-map.ts";+import {+ appBoot,+ appGone,+ appLogs,+ appReporter,+ recentAppChecks,+ waitAppReady,+ type AppErrorSink,+ type Exec,+} from "./shared-deploy-provider.ts";const MACHINES_API_BASE_URL = "https://api.machines.dev/v1";const DEFAULT_REGION = "lhr";@@ -92,6 +102,7 @@ interface FlyMachinesApi {getMachine(appName: string, machineId: string): Promise<FlyMachine | null>;setCordon(appName: string, machineId: string, cordoned: boolean): Promise<void>;destroyMachine(appName: string, machineId: string): Promise<void>;+ exec(appName: string, machineId: string, script: string, timeoutSec: number): ReturnType<Exec>;}interface FlyApiResponse {@@ -102,7 +113,12 @@ interface FlyApiResponse {function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch }): FlyMachinesApi {const fetchImpl = opts.fetchImpl ?? fetch;- const request = async (method: string, path: string, body?: unknown): Promise<FlyApiResponse> => {+ const request = async (+ method: string,+ path: string,+ body?: unknown,+ timeoutMs = API_TIMEOUT_MS,+ ): Promise<FlyApiResponse> => {const res = await fetchImpl(`${MACHINES_API_BASE_URL}${path}`, {method,headers: {@@ -110,7 +126,7 @@ function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch })...(body === undefined ? {} : { "content-type": "application/json" }),},...(body === undefined ? {} : { body: JSON.stringify(body) }),- signal: AbortSignal.timeout(API_TIMEOUT_MS),+ signal: AbortSignal.timeout(timeoutMs),});return { ok: res.ok, status: res.status, text: await res.text() };};@@ -283,6 +299,18 @@ function createFlyMachinesApi(opts: { token: string; fetchImpl?: typeof fetch })if (r.ok || r.status === 404) return;throw failure(`destroy machine ${machineId}`, r);},+ async exec(appName, machineId, script, timeoutSec) {+ const what = `exec in machine ${machineId}`;+ const r = await request(+ "POST",+ `${machine(appName, machineId)}/exec`,+ { cmd: `sh -c ${shq(script)}`, timeout: timeoutSec },+ (timeoutSec + 10) * 1000,+ );+ if (!r.ok) throw failure(what, r);+ const out = parse<{ exit_code?: number; stdout?: string; stderr?: string }>(what, r);+ return { code: out.exit_code ?? 0, stdout: out.stdout ?? "", stderr: out.stderr ?? "" };+ },};}@@ -309,6 +337,7 @@ export interface FlyDeployProviderOptions {appReadyTimeoutMs?: number;pollIntervalMs?: number;fetchImpl?: typeof fetch;+ onError?: AppErrorSink;}export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployProvider {@@ -321,6 +350,11 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPconst appReadyTimeoutMs = opts.appReadyTimeoutMs ?? APP_READY_TIMEOUT_MS;const pollIntervalMs = opts.pollIntervalMs ?? POLL_INTERVAL_MS;const seconds = (ms: number): string => `${Math.round(ms / 1000)}s`;+ const execIn =+ (appName: string, machineId: string): Exec =>+ (script, timeoutSec) =>+ api.exec(appName, machineId, script, timeoutSec);+ const checked = recentAppChecks();function ensureConfigured(): void {if (@@ -418,7 +452,11 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPexec: ["/bin/sh","-lc",- `mkdir -p ${APP_DIR} && tar -xzf ${BUNDLE_GUEST_PATH} -C ${APP_DIR} && cd ${APP_DIR} && exec sh -lc ${shq(version.entrypoint)}`,+ appBoot(+ `cd ${APP_DIR}; ${version.entrypoint}`,+ APP_PORT,+ `mkdir -p ${APP_DIR} && tar -xzf ${BUNDLE_GUEST_PATH} -C ${APP_DIR}`,+ ),],},});@@ -442,7 +480,17 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP);}- async function waitAppReady(appName: string, machineId: string): Promise<void> {+ async function waitServing(+ d: Deployment,+ env: Record<string, string> | undefined,+ appName: string,+ machineId: string,+ ): Promise<void> {+ await waitAppReady(execIn(appName, machineId), {+ appPort: APP_PORT,+ windowSec: Math.ceil(appReadyTimeoutMs / 1000),+ report: appReporter(opts.onError, d, env),+ });const deadline = Date.now() + appReadyTimeoutMs;let machine: FlyMachine | null = null;while (Date.now() < deadline) {@@ -463,12 +511,33 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPreturn {profile: { managedScaleToZero: !!opts.sharedAppName, ...(opts.dataVolumeSizeGb ? { dataDir: "/data" } : {}) },- async resolveEndpoint(d) {+ async resolveEndpoint(d, version) {+ if (!checked.has(d.id)) {+ ensureConfigured();+ const machines = await deploymentMachines(d).catch(+ swallowAs<FlyMachine[]>("fly-deploy: list app machines", []),+ );+ const appName = appNameFor(d);+ const report = appReporter(opts.onError, d, version.env);+ if (+ machines.length === 1 &&+ machines[0]!.state === "started" &&+ (await appGone(execIn(appName, machines[0]!.id), { appPort: APP_PORT, report }))+ )+ return null;+ checked.set(d.id, true);+ }if (!opts.privateTransport) return d.endpoint;- ensureConfigured();return endpointFor(d, await portFor(d));},+ async logs(d, logOpts): Promise<string | null> {+ ensureConfigured();+ const machines = await deploymentMachines(d);+ if (machines.length !== 1 || machines[0]!.state !== "started") return null;+ return appLogs(execIn(appNameFor(d), machines[0]!.id), logOpts.tailLines);+ },+async setAlwaysOn(d, alwaysOn): Promise<void> {ensureConfigured();const appName = appNameFor(d);@@ -487,13 +556,13 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPtry {await api.updateMachine(appName, machine.id, config);await waitStarted(appName, machine.id);- await waitAppReady(appName, machine.id);+ await waitServing(d, config.env, appName, machine.id);await api.setCordon(appName, machine.id, false);if (opts.configStore) await opts.configStore.put(d.id, config);} catch (error) {await api.updateMachine(appName, machine.id, accepted);await waitStarted(appName, machine.id);- await waitAppReady(appName, machine.id);+ await waitServing(d, accepted.env, appName, machine.id);if (opts.configStore) await opts.configStore.put(d.id, accepted);throw error;}@@ -535,14 +604,14 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPtry {await api.updateMachine(appName, previous.id, config);await waitStarted(appName, previous.id);- await waitAppReady(appName, previous.id);+ await waitServing(d, config.env, appName, previous.id);await api.setCordon(appName, previous.id, false);await opts.configStore!.put(d.id, config);} catch (error) {if (accepted) {await api.updateMachine(appName, previous.id, accepted);await waitStarted(appName, previous.id);- await waitAppReady(appName, previous.id);+ await waitServing(d, accepted.env, appName, previous.id);await api.setCordon(appName, previous.id, false);await opts.configStore!.put(d.id, accepted);}@@ -552,7 +621,7 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployPconst created = await api.createMachine(appName, { region, config, skip_service_registration: true });try {await waitStarted(appName, created.id);- await waitAppReady(appName, created.id);+ await waitServing(d, config.env, appName, created.id);await api.setCordon(appName, created.id, false);await opts.configStore!.put(d.id, config);} catch (error) {@@ -573,7 +642,7 @@ export function createFlyDeployProvider(opts: FlyDeployProviderOptions): DeployP});try {await waitStarted(appName, machine.id);- await waitAppReady(appName, machine.id);+ await waitServing(d, version.env, appName, machine.id);} catch (error) {await api.destroyMachine(appName, machine.id)diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.tsindex 85906a039..fe3adb235 100644--- a/src/deploy/porter-deploy-provider.ts+++ b/src/deploy/porter-deploy-provider.ts@@ -3,7 +3,7 @@ import { LRUCache } from "lru-cache";import { NotFoundError } from "porter-sandbox";import type { Deployment, DeploymentVersion } from "./deploy-store.ts";import type { DeployEndpoint, DeployProvider } from "./deploy-provider.ts";-import { appGone, appReporter, supervised, waitAppReady, type AppErrorSink } from "./shared-deploy-provider.ts";+import { appGone, appLaunch, appLogs, appReporter, waitAppReady, type AppErrorSink } from "./shared-deploy-provider.ts";import { normalizeRelPath, posixJoin, readTree } from "./deploy-fs.ts";import { makeTar } from "../sandbox/tar.ts";import {@@ -27,8 +27,6 @@ import { errMessage, swallow, swallowAs } from "../util/errors.ts";const APP_DIR = "/app";const HOME_DIR = "/root";const DATA_DIR = "/data";-const PID_PATH = "/tmp/qm-app.pid";-const LOG_PATH = "/tmp/qm-app.log";const APP_PORT_DEFAULT = 8080;const ENDPOINT_PORT = 443;const APP_READY_WINDOW_SEC_DEFAULT = 60;@@ -148,19 +146,16 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dasync function startApp(d: Deployment, sandboxId: string, version: DeploymentVersion): Promise<void> {const inner = `cd ${shq(APP_DIR)}; ${version.entrypoint}`;- const launch = `sh -c ${shq(supervised(inner, appPort, PID_PATH))} < /dev/null > ${shq(LOG_PATH)} 2>&1 & echo $! > ${shq(PID_PATH)}`;const script = [`mkdir -p ${shq(APP_DIR)} ${shq(DATA_DIR)}`,`export HOME=${shq(HOME_DIR)} PORT=${appPort} DATA_DIR=${shq(DATA_DIR)}`,- `if command -v setsid >/dev/null 2>&1; then setsid ${launch}; else ${launch}; fi`,+ appLaunch(inner, appPort),].join("; ");const r = await execRaw(sandboxId, script, APP_START_EXEC_TIMEOUT_SEC);if (r.code !== 0) throw new Error(`porter deploy app start failed: ${r.stderr.slice(0, 300)}`);await waitAppReady((script, timeoutSec) => execRaw(sandboxId, script, timeoutSec), {appPort,windowSec: readyWindowSec,- pidPath: PID_PATH,- logPath: LOG_PATH,report: appReporter(opts.onError, d, version.env),});}@@ -255,7 +250,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D}const exec = (script: string, timeoutSec: number) => execRaw(stored.sandboxId, script, timeoutSec);const report = appReporter(opts.onError, d, version.env);- if (await appGone(exec, { appPort, pidPath: PID_PATH, logPath: LOG_PATH, report })) return null;+ if (await appGone(exec, { appPort, report })) return null;const endpoint = endpointOf(stored);if (resolveCacheMs > 0) resolveCache.set(d.id, endpoint);return endpoint;@@ -264,9 +259,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dasync logs(d, logOpts): Promise<string | null> {const stored = await liveStored(d);if (!stored) return null;- const lines = Math.max(1, Math.min(2000, Math.floor(logOpts.tailLines)));- const r = await execRaw(stored.sandboxId, `tail -n ${lines} ${shq(LOG_PATH)} 2>/dev/null || true`, 30);- return r.stdout;+ return appLogs((script, timeoutSec) => execRaw(stored.sandboxId, script, timeoutSec), logOpts.tailLines);},destroy: (d) =>diff --git a/src/deploy/shared-deploy-provider.ts b/src/deploy/shared-deploy-provider.tsindex 94dd5f8ec..d6578511f 100644--- a/src/deploy/shared-deploy-provider.ts+++ b/src/deploy/shared-deploy-provider.ts@@ -1,3 +1,4 @@+import { LRUCache } from "lru-cache";import type { ExecResult } from "../sandbox/sandbox.ts";import { shq } from "../util/shell.ts";import { errMessage, swallow } from "../util/errors.ts";@@ -13,8 +14,11 @@ const REPORT_CONTEXT_LINES = 40;const READY_EXEC_GRACE_SEC = 30;const APP_PROBE_TIMEOUT_SEC = 5;const EXITED_MARK = "qm: entrypoint exited";+const APP_CHECK_CACHE_MS = 15_000;+export const APP_PID_PATH = "/tmp/qm-app.pid";+const APP_LOG_PATH = "/tmp/qm-app.log";-type Exec = (script: string, timeoutSec: number) => Promise<Pick<ExecResult, "stdout" | "stderr" | "code">>;+export type Exec = (script: string, timeoutSec: number) => Promise<Pick<ExecResult, "stdout" | "stderr" | "code">>;export type AppErrorSink = (e: { category: string; code: string; message: string; scopeLabel: string },@@ -25,7 +29,7 @@ type Report = (code: string, detail: string) => Error;export function appReporter(onError: AppErrorSink | undefined, d: Deployment, env?: Record<string, string>): Report {const mask = createSecretValueMasker(env);return (code, detail) => {- const error = new Error(redactSecrets(mask(`app ${d.name ?? d.id} (deployment ${d.id}): ${detail}`)));+ const error = new Error(`app ${d.name ?? d.id} (deployment ${d.id}): ${redactSecrets(mask(detail))}`);onError?.({ category: "deploy", code, message: error.message, scopeLabel: d.ownerScopeId }, error);return error;};@@ -34,44 +38,60 @@ export function appReporter(onError: AppErrorSink | undefined, d: Deployment, enexport interface AppReadyOptions {appPort: number;windowSec: number;- pidPath: string;- logPath: string;report?: Report | undefined;}export const listeningOn = (port: number): string =>- `listening() { curl -s -o /dev/null --max-time 1 http://127.0.0.1:${port}/; [ "$?" != "7" ]; }`;+ `listening() { curl -s -o /dev/null --max-time 1 http://127.0.0.1:${port}/; c=$?; [ "$c" != 7 ] && [ "$c" != 127 ]; }`;-const exitedPath = (pidPath: string): string => shq(`${pidPath}.exited`);+const EXITED = shq(`${APP_PID_PATH}.exited`);+const REPORTED = shq(`${APP_LOG_PATH}.reported`);-export function supervised(inner: string, appPort: number, pidPath: string): string {+function supervised(inner: string, appPort: number): string {return [listeningOn(appPort),- `rm -f ${exitedPath(pidPath)}`,+ `rm -f ${EXITED} ${REPORTED}`,"n=1",- `while :; do s=$(date +%s); sh -c ${shq(inner)}; rc=$?; echo "$rc" >> ${exitedPath(pidPath)}`,+ `while :; do s=$(date +%s); sh -c ${shq(inner)}; rc=$?; echo "$rc" >> ${EXITED}`,'echo "qm: entrypoint exited with status $rc, restarting"',"while listening; do sleep 5; done",`[ $(( $(date +%s) - s )) -lt 60 ] || n=1; sleep "$n"; [ "$n" -ge 30 ] || n=$((n * 2)); done`,].join("; ");}-export async function appGone(- exec: Exec,- opts: { appPort: number; pidPath: string; logPath: string; report?: Report | undefined },-): Promise<boolean> {- const log = shq(opts.logPath);- const seen = shq(`${opts.logPath}.reported`);+export function appLaunch(inner: string, appPort: number): string {+ const launch = `sh -c ${shq(supervised(inner, appPort))} < /dev/null > ${shq(APP_LOG_PATH)} 2>&1 & echo $! > ${shq(APP_PID_PATH)}`;+ return `if command -v setsid >/dev/null 2>&1; then setsid ${launch}; else ${launch}; fi`;+}++export function appBoot(inner: string, appPort: number, prep = "true"): string {+ return [+ `echo $$ > ${APP_PID_PATH}`,+ `if { ${prep}; } > ${shq(APP_LOG_PATH)} 2>&1; then ${appLaunch(inner, appPort)}; else echo prep > ${APP_PID_PATH}; fi`,+ `trap 'p=$(cat ${APP_PID_PATH}); kill -TERM -"$p" 2>/dev/null || kill -TERM "$p"; while kill -0 "$p" 2>/dev/null; do sleep 0.2; done; exit 0' INT TERM`,+ "while :; do sleep 3600 & wait $!; done",+ ].join("; ");+}++export const recentAppChecks = (): LRUCache<string, true> => new LRUCache({ max: 500, ttl: APP_CHECK_CACHE_MS });++export async function appLogs(exec: Exec, tailLines: number): Promise<string> {+ const lines = Math.max(1, Math.min(2000, Math.floor(tailLines)));+ return (await exec(`tail -n ${lines} ${shq(APP_LOG_PATH)} 2>/dev/null || true`, 30)).stdout;+}++export async function appGone(exec: Exec, opts: { appPort: number; report?: Report | undefined }): Promise<boolean> {+ const log = shq(APP_LOG_PATH);const probe = [listeningOn(opts.appPort),- `pid=$(cat ${shq(opts.pidPath)} 2>/dev/null || true)`,- `s=$(cat ${seen} 2>/dev/null || true)`,+ `pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true)`,+ `s=$(cat ${REPORTED} 2>/dev/null || true)`,`if ! listening && ! { [ -n "$pid" ] && { kill -0 -"$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null; }; }; then ` +- `[ -n "$pid" ] && [ "$s" != "down $pid" ] && { echo "down $pid" > ${seen}; echo down; tail -c ${REPORT_LOG_BYTES} ${log} 2>/dev/null; }; exit ${EXIT_APP_GONE}; fi`,- `n=$(wc -l < ${exitedPath(opts.pidPath)} 2>/dev/null || echo 0); r=0`,+ `[ -n "$pid" ] && [ "$s" != "down $pid" ] && { echo "down $pid" > ${REPORTED}; echo down; tail -c ${REPORT_LOG_BYTES} ${log} 2>/dev/null; }; exit ${EXIT_APP_GONE}; fi`,+ `n=$(wc -l < ${EXITED} 2>/dev/null || echo 0); r=0`,`[ "\${s% *}" = "$pid" ] && r=\${s#* }`,`[ "$n" -gt "$r" ] || exit 0`,- `echo "$pid $n" > ${seen}; echo $((n - r)); grep -F -B ${REPORT_CONTEXT_LINES} ${shq(EXITED_MARK)} ${log} | tail -c ${REPORT_LOG_BYTES}`,+ `echo "$pid $n" > ${REPORTED}; echo $((n - r)); grep -F -B ${REPORT_CONTEXT_LINES} ${shq(EXITED_MARK)} ${log} | tail -c ${REPORT_LOG_BYTES}`,].join("; ");const r = await exec(probe, APP_PROBE_TIMEOUT_SEC).catch((e) => {swallow("deploy: app liveness probe", e);@@ -99,17 +119,16 @@ export async function appGone(export async function waitAppReady(exec: Exec, opts: AppReadyOptions): Promise<void> {const probe =- `pid=$(cat ${shq(opts.pidPath)} 2>/dev/null || true); died=0; ` +- `end=$(( $(date +%s) + ${opts.windowSec} )); ` +- `while [ "$(date +%s)" -lt "$end" ]; do ` ++ `died=0; end=$(( $(date +%s) + ${opts.windowSec} )); ` ++ `while [ "$(date +%s)" -lt "$end" ]; do pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true); ` +`code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:${opts.appPort}/); ` +`[ "$code" != 000 ] && [ "$code" -lt 500 ] && exit 0; ` +- `{ kill -0 -"$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null; } && [ ! -f ${exitedPath(opts.pidPath)} ] || died=1; ` ++ `[ -z "$pid" ] || { kill -0 -"$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null; } && [ ! -f ${EXITED} ] || died=1; ` +`sleep 0.5; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` +`[ "$code" -ge 500 ] 2>/dev/null && { echo "$code"; exit ${EXIT_APP_SERVER_ERROR}; }; exit 1`;const r = await exec(probe, opts.windowSec + READY_EXEC_GRACE_SEC);if (r.code === 0) return;- const log = await exec(`tail -c ${REPORT_LOG_BYTES} ${shq(opts.logPath)} 2>/dev/null || true`, 30)+ const log = await exec(`tail -c ${REPORT_LOG_BYTES} ${shq(APP_LOG_PATH)} 2>/dev/null || true`, 30).then((out) => `${out.stdout}${out.stderr}`.trim()).catch((e: unknown) => `(the app log could not be read: ${errMessage(e)})`);let why = `app never listened on port ${opts.appPort} within ${opts.windowSec}s`;diff --git a/src/wiring.ts b/src/wiring.tsindex dea433bf7..f729511fb 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1681,13 +1681,14 @@ export function buildApp(: undefined;const buildDeployProvider: Record<Config["deployProvider"], () => DeployProvider> = {aws: buildAwsDeploy,- docker: createDockerDeployProvider,+ docker: () => createDockerDeployProvider({ onError: sandboxOnError }),fly: () =>createFlyDeployProvider({...config.flyDeploy,...(flyTunnel ? { privateTransport: flyTunnel } : {}),configStore: artifactMap<FlyMachineConfig>("fly_deploy_configs"),portStore: artifactMap<string>("fly_deploy_ports"),+ onError: sandboxOnError,}),porter: () =>createPorterDeployProvider({diff --git a/test/docker-deploy-provider.test.ts b/test/docker-deploy-provider.test.tsindex bf63ad8b1..c78dbc61f 100644--- a/test/docker-deploy-provider.test.ts+++ b/test/docker-deploy-provider.test.ts@@ -188,3 +188,30 @@ test("the daemon probe reports a hung daemon as a timeout", async () => {assert.equal(await dockerDaemonFailure({ dockerExec }), "no response within 10s");});++test("a Docker app that only answers 5xx fails publish, reports its output and leaves no container", async () => {+ const calls: string[][] = [];+ const dockerExec: DockerExec = async (args) => {+ calls.push(args);+ const script = args[0] === "exec" ? args.at(-1)! : "";+ if (script.includes("http_code")) return { code: 96, stdout: "500\n", stderr: "" };+ if (script.startsWith("tail -c")) return { code: 0, stdout: "boot: database unreachable", stderr: "" };+ return { code: 0, stdout: "", stderr: "" };+ };+ const reported: Array<{ code: string; message: string }> = [];+ const store = createDeployStore();+ const d = await store.create({+ ownerScopeId: scopeId("personal", "U1"),+ createdBy: "U1",+ entrypoint: "node server.js",+ snapshotDir: "/snap/broken",+ });+ const provider = createDockerDeployProvider({ dockerExec, onError: (e) => reported.push(e) });+ await assert.rejects(provider.apply(d, d.versions[0]!), /answered HTTP 500 on \/ .*database unreachable/s);+ assert.deepEqual(+ reported.map((e) => e.code),+ ["app_start_failed"],+ );+ assert.match(reported[0]!.message, new RegExp(`\\(deployment ${d.id}\\)`), "the record names the deployment");+ assert.equal(calls.at(-2)!.join(" "), `rm -f agent-deploy-${d.id.slice(0, 12)}`);+});diff --git a/test/fly-deploy-provider.test.ts b/test/fly-deploy-provider.test.tsindex cc1e35cf1..5a516e88f 100644--- a/test/fly-deploy-provider.test.ts+++ b/test/fly-deploy-provider.test.ts@@ -50,6 +50,7 @@ interface FakeFlyOptions {cordonStatus?: number;cordonFailsAfterApply?: boolean;uncordonStatus?: number;+ exec?: (script: string) => { exit_code?: number; stdout?: string };}function fakeFly(opts: FakeFlyOptions = {}) {@@ -160,6 +161,10 @@ function fakeFly(opts: FakeFlyOptions = {}) {if (status < 300) cordoned.delete(machineId);return json(status, {});}+ if (method === "POST" && segments.length === 6 && segments[5] === "exec") {+ const script = /^sh -c '(.*)'$/s.exec(JSON.parse(String(init.body)).cmd)![1]!.replaceAll(`'"'"'`, "'");+ return json(200, opts.exec?.(script) ?? {});+ }if (method === "DELETE" && segments.length === 5) {if ((opts.destroyMachineStatus ?? 200) < 300) {machines.delete(machineId);@@ -260,7 +265,7 @@ test("apply: creates an isolated Fly app, injects the snapshot, and returns privassert.equal(config.files[0]!.guest_path, "/app.tar.gz");assert.deepEqual(config.init.exec.slice(0, 2), ["/bin/sh", "-lc"]);assert.match(config.init.exec[2]!, /tar -xzf \/app\.tar\.gz -C \/app/);- assert.match(config.init.exec[2]!, /exec sh -lc 'node server\.js'/);+ assert.match(config.init.exec[2]!, /cd \/app; node server\.js/);const unpacked = await parseTar(gunzipSync(Buffer.from(config.files[0]!.raw_value, "base64")));assert.deepEqual(@@ -417,6 +422,39 @@ test("apply: an entrypoint that exits without binding the port reports why, with);});+test("apply: an app that only answers 5xx fails publish and reports its output", async () => {+ const reported: Array<{ code: string; message: string }> = [];+ const { fetchImpl, machines } = fakeFly({+ exec: (script) =>+ script.includes("http_code") ? { exit_code: 96, stdout: "500\n" } : { stdout: "boot: database unreachable" },+ });+ await assert.rejects(+ provider(fetchImpl, { onError: (e) => reported.push(e) }).apply(+ deployment(ID),+ version(snapshot({ "server.js": "" })),+ ),+ /answered HTTP 500 on \/ .*database unreachable/s,+ );+ assert.deepEqual(+ reported.map((e) => e.code),+ ["app_start_failed"],+ );+ assert.equal(machines.size, 0, "the unhealthy machine is removed");+});++test("a visit finds a Fly app with no supervisor, reports it and asks for a relaunch", async () => {+ const reported: Array<{ code: string; message: string }> = [];+ const { fetchImpl } = fakeFly({+ existingMachines: ["machine-live"],+ exec: (script) => (script.includes(".reported") ? { exit_code: 99, stdout: "down\nfatal: out of widgets" } : {}),+ });+ const d = { ...deployment(ID), endpoint: { host: `${APP}.flycast`, port: 8080 } };+ const deploy = provider(fetchImpl, { onError: (e) => reported.push(e) });+ assert.equal(await deploy.resolveEndpoint!(d, version(snapshot({}))), null);+ assert.equal(reported[0]!.code, "app_down");+ assert.match(reported[0]!.message, /out of widgets/);+});+test("apply: an app that stays up but never passes its service check reports the readiness window", async () => {const { fetchImpl } = fakeFly({ checkStates: ["critical"] });await assert.rejects(