Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fix app readiness and liveness probes found broken by live Fly and AWS QA (#2238)

Regan Bellee6d77560023
diff --git a/docs/error-reporting.md b/docs/error-reporting.md
index d73c9c4d8..33830b200 100644
--- a/docs/error-reporting.md
+++ b/docs/error-reporting.md
@@ -24,4 +24,4 @@ Browser Sentry ignores plain network `TypeError` failures (`Failed to fetch`, `N
## Published apps
-Published-app failures become operator error records in the app owner's scope, so they reach the admin errors view and Sentry: `deploy:app_start_failed` (publish failed, including an app that only answered 5xx on `/` for the readiness window), `deploy:app_restarted` (the supervisor restarted the app since the last visit-time check), and `deploy:app_down` (the app and its supervisor are gone). Each record names the deployment and carries up to 4 KB of the app's recent output, with the app's env values and known secret patterns masked. Every app host (AWS, Porter, Fly, Docker) runs the app under the same supervisor and probes, so the app image must provide `sh` and `curl`.
+Published-app failures become operator error records in the app owner's scope, so they reach the admin errors view and Sentry: `deploy:app_start_failed` (publish failed, including an app that only answered 5xx on `/` for the readiness window), `deploy:app_restarted` (the supervisor restarted the app since the last visit-time check), and `deploy:app_down` (the app and its supervisor are gone). Each record names the deployment and carries up to 4 KB of the app's recent output, with the app's env values and known secret patterns masked. Every app host (AWS, Porter, Fly, Docker) runs the app under the same supervisor and probes, so the app image must provide `sh`; with `curl` or `wget` in the image the readiness check also rejects an app that only answers 5xx, and without either it waits only for the port to listen.
diff --git a/src/deploy/shared-deploy-provider.ts b/src/deploy/shared-deploy-provider.ts
index d6578511f..df65369f5 100644
--- a/src/deploy/shared-deploy-provider.ts
+++ b/src/deploy/shared-deploy-provider.ts
@@ -11,7 +11,8 @@ const EXIT_APP_EXITED = 98;
const EXIT_APP_GONE = 99;
const REPORT_LOG_BYTES = 4000;
const REPORT_CONTEXT_LINES = 40;
-const READY_EXEC_GRACE_SEC = 30;
+const READY_EXEC_WINDOW_SEC = 45;
+const READY_EXEC_GRACE_SEC = 15;
const APP_PROBE_TIMEOUT_SEC = 5;
const EXITED_MARK = "qm: entrypoint exited";
const APP_CHECK_CACHE_MS = 15_000;
@@ -42,7 +43,21 @@ export interface AppReadyOptions {
}
export const listeningOn = (port: number): string =>
- `listening() { curl -s -o /dev/null --max-time 1 http://127.0.0.1:${port}/; c=$?; [ "$c" != 7 ] && [ "$c" != 127 ]; }`;
+ `listening() { grep -q ':${port.toString(16).toUpperCase().padStart(4, "0")} [0-9A-F]*:[0-9A-F]* 0A' /proc/net/tcp /proc/net/tcp6 2>/dev/null; }`;
+
+const ALIVE =
+ 'alive() { _t=$1; [ -n "$_t" ] || return 1; for _f in /proc/[0-9]*/stat; do read -r _s 2>/dev/null < "$_f" || continue; ' +
+ '_p=${_f#/proc/}; _s=${_s##*) }; set -- $_s; [ "$1" != Z ] && { [ "$3" = "$_t" ] || [ "${_p%/stat}" = "$_t" ]; } && return 0; done; return 1; }';
+
+const statusOn = (port: number): string =>
+ `status() { if command -v curl >/dev/null 2>&1; then curl -s --max-time 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:${port}/; ` +
+ `elif command -v wget >/dev/null 2>&1; then wget -S -q -T 2 -t 1 -O /dev/null http://127.0.0.1:${port}/ 2>&1 | awk '$1 ~ /^HTTP\\// { c = $2 } END { print c ? c : "000" }'; fi; }`;
+
+const whole = (log: string): string => {
+ if (Buffer.byteLength(log) < REPORT_LOG_BYTES) return log;
+ const rest = log.slice(log.indexOf("\n") + 1 || log.length);
+ return rest.trim() ? `…\n${rest}` : `(the last line is longer than ${REPORT_LOG_BYTES} bytes and was left out)`;
+};
const EXITED = shq(`${APP_PID_PATH}.exited`);
const REPORTED = shq(`${APP_LOG_PATH}.reported`);
@@ -84,9 +99,10 @@ export async function appGone(exec: Exec, opts: { appPort: number; report?: Repo
const log = shq(APP_LOG_PATH);
const probe = [
listeningOn(opts.appPort),
+ ALIVE,
`pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true)`,
`s=$(cat ${REPORTED} 2>/dev/null || true)`,
- `if ! listening && ! { [ -n "$pid" ] && { kill -0 -"$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null; }; }; then ` +
+ `if ! listening && ! alive "$pid"; then ` +
`[ -n "$pid" ] && [ "$s" != "down $pid" ] && { echo "down $pid" > ${REPORTED}; echo down; tail -c ${REPORT_LOG_BYTES} ${log} 2>/dev/null; }; exit ${EXIT_APP_GONE}; fi`,
`n=$(wc -l < ${EXITED} 2>/dev/null || echo 0); r=0`,
`[ "\${s% *}" = "$pid" ] && r=\${s#* }`,
@@ -98,38 +114,44 @@ export async function appGone(exec: Exec, opts: { appPort: number; report?: Repo
return null;
});
if (r?.code === EXIT_APP_GONE) {
- const [first, ...tail] = r.stdout.trim().split("\n");
+ const [first, ...tail] = r.stdout.split("\n");
if (first === "down")
opts.report?.(
"app_down",
- `app stopped and nothing restarted it; last output:\n${tail.join("\n").trim() || "(none)"}`,
+ `app stopped and nothing restarted it; last output:\n${whole(tail.join("\n")).trim() || "(none)"}`,
);
return true;
}
- const out = r?.code === 0 ? r.stdout.trim() : "";
- if (out) {
+ const out = r?.code === 0 ? r.stdout : "";
+ if (out.trim()) {
const [count, ...rest] = out.split("\n");
opts.report?.(
"app_restarted",
- `app exited ${count} time(s) and was restarted; output before exiting:\n${rest.join("\n")}`,
+ `app exited ${count} time(s) and was restarted; output before exiting:\n${whole(rest.join("\n")).trim()}`,
);
}
return false;
}
export async function waitAppReady(exec: Exec, opts: AppReadyOptions): Promise<void> {
- const probe =
- `died=0; end=$(( $(date +%s) + ${opts.windowSec} )); ` +
+ const probe = (windowSec: number): string =>
+ `${listeningOn(opts.appPort)}; ${statusOn(opts.appPort)}; ${ALIVE}; ` +
+ `died=0; end=$(( $(date +%s) + ${windowSec} )); ` +
`while [ "$(date +%s)" -lt "$end" ]; do pid=$(cat ${shq(APP_PID_PATH)} 2>/dev/null || true); ` +
- `code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:${opts.appPort}/); ` +
- `[ "$code" != 000 ] && [ "$code" -lt 500 ] && exit 0; ` +
- `[ -z "$pid" ] || { kill -0 -"$pid" 2>/dev/null || kill -0 "$pid" 2>/dev/null; } && [ ! -f ${EXITED} ] || died=1; ` +
+ `code=; listening && { code=$(status); case "$code" in ""|[1-4]??) exit 0 ;; esac; }; ` +
+ `[ -z "$pid" ] || alive "$pid" && [ ! -f ${EXITED} ] || died=1; ` +
`sleep 0.5; done; [ "$died" = 1 ] && exit ${EXIT_APP_EXITED}; ` +
`[ "$code" -ge 500 ] 2>/dev/null && { echo "$code"; exit ${EXIT_APP_SERVER_ERROR}; }; exit 1`;
- const r = await exec(probe, opts.windowSec + READY_EXEC_GRACE_SEC);
+ let left = opts.windowSec;
+ let r: Awaited<ReturnType<Exec>>;
+ do {
+ const windowSec = Math.min(READY_EXEC_WINDOW_SEC, left);
+ left -= windowSec;
+ r = await exec(probe(windowSec), windowSec + READY_EXEC_GRACE_SEC);
+ } while (r.code !== 0 && left > 0);
if (r.code === 0) return;
const log = await exec(`tail -c ${REPORT_LOG_BYTES} ${shq(APP_LOG_PATH)} 2>/dev/null || true`, 30)
- .then((out) => `${out.stdout}${out.stderr}`.trim())
+ .then((out) => whole(`${out.stdout}${out.stderr}`).trim())
.catch((e: unknown) => `(the app log could not be read: ${errMessage(e)})`);
let why = `app never listened on port ${opts.appPort} within ${opts.windowSec}s`;
if (r.code === EXIT_APP_EXITED) why = `the version's entrypoint exited without binding port ${opts.appPort}`;
diff --git a/test/aws-deploy-provider.test.ts b/test/aws-deploy-provider.test.ts
index ba83f1a2a..211cad25e 100644
--- a/test/aws-deploy-provider.test.ts
+++ b/test/aws-deploy-provider.test.ts
@@ -585,7 +585,7 @@ test("apply: the readiness gate fails when the launched process dies instead of
);
assert.match(
execs.find(readinessProbe)!,
- /kill -0 -"\$pid"/,
+ /alive "\$pid"/,
"readiness tracks the process this deploy launched, not just the port",
);
});