Python
Servidor: un sondeo que no llega al backend se reintenta en vez de dar el run por perdido
neuregex3d34bc9e6c09
diff --git a/server/history.ts b/server/history.tsindex 7369ac1..fbc8d7a 100644--- a/server/history.ts+++ b/server/history.ts@@ -84,6 +84,42 @@ function normStatus(s: string): Status {return (["queued", "running", "succeeded", "failed", "canceled", "expired"] as string[]).includes(s) ? (s as Status) : "running";}+/** A poll that failed to reach the backend, as opposed to the backend answering that the job is gone or the request is+ * wrong: node's fetch says "fetch failed" (connection reset, DNS, TLS), a proxy answers 502/503/504 while the GPU+ * container starts, or nothing answers in time. */+export function isTransient(e: unknown): boolean {+ const err = e as { message?: string; httpStatus?: number; name?: string; cause?: { code?: string } };+ if (typeof err?.httpStatus === "number") return err.httpStatus >= 500 || err.httpStatus === 429;+ if (/not terminal after/.test(err?.message ?? "")) return false; // the SDK's own wait timeout+ return /fetch failed|ECONN|ETIMEDOUT|EAI_AGAIN|socket hang up|network|timed? ?out|aborted/i.test(`${err?.message ?? ""} ${err?.cause?.code ?? ""}`);+}++const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));++/** job.wait() stops at the first poll that fails to reach the backend; a backend waking up (the Modal container's cold+ * start) drops one or two, so the wait is picked up again, for up to `patience` of continuous failures, before the+ * run is called lost. */+export async function waitPatiently(job: { wait(timeoutMs?: number): Promise<unknown> }, timeoutMs: number, patience = 10 * 60 * 1000,+ log: (msg: string) => void = () => {}): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ let firstFailure: number | null = null;+ let failures = 0;+ for (;;) {+ try {+ await job.wait(Math.max(1000, deadline - Date.now()));+ return;+ } catch (e) {+ if (!isTransient(e) || Date.now() >= deadline) throw e;+ firstFailure ??= Date.now();+ if (Date.now() - firstFailure > patience) throw e;+ failures++;+ const pause = Math.min(15_000, 2_000 * failures);+ log(`poll failed (${(e as Error).message}); trying again in ${pause / 1000}s`);+ await sleep(pause);+ }+ }+}+const secondsBetween = (a: string | Date | null | undefined, b: string | Date | null | undefined) =>a && b ? Math.max(0, Math.round((new Date(b).getTime() - new Date(a).getTime()) / 100) / 10) : null;@@ -190,7 +226,7 @@ export class History {let waitError: unknown = null;try {- await job.wait(6 * 3600 * 1000);+ await waitPatiently(job, 6 * 3600 * 1000, 10 * 60 * 1000, (msg) => console.warn(`[history] ${entry.id}: ${msg}`));this.onContact();} catch (e) {waitError = e;diff --git a/test/history.test.ts b/test/history.test.tsindex a82cce2..c6bd33a 100644--- a/test/history.test.ts+++ b/test/history.test.ts@@ -147,3 +147,22 @@ test("file names are checked before touching the disk", () => {assert.equal(history.remove("../x"), false);} finally { done(); }});++test("a poll that cannot reach the backend is retried; a backend answer that the job is gone is not", async () => {+ const { waitPatiently, isTransient } = await import("../server/history.ts");+ let calls = 0;+ const flaky = { async wait() { calls++; if (calls < 3) throw Object.assign(new TypeError("fetch failed"), { cause: { code: "ECONNRESET" } }); } };+ const log: string[] = [];+ const t0 = Date.now();+ await waitPatiently(flaky, 60_000, 60_000, (m) => log.push(m));+ assert.equal(calls, 3);+ assert.equal(log.length, 2);+ assert.ok(Date.now() - t0 >= 2000, "backs off between attempts");++ const gone = { async wait() { throw Object.assign(new Error("job not found"), { httpStatus: 404 }); } };+ await assert.rejects(() => waitPatiently(gone, 60_000, 60_000), /job not found/);+ const proxy = Object.assign(new Error("Bad Gateway"), { httpStatus: 502 });+ assert.equal(isTransient(proxy), true);+ assert.equal(isTransient(new Error("job x not terminal after 10ms (status=running)")), false);+ assert.equal(isTransient(Object.assign(new Error("bad request"), { httpStatus: 400 })), false);+});