Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(scripts): let download scripts exit after following a redirect (#2410)

Chad Pihac4ae6071e882
diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.js
index a67bb3f0e..7e197b316 100644
--- a/scripts/lib/download-utils.js
+++ b/scripts/lib/download-utils.js
@@ -8,6 +8,13 @@ const MAX_RETRIES = 3;
const RETRY_DELAY = 2000;
const MAX_REDIRECTS = 5;
+// Node's global agent keeps sockets alive, and a socket only returns to the pool (where it
+// stops holding the event loop open) once its response has been read to the end. A redirect
+// or error body we never read pins the socket until the server drops it, often for minutes.
+function discardBody(response) {
+ response.resume();
+}
+
/**
* Fetch JSON from a URL with proper error handling.
* @param {string} url - URL to fetch
@@ -40,6 +47,7 @@ function fetchJson(url, redirectCount = 0) {
https
.get(url, options, (res) => {
if ([301, 302, 303, 307, 308].includes(res.statusCode)) {
+ discardBody(res);
const location = res.headers.location;
if (!location) {
reject(new Error("Redirect without location header"));
@@ -53,6 +61,7 @@ function fetchJson(url, redirectCount = 0) {
}
if (res.statusCode !== 200) {
+ discardBody(res);
reject(new Error(`HTTP ${res.statusCode}`));
return;
}
@@ -69,7 +78,10 @@ function fetchJson(url, redirectCount = 0) {
res.on("error", reject);
})
.on("error", reject)
- .on("timeout", () => reject(new Error("Request timeout")));
+ .on("timeout", function onTimeout() {
+ // Destroying rejects through the "error" handler and frees the stalled socket.
+ this.destroy(new Error("Request timeout"));
+ });
});
}
@@ -161,8 +173,9 @@ function downloadFile(url, dest, retryCount = 0) {
return;
}
- activeRequest = https.get(currentUrl, (response) => {
+ const req = https.get(currentUrl, (response) => {
if ([301, 302, 303, 307, 308].includes(response.statusCode)) {
+ discardBody(response);
const location = response.headers.location;
if (!location) {
cleanup();
@@ -209,12 +222,21 @@ function downloadFile(url, dest, retryCount = 0) {
});
});
- activeRequest.on("error", (err) => {
+ activeRequest = req;
+
+ // A redirect we already followed can still error or time out while its body drains;
+ // that must not tear down the request that replaced it.
+ req.on("error", (err) => {
+ if (req !== activeRequest) return;
cleanup();
reject(err);
});
- activeRequest.setTimeout(REQUEST_TIMEOUT, () => {
+ req.setTimeout(REQUEST_TIMEOUT, () => {
+ if (req !== activeRequest) {
+ req.destroy();
+ return;
+ }
cleanup();
reject(new Error("Connection timed out"));
});
diff --git a/test/scripts/downloadUtils.test.js b/test/scripts/downloadUtils.test.js
new file mode 100644
index 000000000..a51017ce3
--- /dev/null
+++ b/test/scripts/downloadUtils.test.js
@@ -0,0 +1,174 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { spawn } = require("child_process");
+const fs = require("fs");
+const http = require("http");
+const os = require("os");
+const path = require("path");
+
+const DOWNLOAD_UTILS = path.join(__dirname, "..", "..", "scripts", "lib", "download-utils.js");
+const FILE_BODY = Buffer.alloc(64 * 1024, 7);
+// Long enough that only an unreleased socket, never the server, could hold the child open.
+const SERVER_KEEP_ALIVE_MS = 60_000;
+const EXIT_DEADLINE_MS = 5_000;
+
+// Mirrors the Hugging Face and GitHub responses: redirects and errors carry a body.
+function startServer() {
+ let pendingRedirectSocket = null;
+ const server = http.createServer((req, res) => {
+ const redirect = (location) => {
+ res.writeHead(302, { Location: location, "Content-Type": "text/plain" });
+ res.end("Found. Redirecting to " + location + "x".repeat(1024));
+ };
+ switch (req.url) {
+ case "/model.bin":
+ return redirect("/cdn/model.bin");
+ case "/cdn/model.bin":
+ res.writeHead(200, { "Content-Length": FILE_BODY.length });
+ return res.end(FILE_BODY);
+ case "/repos/owner/repo/releases/latest":
+ return redirect("/repos/owner/renamed/releases/latest");
+ case "/repos/owner/renamed/releases/latest":
+ res.writeHead(200, { "Content-Type": "application/json" });
+ return res.end(JSON.stringify({ tag_name: "v1.0.0", html_url: "u", assets: [] }));
+ // A redirect whose body is still arriving when its connection is reset, which only
+ // happens once the client has already moved on to the redirect target.
+ case "/reset/model.bin":
+ res.writeHead(302, { Location: "/reset/cdn/model.bin", "Content-Length": 4096 });
+ res.write("Found.");
+ pendingRedirectSocket = req.socket;
+ return;
+ case "/reset/cdn/model.bin":
+ pendingRedirectSocket.resetAndDestroy();
+ setTimeout(() => {
+ res.writeHead(200, { "Content-Length": FILE_BODY.length });
+ res.end(FILE_BODY);
+ }, 100);
+ return;
+ case "/repos/owner/missing/releases/latest":
+ res.writeHead(404, { "Content-Type": "application/json" });
+ return res.end(JSON.stringify({ message: "Not Found" }));
+ case "/repos/owner/stalled/releases/latest":
+ return;
+ default:
+ res.writeHead(404);
+ return res.end();
+ }
+ });
+ server.keepAliveTimeout = SERVER_KEEP_ALIVE_MS;
+ return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server)));
+}
+
+// Runs `body` in a fresh Node process with https.get pointed at the local server, and
+// resolves with how the process ended. A socket left referenced keeps the child alive.
+// `requestTimeoutMs` shortens the timeout passed in request options (fetchJson's 30s).
+function runChild(port, body, { requestTimeoutMs } = {}) {
+ const script = `
+ const http = require("http");
+ const https = require("https");
+ const requestTimeoutMs = ${JSON.stringify(requestTimeoutMs ?? null)};
+ https.get = (url, ...rest) => {
+ const target = new URL(url);
+ if (requestTimeoutMs && rest[0] && typeof rest[0] === "object") {
+ rest[0] = { ...rest[0], timeout: requestTimeoutMs };
+ }
+ return http.get("http://127.0.0.1:${port}" + target.pathname + target.search, ...rest);
+ };
+ const utils = require(${JSON.stringify(DOWNLOAD_UTILS)});
+ (async () => { ${body} })().catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ });
+ `;
+ return new Promise((resolve) => {
+ const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "pipe"] });
+ let output = "";
+ child.stdout.on("data", (chunk) => (output += chunk));
+ child.stderr.on("data", (chunk) => (output += chunk));
+ const timer = setTimeout(() => child.kill("SIGKILL"), EXIT_DEADLINE_MS);
+ child.on("exit", (code, signal) => {
+ clearTimeout(timer);
+ resolve({ code, signal, output });
+ });
+ });
+}
+
+async function withServer(fn) {
+ const server = await startServer();
+ try {
+ await fn(server.address().port);
+ } finally {
+ server.closeAllConnections();
+ await new Promise((resolve) => server.close(resolve));
+ }
+}
+
+test("downloadFile lets the process exit after following a redirect", async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-utils-"));
+ const dest = path.join(dir, "model.bin");
+ try {
+ await withServer(async (port) => {
+ const result = await runChild(
+ port,
+ `await utils.downloadFile("https://huggingface.co/model.bin", ${JSON.stringify(dest)});`
+ );
+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);
+ assert.equal(result.code, 0, result.output);
+ });
+ assert.deepEqual(fs.readFileSync(dest), FILE_BODY);
+ } finally {
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("fetchLatestRelease lets the process exit after a redirect or an error status", async () => {
+ await withServer(async (port) => {
+ const result = await runChild(
+ port,
+ `
+ const found = await utils.fetchLatestRelease("owner/repo");
+ const missing = await utils.fetchLatestRelease("owner/missing");
+ console.log(JSON.stringify({ tag: found && found.tag, missing }));
+ `
+ );
+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);
+ assert.equal(result.code, 0, result.output);
+ assert.match(result.output, /\{"tag":"v1\.0\.0","missing":null\}/);
+ });
+});
+
+test("downloadFile ignores a reset on the redirect it already followed", async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-utils-"));
+ const dest = path.join(dir, "model.bin");
+ try {
+ await withServer(async (port) => {
+ const result = await runChild(
+ port,
+ `await utils.downloadFile("https://huggingface.co/reset/model.bin", ${JSON.stringify(dest)});`
+ );
+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);
+ assert.equal(result.code, 0, result.output);
+ assert.doesNotMatch(result.output, /Retry/);
+ });
+ assert.deepEqual(fs.readFileSync(dest), FILE_BODY);
+ } finally {
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+test("fetchLatestRelease lets the process exit after a request timeout", async () => {
+ await withServer(async (port) => {
+ const result = await runChild(
+ port,
+ `
+ const stalled = await utils.fetchLatestRelease("owner/stalled");
+ console.log(JSON.stringify({ stalled }));
+ `,
+ { requestTimeoutMs: 200 }
+ );
+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);
+ assert.equal(result.code, 0, result.output);
+ assert.match(result.output, /Request timeout/);
+ assert.match(result.output, /\{"stalled":null\}/);
+ });
+});