Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(scripts): let download scripts exit after following a redirect (#2410)
Chad Pihac4ae6071e882
diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.jsindex a67bb3f0e..7e197b316 100644--- a/scripts/lib/download-utils.js+++ b/scripts/lib/download-utils.js@@ -8,6 +8,13 @@ const MAX_RETRIES = 3;const RETRY_DELAY = 2000;const MAX_REDIRECTS = 5;+// Node's global agent keeps sockets alive, and a socket only returns to the pool (where it+// stops holding the event loop open) once its response has been read to the end. A redirect+// or error body we never read pins the socket until the server drops it, often for minutes.+function discardBody(response) {+ response.resume();+}+/*** Fetch JSON from a URL with proper error handling.* @param {string} url - URL to fetch@@ -40,6 +47,7 @@ function fetchJson(url, redirectCount = 0) {https.get(url, options, (res) => {if ([301, 302, 303, 307, 308].includes(res.statusCode)) {+ discardBody(res);const location = res.headers.location;if (!location) {reject(new Error("Redirect without location header"));@@ -53,6 +61,7 @@ function fetchJson(url, redirectCount = 0) {}if (res.statusCode !== 200) {+ discardBody(res);reject(new Error(`HTTP ${res.statusCode}`));return;}@@ -69,7 +78,10 @@ function fetchJson(url, redirectCount = 0) {res.on("error", reject);}).on("error", reject)- .on("timeout", () => reject(new Error("Request timeout")));+ .on("timeout", function onTimeout() {+ // Destroying rejects through the "error" handler and frees the stalled socket.+ this.destroy(new Error("Request timeout"));+ });});}@@ -161,8 +173,9 @@ function downloadFile(url, dest, retryCount = 0) {return;}- activeRequest = https.get(currentUrl, (response) => {+ const req = https.get(currentUrl, (response) => {if ([301, 302, 303, 307, 308].includes(response.statusCode)) {+ discardBody(response);const location = response.headers.location;if (!location) {cleanup();@@ -209,12 +222,21 @@ function downloadFile(url, dest, retryCount = 0) {});});- activeRequest.on("error", (err) => {+ activeRequest = req;++ // A redirect we already followed can still error or time out while its body drains;+ // that must not tear down the request that replaced it.+ req.on("error", (err) => {+ if (req !== activeRequest) return;cleanup();reject(err);});- activeRequest.setTimeout(REQUEST_TIMEOUT, () => {+ req.setTimeout(REQUEST_TIMEOUT, () => {+ if (req !== activeRequest) {+ req.destroy();+ return;+ }cleanup();reject(new Error("Connection timed out"));});diff --git a/test/scripts/downloadUtils.test.js b/test/scripts/downloadUtils.test.jsnew file mode 100644index 000000000..a51017ce3--- /dev/null+++ b/test/scripts/downloadUtils.test.js@@ -0,0 +1,174 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { spawn } = require("child_process");+const fs = require("fs");+const http = require("http");+const os = require("os");+const path = require("path");++const DOWNLOAD_UTILS = path.join(__dirname, "..", "..", "scripts", "lib", "download-utils.js");+const FILE_BODY = Buffer.alloc(64 * 1024, 7);+// Long enough that only an unreleased socket, never the server, could hold the child open.+const SERVER_KEEP_ALIVE_MS = 60_000;+const EXIT_DEADLINE_MS = 5_000;++// Mirrors the Hugging Face and GitHub responses: redirects and errors carry a body.+function startServer() {+ let pendingRedirectSocket = null;+ const server = http.createServer((req, res) => {+ const redirect = (location) => {+ res.writeHead(302, { Location: location, "Content-Type": "text/plain" });+ res.end("Found. Redirecting to " + location + "x".repeat(1024));+ };+ switch (req.url) {+ case "/model.bin":+ return redirect("/cdn/model.bin");+ case "/cdn/model.bin":+ res.writeHead(200, { "Content-Length": FILE_BODY.length });+ return res.end(FILE_BODY);+ case "/repos/owner/repo/releases/latest":+ return redirect("/repos/owner/renamed/releases/latest");+ case "/repos/owner/renamed/releases/latest":+ res.writeHead(200, { "Content-Type": "application/json" });+ return res.end(JSON.stringify({ tag_name: "v1.0.0", html_url: "u", assets: [] }));+ // A redirect whose body is still arriving when its connection is reset, which only+ // happens once the client has already moved on to the redirect target.+ case "/reset/model.bin":+ res.writeHead(302, { Location: "/reset/cdn/model.bin", "Content-Length": 4096 });+ res.write("Found.");+ pendingRedirectSocket = req.socket;+ return;+ case "/reset/cdn/model.bin":+ pendingRedirectSocket.resetAndDestroy();+ setTimeout(() => {+ res.writeHead(200, { "Content-Length": FILE_BODY.length });+ res.end(FILE_BODY);+ }, 100);+ return;+ case "/repos/owner/missing/releases/latest":+ res.writeHead(404, { "Content-Type": "application/json" });+ return res.end(JSON.stringify({ message: "Not Found" }));+ case "/repos/owner/stalled/releases/latest":+ return;+ default:+ res.writeHead(404);+ return res.end();+ }+ });+ server.keepAliveTimeout = SERVER_KEEP_ALIVE_MS;+ return new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server)));+}++// Runs `body` in a fresh Node process with https.get pointed at the local server, and+// resolves with how the process ended. A socket left referenced keeps the child alive.+// `requestTimeoutMs` shortens the timeout passed in request options (fetchJson's 30s).+function runChild(port, body, { requestTimeoutMs } = {}) {+ const script = `+ const http = require("http");+ const https = require("https");+ const requestTimeoutMs = ${JSON.stringify(requestTimeoutMs ?? null)};+ https.get = (url, ...rest) => {+ const target = new URL(url);+ if (requestTimeoutMs && rest[0] && typeof rest[0] === "object") {+ rest[0] = { ...rest[0], timeout: requestTimeoutMs };+ }+ return http.get("http://127.0.0.1:${port}" + target.pathname + target.search, ...rest);+ };+ const utils = require(${JSON.stringify(DOWNLOAD_UTILS)});+ (async () => { ${body} })().catch((error) => {+ console.error(error);+ process.exitCode = 1;+ });+ `;+ return new Promise((resolve) => {+ const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "pipe"] });+ let output = "";+ child.stdout.on("data", (chunk) => (output += chunk));+ child.stderr.on("data", (chunk) => (output += chunk));+ const timer = setTimeout(() => child.kill("SIGKILL"), EXIT_DEADLINE_MS);+ child.on("exit", (code, signal) => {+ clearTimeout(timer);+ resolve({ code, signal, output });+ });+ });+}++async function withServer(fn) {+ const server = await startServer();+ try {+ await fn(server.address().port);+ } finally {+ server.closeAllConnections();+ await new Promise((resolve) => server.close(resolve));+ }+}++test("downloadFile lets the process exit after following a redirect", async () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-utils-"));+ const dest = path.join(dir, "model.bin");+ try {+ await withServer(async (port) => {+ const result = await runChild(+ port,+ `await utils.downloadFile("https://huggingface.co/model.bin", ${JSON.stringify(dest)});`+ );+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);+ assert.equal(result.code, 0, result.output);+ });+ assert.deepEqual(fs.readFileSync(dest), FILE_BODY);+ } finally {+ fs.rmSync(dir, { recursive: true, force: true });+ }+});++test("fetchLatestRelease lets the process exit after a redirect or an error status", async () => {+ await withServer(async (port) => {+ const result = await runChild(+ port,+ `+ const found = await utils.fetchLatestRelease("owner/repo");+ const missing = await utils.fetchLatestRelease("owner/missing");+ console.log(JSON.stringify({ tag: found && found.tag, missing }));+ `+ );+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);+ assert.equal(result.code, 0, result.output);+ assert.match(result.output, /\{"tag":"v1\.0\.0","missing":null\}/);+ });+});++test("downloadFile ignores a reset on the redirect it already followed", async () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), "download-utils-"));+ const dest = path.join(dir, "model.bin");+ try {+ await withServer(async (port) => {+ const result = await runChild(+ port,+ `await utils.downloadFile("https://huggingface.co/reset/model.bin", ${JSON.stringify(dest)});`+ );+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);+ assert.equal(result.code, 0, result.output);+ assert.doesNotMatch(result.output, /Retry/);+ });+ assert.deepEqual(fs.readFileSync(dest), FILE_BODY);+ } finally {+ fs.rmSync(dir, { recursive: true, force: true });+ }+});++test("fetchLatestRelease lets the process exit after a request timeout", async () => {+ await withServer(async (port) => {+ const result = await runChild(+ port,+ `+ const stalled = await utils.fetchLatestRelease("owner/stalled");+ console.log(JSON.stringify({ stalled }));+ `,+ { requestTimeoutMs: 200 }+ );+ assert.equal(result.signal, null, `process did not exit on its own:\n${result.output}`);+ assert.equal(result.code, 0, result.output);+ assert.match(result.output, /Request timeout/);+ assert.match(result.output, /\{"stalled":null\}/);+ });+});