Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(build): keep HTTP errors out of the setup archive cache
Akshar Patelae7972bd3430
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 57810c005..a5426d569 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -42,7 +42,7 @@ jobs:- 'crates/**'- 'vendor/**'- 'scripts/diagnostic-build.rs'- - 'scripts/setup.js'+ - 'scripts/setup*'- 'Cargo.toml'- 'Cargo.lock'gpui:@@ -273,6 +273,9 @@ jobs:prefix-key: v1-rust-testssave-cache: ${{ github.ref == 'refs/heads/main' }}+ - name: Test native dependency setup+ run: node --test scripts/setup.test.mjs+- name: Run setuprun: node scripts/setup.jsdiff --git a/scripts/setup.js b/scripts/setup.jsindex d61d0898c..c400a297b 100644--- a/scripts/setup.js+++ b/scripts/setup.js@@ -71,9 +71,9 @@ async function main() {if (!(await fileExists(nativeDepsTarPath))) {console.log(`Downloading ${nativeDepsTar}`);- const nativeDepsBytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`)- .then((r) => r.blob())- .then((b) => b.arrayBuffer());+ const nativeDepsBytes = await downloadBytes(+ `${NATIVE_DEPS_URL}/${nativeDepsTar}`,+ );await fs.writeFile(nativeDepsTarPath, Buffer.from(nativeDepsBytes));console.log("Downloaded native deps");downloadedNativeDeps = true;@@ -148,9 +148,7 @@ async function main() {const ffmpegZip = `ffmpeg-${FFMPEG_VERSION}.zip`;const ffmpegZipPath = path.join(targetDir, ffmpegZip);if (!(await fileExists(ffmpegZipPath))) {- const ffmpegZipBytes = await fetch(FFMPEG_ZIP_URL)- .then((r) => r.blob())- .then((b) => b.arrayBuffer());+ const ffmpegZipBytes = await downloadBytes(FFMPEG_ZIP_URL);await fs.writeFile(ffmpegZipPath, Buffer.from(ffmpegZipBytes));console.log(`Downloaded ${ffmpegZip}`);downloadedFfmpeg = true;@@ -273,9 +271,9 @@ async function main() {let downloadedNativeDeps = false;if (!(await fileExists(nativeDepsTarPath))) {console.log(`Downloading ${nativeDepsTar}`);- const bytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`)- .then((r) => r.blob())- .then((b) => b.arrayBuffer());+ const bytes = await downloadBytes(+ `${NATIVE_DEPS_URL}/${nativeDepsTar}`,+ );await fs.writeFile(nativeDepsTarPath, Buffer.from(bytes));console.log("Downloaded native deps");downloadedNativeDeps = true;@@ -441,9 +439,7 @@ async function setupMacOSOnnxRuntime() {if (!(await fileExists(archivePath))) {console.log(`Downloading ${asset.name}`);- const bytes = await fetch(url)- .then((r) => r.blob())- .then((b) => b.arrayBuffer());+ const bytes = await downloadBytes(url);await fs.writeFile(archivePath, Buffer.from(bytes));console.log(`Downloaded ${asset.name}`);} else console.log(`Using cached ${asset.name}`);@@ -495,9 +491,7 @@ async function setupWindowsOnnxRuntime() {if (!(await fileExists(archivePath))) {console.log(`Downloading ${asset.name}`);- const bytes = await fetch(url)- .then((r) => r.blob())- .then((b) => b.arrayBuffer());+ const bytes = await downloadBytes(url);await fs.writeFile(archivePath, Buffer.from(bytes));console.log(`Downloaded ${asset.name}`);} else console.log(`Using cached ${asset.name}`);@@ -706,6 +700,13 @@ async function signMacOSDylib(filePath) {);}+async function downloadBytes(url) {+ const response = await fetch(url);+ if (!response.ok)+ throw new Error(`Failed to download ${url}: HTTP ${response.status}`);+ return response.arrayBuffer();+}+async function fileExists(path) {return await fs.access(path)diff --git a/scripts/setup.test.mjs b/scripts/setup.test.mjsnew file mode 100644index 000000000..2f6f979e6--- /dev/null+++ b/scripts/setup.test.mjs@@ -0,0 +1,180 @@+import assert from "node:assert/strict";+import { execFile as execFileCallback } from "node:child_process";+import { once } from "node:events";+import {+ copyFileSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ rmSync,+ writeFileSync,+} from "node:fs";+import { createServer } from "node:http";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { fileURLToPath } from "node:url";+import { promisify } from "node:util";++const execFile = promisify(execFileCallback);+const repositoryRoot = fileURLToPath(new URL("../", import.meta.url));++test(+ "setup retries failed downloads without poisoning the archive cache",+ {+ skip: process.platform !== "linux" && "Linux setup fixture",+ timeout: 60_000,+ },+ async (t) => {+ const fixture = mkdtempSync(path.join(tmpdir(), "cap-setup-"));+ t.after(() => rmSync(fixture, { recursive: true, force: true }));+ for (const file of [+ "package.json",+ "scripts/setup.js",+ "scripts/linux-bundle-config.mjs",+ "apps/desktop/src-tauri/tauri.conf.json",+ "apps/desktop-gpui/tauri.conf.json",+ ]) {+ const destination = path.join(fixture, file);+ mkdirSync(path.dirname(destination), { recursive: true });+ copyFileSync(path.join(repositoryRoot, file), destination);+ }++ const nativeName = "native-deps-x86_64-linux-gnu.tar.xz";+ const onnxName = "onnxruntime-linux-x64-1.24.3.tgz";+ const nativeUrl = `/spacedriveapp/native-deps/releases/download/v0.26/${nativeName}`;+ const onnxUrl = `/microsoft/onnxruntime/releases/download/v1.24.3/${onnxName}`;+ const nativeLibrary = "synthetic native library\n";+ const onnxLibrary = "synthetic ONNX library\n";+ const archives = new Map();+ for (const [url, entry, contents] of [+ [nativeUrl, "lib/libavcodec.so.61", nativeLibrary],+ [onnxUrl, `${onnxName.slice(0, -4)}/lib/libonnxruntime.so`, onnxLibrary],+ ]) {+ const archive = path.join(fixture, "assets", path.basename(url));+ const directory = `${archive}.contents`;+ mkdirSync(path.dirname(path.join(directory, entry)), { recursive: true });+ writeFileSync(path.join(directory, entry), contents);+ await execFile(+ "tar",+ [url.endsWith(".xz") ? "-cJf" : "-czf", archive, "-C", directory, "."],+ { signal: t.signal },+ );+ archives.set(url, readFileSync(archive));+ }++ let failNative = true;+ const requests = [];+ const server = createServer((request, response) => {+ requests.push(request.url);+ if (failNative && request.url === nativeUrl) {+ response.writeHead(503).end("Service Unavailable\n");+ return;+ }+ const archive = archives.get(request.url);+ response.writeHead(archive ? 200 : 404);+ response.end(archive ?? "Unexpected setup download");+ });+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ t.after(async () => {+ server.closeAllConnections();+ await promisify(server.close.bind(server))();+ });+ const origin = `http://127.0.0.1:${server.address().port}`;+ const preload = path.join(fixture, "http-fixture.mjs");+ writeFileSync(+ preload,+ `const networkFetch = globalThis.fetch;+globalThis.fetch = (input, options) => {+ const url = new URL(input);+ if (url.origin !== "https://github.com")+ throw new Error(\`Unexpected external request: \${url}\`);+ return networkFetch(new URL(url.pathname, ${JSON.stringify(origin)}), options);+};+`,+ );+ const runSetup = () =>+ execFile(process.execPath, ["--import", preload, "scripts/setup.js"], {+ cwd: fixture,+ signal: t.signal,+ timeout: 20_000,+ env: {+ ...process.env,+ NODE_OPTIONS: "",+ RUST_TARGET_TRIPLE: "x86_64-unknown-linux-gnu",+ CAP_USE_SCCACHE: "0",+ TMPDIR: fixture,+ },+ }).then(+ ({ stdout, stderr }) => ({ code: 0, stdout, stderr }),+ (error) => {+ assert.equal(typeof error.code, "number", error.message);+ return error;+ },+ );++ const cache = path.join(fixture, "target", `v0.26-${nativeName}`);+ const failed = await runSetup();+ assert.equal(failed.code, 1, failed.stderr);+ assert.deepEqual(requests, [nativeUrl]);+ assert.equal(+ existsSync(cache),+ false,+ "failed downloads must not be cached",+ );+ assert.match(failed.stderr, /HTTP 503/);++ failNative = false;+ for (const expectedRequests of [[nativeUrl, onnxUrl], []]) {+ requests.length = 0;+ const result = await runSetup();+ assert.equal(result.code, 0, result.stderr);+ assert.deepEqual(requests, expectedRequests);+ assert.deepEqual(readFileSync(cache), archives.get(nativeUrl));+ for (const [name, contents] of [+ ["libavcodec.so.61", nativeLibrary],+ ["libonnxruntime.so", onnxLibrary],+ ["libonnxruntime.so.1", onnxLibrary],+ ]) {+ for (const directory of [+ "target/debug",+ "target/release",+ "target/x86_64-unknown-linux-gnu/debug",+ "target/x86_64-unknown-linux-gnu/release",+ ]) {+ assert.equal(+ readFileSync(path.join(fixture, directory, name), "utf8"),+ contents,+ );+ }+ for (const directory of [+ "apps/desktop/src-tauri",+ "apps/desktop-gpui",+ ]) {+ const config = JSON.parse(+ readFileSync(+ path.join(fixture, directory, "tauri.linux.conf.json"),+ "utf8",+ ),+ );+ const source = config.bundle.linux.deb.files[`/usr/lib/cap/${name}`];+ assert.equal(+ readFileSync(path.resolve(fixture, directory, source), "utf8"),+ contents,+ );+ }+ }+ const cargo = readFileSync(+ path.join(fixture, ".cargo/config.toml"),+ "utf8",+ );+ assert.match(cargo, /\[target\.x86_64-unknown-linux-gnu\]/);+ assert.match(+ cargo,+ /ORT_DYLIB_PATH = .*native-deps\/onnxruntime\/lib\/libonnxruntime\.so/,+ );+ }+ },+);