Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(build): keep HTTP errors out of the setup archive cache

Akshar Patelae7972bd3430
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 57810c005..a5426d569 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,7 +42,7 @@ jobs:
- 'crates/**'
- 'vendor/**'
- 'scripts/diagnostic-build.rs'
- - 'scripts/setup.js'
+ - 'scripts/setup*'
- 'Cargo.toml'
- 'Cargo.lock'
gpui:
@@ -273,6 +273,9 @@ jobs:
prefix-key: v1-rust-tests
save-cache: ${{ github.ref == 'refs/heads/main' }}
+ - name: Test native dependency setup
+ run: node --test scripts/setup.test.mjs
+
- name: Run setup
run: node scripts/setup.js
diff --git a/scripts/setup.js b/scripts/setup.js
index d61d0898c..c400a297b 100644
--- a/scripts/setup.js
+++ b/scripts/setup.js
@@ -71,9 +71,9 @@ async function main() {
if (!(await fileExists(nativeDepsTarPath))) {
console.log(`Downloading ${nativeDepsTar}`);
- const nativeDepsBytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`)
- .then((r) => r.blob())
- .then((b) => b.arrayBuffer());
+ const nativeDepsBytes = await downloadBytes(
+ `${NATIVE_DEPS_URL}/${nativeDepsTar}`,
+ );
await fs.writeFile(nativeDepsTarPath, Buffer.from(nativeDepsBytes));
console.log("Downloaded native deps");
downloadedNativeDeps = true;
@@ -148,9 +148,7 @@ async function main() {
const ffmpegZip = `ffmpeg-${FFMPEG_VERSION}.zip`;
const ffmpegZipPath = path.join(targetDir, ffmpegZip);
if (!(await fileExists(ffmpegZipPath))) {
- const ffmpegZipBytes = await fetch(FFMPEG_ZIP_URL)
- .then((r) => r.blob())
- .then((b) => b.arrayBuffer());
+ const ffmpegZipBytes = await downloadBytes(FFMPEG_ZIP_URL);
await fs.writeFile(ffmpegZipPath, Buffer.from(ffmpegZipBytes));
console.log(`Downloaded ${ffmpegZip}`);
downloadedFfmpeg = true;
@@ -273,9 +271,9 @@ async function main() {
let downloadedNativeDeps = false;
if (!(await fileExists(nativeDepsTarPath))) {
console.log(`Downloading ${nativeDepsTar}`);
- const bytes = await fetch(`${NATIVE_DEPS_URL}/${nativeDepsTar}`)
- .then((r) => r.blob())
- .then((b) => b.arrayBuffer());
+ const bytes = await downloadBytes(
+ `${NATIVE_DEPS_URL}/${nativeDepsTar}`,
+ );
await fs.writeFile(nativeDepsTarPath, Buffer.from(bytes));
console.log("Downloaded native deps");
downloadedNativeDeps = true;
@@ -441,9 +439,7 @@ async function setupMacOSOnnxRuntime() {
if (!(await fileExists(archivePath))) {
console.log(`Downloading ${asset.name}`);
- const bytes = await fetch(url)
- .then((r) => r.blob())
- .then((b) => b.arrayBuffer());
+ const bytes = await downloadBytes(url);
await fs.writeFile(archivePath, Buffer.from(bytes));
console.log(`Downloaded ${asset.name}`);
} else console.log(`Using cached ${asset.name}`);
@@ -495,9 +491,7 @@ async function setupWindowsOnnxRuntime() {
if (!(await fileExists(archivePath))) {
console.log(`Downloading ${asset.name}`);
- const bytes = await fetch(url)
- .then((r) => r.blob())
- .then((b) => b.arrayBuffer());
+ const bytes = await downloadBytes(url);
await fs.writeFile(archivePath, Buffer.from(bytes));
console.log(`Downloaded ${asset.name}`);
} else console.log(`Using cached ${asset.name}`);
@@ -706,6 +700,13 @@ async function signMacOSDylib(filePath) {
);
}
+async function downloadBytes(url) {
+ const response = await fetch(url);
+ if (!response.ok)
+ throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
+ return response.arrayBuffer();
+}
+
async function fileExists(path) {
return await fs
.access(path)
diff --git a/scripts/setup.test.mjs b/scripts/setup.test.mjs
new file mode 100644
index 000000000..2f6f979e6
--- /dev/null
+++ b/scripts/setup.test.mjs
@@ -0,0 +1,180 @@
+import assert from "node:assert/strict";
+import { execFile as execFileCallback } from "node:child_process";
+import { once } from "node:events";
+import {
+ copyFileSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+
+const execFile = promisify(execFileCallback);
+const repositoryRoot = fileURLToPath(new URL("../", import.meta.url));
+
+test(
+ "setup retries failed downloads without poisoning the archive cache",
+ {
+ skip: process.platform !== "linux" && "Linux setup fixture",
+ timeout: 60_000,
+ },
+ async (t) => {
+ const fixture = mkdtempSync(path.join(tmpdir(), "cap-setup-"));
+ t.after(() => rmSync(fixture, { recursive: true, force: true }));
+ for (const file of [
+ "package.json",
+ "scripts/setup.js",
+ "scripts/linux-bundle-config.mjs",
+ "apps/desktop/src-tauri/tauri.conf.json",
+ "apps/desktop-gpui/tauri.conf.json",
+ ]) {
+ const destination = path.join(fixture, file);
+ mkdirSync(path.dirname(destination), { recursive: true });
+ copyFileSync(path.join(repositoryRoot, file), destination);
+ }
+
+ const nativeName = "native-deps-x86_64-linux-gnu.tar.xz";
+ const onnxName = "onnxruntime-linux-x64-1.24.3.tgz";
+ const nativeUrl = `/spacedriveapp/native-deps/releases/download/v0.26/${nativeName}`;
+ const onnxUrl = `/microsoft/onnxruntime/releases/download/v1.24.3/${onnxName}`;
+ const nativeLibrary = "synthetic native library\n";
+ const onnxLibrary = "synthetic ONNX library\n";
+ const archives = new Map();
+ for (const [url, entry, contents] of [
+ [nativeUrl, "lib/libavcodec.so.61", nativeLibrary],
+ [onnxUrl, `${onnxName.slice(0, -4)}/lib/libonnxruntime.so`, onnxLibrary],
+ ]) {
+ const archive = path.join(fixture, "assets", path.basename(url));
+ const directory = `${archive}.contents`;
+ mkdirSync(path.dirname(path.join(directory, entry)), { recursive: true });
+ writeFileSync(path.join(directory, entry), contents);
+ await execFile(
+ "tar",
+ [url.endsWith(".xz") ? "-cJf" : "-czf", archive, "-C", directory, "."],
+ { signal: t.signal },
+ );
+ archives.set(url, readFileSync(archive));
+ }
+
+ let failNative = true;
+ const requests = [];
+ const server = createServer((request, response) => {
+ requests.push(request.url);
+ if (failNative && request.url === nativeUrl) {
+ response.writeHead(503).end("Service Unavailable\n");
+ return;
+ }
+ const archive = archives.get(request.url);
+ response.writeHead(archive ? 200 : 404);
+ response.end(archive ?? "Unexpected setup download");
+ });
+ server.listen(0, "127.0.0.1");
+ await once(server, "listening");
+ t.after(async () => {
+ server.closeAllConnections();
+ await promisify(server.close.bind(server))();
+ });
+ const origin = `http://127.0.0.1:${server.address().port}`;
+ const preload = path.join(fixture, "http-fixture.mjs");
+ writeFileSync(
+ preload,
+ `const networkFetch = globalThis.fetch;
+globalThis.fetch = (input, options) => {
+ const url = new URL(input);
+ if (url.origin !== "https://github.com")
+ throw new Error(\`Unexpected external request: \${url}\`);
+ return networkFetch(new URL(url.pathname, ${JSON.stringify(origin)}), options);
+};
+`,
+ );
+ const runSetup = () =>
+ execFile(process.execPath, ["--import", preload, "scripts/setup.js"], {
+ cwd: fixture,
+ signal: t.signal,
+ timeout: 20_000,
+ env: {
+ ...process.env,
+ NODE_OPTIONS: "",
+ RUST_TARGET_TRIPLE: "x86_64-unknown-linux-gnu",
+ CAP_USE_SCCACHE: "0",
+ TMPDIR: fixture,
+ },
+ }).then(
+ ({ stdout, stderr }) => ({ code: 0, stdout, stderr }),
+ (error) => {
+ assert.equal(typeof error.code, "number", error.message);
+ return error;
+ },
+ );
+
+ const cache = path.join(fixture, "target", `v0.26-${nativeName}`);
+ const failed = await runSetup();
+ assert.equal(failed.code, 1, failed.stderr);
+ assert.deepEqual(requests, [nativeUrl]);
+ assert.equal(
+ existsSync(cache),
+ false,
+ "failed downloads must not be cached",
+ );
+ assert.match(failed.stderr, /HTTP 503/);
+
+ failNative = false;
+ for (const expectedRequests of [[nativeUrl, onnxUrl], []]) {
+ requests.length = 0;
+ const result = await runSetup();
+ assert.equal(result.code, 0, result.stderr);
+ assert.deepEqual(requests, expectedRequests);
+ assert.deepEqual(readFileSync(cache), archives.get(nativeUrl));
+ for (const [name, contents] of [
+ ["libavcodec.so.61", nativeLibrary],
+ ["libonnxruntime.so", onnxLibrary],
+ ["libonnxruntime.so.1", onnxLibrary],
+ ]) {
+ for (const directory of [
+ "target/debug",
+ "target/release",
+ "target/x86_64-unknown-linux-gnu/debug",
+ "target/x86_64-unknown-linux-gnu/release",
+ ]) {
+ assert.equal(
+ readFileSync(path.join(fixture, directory, name), "utf8"),
+ contents,
+ );
+ }
+ for (const directory of [
+ "apps/desktop/src-tauri",
+ "apps/desktop-gpui",
+ ]) {
+ const config = JSON.parse(
+ readFileSync(
+ path.join(fixture, directory, "tauri.linux.conf.json"),
+ "utf8",
+ ),
+ );
+ const source = config.bundle.linux.deb.files[`/usr/lib/cap/${name}`];
+ assert.equal(
+ readFileSync(path.resolve(fixture, directory, source), "utf8"),
+ contents,
+ );
+ }
+ }
+ const cargo = readFileSync(
+ path.join(fixture, ".cargo/config.toml"),
+ "utf8",
+ );
+ assert.match(cargo, /\[target\.x86_64-unknown-linux-gnu\]/);
+ assert.match(
+ cargo,
+ /ORT_DYLIB_PATH = .*native-deps\/onnxruntime\/lib\/libonnxruntime\.so/,
+ );
+ }
+ },
+);