YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
cache macOS native dependencies across worktrees (#5266)
Ezra Ellettef2e91504d5a9
diff --git a/.gitignore b/.gitignoreindex 317a8e72f..d254d3c49 100644--- a/.gitignore+++ b/.gitignore@@ -83,6 +83,7 @@ crates/screenpipe-audio/openblas-0.3.31-x64.zipcrates/screenpipe-audio/openblas-0.3.31-woa64-dll.zipapps/screenpipe-app-tauri/src-tauri/openblas/apps/screenpipe-app-tauri/src-tauri/mlx.metallib+apps/screenpipe-app-tauri/src-tauri/mlx.metallib-*apps/screenpipe-app-tauri/src-tauri/libonnxruntime.dylib.serena/.worktrees/diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex b951ef3c7..6fc059997 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -54,6 +54,24 @@ before you begin:bun tauri build --features metal```+#### sharing downloaded dependencies across worktrees++The desktop prebuild keeps immutable native downloads such as FFmpeg, FFprobe,+the Bun sidecars, MLX metallib, and Windows OpenBLAS in a machine-wide cache.+The default is `~/.cache/screenpipe/native-deps`. To put it elsewhere, set the+same absolute path in every worktree environment:++```bash+export SCREENPIPE_NATIVE_CACHE_DIR="$HOME/.cache/screenpipe/native-deps"+export SCREENPIPE_FRONTEND_CACHE_DIR="$HOME/.cache/screenpipe/frontend-out"+```++Cache entries are versioned, validated, and protected by per-artifact locks, so+concurrent worktree builds download each artifact at most once. Set either+variable to `off` to disable that cache. Keep Cargo's `target` directory local+to each worktree when running concurrent builds; do not set a shared+`CARGO_TARGET_DIR`.+### windowsThese steps build the local NSIS installer (`*-setup.exe`). The default Tauri@@ -136,11 +154,11 @@ next to the released app without taking over its identity or updater settings.- Plan for at least 10 GB of free disk space. In a clean x64 worktree the app-localCargo target alone can exceed 5 GB; `node_modules`, sidecars, and global Cargocaches need additional space.-- A fresh Git worktree shares Git objects and the global Cargo registry, but not- `node_modules`, downloaded sidecars, or `src-tauri/target`. A first installer- build can therefore take tens of minutes. Rebuilds in the same worktree reuse- dependency caches, but regenerating source, relinking the app, and compressing- the installer can still take several minutes.+- A fresh Git worktree shares Git objects, the global Cargo registry, and the+ machine-wide native/frontend caches, but not `node_modules` or+ `src-tauri/target`. The native files are restored from cache without another+ download. Cargo compilation, relinking, and installer compression remain+ worktree-local and can still take several minutes.- `failed to get npm global prefix` is a non-fatal Bun-discovery fallback when itis immediately followed by `found bun` and `bun binary copied successfully`.- The local development installer is unsigned. Official Windows release buildsdiff --git a/apps/screenpipe-app-tauri/scripts/native_dependency_cache.js b/apps/screenpipe-app-tauri/scripts/native_dependency_cache.jsindex eaa5c2513..8f113b074 100644--- a/apps/screenpipe-app-tauri/scripts/native_dependency_cache.js+++ b/apps/screenpipe-app-tauri/scripts/native_dependency_cache.js@@ -19,9 +19,9 @@ async function exists(filePath) {}}-async function isValid(directory, validate) {+async function isValid(artifactPath, validate) {try {- return (await exists(directory)) && (await validate(directory))+ return (await exists(artifactPath)) && (await validate(artifactPath))} catch {return false}@@ -31,14 +31,14 @@ function temporarySibling(destination, label) {return `${destination}.${label}-${process.pid}-${randomUUID()}`}-async function publishDirectory(source, destination, validate) {+async function publishArtifact(source, destination, validate, artifactType) {const stage = temporarySibling(destination, 'tmp')await fs.rm(stage, { recursive: true, force: true })try {await fs.mkdir(path.dirname(destination), { recursive: true })- await fs.cp(source, stage, { recursive: true })+ await fs.cp(source, stage, { recursive: artifactType === 'directory' })if (!(await isValid(stage, validate))) {- throw new Error(`native dependency staging directory failed validation: ${stage}`)+ throw new Error(`native dependency staging artifact failed validation: ${stage}`)}await fs.rm(destination, { recursive: true, force: true })await fs.rename(stage, destination)@@ -88,17 +88,18 @@ export function getNativeDependencyCacheRoot() {* @param {{* cacheKey: string,* destination: string,- * validate: (directory: string) => Promise<boolean>,- * populate: (directory: string) => Promise<void>,+ * validate: (artifactPath: string) => Promise<boolean>,+ * populate: (artifactPath: string) => Promise<void>,* cacheRoot?: string | null,* }} options* @returns {Promise<string>}*/-export async function ensureCachedDirectory({+async function ensureCachedArtifact({cacheKey,destination,validate,populate,+ artifactType,cacheRoot = getNativeDependencyCacheRoot(),}) {const destinationValid = await isValid(destination, validate)@@ -108,7 +109,8 @@ export async function ensureCachedDirectory({const stage = temporarySibling(destination, 'populate')await fs.rm(stage, { recursive: true, force: true })try {- await fs.mkdir(stage, { recursive: true })+ if (artifactType === 'directory') await fs.mkdir(stage, { recursive: true })+ else await fs.mkdir(path.dirname(stage), { recursive: true })await populate(stage)if (!(await isValid(stage, validate))) {throw new Error(`native dependency population failed validation: ${cacheKey}`)@@ -135,13 +137,14 @@ export async function ensureCachedDirectory({if (!(await isValid(cacheDirectory, validate))) {if (destinationValid) {console.log(`seeding native dependency cache from worktree: ${cacheKey}`)- await publishDirectory(destination, cacheDirectory, validate)+ await publishArtifact(destination, cacheDirectory, validate, artifactType)} else {console.log(`populating native dependency cache: ${cacheKey}`)const stage = temporarySibling(cacheDirectory, 'populate')await fs.rm(stage, { recursive: true, force: true })try {- await fs.mkdir(stage, { recursive: true })+ if (artifactType === 'directory') await fs.mkdir(stage, { recursive: true })+ else await fs.mkdir(path.dirname(stage), { recursive: true })await populate(stage)if (!(await isValid(stage, validate))) {throw new Error(`native dependency population failed validation: ${cacheKey}`)@@ -160,7 +163,39 @@ export async function ensureCachedDirectory({if (!destinationValid) {console.log(`restoring native dependency from cache: ${cacheKey}`)- await publishDirectory(cacheDirectory, destination, validate)+ await publishArtifact(cacheDirectory, destination, validate, artifactType)}return destination}++/**+ * Materialize a versioned native dependency directory into a worktree.+ *+ * @param {{+ * cacheKey: string,+ * destination: string,+ * validate: (directory: string) => Promise<boolean>,+ * populate: (directory: string) => Promise<void>,+ * cacheRoot?: string | null,+ * }} options+ * @returns {Promise<string>}+ */+export async function ensureCachedDirectory(options) {+ return ensureCachedArtifact({ ...options, artifactType: 'directory' })+}++/**+ * Materialize a versioned native dependency file into a worktree.+ *+ * @param {{+ * cacheKey: string,+ * destination: string,+ * validate: (filePath: string) => Promise<boolean>,+ * populate: (filePath: string) => Promise<void>,+ * cacheRoot?: string | null,+ * }} options+ * @returns {Promise<string>}+ */+export async function ensureCachedFile(options) {+ return ensureCachedArtifact({ ...options, artifactType: 'file' })+}diff --git a/apps/screenpipe-app-tauri/scripts/native_dependency_cache.test.js b/apps/screenpipe-app-tauri/scripts/native_dependency_cache.test.jsindex f9674517e..baa4f8535 100644--- a/apps/screenpipe-app-tauri/scripts/native_dependency_cache.test.js+++ b/apps/screenpipe-app-tauri/scripts/native_dependency_cache.test.js@@ -6,7 +6,7 @@ import { afterEach, describe, expect, test } from 'bun:test'import fs from 'fs/promises'import os from 'os'import path from 'path'-import { ensureCachedDirectory } from './native_dependency_cache.js'+import { ensureCachedDirectory, ensureCachedFile } from './native_dependency_cache.js'const temporaryRoots = []@@ -140,3 +140,58 @@ describe('ensureCachedDirectory', () => {expect(await validate(destination)).toBe(true)})})++describe('ensureCachedFile', () => {+ test('populates once and restores multiple worktrees', async () => {+ const root = await temporaryRoot()+ const cacheRoot = path.join(root, 'cache')+ let populations = 0+ const validateFile = async (filePath) =>+ (await fs.readFile(filePath, 'utf8').catch(() => '')) === 'cached binary'+ const populate = async (filePath) => {+ populations += 1+ await fs.writeFile(filePath, 'cached binary', { mode: 0o755 })+ }++ for (const name of ['worktree-a', 'worktree-b']) {+ await ensureCachedFile({+ cacheKey: 'binary-v1-aarch64',+ destination: path.join(root, name, 'binary'),+ validate: validateFile,+ populate,+ cacheRoot,+ })+ }++ expect(populations).toBe(1)+ expect(await validateFile(path.join(root, 'worktree-a', 'binary'))).toBe(true)+ expect(await validateFile(path.join(root, 'worktree-b', 'binary'))).toBe(true)+ })++ test('serializes concurrent file population', async () => {+ const root = await temporaryRoot()+ const cacheRoot = path.join(root, 'cache')+ let populations = 0+ const validateFile = async (filePath) =>+ (await fs.readFile(filePath, 'utf8').catch(() => '')) === 'cached binary'+ const populate = async (filePath) => {+ populations += 1+ await new Promise((resolve) => setTimeout(resolve, 50))+ await fs.writeFile(filePath, 'cached binary', { mode: 0o755 })+ }++ await Promise.all(+ ['worktree-a', 'worktree-b'].map((name) =>+ ensureCachedFile({+ cacheKey: 'binary-v1-aarch64',+ destination: path.join(root, name, 'binary'),+ validate: validateFile,+ populate,+ cacheRoot,+ }),+ ),+ )++ expect(populations).toBe(1)+ })+})diff --git a/apps/screenpipe-app-tauri/scripts/pre_build.js b/apps/screenpipe-app-tauri/scripts/pre_build.jsindex 09bea82ea..19adad149 100644--- a/apps/screenpipe-app-tauri/scripts/pre_build.js+++ b/apps/screenpipe-app-tauri/scripts/pre_build.js@@ -9,7 +9,7 @@ import os from 'os'import path from 'path'import { setupOpenBlas } from './setup_openblas.js'import { downloadFile, find7z } from './find_tools.js'-import { ensureCachedDirectory } from './native_dependency_cache.js'+import { ensureCachedDirectory, ensureCachedFile } from './native_dependency_cache.js'const originalCWD = process.cwd()// Change CWD to src-tauri@@ -219,8 +219,8 @@ async function copyBunBinary() {const releaseTarget = process.env.SCREENPIPE_RELEASE_TARGET;const archMap = [- { target: 'aarch64-apple-darwin', url: `https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-darwin-aarch64.zip`, dest: bunDest1, label: 'aarch64' },- { target: 'x86_64-apple-darwin', url: `https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-darwin-x64.zip`, dest: bunDest2, label: 'x64' },+ { target: 'aarch64-apple-darwin', expectedArch: 'arm64', url: `https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-darwin-aarch64.zip`, dest: bunDest1, label: 'aarch64' },+ { target: 'x86_64-apple-darwin', expectedArch: 'x86_64', url: `https://github.com/oven-sh/bun/releases/download/bun-v${bunVersion}/bun-darwin-x64.zip`, dest: bunDest2, label: 'x64' },];// In CI we set SCREENPIPE_RELEASE_TARGET per-matrix-entry and only need@@ -234,28 +234,15 @@ async function copyBunBinary() {throw new Error(`unknown SCREENPIPE_RELEASE_TARGET for macOS: ${releaseTarget}`);}- for (const { url, dest, label } of wanted) {- if (await fs.exists(dest)) {- console.log(`bun ${label} binary already exists, skipping download.`);- continue;- }- console.log(`downloading bun v${bunVersion} for macOS ${label}...`);- const tmpZip = path.join(cwd, `bun-darwin-${label}.zip`);- const tmpDir = path.join(cwd, `bun-darwin-${label}-tmp`);- try {- await downloadFile(url, tmpZip, { retries: 10, timeoutMs: 120000 });- await $`unzip -o ${tmpZip} -d ${tmpDir}`;- // The zip contains a folder like bun-darwin-aarch64/bun or bun-darwin-x64/bun- const entries = await fs.readdir(tmpDir);- const extractedBun = path.join(tmpDir, entries[0], 'bun');- await copyFile(extractedBun, dest);- console.log(`bun ${label} binary installed to ${dest}`);- await fs.rm(tmpZip, { force: true });- await fs.rm(tmpDir, { recursive: true, force: true });- } catch (error) {- console.error(`failed to download bun ${label}:`, error);- process.exit(1);- }+ for (const { url, dest, label, target, expectedArch } of wanted) {+ await ensureCachedFile({+ cacheKey: `bun-macos-${target}-v${bunVersion}`,+ destination: dest,+ validate: (filePath) => validateMacosBinary(filePath, expectedArch),+ populate: (filePath) => populateZipBinary(url, filePath, 'bun'),+ })+ await fs.chmod(dest, 0o755)+ console.log(`bun ${label} binary ready at ${dest}`)}return;}@@ -283,46 +270,79 @@ async function copyFile(src, dest) {await fs.chmod(dest, 0o755); // ensure the binary is executable}-async function ensureMacosMlxMetallibSidecar() {- const releaseTarget = process.env.SCREENPIPE_RELEASE_TARGET;- if (releaseTarget !== 'aarch64-apple-darwin') return;-- const minSize = 1_000_000; // real metallib is ~84MB- const baseMetallib = path.join(cwd, 'mlx.metallib');- const sidecarMetallib = path.join(cwd, 'mlx.metallib-aarch64-apple-darwin');+async function validateFileSize(filePath, minSize) {+ try {+ const stat = await fs.stat(filePath)+ return stat.isFile() && stat.size >= minSize+ } catch {+ return false+ }+}- const fileSize = async (filePath) => {- try {- return (await fs.stat(filePath)).size;- } catch {- return 0;- }- };+async function validateMacosBinary(filePath, expectedArch) {+ if (!(await validateFileSize(filePath, 1_000_000))) return false+ try {+ const fileOutput = await runWithTimeout(['file', filePath], { label: `file ${path.basename(filePath)}` })+ return new RegExp(`\\b${expectedArch}\\b`).test(fileOutput)+ } catch {+ return false+ }+}- if ((await fileSize(sidecarMetallib)) >= minSize) {- if ((await fileSize(baseMetallib)) < minSize) {- await fs.copyFile(sidecarMetallib, baseMetallib);+async function findFileByName(root, fileName) {+ const pending = [root]+ while (pending.length > 0) {+ const directory = pending.pop()+ for (const entry of await fs.readdir(directory, { withFileTypes: true })) {+ const entryPath = path.join(directory, entry.name)+ if (entry.isDirectory()) pending.push(entryPath)+ else if (entry.isFile() && entry.name === fileName) return entryPath}- console.log('mlx.metallib sidecar already exists for tauri externalBin.');- return;}+ return null+}- if ((await fileSize(baseMetallib)) >= minSize) {- await fs.copyFile(baseMetallib, sidecarMetallib);- await fs.chmod(sidecarMetallib, 0o755);- console.log('copied mlx.metallib to aarch64 tauri externalBin sidecar.');- return;+async function populateZipBinary(url, destination, binaryName) {+ const archive = `${destination}.zip`+ const extractDirectory = `${destination}.extract`+ try {+ await downloadFile(url, archive, { retries: 10, timeoutMs: 900000 })+ await fs.mkdir(extractDirectory, { recursive: true })+ await $`unzip -o ${archive} -d ${extractDirectory}`+ const extractedBinary = await findFileByName(extractDirectory, binaryName)+ if (!extractedBinary) throw new Error(`${binaryName} was not found in ${url}`)+ await copyFile(extractedBinary, destination)+ } finally {+ await fs.rm(archive, { force: true })+ await fs.rm(extractDirectory, { recursive: true, force: true })}+}- console.log('downloading mlx.metallib for macOS aarch64 tauri externalBin...');- await downloadFile(config.macos.mlxMetallibUrl, sidecarMetallib, { retries: 10, timeoutMs: 900000 });- await fs.chmod(sidecarMetallib, 0o755);- await fs.copyFile(sidecarMetallib, baseMetallib);- const size = await fileSize(sidecarMetallib);- if (size < minSize) {- throw new Error(`downloaded mlx.metallib sidecar is too small: ${size} bytes`);+async function ensureMacosMlxMetallibSidecar() {+ const releaseTarget = process.env.SCREENPIPE_RELEASE_TARGET+ const target = releaseTarget || (process.arch === 'arm64' ? 'aarch64-apple-darwin' : 'x86_64-apple-darwin')+ if (target !== 'aarch64-apple-darwin') return++ const minSize = 1_000_000 // real metallib is ~84MB+ const baseMetallib = path.join(cwd, 'mlx.metallib')+ const sidecarMetallib = path.join(cwd, 'mlx.metallib-aarch64-apple-darwin')++ // Cargo's build script may already have downloaded the unsuffixed copy in an+ // older worktree. Promote it first so ensureCachedFile can seed the shared cache.+ if (!(await validateFileSize(sidecarMetallib, minSize)) && (await validateFileSize(baseMetallib, minSize))) {+ await fs.copyFile(baseMetallib, sidecarMetallib)}- console.log(`mlx.metallib sidecar installed to ${sidecarMetallib}`);++ await ensureCachedFile({+ cacheKey: 'mlx-metallib-macos-aarch64-v0.2.0',+ destination: sidecarMetallib,+ validate: (filePath) => validateFileSize(filePath, minSize),+ populate: (filePath) =>+ downloadFile(config.macos.mlxMetallibUrl, filePath, { retries: 10, timeoutMs: 900000 }),+ })+ await fs.chmod(sidecarMetallib, 0o755)+ await fs.copyFile(sidecarMetallib, baseMetallib)+ console.log(`mlx.metallib sidecar ready at ${sidecarMetallib}`)}async function linkSystemBinary(binaryName, destination) {@@ -866,6 +886,50 @@ async function setupWindowsFfmpeg(sevenZ) {})}+async function setupMacosFfmpegSidecars() {+ const sidecars = [+ {+ name: 'ffmpeg-aarch64-apple-darwin',+ binaryName: 'ffmpeg',+ expectedArch: 'arm64',+ url: config.macos.ffmpegUrlArm,+ cacheKey: 'ffmpeg-macos-aarch64-osxexperts-7',+ },+ {+ name: 'ffprobe-aarch64-apple-darwin',+ binaryName: 'ffprobe',+ expectedArch: 'arm64',+ url: config.macos.ffprobeUrlArm,+ cacheKey: 'ffprobe-macos-aarch64-osxexperts-7.1',+ },+ {+ name: 'ffmpeg-x86_64-apple-darwin',+ binaryName: 'ffmpeg',+ expectedArch: 'x86_64',+ url: config.macos.ffmpegUrlx86_64,+ cacheKey: 'ffmpeg-macos-x86_64-osxexperts-8.0',+ },+ {+ name: 'ffprobe-x86_64-apple-darwin',+ binaryName: 'ffprobe',+ expectedArch: 'x86_64',+ url: config.macos.ffprobeUrlx86_64,+ cacheKey: 'ffprobe-macos-x86_64-osxexperts-7.1',+ },+ ]++ for (const sidecar of sidecars) {+ const destination = path.join(cwd, sidecar.name)+ await ensureCachedFile({+ cacheKey: sidecar.cacheKey,+ destination,+ validate: (filePath) => validateMacosBinary(filePath, sidecar.expectedArch),+ populate: (filePath) => populateZipBinary(sidecar.url, filePath, sidecar.binaryName),+ })+ await fs.chmod(destination, 0o755)+ }+}+/* ########## Windows ########## */if (platform == 'windows') {const sevenZ = await find7z();@@ -910,37 +974,7 @@ if (platform == 'macos') {// and ffmpeg SIGABRTs on every invocation. osxexperts.net binaries// are statically linked and have zero external deps — safe to copy// into any .app.- if (!(await fs.exists(`ffmpeg-aarch64-apple-darwin`))) {- await $`wget --no-config ${config.macos.ffmpegUrlArm} -O ffmpeg-aarch64.zip`;- await $`unzip -o ffmpeg-aarch64.zip -d ffmpeg-aarch64`;- await $`cp ffmpeg-aarch64/ffmpeg ffmpeg-aarch64-apple-darwin`;- await $`rm ffmpeg-aarch64.zip`;- await $`rm -rf ffmpeg-aarch64`;- }-- if (!(await fs.exists(`ffprobe-aarch64-apple-darwin`))) {- await $`wget --no-config ${config.macos.ffprobeUrlArm} -O ffprobe-aarch64.zip`;- await $`unzip -o ffprobe-aarch64.zip -d ffprobe-aarch64`;- await $`cp ffprobe-aarch64/ffprobe ffprobe-aarch64-apple-darwin`;- await $`rm ffprobe-aarch64.zip`;- await $`rm -rf ffprobe-aarch64`;- }-- if (!(await fs.exists(`ffmpeg-x86_64-apple-darwin`))) {- await $`wget --no-config ${config.macos.ffmpegUrlx86_64} -O ffmpeg-x86_64.zip`;- await $`unzip -o ffmpeg-x86_64.zip -d ffmpeg-x86_64`;- await $`cp ffmpeg-x86_64/ffmpeg ffmpeg-x86_64-apple-darwin`;- await $`rm ffmpeg-x86_64.zip`;- await $`rm -rf ffmpeg-x86_64`;- }-- if (!(await fs.exists(`ffprobe-x86_64-apple-darwin`))) {- await $`wget --no-config ${config.macos.ffprobeUrlx86_64} -O ffprobe-x86_64.zip`;- await $`unzip -o ffprobe-x86_64.zip -d ffprobe-x86_64`;- await $`cp ffprobe-x86_64/ffprobe ffprobe-x86_64-apple-darwin`;- await $`rm ffprobe-x86_64.zip`;- await $`rm -rf ffprobe-x86_64`;- }+ await setupMacosFfmpegSidecars()await ensureMacosMlxMetallibSidecar();