Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: exclude regenerable churn from Time Machine backups (#13502)
Adam Eivya3915346f9f3
diff --git a/docs/SETUP.md b/docs/SETUP.mdindex 2f2a8f7fe2..2909e09be9 100644--- a/docs/SETUP.md+++ b/docs/SETUP.md@@ -43,6 +43,8 @@ For endpoint overrides and readiness checks, see`npm run setup` is the non-wrapper equivalent. All three paths install dependencies, provision PostgreSQL, prepare runtime data and the managed browser, ask about a local LLM when an interactive terminal is available, safely attempt Tailscale certificate provisioning, and print the remaining setup walkthrough.+On macOS, setup also adds fixed-path Time Machine exclusions for regenerable caches, agent worktrees, cloned repos, training runs, model weights, runtimes, dependencies, and scratch directories. Creative media, deployed LoRAs, personal records, and checkout recovery archives stay eligible for backup. This uses `sudo tmutil addexclusion -p` (macOS also requires Full Disk Access for the invoking terminal); if sudo is declined or unavailable, setup warns and continues, retrying on the next setup (or `node scripts/setup-timemachine.js`). Set `PORTOS_SKIP_TIMEMACHINE_EXCLUDE=1` to opt out. Existing user exclusions are never removed; an already-excluded creative directory must be restored to backup by the operator.+## Network setupThe network sequence is:diff --git a/scripts/lib/timeMachineExclude.js b/scripts/lib/timeMachineExclude.jsnew file mode 100644index 0000000000..a3c6a4f2c7--- /dev/null+++ b/scripts/lib/timeMachineExclude.js@@ -0,0 +1,78 @@+/** Fixed paths survive deletion/recreation; never derive these from Spotlight's archive exclusions. */+import { spawnSync } from 'node:child_process';+import { resolve } from 'node:path';++// Separate from rsync's glob policy: tmutil cannot exclude regenerable files+// inside a creative directory without also losing its irreplaceable contents.+export const TIME_MACHINE_EXCLUDED_DIRS = [+ 'data/browser-profile',+ 'data/cache',+ 'data/browser-downloads',+ 'data/code-animation-workspaces',+ 'data/code-animation-exports',+ 'data/composition-proofs',+ 'data/music-video-compositions',+ 'data/music-video-film-look',+ 'data/music-video-stats-outro',+ 'data/music-video-segments',+ 'data/music-video-song-renders',+ 'data/supercollider',+ 'data/federated-media-inbox',+ 'data/cos/slashdo-resolved',+ 'data/cos/reference-repos',+ 'data/cos/worktrees',+ 'data/cos/initialized-checkout-fixtures',+ 'data/autofixer/worktrees',+ 'data/backup-admission',+ 'data/model-tests/sandboxes',+ 'data/image-thumbnails',+ 'data/video-thumbnails',+ 'data/python',+ 'data/repos',+ // Deployed adapters live in data/loras, which deliberately stays backed up.+ 'data/training-runs',+ 'data/venvs',+ 'data/jev/corpora',+ 'data/jev/embeddings',+ 'tmp',+ 'models',+ 'node_modules',+ 'client/node_modules',+ 'server/node_modules',+ 'autofixer/node_modules',+ 'scripts/node_modules',+ 'browser/node_modules',+];++/** Best-effort setup: no directory creation, exclusion removal, or install failure. */+export function excludeFromTimeMachine(installRoot, {+ platform = process.platform,+ run = spawnSync,+ env = process.env,+} = {}) {+ if (platform !== 'darwin') return { status: 'unsupported' };+ if (env.PORTOS_SKIP_TIMEMACHINE_EXCLUDE === '1') return { status: 'skipped' };+ const result = { status: 'present', added: 0, excluded: 0, failures: [] };+ const options = { encoding: 'utf8', stdio: ['inherit', 'pipe', 'pipe'], timeout: 30_000 };+ for (const relativePath of TIME_MACHINE_EXCLUDED_DIRS) {+ const path = resolve(installRoot, relativePath);+ try {+ const check = run('tmutil', ['isexcluded', path], options);+ if (check.status === 0 && check.stdout?.includes('[Excluded]')) {+ result.excluded++;+ continue;+ }+ const add = run('sudo', ['--', 'tmutil', 'addexclusion', '-p', path], options);+ if (add.status === 0) {+ result.added++;+ } else {+ result.failures.push({ path, error: add.error?.message || add.stderr?.trim() || `sudo exited with status ${add.status}` });+ }+ } catch (error) {+ // An unavailable process runner is also a best-effort setup failure.+ result.failures.push({ path, error: error.message });+ }+ }+ result.status = result.failures.length ? 'partial' : result.added ? 'updated' : 'present';+ return result;+}diff --git a/scripts/lib/timeMachineExclude.test.js b/scripts/lib/timeMachineExclude.test.jsnew file mode 100644index 0000000000..bc650e140a--- /dev/null+++ b/scripts/lib/timeMachineExclude.test.js@@ -0,0 +1,98 @@+import { existsSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { describe, expect, it, vi } from 'vitest';+import { setupTimeMachine } from '../setup-timemachine.js';+import { TIME_MACHINE_EXCLUDED_DIRS } from './timeMachineExclude.js';++const root = resolve(tmpdir(), 'portos-time-machine-fixture-not-created');+const dataPaths = [+ 'browser-profile', 'cache', 'browser-downloads', 'code-animation-workspaces',+ 'code-animation-exports', 'composition-proofs', 'music-video-compositions',+ 'music-video-film-look', 'music-video-stats-outro', 'music-video-segments',+ 'music-video-song-renders', 'supercollider', 'federated-media-inbox',+ 'cos/slashdo-resolved', 'cos/reference-repos', 'cos/worktrees',+ 'cos/initialized-checkout-fixtures', 'autofixer/worktrees', 'backup-admission',+ 'model-tests/sandboxes', 'image-thumbnails', 'video-thumbnails', 'python',+ 'repos', 'training-runs', 'venvs', 'jev/corpora', 'jev/embeddings',+];+const rootPaths = [+ 'tmp', 'models', 'node_modules', 'client/node_modules', 'server/node_modules',+ 'autofixer/node_modules', 'scripts/node_modules', 'browser/node_modules',+];+const preservedData = [+ 'images', 'videos', 'music', 'music-video', 'sprites', 'image-to-3d',+ 'lora-datasets', 'loras', 'brain', 'writers-room', 'screenshots', 'uploads',+ 'audio', 'compositions', 'voice-profiles', 'cos/agents', 'cos/memory',+ 'cos/attachments', 'cos/held-checkout-archives', 'cos/branch-retirement-archives',+ 'cos/initialized-checkout-retirements', 'jev/heads',+];+const setup = (options = {}) => setupTimeMachine(root, {+ platform: 'darwin', env: {}, log: vi.fn(), ...options,+});++describe('Time Machine setup backup boundary', () => {+ it('does not spawn on unsupported platforms or when opted out', () => {+ const run = vi.fn();+ expect(setup({ platform: 'linux', run }).status).toBe('unsupported');+ expect(setup({ env: { PORTOS_SKIP_TIMEMACHINE_EXCLUDE: '1' }, run }).status).toBe('skipped');+ expect(run).not.toHaveBeenCalled();+ });++ it('records exactly the fixed paths once, preserves creative/recovery trees, and never creates missing paths', () => {+ const excluded = new Set([join(root, 'data/cache')]);+ const calls = [];+ const run = vi.fn((command, args) => {+ calls.push([command, ...args]);+ const path = args.at(-1);+ if (command === 'tmutil') return { status: 0, stdout: `${excluded.has(path) ? '[Excluded]' : '[Included]'} ${path}` };+ excluded.add(path);+ return { status: 0 };+ });+ const expected = [...dataPaths.map(path => `data/${path}`), ...rootPaths];+ expect(existsSync(root)).toBe(false);+ expect(TIME_MACHINE_EXCLUDED_DIRS.slice().sort()).toEqual(expected.slice().sort());+ const result = setup({ run });+ expect(result).toEqual({ status: 'updated', added: expected.length - 1, excluded: 1, failures: [] });+ const additions = calls.filter(call => call[0] === 'sudo');+ expect(additions).toEqual(expected.filter(path => path !== 'data/cache')+ .map(path => ['sudo', '--', 'tmutil', 'addexclusion', '-p', resolve(root, path)]));+ for (const path of preservedData.map(path => `data/${path}`)) {+ expect(expected.some(exclude => path === exclude || path.startsWith(`${exclude}/`)), path).toBe(false);+ expect(excluded.has(resolve(root, path)), path).toBe(false);+ }+ calls.length = 0;+ expect(setup({ run })).toEqual({ status: 'present', added: 0, excluded: expected.length, failures: [] });+ expect(calls.every(call => call[0] === 'tmutil')).toBe(true);+ expect(existsSync(root)).toBe(false);+ });++ it('warns on rejected sudo, reports failures without throwing, and converges on a later successful setup', () => {+ const log = vi.fn();+ const run = vi.fn((command) => command === 'tmutil'+ ? { status: 0, stdout: '[Included]' }+ : { status: 1, stderr: 'sudo: a password is required' });+ const result = setup({ run, log });+ expect(result.status).toBe('partial');+ expect(result.added).toBe(0);+ expect(result.failures).toHaveLength(TIME_MACHINE_EXCLUDED_DIRS.length);+ expect(result.failures[0].error).toBe('sudo: a password is required');+ expect(log.mock.calls.flat().join('\n')).toContain('Setup will continue');+ run.mockImplementation(command => command === 'tmutil'+ ? { status: 0, stdout: '[Included]' } : { status: 0 });+ expect(setup({ run }).status).toBe('updated');+ });++ it('retains missing-path and process errors as retryable setup warnings', () => {+ const run = vi.fn((command) => command === 'tmutil'+ ? { status: 1, stdout: '[Excluded]', stderr: 'No such file or directory' }+ : { status: null, error: new Error('spawn sudo ENOENT') });+ const result = setup({ run });+ expect(result.status).toBe('partial');+ expect(result.excluded).toBe(0);+ expect(result.failures[0].error).toBe('spawn sudo ENOENT');+ expect(run.mock.calls.some(([command]) => command === 'sudo')).toBe(true);+ run.mockImplementation(() => { throw new Error('runner unavailable'); });+ expect(setup({ run }).failures[0].error).toBe('runner unavailable');+ });+});diff --git a/scripts/setup-data.js b/scripts/setup-data.jsindex c287a9d22d..f10a62f377 100644--- a/scripts/setup-data.js+++ b/scripts/setup-data.js@@ -9,6 +9,7 @@ import { mergeJsonStarterTargets } from './lib/mergeJsonStarter.js';import { rewriteAppsPortosRoot } from './lib/rewriteAppsPortosRoot.js';import { adoptLegacyAppliedList } from './lib/migrationLedger.js';import { setupSpotlight } from './setup-spotlight.js';+import { setupTimeMachine } from './setup-timemachine.js';const __dirname = dirname(fileURLToPath(import.meta.url));const rootDir = join(__dirname, '..');@@ -48,6 +49,7 @@ const rewritePortosRootPlaceholder = () => {const { created } = seedDataDir({ referenceDir, dataDir, log: console.log });rewritePortosRootPlaceholder();setupSpotlight(dataDir);+setupTimeMachine(rootDir);console.log(created? '✅ Data directory created': '✅ Data directory already exists, ensured subdirectories and files');diff --git a/scripts/setup-timemachine.js b/scripts/setup-timemachine.jsnew file mode 100644index 0000000000..7b184eb956--- /dev/null+++ b/scripts/setup-timemachine.js@@ -0,0 +1,20 @@+#!/usr/bin/env node+/** Add fixed-path Time Machine exclusions for regenerable churn on macOS. */+import { dirname, resolve } from 'node:path';+import { fileURLToPath } from 'node:url';+import { isDirectlyInvoked } from './lib/directInvocation.js';+import { excludeFromTimeMachine } from './lib/timeMachineExclude.js';++const installRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');++export function setupTimeMachine(root = installRoot, { log = console.log, ...options } = {}) {+ const result = excludeFromTimeMachine(root, options);+ if (result.status === 'updated') log(`💾 Added ${result.added} fixed-path Time Machine exclusions for regenerable PortOS directories`);+ if (result.status === 'partial') {+ for (const failure of result.failures) log(`⚠️ Time Machine exclusion failed for ${failure.path}: ${failure.error}`);+ log('⚠️ Setup will continue. Fixed-path Time Machine exclusions require sudo; rerun node scripts/setup-timemachine.js to retry.');+ }+ return result;+}++if (isDirectlyInvoked(import.meta.url)) setupTimeMachine();