Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12734]): replace cache-only worktree node_modules and document external worktree links
[._.]/ Adam Eivy55a33268d368
diff --git a/AGENTS.md b/AGENTS.mdindex d685a075ef..fc8b071150 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -135,7 +135,7 @@ The server is always user-facing on `:5555` (HTTP or HTTPS). The client runs on**Ports.** PortOS owns the 5553–5569 band (core services and the extension band; see `docs/PORTS.md`). System PostgreSQL is on 5432, Docker PostgreSQL on 5561. When HTTPS is on, local curl/scripts hit the loopback-only HTTP mirror on `:5553` to skip the cert warning. Define ports in the top-level `PORTS` object in `ecosystem.config.cjs` (re-exported at `server/lib/ports.js`). Guide and diagram: `docs/PORTS.md`.-**Never run `npm ci` or `npm install` from inside an ordinary CoS worktree (`data/cos/worktrees/*`).** Ordinary worktrees get the primary checkout's `node_modules` (root, `client/`, `server/`) by symlink; npm follows the symlink and empties the PRIMARY checkout's real `node_modules`, which takes every agent spawn down with `posix_spawn failed: No such file or directory` until someone reinstalls. If an ordinary worktree is missing dependencies, use the linked workspace binaries directly (`server/node_modules/.bin/vitest run <files>`) — never install. **The `dependency-updates` scheduled task is the exception:** its CoS-managed worktree receives no dependency symlinks, so package-manager installs and updates stay inside that worktree. Do not use the source checkout's dependency paths for that task. Recovery and mechanism: `docs/TROUBLESHOOTING.md` ("Every Agent Spawn Fails…"); the runner names this fault in `server/lib/ptySpawnDiagnostics.js`.+**Never run `npm ci` or `npm install` from inside an ordinary CoS worktree (`data/cos/worktrees/*`).** Ordinary worktrees get the primary checkout's `node_modules` (root, `client/`, `server/`) by symlink; npm follows the symlink and empties the PRIMARY checkout's real `node_modules`, which takes every agent spawn down with `posix_spawn failed: No such file or directory` until someone reinstalls. If an ordinary worktree is missing dependencies, use the linked workspace binaries directly (`server/node_modules/.bin/vitest run <files>`) — never install. **The `dependency-updates` scheduled task is the exception:** its CoS-managed worktree receives no dependency symlinks, so package-manager installs and updates stay inside that worktree. Do not use the source checkout's dependency paths for that task. Any worktree PortOS did not create runs `node scripts/link-worktree-deps.js <primary-checkout> <worktree>` immediately after `git worktree add`, before any test or pregate. Recovery and mechanism: `docs/TROUBLESHOOTING.md` ("Every Agent Spawn Fails…"); the runner names this fault in `server/lib/ptySpawnDiagnostics.js`.**Backup excludes.** `DEFAULT_EXCLUDES` in `server/services/backup.js` is **rsync filter syntax — every path must be anchored with a leading `/`.** An unanchored pattern matches at any depth and silently drops unrelated user data; that is a data-loss bug. See `docs/BACKUP.md` for the `overridable` tiers and `computeEffectiveExcludes()`.diff --git a/scripts/link-worktree-deps.test.js b/scripts/link-worktree-deps.test.jsindex 4de27cde58..89d5b3025e 100644--- a/scripts/link-worktree-deps.test.js+++ b/scripts/link-worktree-deps.test.js@@ -76,6 +76,47 @@ describe('link-worktree-deps script', () => {expect(readFileSync(join(linked, '@acme', 'core', 'marker.txt'), 'utf8')).toBe('fresh');});+ it('replaces a cache-only root node_modules with a link to source dependencies', () => {+ root = mkdtempSync(join(tmpdir(), 'portos-link-worktree-deps-'));+ const source = join(root, 'source');+ const worktree = join(root, 'worktree');+ mkdirSync(join(source, 'node_modules'), { recursive: true });+ writeFileSync(join(source, 'node_modules', 'real-pkg.txt'), 'real');++ // Simulate vitest creating a cache-only node_modules before linking+ mkdirSync(join(worktree, 'node_modules', '.vite'), { recursive: true });+ mkdirSync(join(worktree, 'node_modules', '.vitest-cache'), { recursive: true });++ const result = runScript(source, worktree);++ expect(result.status).toBe(0);+ expect(lstatSync(join(worktree, 'node_modules')).isSymbolicLink()).toBe(true);+ expect(readlinkSync(join(worktree, 'node_modules'))).toBe(join(source, 'node_modules'));+ expect(readFileSync(join(worktree, 'node_modules', 'real-pkg.txt'), 'utf8')).toBe('real');+ });++ it('never removes a worktree node_modules holding real packages and logs a warning', () => {+ root = mkdtempSync(join(tmpdir(), 'portos-link-worktree-deps-'));+ const source = join(root, 'source');+ const worktree = join(root, 'worktree');+ mkdirSync(join(source, 'node_modules'), { recursive: true });+ writeFileSync(join(source, 'node_modules', 'source-pkg.txt'), 'source');++ // Real package directory inside worktree node_modules+ mkdirSync(join(worktree, 'node_modules', 'some-package'), { recursive: true });+ writeFileSync(join(worktree, 'node_modules', 'some-package', 'index.js'), 'custom');++ const result = runScript(source, worktree);++ expect(result.status).toBe(0);+ expect(lstatSync(join(worktree, 'node_modules')).isSymbolicLink()).toBe(false);+ expect(existsSync(join(worktree, 'node_modules', 'some-package', 'index.js'))).toBe(true);+ expect(result.stdout).toContain('⚠️');+ expect(result.stdout).toContain(join(worktree, 'node_modules'));+ // Source checkout real node_modules is never touched+ expect(existsSync(join(source, 'node_modules', 'source-pkg.txt'))).toBe(true);+ });+it('exits non-zero without both paths', () => {const result = runScript('/only-one-path');expect(result.status).toBe(1);diff --git a/scripts/pregate.js b/scripts/pregate.jsindex 30d66432ef..02179f46f8 100644--- a/scripts/pregate.js+++ b/scripts/pregate.js@@ -676,12 +676,21 @@ async function main() {const dependencyProblems = dependencyConsistencyProblems(REPO_ROOT);if (dependencyProblems.length > 0) {+ const gitCommonDir = git(['rev-parse', '--git-common-dir']);+ const gitDir = git(['rev-parse', '--git-dir']);+ const isLinkedWorktree = resolve(REPO_ROOT, gitCommonDir) !== resolve(REPO_ROOT, gitDir);+ const primaryCheckout = isLinkedWorktree ? resolve(REPO_ROOT, gitCommonDir, '..') : null;+console.error('❌ Dependency consistency preflight failed; lint/tests have not run.');for (const { label, linked, owner, issues } of dependencyProblems) {console.error(` ${label} (${linked ? 'linked' : 'owned'} node_modules):\n${issues.map(issue => ` ${issue}`).join('\n')}`);- console.error(linked- ? ` Reconcile dependencies in the owning source checkout (${owner}), with a full development install (npm ci --include=dev), or validate in a separate clone with its own locked installs. Never run npm install/ci through worktree links.`- : ` Reconcile this checkout's dependencies from its committed lockfiles (npm ci --include=dev in the owning workspace), then rerun pregate.`);+ if (isLinkedWorktree && (!linked || issues.some(i => i.includes('missing') || i.includes('unreadable')))) {+ console.error(` Worktree dependency dir is missing or unlinked for ${label}. Re-link with: node scripts/link-worktree-deps.js ${primaryCheckout || '<primary-checkout>'} ${REPO_ROOT}`);+ } else {+ console.error(linked+ ? ` Reconcile dependencies in the owning source checkout (${owner}), with a full development install (npm ci --include=dev), or validate in a separate clone with its own locked installs. Never run npm install/ci through worktree links.`+ : ` Reconcile this checkout's dependencies from its committed lockfiles (npm ci --include=dev in the owning workspace), then rerun pregate.`);+ }}process.exit(1);}diff --git a/server/services/worktreeManager.js b/server/services/worktreeManager.jsindex 2ec0606a1b..bb0326071a 100644--- a/server/services/worktreeManager.js+++ b/server/services/worktreeManager.js@@ -30,6 +30,7 @@ import { provisionWorkspaceNodeModules, removeProvisionedNodeModules } from '../export { isHumanClaimWorktree } from '../lib/worktreeOwnership.js';const WORKTREES_DIR = PATHS.worktrees;+const CACHE_ENTRIES = new Set(['.vite', '.vitest-cache', '.cache']);// `git worktree list --porcelain` reports POSIX separators on every platform —// `H:/repo/data/cos/worktrees/agent-x` — while WORKTREES_DIR is built with// `join` and is backslash-separated on Windows. So a git-reported path is never@@ -81,8 +82,22 @@ export async function linkWorktreeDependencies(sourceWorkspace, worktreePath) {const targetPath = join(worktreePath, relativePath);const sourceExists = await stat(sourcePath).then(info => info.isDirectory()).catch(() => false);if (!sourceExists) return;- const targetExists = await lstat(targetPath).then(() => true).catch(() => false);- if (targetExists) return;+ const targetStat = await lstat(targetPath).catch(() => null);+ if (targetStat) {+ const isSymlink = typeof targetStat.isSymbolicLink === 'function' ? targetStat.isSymbolicLink() : false;+ const isDir = typeof targetStat.isDirectory === 'function' ? targetStat.isDirectory() : !isSymlink;+ if (!isSymlink && isDir) {+ const entries = await readdir(targetPath).catch(() => null);+ if (entries !== null && entries.length > 0 && entries.every(name => CACHE_ENTRIES.has(name))) {+ await rm(targetPath, { recursive: true, force: true });+ } else {+ console.log(`⚠️ linkWorktreeDependencies: skipping existing ${targetPath} (not a cache-only directory)`);+ return;+ }+ } else {+ return;+ }+ }if (relativePath === 'node_modules' && await provisionWorkspaceNodeModules(sourceWorkspace, worktreePath)) return;await symlink(sourcePath, targetPath, 'dir');}));