Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.8 KB
Raw Blame
/**
 * Ensures all workspace dependencies are installed before starting.
 * Repairs missing or stale dependencies only in workspaces that own their tree.
 * Handles ENOTEMPTY npm bug by retrying with clean node_modules.
 */
import { existsSync, rmSync, readFileSync, writeFileSync, mkdirSync, statSync } from 'fs';
import { execFileSync } from 'child_process';
import { createHash } from 'crypto';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { rebuildTrusted } from './trusted-rebuilds.js';
import { isDirectlyInvoked } from './lib/directInvocation.js';
import { dependencyWorkspaces, hasLinkedDependencies } from './lib/workspaceDependencies.js';
// Node refuses to spawn npm's `.cmd` shim under `shell:false` (CVE-2024-27980),
// so every npm spawn goes through this wrap. Safe to import before `npm install`
// has ever run — bufferedSpawn's whole import graph is Node builtins only.
import { prepareCliSpawn } from '../server/lib/bufferedSpawn.js';

const npmSpawn = (args, options) => {
  const { command, args: spawnArgs } = prepareCliSpawn('npm', args);
  return execFileSync(command, spawnArgs, options);
};

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');

// Persisted dependency-input hashes (per workspace) from the last successful install.
// Both package.json and package-lock.json select the installed tree.
// A changed hash means an input moved since we last resolved the tree. An
// in-place `npm install` over the existing node_modules could leave a stale /
// duplicated tree (e.g. a react@18 copy lingering beside react@19 after a major
// bump) — which builds fine but throws "Invalid hook call" at runtime. When the
// hash changes we wipe node_modules first and reinstall from scratch instead.
// This mirrors update.sh's pull-diff clean-reinstall for the manual
// `git pull` + `npm start` path, which has no pull context to diff against.
const HASH_FILE = join(ROOT, 'data', 'deps-hashes.json');

export const WORKSPACES = dependencyWorkspaces(ROOT);

function dependencyHash(dir) {
  const pkgPath = join(dir, 'package.json');
  if (!existsSync(pkgPath)) return null;
  const lockPath = join(dir, 'package-lock.json');
  // Legacy receipts hashed only package.json and cannot prove which lockfile
  // was installed. The v2 prefix forces one reconciliation on upgrade.
  const hash = createHash('sha256').update(readFileSync(pkgPath)).update('\0');
  hash.update(existsSync(lockPath) ? readFileSync(lockPath) : '<no-lockfile>');
  return `v2:${hash.digest('hex')}`;
}

function loadHashes() {
  try {
    return JSON.parse(readFileSync(HASH_FILE, 'utf8'));
  } catch {
    return {};
  }
}

function saveHashes(hashes) {
  try {
    mkdirSync(dirname(HASH_FILE), { recursive: true });
    writeFileSync(HASH_FILE, JSON.stringify(hashes, null, 2));
  } catch (err) {
    // Non-fatal — the worst case is we re-evaluate the hash next boot.
    console.error(`⚠️  Could not persist deps hashes: ${err.message ?? err}`);
  }
}

// Filesystem fallback for the no-baseline case (first run after this feature
// lands, or a fresh manual checkout): npm writes node_modules/.package-lock.json
// at the end of every install, so its mtime is the last-install time. If
// either dependency input was modified more recently — e.g. a `git pull`
// brought a new lockfile over a still-present node_modules — clean-reinstall
// even though we have no stored hash to compare against.
// Returns false when we can't tell (missing marker, stat error) so we never
// wipe a tree we can't prove is stale.
function dependenciesNewerThanInstall(dir) {
  const markerPath = join(dir, 'node_modules', '.package-lock.json');
  const installMarker = existsSync(markerPath) ? markerPath : join(dir, 'node_modules');
  try {
    const installedAt = statSync(installMarker).mtimeMs;
    return ['package.json', 'package-lock.json'].some((name) => {
      const input = join(dir, name);
      return existsSync(input) && statSync(input).mtimeMs > installedAt;
    });
  } catch {
    return false;
  }
}

// Wipe `node_modules` ONLY. Every workspace lockfile ensure-deps touches (root,
// client, server, autofixer) is tracked, so `npm install` re-resolves from the
// committed lock — which is the state we want. Deleting it would let a clean
// reinstall silently float transitive versions past the `overrides` pins.
export function cleanWorkspaceDeps(dir) {
  try {
    rmSync(join(dir, 'node_modules'), { recursive: true, force: true });
  } catch { /* best effort */ }
}

// The trusted install-script allowlist lives in scripts/trusted-rebuilds.js —
// shared with the root `setup` script, setup.ps1, update.sh, update.ps1 and CI so
// the list has exactly one home.
// Every workspace pins `ignore-scripts=true` in its own .npmrc, so a plain
// `npm install` leaves these native deps unbuilt and the server crashes on a
// missing node-pty binding. Run after every (re)install — not just the
// clean-reinstall path — so a fresh install into a missing node_modules is
// equally whole.

function install(dir, label) {
  try {
    // This repairs an installed tree; it never authors dependencies. Keep the
    // committed lockfile byte-stable even when the local npm writer is older.
    npmSpawn(['install', '--no-save'], { cwd: dir, stdio: 'inherit', windowsHide: true });
    return rebuildTrusted(dir, label);
  } catch (err) {
    console.error(`⚠️  npm install failed for ${label}: ${err.message ?? err}`);
    console.log(`⚠️  Cleaning node_modules and retrying...`);
    try {
      rmSync(join(dir, 'node_modules'), { recursive: true, force: true });
    } catch (cleanupErr) {
      console.error(`❌ Failed to clean node_modules for ${label}: ${cleanupErr.message}`);
      return false;
    }
    try {
      npmSpawn(['install', '--no-save'], { cwd: dir, stdio: 'inherit', windowsHide: true });
      return rebuildTrusted(dir, label);
    } catch (retryErr) {
      console.error(`❌ npm install failed for ${label} after retry: ${retryErr.message ?? retryErr}`);
      return false;
    }
  }
}

// Import-safe driver: the module exposes its helpers to scripts/ensure-deps.test.js,
// and only performs installs when run as `node scripts/ensure-deps.js`.
function main() {
  // Ordinary CoS worktrees borrow the source checkout's dependencies. Refuse
  // the entire repair before any install, cleanup or receipt write;
  // checking inside the repair loop could already mutate an earlier workspace.
  // lstat also catches dangling links and Windows directory junctions.
  for (const { dir, label } of WORKSPACES) {
    if (hasLinkedDependencies(dir)) {
      console.error(`⛔ Refusing dependency repair: ${label}/node_modules is linked. Run startup in the source checkout; use its workspace binaries for worktree tests.`);
      process.exitCode = 1;
      return;
    }
  }

  const storedHashes = loadHashes();
  let hashesDirty = false;
  let needed = false;

  for (const { dir, label } of WORKSPACES) {
    const currentHash = dependencyHash(dir);
    const nodeModulesMissing = !existsSync(join(dir, 'node_modules'));
    const storedHash = storedHashes[label];
    // With a stored baseline, a differing hash means a dependency input moved
    // since the last install. Without one (first run after this feature lands, or a fresh
    // manual checkout), fall back to the install-marker mtime so a `git pull` +
    // `npm start` catches a changed manifest or lockfile over a present
    // node_modules instead of silently seeding the stale tree.
    const depsChanged = storedHash != null
      ? currentHash != null && storedHash !== currentHash
      : !nodeModulesMissing && dependenciesNewerThanInstall(dir);

    if (nodeModulesMissing || depsChanged) {
      if (depsChanged) {
        // Clean whenever dependency inputs changed — even if node_modules is
        // already gone — so npm rebuilds the tree from the committed lockfile instead of
        // layering onto a tree resolved against the previous manifest.
        console.log(`🧹 ${label} dependency inputs changed since last install — clean reinstall...`);
        cleanWorkspaceDeps(dir);
      } else {
        console.log(`📦 Missing node_modules for ${label} — installing...`);
      }
      if (!install(dir, label)) process.exit(1);
      needed = true;
    }

    if (currentHash != null && storedHashes[label] !== currentHash) {
      storedHashes[label] = currentHash;
      hashesDirty = true;
    }
  }

  // Verify critical packages exist even if node_modules dirs were present
  // Grouped by workspace to avoid redundant installs
  const criticalPackages = [
    { dir: ROOT, label: 'root', pkg: 'pm2/package.json' },
    { dir: join(ROOT, 'client'), label: 'client', pkg: 'vite/bin/vite.js' },
    { dir: join(ROOT, 'server'), label: 'server', pkg: 'express/package.json' },
    { dir: join(ROOT, 'server'), label: 'server', pkg: 'pg/package.json' },
  ];

  const criticalByDir = new Map();
  for (const { dir, label, pkg } of criticalPackages) {
    if (!criticalByDir.has(dir)) criticalByDir.set(dir, { label, pkgs: [] });
    criticalByDir.get(dir).pkgs.push(pkg);
  }

  for (const [dir, { label, pkgs }] of criticalByDir) {
    const missing = pkgs.filter(pkg => !existsSync(join(dir, 'node_modules', ...pkg.split('/'))));
    if (!missing.length) continue;

    console.log(`📦 Missing ${missing.map(p => p.split('/')[0]).join(', ')} in ${label} — reinstalling deps...`);
    if (!install(dir, label)) process.exit(1);
    needed = true;

    const stillMissing = pkgs.filter(pkg => !existsSync(join(dir, 'node_modules', ...pkg.split('/'))));
    if (stillMissing.length) {
      console.error(`❌ Still missing in ${label} after reinstall: ${stillMissing.map(p => p.split('/')[0]).join(', ')}`);
      process.exit(1);
    }
  }

  if (hashesDirty) saveHashes(storedHashes);

  if (needed) console.log('✅ Dependencies verified');
}

if (isDirectlyInvoked(import.meta.url)) main();