Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.7 KB
Raw Blame
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { spawn, execFileSync } from 'child_process';
import { mkdirSync, writeFileSync, copyFileSync, readFileSync, existsSync, chmodSync, rmSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { makeGitSandbox, destroyGitSandbox, SKIP_HEAVY_INTEGRATION } from '../server/lib/gitTestRepo.js';

const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const UPDATE_SH = join(REPO_ROOT, 'update.sh');

// Everything update.sh shells out to between the pm2 delete and the pm2 start.
// Stubbing them lets the success path run end to end offline, which is the only
// way to prove the exit trap does NOT start the apps a second time.
const STUB_SCRIPTS = [
  'setup-data.js', 'setup-db.js', 'setup-browser.js', 'setup-ghostty.js',
  'setup-cert.js', 'setup-guide.js', 'run-migrations.js',
  'verify-server-health.js', 'print-access-url.js', 'open-ui-in-browser.js'
];

/**
 * A throwaway checkout of update.sh with `npm`, `npx` and `pm2` shimmed, so the
 * assertions below are what the script actually does to PM2 rather than a grep
 * for the guard's source text. The rationale for the guard itself lives with it
 * in update.sh.
 *
 * @param {{origin?: boolean, failAfterDelete?: boolean, npmShim?: string,
 *   forceClean?: boolean, pm2OnPath?: boolean, healthy?: boolean}} options
 *   origin:false leaves the checkout with no upstream, so the git-pull step
 *   fails before anything is deleted. failAfterDelete fails the first step AFTER
 *   the pm2 delete that does not also wipe node_modules. npmShim picks whether
 *   `npm`/`npx` succeed ('ok'), fail ('fail' — which makes safe_install wipe
 *   node_modules first) or stall ('slow', so a signal can arrive mid-window).
 *   forceClean makes safe_install wipe root node_modules regardless of the diff,
 *   pm2OnPath supplies a fallback pm2 for when that wipe removes the local one,
 *   and healthy:false makes the health probe report the server never came back.
 */
async function makeSandbox({
  origin = true, failAfterDelete = true, npmShim = 'ok',
  forceClean = false, pm2OnPath = false, healthy = true, stallDelete = false, nodeVersion = null, originAhead = false, peerTarget = false
} = {}) {
  const { scratch, repo } = await makeGitSandbox({ origin, prefix: 'portos-update-guard-' });
  const bin = join(scratch, 'bin');
  const calls = join(scratch, 'pm2-calls.log');
  const releaseFile = join(scratch, 'pm2-delete-stalled');

  mkdirSync(bin, { recursive: true });
  mkdirSync(join(repo, 'scripts'), { recursive: true });
  mkdirSync(join(repo, 'node_modules', 'pm2', 'bin'), { recursive: true });

  copyFileSync(UPDATE_SH, join(repo, 'update.sh'));
  chmodSync(join(repo, 'update.sh'), 0o755);
  writeFileSync(join(repo, 'package.json'), JSON.stringify({ name: 'sandbox', version: '0.0.0' }));
  writeFileSync(join(repo, 'ecosystem.config.cjs'), 'module.exports = { apps: [] };\n');
  // Mirror the production ignore rule so update.sh's own runtime log does not
  // turn an otherwise clean synthetic checkout into a dirty one.
  writeFileSync(join(repo, '.gitignore'), 'data/*\n');

  // Records every pm2 invocation the script makes, in order. When stallDelete is
  // set the `delete` call logs and then BLOCKS until the test deletes the release
  // file, so a signal can be delivered while the delete is still running — the
  // window a latch armed after the delete would miss.
  writeFileSync(join(repo, 'node_modules', 'pm2', 'package.json'), JSON.stringify({ name: 'pm2', version: '0.0.0' }));
  writeFileSync(
    join(repo, 'node_modules', 'pm2', 'bin', 'pm2'),
    `const fs = require('fs');
const args = process.argv.slice(2).join(' ');
fs.appendFileSync(${JSON.stringify(calls)}, args + '\\n');
if (${stallDelete} && args.startsWith('delete')) {
  const release = ${JSON.stringify(releaseFile)};
  fs.writeFileSync(release, 'stalled');
  while (fs.existsSync(release)) { try { require('child_process').execFileSync('sleep', ['0.05']); } catch { break; } }
}
`
  );

  // The REAL runtime gate (the pulled target's own copy), so the preflight
  // exercises the shared requirement rather than a stub with a copied floor.
  mkdirSync(join(repo, 'scripts', 'lib'), { recursive: true });
  copyFileSync(join(REPO_ROOT, 'scripts', 'checkNodeVersion.js'), join(repo, 'scripts', 'checkNodeVersion.js'));
  copyFileSync(join(REPO_ROOT, 'scripts', 'lib', 'directInvocation.js'), join(repo, 'scripts', 'lib', 'directInvocation.js'));
  // Make the interpreter report an older runtime to that gate (and only to it).
  let nodeOptions = '';
  if (nodeVersion) {
    const preload = join(scratch, 'fake-node-version.mjs');
    writeFileSync(preload, `Object.defineProperty(process.versions, 'node', { value: ${JSON.stringify(nodeVersion)} });\n`);
    nodeOptions = `--import=${preload}`;
  }

  writeFileSync(join(repo, 'scripts', 'SLASHDO_VERSION'), '0.0.0\n');
  writeFileSync(join(repo, 'scripts', 'trusted-rebuilds.js'), `process.exit(${failAfterDelete ? 1 : 0});\n`);
  for (const stub of STUB_SCRIPTS) {
    writeFileSync(join(repo, 'scripts', stub), 'process.exit(0);\n');
  }
  writeFileSync(join(repo, 'scripts', 'verify-server-health.js'), `process.exit(${healthy ? 0 : 1});\n`);
  // Non-zero means "the daemon is already ours", which skips the co-located
  // `pm2 update` restart — the branch a healthy install takes.
  writeFileSync(join(repo, 'scripts', 'pm2-daemon-refresh.js'), 'process.exit(1);\n');

  // The workspaces safe_install cd's into, and the dependency it sanity-checks.
  for (const ws of ['client', 'server', 'autofixer']) {
    mkdirSync(join(repo, ws), { recursive: true });
    writeFileSync(join(repo, ws, 'package.json'), JSON.stringify({ name: ws, version: '0.0.0' }));
  }
  mkdirSync(join(repo, 'client', 'node_modules', 'vite', 'bin'), { recursive: true });
  writeFileSync(join(repo, 'client', 'node_modules', 'vite', 'bin', 'vite.js'), '');

  // update.sh only ever calls these as bare commands, so a PATH shim covers
  // every install and the slash-do refresh without touching the network.
  const npmBody = { ok: 'exit 0', fail: 'exit 1', slow: 'sleep 2\nexit 0' }[npmShim];
  for (const shim of ['npm', 'npx']) {
    writeFileSync(join(bin, shim), `#!/bin/sh\n${npmBody}\n`);
    chmodSync(join(bin, shim), 0o755);
  }
  // The success case is the only one that runs past trusted-rebuilds, so it
  // reaches update.sh's ffmpeg step — which on a machine without ffmpeg would
  // run a real `brew install`, or on Linux CI a passwordless `sudo apt-get
  // install` that mutates the runner. update.sh only probes `command -v`.
  writeFileSync(join(bin, 'ffmpeg'), '#!/bin/sh\nexit 0\n');
  chmodSync(join(bin, 'ffmpeg'), 0o755);
  // A pm2 the recovery can still reach after safe_install wipes the local one.
  if (pm2OnPath) {
    writeFileSync(join(bin, 'pm2'), `#!/bin/sh\nprintf '%s\\n' "$*" >> ${JSON.stringify(calls)}\n`);
    chmodSync(join(bin, 'pm2'), 0o755);
  }

  // The updater now refuses dirty checkouts instead of stashing them. Commit
  // the synthetic fixture so this test still exercises the headless recovery
  // window rather than the dirty-checkout guard.
  execFileSync('git', ['add', '.'], { cwd: repo });
  execFileSync('git', ['commit', '-m', 'fixture'], { cwd: repo, stdio: 'ignore' });

  // Leave the upstream one commit ahead of the checkout, so the update has a real
  // pull to perform and a refusal can be shown to have left HEAD where it was.
  let aheadSha = null;
  if (originAhead) {
    writeFileSync(join(repo, 'ahead.txt'), 'ahead\n');
    execFileSync('git', ['add', 'ahead.txt'], { cwd: repo });
    execFileSync('git', ['commit', '-m', 'ahead'], { cwd: repo, stdio: 'ignore' });
    aheadSha = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim();
    execFileSync('git', ['push', 'origin', 'main'], { cwd: repo, stdio: 'ignore' });
    execFileSync('git', ['reset', '--hard', 'HEAD~1'], { cwd: repo, stdio: 'ignore' });
  }
  const headSha = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim();

  return { scratch, repo, bin, calls, forceClean, releaseFile, nodeOptions, headSha, aheadSha, peerTarget };
}

// The three runs share no state, so they go out concurrently rather than
// serializing three full update scripts.
function runUpdate(sandbox, { onStart } = {}) {
  return new Promise((resolve) => {
    const env = { ...process.env, PATH: `${sandbox.bin}:${process.env.PATH}` };
    if (sandbox.forceClean) env.PORTOS_FORCE_CLEAN_WORKSPACES = '.';
    if (sandbox.nodeOptions) env.NODE_OPTIONS = sandbox.nodeOptions;
    if (sandbox.peerTarget) {
      env.PORTOS_PEER_UPDATE_TARGET_SHA = sandbox.aheadSha;
      env.PORTOS_PEER_UPDATE_EXPECTED_SHA = sandbox.headSha;
    }
    const child = spawn('bash', [join(sandbox.repo, 'update.sh')], { cwd: sandbox.repo, env });
    let stdout = '';
    child.stdout.on('data', (d) => { stdout += d; });
    child.stderr.on('data', () => {});
    child.on('close', (status) => resolve({ status, stdout }));
    onStart?.(child);
  });
}

// Resolves once a sentinel appears, so a test can act at a precise point in the
// window instead of guessing at a delay.
function waitForFile(path, timeoutMs = 30000) {
  const deadline = Date.now() + timeoutMs;
  return new Promise((resolve, reject) => {
    const poll = () => {
      if (existsSync(path)) return resolve();
      if (Date.now() > deadline) return reject(new Error(`timed out waiting for ${path}`));
      setTimeout(poll, 25);
    };
    poll();
  });
}

const pm2Calls = (sandbox) =>
  (existsSync(sandbox.calls) ? readFileSync(sandbox.calls, 'utf8') : '').split('\n').filter(Boolean);

describe.skipIf(process.platform === 'win32' || SKIP_HEAVY_INTEGRATION)('update.sh headless-install guard', () => {
  const sandboxes = {};
  const results = {};

  beforeAll(async () => {
    const cases = {
      failed: {},
      clean: { failAfterDelete: false },
      preDelete: { origin: false },
      unhealthy: { healthy: false },
      // A target whose Node requirement the installed runtime no longer meets.
      oldNode: { nodeVersion: '22.12.0' },
      // Same, with a pull pending: the refusal must come before HEAD moves, on
      // both the upstream and the peer-target (ff-only merge) paths.
      oldNodeAhead: { nodeVersion: '22.12.0', origin: true, originAhead: true },
      oldNodePeer: { nodeVersion: '22.12.0', origin: true, originAhead: true, peerTarget: true },
      cleanAhead: { nodeVersion: null, failAfterDelete: false, origin: true, originAhead: true },
      // The failure the guard's own comment names: both npm installs fail, and
      // safe_install wiped root node_modules — pm2 included — on the way.
      pm2Wiped: { npmShim: 'fail', forceClean: true, pm2OnPath: true }
    };
    await Promise.all(Object.entries(cases).map(async ([name, options]) => {
      sandboxes[name] = await makeSandbox(options);
    }));
    await Promise.all(Object.keys(cases).map(async (name) => {
      results[name] = await runUpdate(sandboxes[name]);
    }));
  }, 180000);

  afterAll(async () => {
    await Promise.all(Object.values(sandboxes).map(box => destroyGitSandbox(box.scratch)));
  });

  it('restarts the PM2 apps it deleted when a later step aborts the update', () => {
    const calls = pm2Calls(sandboxes.failed);
    const deleteAt = calls.findIndex(c => c.startsWith('delete ecosystem.config.cjs'));
    const startAt = calls.findIndex(c => c.startsWith('start ecosystem.config.cjs'));
    expect(deleteAt, `pm2 calls were: ${JSON.stringify(calls)}`).toBeGreaterThan(-1);
    expect(startAt, `pm2 calls were: ${JSON.stringify(calls)}`).toBeGreaterThan(deleteAt);
  });

  it('still reports the update as failed after recovering', () => {
    expect(results.failed.status).not.toBe(0);
    expect(results.failed.stdout).toContain('STEP:restart:warning:');
  });

  it('starts the apps exactly once on an update that succeeds', () => {
    expect(results.clean.status, results.clean.stdout).toBe(0);
    expect(pm2Calls(sandboxes.clean).filter(c => c.startsWith('start ecosystem.config.cjs'))).toHaveLength(1);
    expect(results.clean.stdout).toContain('STEP:restart:done:');
    expect(results.clean.stdout).not.toContain('STEP:restart:warning:');
  });

  it('does not claim a recovery the health probe never confirmed', () => {
    expect(results.unhealthy.status).not.toBe(0);
    expect(results.unhealthy.stdout).toContain('STEP:restart:error:');
    expect(results.unhealthy.stdout).not.toContain('STEP:restart:warning:');
  });

  it('restarts through a pm2 the failed install did not delete', () => {
    // safe_install wipes root node_modules (pm2 is a ROOT dependency) before it
    // retries, so a recovery hardcoded to ./node_modules/pm2/bin/pm2 would be a
    // no-op on the most likely failure of all.
    const calls = pm2Calls(sandboxes.pm2Wiped);
    const deleteAt = calls.findIndex(c => c.startsWith('delete ecosystem.config.cjs'));
    const startAt = calls.findIndex(c => c.startsWith('start ecosystem.config.cjs'));
    expect(deleteAt, `pm2 calls were: ${JSON.stringify(calls)}`).toBeGreaterThan(-1);
    expect(startAt, `pm2 calls were: ${JSON.stringify(calls)}`).toBeGreaterThan(deleteAt);
    expect(results.pm2Wiped.status).not.toBe(0);
  });

  it('restarts the apps when the update is killed during the delete itself', async () => {
    // Two guards at once: without the TERM/INT/HUP traps bash runs NO exit trap
    // for a fatal signal, and with the latch armed AFTER the delete the trap
    // would see it unset — bash only runs a pending trap between statements, so
    // a signal delivered while `pm2 delete` is still running lands here.
    const box = await makeSandbox({ stallDelete: true });
    try {
      const result = await runUpdate(box, {
        onStart: (child) => {
          waitForFile(box.releaseFile)
            .then(() => {
              child.kill('SIGTERM');
              rmSync(box.releaseFile, { force: true });
            })
            .catch(() => child.kill('SIGKILL'));
        }
      });
      const calls = pm2Calls(box);
      const deleteAt = calls.findIndex(c => c.startsWith('delete ecosystem.config.cjs'));
      const startAt = calls.findIndex(c => c.startsWith('start ecosystem.config.cjs'));
      expect(startAt, `pm2 calls were: ${JSON.stringify(calls)}`).toBeGreaterThan(deleteAt);
      expect(result.status).toBe(143);
    } finally {
      await destroyGitSandbox(box.scratch);
    }
  }, 120000);

  it('refuses an unsupported Node runtime before stopping apps or installing', () => {
    const { status, stdout } = results.oldNode;
    expect(status).toBe(1);
    expect(stdout).toContain('STEP:runtime-preflight:failed:');
    expect(stdout).toContain('Node.js ^22.22.2');
    expect(stdout).not.toContain('STEP:pm2-stop:');
    expect(stdout).not.toContain('STEP:npm-install:');
    expect(pm2Calls(sandboxes.oldNode)).toEqual([]);
  });

  it.each(['oldNodeAhead', 'oldNodePeer'])('refuses an unsupported runtime before the checkout moves (%s)', (name) => {
    const { status, stdout } = results[name];
    const head = () => execFileSync('git', ['rev-parse', 'HEAD'], { cwd: sandboxes[name].repo, encoding: 'utf8' }).trim();
    expect(status).toBe(1);
    expect(stdout).toContain('STEP:runtime-preflight:failed:');
    expect(stdout).toContain('Node.js ^22.22.2');
    expect(stdout).not.toContain('STEP:git-pull:done:');
    expect(stdout).not.toContain('STEP:submodules:');
    expect(head()).toBe(sandboxes[name].headSha);
    expect(existsSync(join(sandboxes[name].repo, 'ahead.txt'))).toBe(false);
    expect(pm2Calls(sandboxes[name])).toEqual([]);
  });

  it('still pulls and updates when the target supports the runtime', () => {
    const { repo, aheadSha } = sandboxes.cleanAhead;
    expect(results.cleanAhead.status, results.cleanAhead.stdout).toBe(0);
    expect(execFileSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).trim()).toBe(aheadSha);
  });

  it('passes the runtime preflight on a supported runtime without changing the flow', () => {
    expect(results.clean.stdout).toContain('STEP:runtime-preflight:done:');
    expect(results.clean.stdout.indexOf('STEP:runtime-preflight:done:'))
      .toBeLessThan(results.clean.stdout.indexOf('STEP:pm2-stop:running:'));
  });

  it('does not touch PM2 when the update aborts before the delete', () => {
    expect(results.preDelete.status).not.toBe(0);
    expect(pm2Calls(sandboxes.preDelete)).toEqual([]);
  });
});

/**
 * update.ps1 is the Windows half of the same bracket and cannot be executed
 * here, so guard the invariant that makes its recovery reachable: PowerShell's
 * `exit` inside a function terminates the whole script WITHOUT raising a
 * terminating error, bypassing both Stop-UpdateScript and the script-scope trap.
 * So every fatal exit in the file must route through Stop-UpdateScript. Scanning
 * the whole file rather than the delete→start line window is the point: the
 * exit that actually caused this bug lives in Safe-Install, which is DEFINED
 * above the delete and CALLED below it.
 */
describe('update.ps1 headless-install guard', () => {
  const ps1 = readFileSync(join(REPO_ROOT, 'update.ps1'), 'utf8').split('\n');
  const lineOf = (needle) => ps1.findIndex(line => line.includes(needle));

  // The only two script-terminating exits that may bypass the recovery: the one
  // Stop-UpdateScript itself performs (after running it), and the final status.
  const SANCTIONED_EXITS = ['exit $Code', 'exit $verifyFailed'];

  it('routes every fatal exit through the recovery', () => {
    const exits = ps1
      .map((line, i) => ({ line: line.trim(), number: i + 1 }))
      .filter(({ line }) => /(^|[{;]\s*)exit\b/i.test(line) || /\[Environment\]::Exit\(/i.test(line));

    const offenders = exits.filter(({ line }) => !SANCTIONED_EXITS.includes(line));
    expect(offenders, `exit(s) bypassing Restore-Pm2Apps: ${JSON.stringify(offenders)}`).toEqual([]);
    // ...and both sanctioned exits must still be there, so the guard can't be
    // "satisfied" by deleting the exit inside Stop-UpdateScript.
    expect([...new Set(exits.map(e => e.line))].sort()).toEqual([...SANCTIONED_EXITS].sort());
  });

  it('keeps every Resolve-Pm2Command branch a plain array collected by @()', () => {
    // `return` ENUMERATES an array into the output stream, so a single-element
    // branch arrives as a bare string and `$pm2 + @('start', …)` concatenates
    // into one garbage token — the PATH fallback would never start pm2. The
    // call-site @() fixes that, but only if every branch returns a PLAIN array:
    // a `,@(…)` wrapper emits the array as ONE stream item and @() nests rather
    // than flattens it, so $CmdArgs[0] is an object[] instead of the executable.
    expect(ps1.some(line => line.includes('$pm2 = @(Resolve-Pm2Command)'))).toBe(true);

    const body = ps1.slice(lineOf('function Resolve-Pm2Command'), lineOf('function Restore-Pm2Apps'));
    const returns = body.map(line => line.trim()).filter(line => line.startsWith('return'));
    expect(returns.length).toBeGreaterThan(2);
    expect(returns.filter(line => /return\s*,/.test(line)), 'a ,@() return nests instead of flattening').toEqual([]);
    expect(returns.every(line => /^return\s+@\(/.test(line))).toBe(true);
  });

  it('runs the target Node gate before the PM2 delete and any install', () => {
    const gate = lineOf('if (Test-Path "scripts/checkNodeVersion.js")');
    expect(gate).toBeGreaterThan(-1);
    expect(gate).toBeLessThan(lineOf('pm2 delete ecosystem.config.cjs --silent'));
    expect(gate).toBeLessThan(lineOf('Safe-Install -Dir "."'));
    expect(ps1.slice(gate, gate + 8).join('\n')).toContain('Stop-UpdateScript 1');
  });

  it('evaluates the target Node gate from git archive before HEAD moves', () => {
    const archive = lineOf('git archive --output=$gateArchive $preflightTarget');
    expect(archive).toBeGreaterThan(-1);
    expect(archive).toBeLessThan(lineOf('Invoke-Logged git merge --ff-only'));
    expect(archive).toBeLessThan(lineOf('Invoke-Logged git pull --rebase'));
    expect(archive).toBeLessThan(lineOf('Step "submodules" "running"'));
    expect(ps1.slice(archive, archive + 25).join('\n')).toContain('Stop-UpdateScript 1');
  });

    it('arms the latch before the delete, not after', () => {
    const armed = lineOf('$script:Pm2AppsDown = $true');
    const deleted = lineOf('pm2 delete ecosystem.config.cjs --silent');
    expect(armed).toBeGreaterThan(-1);
    expect(armed).toBeLessThan(deleted);
  });

  it('defines the recovery before every call site that depends on it', () => {
    const definedAt = lineOf('function Stop-UpdateScript');
    expect(definedAt).toBeGreaterThan(-1);
    const firstCall = ps1.findIndex(line => line.includes('Stop-UpdateScript ') && !line.includes('function '));
    expect(firstCall).toBeGreaterThan(definedAt);
    expect(lineOf('function Restore-Pm2Apps')).toBeLessThan(lineOf('$script:Pm2AppsDown = $true'));
    expect(lineOf('trap {')).toBeLessThan(lineOf('$script:Pm2AppsDown = $true'));
  });
});