Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.1 KB
Raw Blame
/**
 * Install-sync detection — answers "is the checked-out code ahead of what's
 * actually running / installed?" so the UI can tell a user who did a bare
 * `git pull` (without ./update.sh) that their install is half-updated.
 *
 * Five independent signals (issue #1779):
 *   1. running-stale-code   — the server process booted at an older commit than
 *      what's now on disk (a `git pull` advanced HEAD but nothing restarted).
 *   2. stale-deps           — a workspace's package.json/lockfile is newer than
 *      npm's install receipt (`node_modules/.package-lock.json`), or the
 *      receipt is missing entirely (deps never installed). `update.sh` runs
 *      `npm install`; a bare pull does not.
 *   3. stale-build          — the served client bundle (`client/dist`) is older
 *      than the client source (a pull touched the UI but it was never rebuilt).
 *   4. pending-migrations   — migration files exist on disk that aren't in the
 *      applied-list. (Boot normally applies these, so a non-zero count means a
 *      pull landed new migrations and the server hasn't restarted yet.)
 *   5. stale-submodules     — an initialized checkout differs from the commit
 *      pinned by the parent, or a declared submodule has not been initialized.
 *
 * All five self-clear after a proper `update.sh` cycle (install bumps the npm
 * receipt mtime, build bumps the dist mtime, boot applies migrations and
 * captures the new commit) — so no new on-disk marker format is introduced and
 * existing installs get accurate detection immediately, with no migration.
 *
 * Detection is deliberately mtime/receipt-based rather than reading a marker
 * `update.sh` would have to write: that keeps it backward-compatible with every
 * install that updated before this shipped.
 */

import { join } from 'path';
import { stat, readdir } from 'fs/promises';
import { PATHS } from '../lib/fileUtils.js';
import { execGit } from '../lib/execGit.js';
import { parseSubmoduleStatusLine } from '../lib/gitOutputParsers.js';
import { listPendingMigrations } from '../../scripts/lib/migrationLedger.js';
import { isWorktreeRoot } from '../lib/dataRoot.js';

// The commit the running process was launched at. Captured once at boot (see
// captureBootCommit, called from server/index.js). null when not yet captured
// or when HEAD couldn't be read (tarball install / not a git repo).
let bootCommit = null;

// Workspaces `update.sh` installs (safe_install . client server autofixer).
// Order is cosmetic; '.' renders as 'root'.
const DEP_WORKSPACES = ['.', 'client', 'server', 'autofixer'];

// Directories never worth walking for client-source freshness.
const WALK_SKIP_DIRS = new Set(['node_modules', 'dist', '.git']);

// Files created by the host OS rather than consumed by the client build.
// Keep this narrow: hidden files such as .env can be real Vite build inputs.
const CLIENT_NON_BUILD_FILES = new Set(['.DS_Store']);

// The tracked lockfile and npm's receipt (node_modules/.package-lock.json) are
// written by the SAME `npm install`, milliseconds apart, in an order npm does
// not guarantee. Only treat the lockfile as "newer" when it leads the receipt
// by more than this slack — so install-time jitter never produces a permanent
// false "out of sync", while a real pulled-but-uninstalled lock (minutes/hours
// ahead) still trips. A genuine stale lock is never this close.
const LOCKFILE_NEWER_SLACK_MS = 60 * 1000;

/**
 * Capture the commit the process is running. Idempotent — only the first
 * successful read sticks, so a later on-disk `git pull` (which we WANT to
 * detect) can't overwrite the boot value. Call once from boot.
 */
export async function captureBootCommit({ rootDir = PATHS.root, getCommit = gitRevParseHead } = {}) {
  if (bootCommit) return bootCommit;
  const sha = await getCommit(rootDir).catch(() => null);
  if (sha) bootCommit = sha;
  return bootCommit;
}

/** Current boot commit (for inspection / tests). */
export function getBootCommit() {
  return bootCommit;
}

/** Test-only: override/reset the captured boot commit. */
export function __setBootCommitForTest(value) {
  bootCommit = value;
}

async function gitRevParseHead(rootDir) {
  const { stdout, exitCode } = await execGit(['rev-parse', 'HEAD'], rootDir, { ignoreExitCode: true });
  if (exitCode !== 0) return null;
  const sha = stdout.trim();
  return sha || null;
}

/**
 * Is `ancestor` an ancestor of `descendant`? Used to confirm the on-disk HEAD
 * is strictly AHEAD of the boot commit (a real pull-forward) rather than merely
 * different (e.g. the process booted on a feature branch and someone checked out
 * main) — which would otherwise produce a false "stale code" alarm in dev.
 */
async function gitIsAncestor(ancestor, descendant, rootDir) {
  const { exitCode } = await execGit(
    ['merge-base', '--is-ancestor', ancestor, descendant],
    rootDir,
    { ignoreExitCode: true }
  );
  return exitCode === 0;
}

/** mtime in ms for a path, or null when it doesn't exist / can't be stat'd. */
async function statMtimeMs(path) {
  const s = await stat(path).catch(() => null);
  return s ? s.mtimeMs : null;
}

/**
 * Walk client source looking for any file newer than `buildMtimeMs`,
 * short-circuiting on the first hit. Skips node_modules/dist/.git. Returns true
 * as soon as a newer source file is found, false if the build is current.
 */
async function isClientSourceNewer(rootDir, buildMtimeMs, { statMtime = statMtimeMs } = {}) {
  const clientDir = join(rootDir, 'client');
  const stack = [];

  // Every non-OS-metadata file directly under client/ is a build input or config
  // (index.html, package.json, vite.config.js, postcss.config.js,
  // tailwind/tsconfig, …). Stat them all rather than enumerating filenames, so a
  // config-only change still marks the build stale without a list that rots as
  // configs are added.
  // Directories here are handled below: src/public are walked recursively;
  // node_modules/dist/.git are skipped.
  const rootEntries = await readdir(clientDir, { withFileTypes: true }).catch(() => []);
  for (const entry of rootEntries) {
    if (entry.isDirectory()) {
      // Both client/src AND client/public feed the build — Vite copies public/
      // verbatim into dist/, so a public-asset-only change (e.g. a swapped
      // favicon) still requires a rebuild.
      if (entry.name === 'src' || entry.name === 'public') stack.push(join(clientDir, entry.name));
      continue;
    }
    if (CLIENT_NON_BUILD_FILES.has(entry.name)) continue;
    const m = await statMtime(join(clientDir, entry.name));
    if (m != null && m > buildMtimeMs) return true;
  }

  while (stack.length) {
    const dir = stack.pop();
    const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
    for (const entry of entries) {
      if (entry.isDirectory()) {
        if (!WALK_SKIP_DIRS.has(entry.name)) stack.push(join(dir, entry.name));
        continue;
      }
      if (CLIENT_NON_BUILD_FILES.has(entry.name)) continue;
      const m = await statMtime(join(dir, entry.name));
      if (m != null && m > buildMtimeMs) return true;
    }
  }
  return false;
}

/**
 * Classify each workspace's dependency freshness by comparing its manifest /
 * lockfile mtime against npm's install receipt (`node_modules/.package-lock.json`,
 * rewritten on every `npm install`). A workspace with no package.json is skipped
 * (not present in this install).
 */
async function detectStaleDeps(rootDir, { statMtime = statMtimeMs } = {}) {
  const workspaces = [];
  for (const ws of DEP_WORKSPACES) {
    const manifestMtime = await statMtime(join(rootDir, ws, 'package.json'));
    if (manifestMtime == null) continue; // workspace absent

    const receiptMtime = await statMtime(join(rootDir, ws, 'node_modules', '.package-lock.json'));
    const lockMtime = await statMtime(join(rootDir, ws, 'package-lock.json'));

    let stale = false;
    let reason = null;
    if (receiptMtime == null) {
      stale = true;
      reason = 'not-installed';
    } else if (manifestMtime > receiptMtime) {
      stale = true;
      reason = 'manifest-newer';
    } else if (lockMtime != null && lockMtime - receiptMtime > LOCKFILE_NEWER_SLACK_MS) {
      stale = true;
      reason = 'lockfile-newer';
    }
    workspaces.push({ name: ws === '.' ? 'root' : ws, stale, reason });
  }
  return { stale: workspaces.some(w => w.stale), workspaces };
}

/**
 * Compare every recursive submodule checkout with the gitlink pinned by the
 * parent repository. Git prefixes an in-sync checkout with a space, while
 * `+`, `-`, and `U` mean a different commit, uninitialized, and conflicted.
 *
 * `stale: null` is intentionally distinct from a successful empty/in-sync
 * result: an unavailable git checkout must not be reported as authoritative.
 */
async function detectSubmodules(rootDir, {
  getStatus = () => execGit(
    ['submodule', 'status', '--recursive'],
    rootDir,
    { ignoreExitCode: true }
  ),
  isAheadOfPin = async ({ commit, path }) => {
    const pinned = await execGit(
      ['rev-parse', `HEAD:${path}`],
      rootDir,
      { ignoreExitCode: true }
    );
    const pinnedSha = pinned.exitCode === 0 ? pinned.stdout.trim() : '';
    if (!pinnedSha) return false;
    const ancestor = await execGit(
      ['merge-base', '--is-ancestor', pinnedSha, commit],
      join(rootDir, path),
      { ignoreExitCode: true }
    );
    return ancestor.exitCode === 0;
  }
} = {}) {
  const result = await getStatus();
  if (result?.exitCode !== 0 || typeof result?.stdout !== 'string') {
    return { stale: null, paths: null };
  }

  // Split before trimming: the leading space is the authoritative in-sync
  // status character. A successful empty result means there are no submodules.
  const lines = result.stdout.split('\n').filter(line => line.trimEnd());
  const parsed = lines.map(parseSubmoduleStatusLine);
  if (parsed.some(entry => entry === null)) {
    return { stale: null, paths: null };
  }

  const paths = [];
  for (const entry of parsed) {
    if (entry.statusChar === ' ') continue;
    // The Submodules tab intentionally supports checking out the latest remote
    // commit without committing the parent pointer. Git reports that as `+`,
    // but it is not a half-finished update when the checkout descends from the
    // pin. A behind/divergent `+`, `-` (uninitialized), or `U` (conflicted)
    // still needs Reconcile. Comparison failures stay conservative (stale).
    if (entry.statusChar === '+' && await isAheadOfPin(entry).catch(() => false)) continue;
    paths.push(entry.path);
  }
  return { stale: paths.length > 0, paths };
}

/**
 * Compute the full install-sync picture. Every external dependency is
 * injectable so the detection logic is unit-testable without touching real
 * git/fs. Returns a plain object safe to splice into /api/update/status.
 *
 * Best-effort by contract: callers wrap this in `.catch(() => null)` so a git
 * or fs hiccup never blocks the status response.
 */
export async function getInstallState({
  rootDir = PATHS.root,
  // The migration ledger lives under the DATA install root (data/migrations.applied.json),
  // which boot writes via the PORTOS_DATA_ROOT-resolved root — not the code checkout.
  // Read it from the same place so a pinned-data-root worktree boot doesn't report every
  // already-applied migration as pending (#1947). Equals rootDir for a normal install.
  migrationRootDir = PATHS.installRoot,
  boot = bootCommit,
  getCurrentCommit = () => gitRevParseHead(rootDir),
  isAncestor = (a, b) => gitIsAncestor(a, b, rootDir),
  statMtime = statMtimeMs,
  clientSourceNewer = (buildMs) => isClientSourceNewer(rootDir, buildMs, { statMtime }),
  // Mirror the boot backstop (#1947): when the data root is a CoS agent worktree
  // checkout, boot SKIPS migrations and never writes data/migrations.applied.json
  // there — so scanning against that ledger-less tree would report every
  // migration as pending and flag the install outOfSync on every worktree boot.
  // Short-circuit to zero pending instead, matching what boot actually did.
  listPending = () => isWorktreeRoot(migrationRootDir)
    ? Promise.resolve([])
    : listPendingMigrations({ rootDir: migrationRootDir }),
  // A CoS worktree cannot switch to its own main branch, and intentionally
  // leaves submodules uninitialized. Offering Reconcile there would be a
  // permanent false action, so preserve unknown rather than reporting stale.
  getSubmoduleState = () => isWorktreeRoot(rootDir)
    ? Promise.resolve({ stale: null, paths: null })
    : detectSubmodules(rootDir),
} = {}) {
  const currentCommit = await getCurrentCommit().catch(() => null);

  // Running stale code only when the on-disk HEAD is strictly ahead of the
  // boot commit (boot is an ancestor of current). "Merely different" (branch
  // switch, rollback) is NOT flagged — it's not the half-updated state we warn
  // about, and flagging it would cry wolf on every dev branch checkout.
  let runningStaleCode = false;
  if (boot && currentCommit && boot !== currentCommit) {
    runningStaleCode = await isAncestor(boot, currentCommit).catch(() => false);
  }

  const staleDeps = await detectStaleDeps(rootDir, { statMtime });

  // Stale build: null (unknown) when there's no built bundle — that's dev mode
  // (`npm run dev` serves via Vite) or a never-built install, neither of which
  // should raise a false "rebuild needed" alarm.
  const buildMtime = await statMtime(join(rootDir, 'client', 'dist', 'index.html'));
  const staleBuild = buildMtime == null ? null : await clientSourceNewer(buildMtime).catch(() => null);

  const pendingFiles = await listPending().catch(() => []);
  const pendingMigrations = { count: pendingFiles.length, files: pendingFiles };

  const submodules = await getSubmoduleState()
    .catch(() => ({ stale: null, paths: null }));

  const outOfSync =
    runningStaleCode ||
    staleDeps.stale ||
    staleBuild === true ||
    pendingMigrations.count > 0 ||
    submodules.stale === true;

  return {
    bootCommit: boot || null,
    currentCommit: currentCommit || null,
    runningStaleCode,
    staleDeps,
    staleBuild,
    pendingMigrations,
    submodules,
    outOfSync
  };
}

// Exported for unit tests of the individual detectors.
export const __internal = { detectStaleDeps, detectSubmodules, isClientSourceNewer, gitRevParseHead, gitIsAncestor };