Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.4 KB
Raw Blame
import { join } from 'path';
import { readFile, unlink } from 'fs/promises';
import { EventEmitter } from 'events';
import { noteSystemActivity } from './systemActivityNotify.js';
import { readJSONFile, PATHS, ensureDir, atomicWrite } from '../lib/fileUtils.js';
import { createMutex } from '../lib/asyncMutex.js';
import { isPlainObject } from '../lib/objects.js';
import { getOriginInfo, UPSTREAM_OWNER, UPSTREAM_REPO, UPSTREAM_FULL_NAME } from '../lib/gitRemote.js';
import { compareSemver } from '../lib/versionUtils.js';
import { getInstallState } from './installState.js';
import { execGh } from './github.js';

// Re-exported for back-compat: callers/tests historically import compareSemver
// from this module. The implementation now lives in lib/versionUtils.js so the
// local-LLM update detector can share it.
export { compareSemver };

const UPDATE_FILE = join(PATHS.data, 'update.json');
const CHECK_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes
const STARTUP_DELAY_MS = 10 * 1000; // 10 seconds

/**
 * How long a `lastForkSync` record counts as "fresh enough" to skip the
 * fork-sync gate on /api/update/execute. Exported so the route + UI agree on
 * a single number — the UI doesn't need to re-implement the time math; it
 * reads `status.forkSyncFresh` directly.
 */
export const FORK_SYNC_FRESHNESS_MS = 10 * 60 * 1000;

export const updateEvents = new EventEmitter();

const withLock = createMutex();

// ─── Synchronous in-memory mirror of the persisted `updateInProgress` flag ───
//
// The flag itself lives in `data/update.json` and is only readable through an
// `await` (loadState). The CoS spawn engines need the answer SYNCHRONOUSLY, at
// the top of a spawn, to hold a task rather than launch an agent into a process
// `update.sh` is about to `pm2 delete` (issue #4124). Awaiting a disk read there
// would both slow the hot dispatch path and re-open the very window the gate
// exists to close, so this module keeps a boolean in lockstep with every write
// it makes to the persisted flag.
//
// Semantics — deliberately PROCESS-LOCAL, and NOT hydrated from disk on boot:
//   - It flips only where this module actually WRITES `state.updateInProgress`
//     (setUpdateInProgress, recordUpdateResult, clearStaleUpdateInProgress's
//     clear branch). A `setUpdateInProgress(true)` that loses the atomic
//     check-and-set writes nothing, so it leaves the mirror alone.
//   - It starts `false` on every boot because a *running* server process is by
//     definition post-restart: `update.sh` pm2-deletes the old process before it
//     starts a new one, so nothing this process spawns can be severed by an
//     update that was already underway. Hydrating a persisted `true` (which
//     survives a killed-mid-update cycle until `clearStaleUpdateInProgress`
//     ages it out at 30 minutes) would instead wedge CoS for half an hour.
let updateInProgressMirror = false;

/**
 * Is a self-update running in THIS process right now?
 *
 * Synchronous, allocation-free, and safe to call on every spawn attempt.
 * `true` from the moment `/api/update/execute` acquires the update lock until
 * the update settles (`recordUpdateResult`) or is released — i.e. exactly the
 * window in which `update.sh` may pm2-restart the server out from under a
 * freshly spawned agent.
 */
export function isUpdateInProgress() {
  return updateInProgressMirror;
}

let schedulerInterval = null;
let startupTimeout = null;

const defaultState = () => ({
  lastCheck: null,
  latestRelease: null,
  ignoredVersions: [],
  updateInProgress: false,
  updateStartedAt: null,
  lastUpdateResult: null,
  lastForkSync: null,
  autoUpdate: null
});

/**
 * Read the current version from the root package.json.
 * Re-reads on each call so it picks up changes after updates.
 */
export async function getCurrentVersion() {
  const pkgPath = join(PATHS.root, 'package.json');
  const pkg = await readJSONFile(pkgPath, { version: '0.0.0' });
  const version = (typeof pkg.version === 'string' && pkg.version) ? pkg.version : '0.0.0';
  return version;
}

async function loadState() {
  await ensureDir(PATHS.data);
  // Ignored versions and update outcomes are durable user state, not a rebuildable cache.
  const raw = await readJSONFile(UPDATE_FILE, defaultState(), { allowArray: false, strict: true });
  const defaults = defaultState();
  const stateFromFile = isPlainObject(raw) ? raw : {};
  return {
    ...defaults,
    ...stateFromFile,
    ignoredVersions: Array.isArray(stateFromFile.ignoredVersions) ? stateFromFile.ignoredVersions : defaults.ignoredVersions,
    updateInProgress: typeof stateFromFile.updateInProgress === 'boolean' ? stateFromFile.updateInProgress : defaults.updateInProgress,
  };
}

async function saveState(state) {
  await ensureDir(PATHS.data);
  await atomicWrite(UPDATE_FILE, JSON.stringify(state, null, 2) + '\n');
}

/**
 * Inspect the local git origin remote and return its classification.
 * Pure read — never mutates state.
 */
export async function getRemoteInfo() {
  return getOriginInfo();
}

/**
 * Sync the user's GitHub fork from the upstream atomantic/PortOS repo via
 * `gh repo sync owner/fork`. Default fast-forward only — `gh` refuses to
 * overwrite divergent fork history unless `--force` is passed, so this is
 * non-destructive by design.
 *
 * Returns { synced, alreadyUpToDate, fullName, source, mergedBranch, message }.
 *
 * Error handling — by layer:
 *   - This function throws plain `Error` for pre-flight refusals (no origin,
 *     not GitHub, already upstream, not a fork).
 *   - `execGh` rejections (gh CLI failure including the diverged-fork case)
 *     bubble up unchanged.
 *   - The /api/update/sync-fork route runs an upfront 400 gate against the
 *     same conditions (NO_ORIGIN / NOT_GITHUB / ALREADY_UPSTREAM / NOT_A_FORK)
 *     so the pre-flight throws above are defense-in-depth and would normally
 *     be unreachable from the route. The route only string-matches `err.message`
 *     to distinguish the gh diverged-fork case (409 FORK_DIVERGED) from other
 *     gh failures (502 FORK_SYNC_FAILED).
 */
export async function syncFork({ branch, remoteInfo } = {}) {
  // Accept a pre-fetched remoteInfo from the route so we don't spawn `git
  // remote get-url origin` twice per /sync-fork call (and so a TOCTOU race
  // between the route's gate and ours can't surface as confusing messaging).
  const info = remoteInfo || await getRemoteInfo();
  if (!info.hasOrigin) {
    throw new Error('No git origin remote — cannot sync fork.');
  }
  if (!info.isGithub) {
    throw new Error(`Origin remote is not on GitHub (host: ${info.host || 'unknown'}). Fork sync is GitHub-only.`);
  }
  if (info.isUpstream) {
    throw new Error(`Origin is already the upstream ${UPSTREAM_FULL_NAME} — nothing to sync.`);
  }
  if (!info.isFork) {
    // Catches a GitHub remote that points at a repo with a different name
    // (e.g. someone forked-and-renamed). `gh repo sync` would fail with a
    // confusing 502; surface a clear refusal instead.
    throw new Error(
      `Origin ${info.fullName} is not a fork of ${UPSTREAM_FULL_NAME} ` +
      `(repo name differs). Fork sync requires the origin to be a GitHub fork named "${UPSTREAM_REPO}".`
    );
  }

  const targetBranch = branch || 'main';
  const args = ['repo', 'sync', info.fullName, '--source', UPSTREAM_FULL_NAME, '--branch', targetBranch];
  const stdout = await execGh(args);

  // `gh repo sync` prints e.g. "✓ Synced the "main" branch from atomantic/PortOS to owner/PortOS"
  // or "✓ Repository is up to date with atomantic/PortOS"
  const alreadyUpToDate = /up to date/i.test(stdout);

  await withLock(async () => {
    const state = await loadState();
    state.lastForkSync = {
      fullName: info.fullName,
      source: UPSTREAM_FULL_NAME,
      branch: targetBranch,
      alreadyUpToDate,
      syncedAt: new Date().toISOString(),
      message: stdout.trim()
    };
    await saveState(state);
  });

  return {
    synced: true,
    alreadyUpToDate,
    fullName: info.fullName,
    source: UPSTREAM_FULL_NAME,
    mergedBranch: targetBranch,
    message: stdout.trim()
  };
}

/**
 * Check GitHub for the latest release and compare to current version.
 * Always polls the upstream atomantic/PortOS releases so users running
 * from a fork still see upstream version availability. Pull/checkout
 * behavior is unchanged — update.sh still pulls from origin.
 *
 * @param {{ manual?: boolean }} [options] - `manual: true` (the `POST
 *   /api/update/check` "check now" route) always makes a real attempt: a
 *   user who explicitly asked for a check must never be silently swallowed
 *   by the background scheduler's backoff cooldown from an earlier failure
 *   it hit on its own. Only the unattended 30-min scheduler tick opts into
 *   that backoff.
 */
export async function checkForUpdate({ manual = false } = {}) {
  return withLock(async () => {
    const state = await loadState();
    const currentVersion = await getCurrentVersion();

    // `backoffKey` opts the unattended periodic read into execGh's
    // consecutive-failure backoff — the 30-min scheduler retries a failure
    // on its very next tick with no cooldown of its own, which piled up
    // alongside branch-reconcile's identical gap during a real `gh` blip
    // (both logged the same incident). `backoffMaxMs` must exceed
    // CHECK_INTERVAL_MS (30 min) — execGh's default 15-min cap always expires
    // before this scheduler's own next tick, so it would never actually skip
    // a scheduled attempt; a comfortable margin above the interval is what
    // lets repeated failures widen the gap between real attempts.
    const raw = await execGh(
      ['api', `repos/${UPSTREAM_OWNER}/${UPSTREAM_REPO}/releases/latest`],
      undefined,
      manual ? {} : { backoffKey: 'update-check', backoffMaxMs: CHECK_INTERVAL_MS * 3 }
    );
    let data;
    try { data = JSON.parse(raw); } catch { throw new Error(`Failed to parse GitHub release response: ${raw.slice(0, 200)}`); }
    const release = {
      version: data.tag_name?.replace(/^v/, '') || '0.0.0',
      tag: data.tag_name || '',
      url: data.html_url || '',
      publishedAt: data.published_at || '',
      body: data.body || ''
    };

    state.lastCheck = new Date().toISOString();
    state.latestRelease = release;
    await saveState(state);

    const isNewer = compareSemver(release.version, currentVersion) > 0;
    const isIgnored = state.ignoredVersions.includes(release.version);

    updateEvents.emit('update:checked', {
      currentVersion,
      latestRelease: release,
      updateAvailable: isNewer && !isIgnored
    });

    if (isNewer && !isIgnored) {
      updateEvents.emit('update:available', {
        currentVersion,
        latestVersion: release.version,
        latestRelease: release
      });
    }

    return {
      currentVersion,
      latestRelease: release,
      updateAvailable: isNewer && !isIgnored,
      isIgnored
    };
  });
}

/**
 * Get the current update status without checking GitHub.
 * Includes `remoteInfo` so the UI can render fork-aware messaging.
 * Remote inspection is best-effort — a failure (no git, no origin, etc.)
 * never blocks the status response.
 */
export async function getUpdateStatus() {
  const state = await loadState();
  const currentVersion = await getCurrentVersion();
  const latestVersion =
    state.latestRelease && typeof state.latestRelease.version === 'string'
      ? state.latestRelease.version
      : null;
  const isNewer = latestVersion
    ? compareSemver(latestVersion, currentVersion) > 0
    : false;
  const isIgnored = latestVersion
    ? state.ignoredVersions.includes(latestVersion)
    : false;
  const remoteInfo = await getRemoteInfo().catch(() => null);

  // Single source of truth for the freshness check — the UI reads this
  // directly instead of re-implementing the time math + fullName match.
  // GitHub owner/repo names are case-insensitive, so the fullName match
  // must be too (otherwise a remote like `ALICE/PortOS` vs a stored
  // `alice/PortOS` falsely flips this to false).
  const lastSync = state.lastForkSync;
  const forkSyncFresh = !!(
    lastSync &&
    remoteInfo?.fullName &&
    typeof lastSync.fullName === 'string' &&
    lastSync.fullName.toLowerCase() === remoteInfo.fullName.toLowerCase() &&
    (Date.now() - new Date(lastSync.syncedAt).getTime()) < FORK_SYNC_FRESHNESS_MS
  );

  // Install-sync state — "is the checked-out code ahead of what's running /
  // installed?" (issue #1779). Best-effort: a git/fs hiccup here must never
  // block the status response, so failures collapse to null.
  const installState = await getInstallState().catch(() => null);

  return {
    currentVersion,
    ...state,
    updateAvailable: isNewer && !isIgnored,
    remoteInfo,
    upstream: { owner: UPSTREAM_OWNER, repo: UPSTREAM_REPO, fullName: UPSTREAM_FULL_NAME },
    forkSyncFresh,
    forkSyncWindowMs: FORK_SYNC_FRESHNESS_MS,
    installState
  };
}

/**
 * The unattended auto-updater's own durable record — what it last did, when,
 * and why it last stood down. Stored beside the rest of the update state
 * because it is read with it (one file read serves `GET /api/update/status`)
 * and because it must survive the restart the update itself causes.
 *
 * Runtime, NOT configuration: the enabled flag, the channel, and the minimum
 * interval live in `settings.autoUpdate`, which is where a user edits them.
 */
export async function getAutoUpdateRuntime() {
  const state = await loadState();
  return normalizeAutoUpdateRuntime(state.autoUpdate);
}

/**
 * Everything the auto-update tick's CHEAP gates need, from one file read.
 *
 * The cooldown rejects 71 of every 72 ticks at the default 6h interval, and it
 * only needs the runtime record, the last update's completion, and whether an
 * update is already running — none of which requires `getUpdateStatus()`, whose
 * `getInstallState()` walks every file under `client/src`. Reading them together
 * also collapses what used to be three separate loads of `update.json` per tick.
 */
export async function getAutoUpdateGateState() {
  const state = await loadState();
  return {
    runtime: normalizeAutoUpdateRuntime(state.autoUpdate),
    lastUpdateResult: state.lastUpdateResult,
    updateInProgress: state.updateInProgress === true,
  };
}

/** Merge a patch into the runtime record. Returns the merged result. */
export async function recordAutoUpdateRuntime(patch) {
  return withLock(async () => {
    const state = await loadState();
    const next = { ...normalizeAutoUpdateRuntime(state.autoUpdate), ...patch };
    state.autoUpdate = next;
    await saveState(state);
    return next;
  });
}

function normalizeAutoUpdateRuntime(raw) {
  const source = isPlainObject(raw) ? raw : {};
  return {
    // When the feature was switched on with no prior update to measure from.
    // Persisted once and never re-stamped, so an install that reboots often
    // still reaches its first automatic update — re-arming on every boot would
    // push the deadline forever out of reach.
    armedAt: typeof source.armedAt === 'string' ? source.armedAt : null,
    lastAttemptAt: typeof source.lastAttemptAt === 'string' ? source.lastAttemptAt : null,
    lastRunAt: typeof source.lastRunAt === 'string' ? source.lastRunAt : null,
    lastOutcome: typeof source.lastOutcome === 'string' ? source.lastOutcome : null,
    lastSkip: isPlainObject(source.lastSkip) ? source.lastSkip : null,
    // When a repo-repair agent was last DISPATCHED (queued), not when it
    // finished — `autoUpdateScheduler.repairDispatchDue` reads this to bound
    // a stand-down repair agent to one dispatch per cooldown window instead
    // of one per 5-minute tick. Kept OUT of `updateBaselineAt` on purpose: a
    // checkout the repair agent fixes must still update on the very next
    // idle tick, not wait out the same interval a second time. See root
    // AGENTS.md "AI Provider Usage Policy".
    repairQueuedAt: typeof source.repairQueuedAt === 'string' ? source.repairQueuedAt : null,
  };
}

/**
 * Add a version to the ignore list.
 */
export async function ignoreVersion(version) {
  return withLock(async () => {
    const state = await loadState();
    if (!state.ignoredVersions.includes(version)) {
      state.ignoredVersions.push(version);
      await saveState(state);
      noteSystemActivity('update', 'ignored');
    }
    return state;
  });
}

/**
 * Clear all ignored versions.
 */
export async function clearIgnored() {
  return withLock(async () => {
    const state = await loadState();
    state.ignoredVersions = [];
    await saveState(state);
    noteSystemActivity('update', 'ignored-cleared');
    return state;
  });
}

/**
 * Mark update as in progress or completed in state file.
 * When setting to true, atomically rejects if already in progress (returns false).
 * Returns true if the flag was set successfully.
 */
export async function setUpdateInProgress(inProgress) {
  return withLock(async () => {
    const state = await loadState();
    if (inProgress && state.updateInProgress) return false;
    state.updateInProgress = inProgress;
    state.updateStartedAt = inProgress ? new Date().toISOString() : null;
    await saveState(state);
    // Mirror BEFORE returning so the caller's next synchronous statement — and
    // any spawn engine that runs before the next tick — already sees the gate.
    updateInProgressMirror = inProgress;
    // `false` here is an explicit lock release with no result record — the
    // caller aborted the attempt. A finished attempt notes completion or
    // failure from `recordUpdateResult` instead.
    noteSystemActivity('update', inProgress ? 'start' : 'cancellation');
    return true;
  });
}

/**
 * Record the result of an update attempt.
 */
export async function recordUpdateResult(result) {
  return withLock(async () => {
    const state = await loadState();
    state.updateInProgress = false;
    state.updateStartedAt = null;
    state.lastUpdateResult = result;
    await saveState(state);
    updateInProgressMirror = false;
    noteSystemActivity('update', result?.success === true ? 'completion' : 'failure');
  });
}

/**
 * Process the update-completion marker an `update.sh` cycle leaves behind.
 *
 * After a self-update restarts the server, the previous process writes
 * `data/update-complete.json` ({ version, completedAt }); on boot we read it,
 * record success/failure against the now-running version, and remove the
 * marker. Extracted from a bootstrap IIFE in `server/index.js` so the update
 * lifecycle lives in one service (and can be unit-tested) rather than inline at
 * the boot site. Best-effort and self-contained — never throws; every failure
 * branch removes the marker so a corrupt/mismatched file can't wedge boot into
 * re-processing it forever.
 */
export async function processUpdateMarker() {
  const updateMarkerPath = join(PATHS.data, 'update-complete.json');
  const removeMarker = () => unlink(updateMarkerPath).catch(e => {
    if (e?.code !== 'ENOENT') console.error(`❌ Failed to remove update marker: ${e.message}`);
  });

  let raw;
  try { raw = await readFile(updateMarkerPath, 'utf-8'); }
  catch (err) {
    if (err?.code === 'ENOENT') return; // No marker = no recent update
    console.error(`❌ Failed to read update marker: ${err?.message ?? err}`);
    return removeMarker();
  }

  let marker;
  try { marker = JSON.parse(raw); }
  catch (err) {
    console.error(`❌ Corrupted update marker (invalid JSON): ${err?.message ?? err}`);
    return removeMarker();
  }

  if (!marker.version || !marker.completedAt) {
    console.error(`❌ Update marker missing required fields (version: ${marker.version}, completedAt: ${marker.completedAt})`);
    return removeMarker();
  }

  const runningVersion = await getCurrentVersion();
  if (marker.version !== runningVersion) {
    console.error(`❌ Update marker version (${marker.version}) doesn't match running version (${runningVersion}) — recording as failed`);
    await recordUpdateResult({ version: marker.version, success: false, completedAt: marker.completedAt, log: `Version mismatch: expected ${marker.version}, running ${runningVersion}` })
      .catch(e => console.error(`❌ Failed to record update result: ${e.message}`));
    return removeMarker();
  }

  console.log(`✅ Update to v${marker.version} completed at ${marker.completedAt}`);
  await recordUpdateResult({ version: marker.version, success: true, completedAt: marker.completedAt, log: '' })
    .catch(e => console.error(`❌ Failed to record update result: ${e.message}`));
  return removeMarker();
}

const STALE_UPDATE_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes

/**
 * Clear stale updateInProgress flag on boot.
 * If the server was killed mid-update (before recordUpdateResult ran),
 * updateInProgress stays true and blocks future updates indefinitely.
 * This detects that condition via updateStartedAt age and clears it.
 */
export async function clearStaleUpdateInProgress() {
  return withLock(async () => {
    const state = await loadState();
    if (!state.updateInProgress) return false;

    const startedAtMs = state.updateStartedAt ? new Date(state.updateStartedAt).getTime() : NaN;
    const hasValidTimestamp = Number.isFinite(startedAtMs);
    const ageMs = hasValidTimestamp ? Date.now() - startedAtMs : null;

    // If no valid timestamp or older than timeout, treat as stale
    if (!hasValidTimestamp || ageMs > STALE_UPDATE_TIMEOUT_MS) {
      const ageStr = ageMs !== null ? `${Math.round(ageMs / 60000)}min` : 'unknown';
      console.log(`🧹 Clearing stale updateInProgress (started ${state.updateStartedAt ?? 'unknown'}, age ${ageStr})`);
      state.updateInProgress = false;
      state.updateStartedAt = null;
      state.lastUpdateResult = {
        version: state.latestRelease?.version ?? 'unknown',
        success: false,
        completedAt: new Date().toISOString(),
        log: 'Cleared stale update lock on boot — server was likely killed mid-update'
      };
      await saveState(state);
      updateInProgressMirror = false;
      noteSystemActivity('update', 'failure');
      return true;
    }

    return false;
  });
}

/**
 * Start the periodic update checker.
 */
export function startUpdateScheduler() {
  if (startupTimeout || schedulerInterval) return;

  // Initial check after startup delay
  startupTimeout = setTimeout(() => {
    startupTimeout = null;
    checkForUpdate().catch(err => {
      console.log(`🔌 Update check failed: ${err.message}`);
    });
  }, STARTUP_DELAY_MS);

  // Periodic checks
  schedulerInterval = setInterval(() => {
    checkForUpdate().catch(err => {
      console.log(`🔌 Update check failed: ${err.message}`);
    });
  }, CHECK_INTERVAL_MS);

  console.log(`🔄 Update scheduler started (every ${CHECK_INTERVAL_MS / 60000}min)`);
}