Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

11.7 KB
Raw Blame
/**
 * Rename each sprite character's native animation-run storage from the
 * vendor-named `grok/<runId>/` layout to the vendor-NEUTRAL `runs/<runId>/`
 * layout, and rewrite every path that referenced it.
 *
 * Why:
 *   PortOS's own walk generations were stored under `data/sprites/<id>/grok/`
 *   — baking one provider (grok) into the on-disk contract. As other agent
 *   services / generation methods come online, the storage layout must be
 *   vendor-neutral: runs live under `runs/`, and the provider is recorded as
 *   the run record's `provider` field, not encoded in the path. (The importer
 *   already used `runs/` for source-pipeline runs; this unifies both.)
 *
 * What moves on disk (per character record):
 *
 *     before:                              after:
 *     data/sprites/<id>/                   data/sprites/<id>/
 *     └── grok/<runId>/…                   └── runs/<runId>/…
 *
 * What is rewritten (the embedded `grok/…` path strings become `runs/…`):
 *   - runs/<runId>/animation-run.json        (run records — not sha-pinned)
 *   - runs/<runId>/generated/*-manifest.json (postprocess manifests)
 *   - walk/<id>-walk-selection-v1.json       (per-direction runPath/runManifest)
 *   - walk/<id>-walk-set-v1.json             (finalized selection copy)
 *   - runtime/vN/ atlas manifests            (compiled-atlas provenance)
 *
 * Content-sha CASCADE (the delicate part): image-byte shas never change (a
 * frame PNG is byte-identical after a directory move), but the JSON manifests
 * that EMBED paths do change, and several are sha-pinned. So after rewriting we
 * recompute, in dependency order:
 *   - each rewritten postprocess manifest → its selection/walk-set entry's
 *     `runManifestSha256`
 *   - the rewritten selection file        → the walk-set's `selectionSha256`
 *   - the rewritten walk-set file         → the runtime pointer's `walkSetSha256`
 *   - the rewritten current atlas manifest → the pointer's `manifestSha256`
 * atlas.js's compile idempotency keys on `walkSetSha256`, so updating the
 * pointer keeps a post-migration recompile a no-op (it never re-reads the
 * immutable atlas PNG, whose bytes are unchanged).
 *
 * NOT touched: imported/redraw records (their paths are `runs/`, `art-source/`,
 * or `imagegen/`, never `grok/`); the atlas PNGs; publications.json (its paths
 * are `runtime/…`, no `grok/`); reference anchors (`reference/…`).
 *
 * Idempotency: gated per record on the presence of a `grok/` directory OR any
 * residual `grok/` string in the selection/walk-set/runtime-atlas manifests
 * (crash-recovery of a run that moved the dirs but not all of the strings —
 * including a crash between the walk-set rewrite and the runtime-manifest
 * rewrite, which would otherwise read as "already migrated" and strand the
 * runtime manifests). A second run finds none of these and skips. A
 * move-target collision (`runs/<runId>` already exists) throws for that
 * record — the run is left in `grok/` (still readable via the layout-
 * agnostic reader) and the migration stays PENDING for a retry, mirroring
 * migration 200's per-record aggregate-throw.
 */

import { readdir, readFile, writeFile, rename, rm, mkdir, stat } from 'fs/promises';
import { join } from 'path';
import { createHash } from 'crypto';

const LEGACY_DIR = 'grok';
const NEUTRAL_DIR = 'runs';

const exists = (abs) => stat(abs).then(() => true, () => false);
const sha256File = async (abs) => createHash('sha256').update(await readFile(abs)).digest('hex');
const readJson = async (abs) => { try { return JSON.parse(await readFile(abs, 'utf8')); } catch { return null; } };
// Atomic write (temp + rename) so a crash mid-migration can never leave a torn,
// unparseable state file (selection / walk-set / pointer) behind — the prior
// file stays intact and the idempotent gate re-triggers cleanly on the retry.
// Inlined rather than importing server/lib/fileUtils so the migration keeps its
// minimal fs+crypto dependency surface.
const writeJson = async (abs, obj) => {
  const tmp = `${abs}.202.tmp`;
  await writeFile(tmp, `${JSON.stringify(obj, null, 2)}\n`);
  await rename(tmp, abs);
};

// Rewrite a record-relative path VALUE that starts with `grok/`. Only the
// path prefix is matched, so provenance strings that merely CONTAIN "grok"
// (e.g. the run kind `grok-game-animation-frames-run`) are left intact.
const neutralize = (s) => (typeof s === 'string' && s.startsWith(`${LEGACY_DIR}/`) ? `${NEUTRAL_DIR}/${s.slice(LEGACY_DIR.length + 1)}` : s);
const deepNeutralize = (v) => {
  if (typeof v === 'string') return neutralize(v);
  if (Array.isArray(v)) return v.map(deepNeutralize);
  if (v && typeof v === 'object') {
    const out = {};
    for (const k of Object.keys(v)) out[k] = deepNeutralize(v[k]);
    return out;
  }
  return v;
};

// Recursively collect *.json files under `dir` (absolute paths). Missing dir → [].
async function listJsonFiles(dir) {
  let entries;
  try {
    entries = await readdir(dir, { withFileTypes: true });
  } catch {
    return [];
  }
  const out = [];
  for (const entry of entries) {
    const abs = join(dir, entry.name);
    if (entry.isDirectory()) out.push(...await listJsonFiles(abs));
    else if (entry.isFile() && entry.name.endsWith('.json')) out.push(abs);
  }
  return out;
}

// True if any compiled-atlas manifest under runtime/vN/ still embeds a
// `grok/` path. Guards a crash between step 4 (walk-set rewritten, so
// selText/wsText already read clean) and step 5 (runtime manifests) from
// making the top-of-record gate see "already migrated" and skip the record
// forever, leaving stale grok/ provenance in the runtime manifests.
async function runtimeHasGrokResidue(runtimeDir) {
  for (const vDir of await readdir(runtimeDir, { withFileTypes: true }).catch(() => [])) {
    if (!vDir.isDirectory() || !/^v\d+$/.test(vDir.name)) continue;
    for (const abs of await listJsonFiles(join(runtimeDir, vDir.name))) {
      const text = await readFile(abs, 'utf8').catch(() => '');
      if (text.includes(`${LEGACY_DIR}/`)) return true;
    }
  }
  return false;
}

// Rewrite `grok/`→`runs/` in one JSON file, writing only when it changed.
async function neutralizeJsonFile(abs) {
  const obj = await readJson(abs);
  if (!obj) return;
  const neu = deepNeutralize(obj);
  if (JSON.stringify(neu) !== JSON.stringify(obj)) await writeJson(abs, neu);
}

// Rewrite the per-direction entries of a selection/walk-set object in place,
// recomputing `runManifestSha256` for every entry whose manifest we actually
// moved (a `grok/`-origin entry). Returns whether anything changed.
async function neutralizeDirectionEntries(recDir, directions) {
  let changed = false;
  for (const entry of Object.values(directions || {})) {
    if (!entry || typeof entry !== 'object') continue;
    const movedManifest = typeof entry.runManifest === 'string' && entry.runManifest.startsWith(`${LEGACY_DIR}/`);
    const movedPath = typeof entry.runPath === 'string' && entry.runPath.startsWith(`${LEGACY_DIR}/`);
    if (movedPath) { entry.runPath = neutralize(entry.runPath); changed = true; }
    if (movedManifest) {
      entry.runManifest = neutralize(entry.runManifest);
      // The manifest file was already moved + rewritten, so hash the new bytes.
      entry.runManifestSha256 = await sha256File(join(recDir, entry.runManifest));
      changed = true;
    }
  }
  return changed;
}

async function migrateRecord(recDir, recordId) {
  const grokDir = join(recDir, LEGACY_DIR);
  const runsDir = join(recDir, NEUTRAL_DIR);
  const selAbs = join(recDir, 'walk', `${recordId}-walk-selection-v1.json`);
  const wsAbs = join(recDir, 'walk', `${recordId}-walk-set-v1.json`);
  const ptrAbs = join(recDir, 'runtime', 'current.json');
  const runtimeDir = join(recDir, 'runtime');

  const hasGrokDir = await exists(grokDir);
  const selText = await readFile(selAbs, 'utf8').catch(() => '');
  const wsText = await readFile(wsAbs, 'utf8').catch(() => '');
  if (
    !hasGrokDir &&
    !selText.includes(`${LEGACY_DIR}/`) &&
    !wsText.includes(`${LEGACY_DIR}/`) &&
    !(await runtimeHasGrokResidue(runtimeDir))
  ) {
    return false; // nothing native/grok here (imported/redraw-only, or already migrated)
  }

  // 1. Move every grok/<runId> directory to runs/<runId>. A collision means a
  //    runs/<runId> already exists — refuse rather than merge/overwrite.
  if (hasGrokDir) {
    await mkdir(runsDir, { recursive: true });
    for (const entry of await readdir(grokDir, { withFileTypes: true })) {
      if (!entry.isDirectory()) continue;
      const dest = join(runsDir, entry.name);
      if (await exists(dest)) {
        throw new Error(`run dir collision for ${recordId}: ${NEUTRAL_DIR}/${entry.name} already exists — left ${LEGACY_DIR}/${entry.name} in place`);
      }
      await rename(join(grokDir, entry.name), dest);
    }
    await rm(grokDir, { recursive: true, force: true }); // now empty
  }

  // 2. Rewrite every run record + postprocess manifest now under runs/.
  for (const abs of await listJsonFiles(runsDir)) await neutralizeJsonFile(abs);

  // 3. Selection → recompute per-entry runManifestSha256 from the moved manifests.
  const selection = await readJson(selAbs);
  if (selection?.directions) {
    await neutralizeDirectionEntries(recDir, selection.directions);
    await writeJson(selAbs, selection);
  }
  const newSelectionSha = await exists(selAbs) ? await sha256File(selAbs) : null;

  // 4. Walk-set → same per-entry recompute, plus the selection-content sha.
  const walkSet = await readJson(wsAbs);
  if (walkSet) {
    if (walkSet.directions) await neutralizeDirectionEntries(recDir, walkSet.directions);
    if (newSelectionSha && typeof walkSet.selectionSha256 === 'string') walkSet.selectionSha256 = newSelectionSha;
    await writeJson(wsAbs, walkSet);
  }
  const newWalkSetSha = await exists(wsAbs) ? await sha256File(wsAbs) : null;

  // 5. Compiled-atlas manifests (provenance) + the runtime pointer's shas.
  for (const vDir of await readdir(runtimeDir, { withFileTypes: true }).catch(() => [])) {
    if (!vDir.isDirectory() || !/^v\d+$/.test(vDir.name)) continue;
    for (const abs of await listJsonFiles(join(runtimeDir, vDir.name))) await neutralizeJsonFile(abs);
  }
  const pointer = await readJson(ptrAbs);
  if (pointer) {
    if (newWalkSetSha && typeof pointer.walkSetSha256 === 'string') pointer.walkSetSha256 = newWalkSetSha;
    if (typeof pointer.manifestPath === 'string' && await exists(join(recDir, pointer.manifestPath))) {
      pointer.manifestSha256 = await sha256File(join(recDir, pointer.manifestPath));
    }
    await writeJson(ptrAbs, pointer);
  }
  return true;
}

export default {
  up: async ({ rootDir }) => {
    const spritesDir = join(rootDir, 'data', 'sprites');
    let records;
    try {
      records = await readdir(spritesDir, { withFileTypes: true });
    } catch {
      return { ok: true, migrated: 0 }; // no sprites tree on this install
    }

    let migrated = 0;
    const failures = [];
    // Sequential + per-record try/catch: one record's collision must not abort
    // the rest, and an aggregate throw keeps the migration PENDING for retry
    // (the healthy records already migrated become gate no-ops next boot).
    for (const rec of records) {
      if (!rec.isDirectory()) continue;
      try {
        if (await migrateRecord(join(spritesDir, rec.name), rec.name)) {
          migrated++;
          console.log(`🧭 migration 202: sprite ${rec.name} → runs/ layout`);
        }
      } catch (err) {
        failures.push(rec.name);
        console.error(`❌ migration 202 (${rec.name}): ${err.message}`);
      }
    }
    if (failures.length > 0) {
      throw new Error(`migration 202: ${failures.length} sprite record(s) failed to migrate (${failures.join(', ')}) — resolve the collision(s) and reboot to retry; the rest migrated.`);
    }
    return { ok: true, migrated };
  },
};