Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

33.2 KB
Raw Blame
/**
 * Sprites — production-asset importer (issue #2895, phase 1).
 *
 * Ingests the approved/final artifacts of an ElsewhereAcres-layout sprite
 * pipeline into data/sprites/, creating/refreshing a sprite record per
 * subject. Expected source layout (all paths relative to `sourceRoot`):
 *
 *   art-pipeline/characters/<id>.json            character specs
 *   art-source/sprites/<id>/reference/           locked main + anchors (+ candidates/, skipped)
 *   art-source/sprites/<id>/walk/                finalized walk-set manifests
 *   art-source/sprites/<id>/grok|runs/<run-id>/  animation runs (only files the walk set references)
 *   art-source/sprites/<id>/runtime/vN/          immutable published atlas archives
 *   art-source/sprites/<id>/imagegen/vN/         approved redraw sources
 *   art-pipeline/catalog/runtime-selection.json  current published-atlas pointer
 *   game/assets/sprites/<family>/                props atlas families (PNG + README)
 *
 * Only approved/final artifacts import: reference candidates and extracted
 * frame intermediates stay behind, while a run's source clip does come across
 * (#2984 — see SOURCE_CLIP_REF). Where a source manifest carries a
 * sha256 for a file, the copied bytes are verified against it — a mismatch is
 * recorded per-item (the import continues) so one corrupt file can't abort a
 * whole tree. Runs on demand only (a user action) — never at boot.
 */

import { join, basename, dirname } from 'path';
import { readdir, realpath } from 'fs/promises';
import { PATHS, ensureDir, sha256File, atomicWrite, pathExists, readJSONFile, expandHome, copyFileGuarded, rmGuarded } from '../../lib/fileUtils.js';
import { isWithinRoot } from '../../lib/workspaceRoots.js';
import { ServerError } from '../../lib/errorHandler.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { upsertImportedRecord } from './records.js';
import {
  spriteDir,
  RUN_DIR_MATCH,
  altRunLayoutPath,
  canonicalizeImportedRunPath,
  canonicalizeImportedRunPathValue,
  deepCanonicalizeImportedRunPaths,
  SOURCE_CLIP_NAME,
  toRecordRelativeAssetPath,
} from './paths.js';
import { isValidSpriteId } from './recordsLogic.js';

// The source pipeline keys everything on magenta; imported characters carry it
// so later re-renders stay consistent with their existing anchors/strips.
const LEGACY_CHROMA_KEY = '#FF00FF';

const PROP_FILE_EXTENSIONS = ['.png', '.md'];

// Absent-or-corrupt manifest reads collapse to null — the importer treats both
// as "nothing to import from this manifest."
const readJson = (p) => readJSONFile(p, null, { logError: false });

// The selected repository is the read boundary, including metadata and assets
// reached through manifests. Check at use time and return the canonical path:
// later copies must not follow the original (possibly redirected) symlink.
// Missing optional artifacts retain their existing null/skip behavior.
async function sourcePath(sourceRoot, candidate) {
  const resolved = await realpath(candidate).catch((err) => {
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;
    throw err;
  });
  // Both paths are realpath results. Do not fold case based on the OS: a
  // case-sensitive volume may have a distinct sibling with the same spelling.
  if (resolved && !isWithinRoot(resolved, sourceRoot)) {
    throw new ServerError('Sprite import source path resolves outside the selected source root', {
      status: 400, code: 'SOURCE_OUTSIDE_ROOT',
    });
  }
  return resolved;
}

async function readSourceJson(sourceRoot, candidate) {
  const checked = await sourcePath(sourceRoot, candidate);
  return checked ? readJson(checked) : null;
}

async function copySourceFile(sourceRoot, candidate, dest) {
  const checked = await sourcePath(sourceRoot, candidate);
  if (!checked) return false;
  await copyFileGuarded(checked, dest);
  return true;
}

async function charSourcePath(sourceRoot, srcCharDir, rel) {
  const direct = await sourcePath(sourceRoot, join(srcCharDir, rel));
  if (direct) return direct;
  const alt = altRunLayoutPath(rel);
  return alt ? sourcePath(sourceRoot, join(srcCharDir, alt)) : null;
}

/**
 * Recursively copy `srcDir` into `destDir`, filtering by `shouldCopy(relPath,
 * isDir)`. Returns the list of copied file paths relative to destDir.
 */
async function copyTree(sourceRoot, srcDir, destDir, shouldCopy = () => true) {
  const copied = [];
  const ensuredDirs = new Set();
  async function walk(current, rel) {
    const checked = await sourcePath(sourceRoot, current);
    if (!checked) return;
    let entries;
    try {
      entries = await readdir(checked, { withFileTypes: true });
    } catch {
      return;
    }
    for (const entry of entries) {
      if (entry.name.startsWith('.')) continue;
      const entryRel = rel ? `${rel}/${entry.name}` : entry.name;
      if (!shouldCopy(entryRel, entry.isDirectory())) continue;
      const srcPath = join(current, entry.name);
      if (entry.isDirectory()) {
        await walk(srcPath, entryRel);
      } else if (entry.isFile()) {
        const destPath = join(destDir, entryRel);
        const parent = dirname(destPath);
        if (!ensuredDirs.has(parent)) {
          await ensureDir(parent);
          ensuredDirs.add(parent);
        }
        if (await copySourceFile(sourceRoot, srcPath, destPath)) copied.push(entryRel);
      }
    }
  }
  await walk(srcDir, '');
  return copied;
}

/**
 * Verify hash-pinned files against the sha256s the source manifests declare.
 * Only files copied by THIS run count — a stale copy left in the destination
 * by an earlier import must not vouch for a file the current source no
 * longer provides. Duplicate expectations (the same file pinned by more than
 * one manifest) collapse to one check per (path, hash).
 */
async function verifyHashes(destDir, expectations, errors, copiedThisRun) {
  const seen = new Set();
  const unique = expectations.filter(({ relPath, sha256 }) => {
    if (typeof sha256 !== 'string' || !sha256) return false;
    const key = `${relPath}\n${sha256}`;
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
  const outcomes = await Promise.all(
    unique.map(async ({ relPath, sha256 }) => {
      if (!copiedThisRun.has(relPath)) return { error: `missing from source: ${relPath}` };
      if ((await sha256File(join(destDir, relPath))) === sha256) return { verified: true };
      // A still that fails its pin stays on disk with an error recorded — the
      // record is still browsable and the mismatch is visible. A source CLIP
      // is different: it is the INPUT to re-deriving a cycle (#2984), so bytes
      // the manifest disowns would let a later re-derive silently emit frames
      // nobody approved. Drop it — that run just stays non-re-derivable until
      // a clean re-import.
      if (SOURCE_CLIP_REF.test(relPath)) {
        await rmGuarded(join(destDir, relPath), { force: true });
        return { dropped: relPath };
      }
      return { error: `sha256 mismatch: ${relPath}` };
    }),
  );
  for (const o of outcomes) if (o.error) errors.push(o.error);
  // Collapse drops by PATH, not by outcome: one clip pinned by two manifests
  // that disagree on its hash yields two mismatching expectations for the same
  // file, and counting both would decrement the caller's file total twice for a
  // single deleted clip (and report the same run twice).
  const dropped = [...new Set(outcomes.filter((o) => o.dropped).map((o) => o.dropped))];
  for (const relPath of dropped) {
    errors.push(`run ${relPath.split('/')[1]}: source clip failed sha256 verification — not imported`);
  }
  // The caller subtracts `dropped` from its file count: a clip removed here was
  // counted when it was copied, and an import summary must not claim a file it
  // deliberately deleted.
  return { verified: outcomes.filter((o) => o.verified).length, dropped: dropped.length };
}

// Manifest paths reference files repo-relative (art-source/sprites/<id>/…) or
// occasionally bare. Normalize to a path relative to the character's source
// dir so it can be re-anchored under data/sprites/<id>/. Manifest content is
// request-influenced data feeding join() as a WRITE destination, so a `..`
// segment (or empty segment) rejects the whole path — a corrupt/crafted
// manifest must not be able to copy outside data/sprites/.
function relToCharacterDir(manifestPath, characterId) {
  if (typeof manifestPath !== 'string' || !manifestPath) return null;
  // Anchor to the SOURCE tree — a published game-tree path
  // (game/assets/sprites/<id>/…) or engine resource URI also contains
  // "sprites/<id>/" but is not a character-dir asset.
  const marker = `art-source/sprites/${characterId}/`;
  const idx = manifestPath.indexOf(marker);
  const rel = idx >= 0 ? manifestPath.slice(idx + marker.length) : manifestPath.replace(/^\/+/, '');
  // A repo-anchored path OUTSIDE the character dir (contract docs, pipeline
  // catalogs) is provenance, not a character asset — don't misread it as
  // character-relative and then report it "missing".
  if (idx < 0 && /^(art-pipeline|art-source|game)\//.test(rel)) return null;
  // Split on BOTH separators — on Windows a `..\` segment would otherwise ride
  // through the check as one opaque segment and still traverse in join().
  if (!rel || rel.split(/[\\/]/).some((seg) => seg === '..' || seg === '')) return null;
  return rel;
}

// Asset references worth importing from a run's manifests: images, previews,
// and packaged manifests — never raw/extracted intermediates.
const ASSET_REF = /\.(png|gif|json)$/i;
// A run's i2v source clip — the ONE video that imports (#2984). Frame counts
// became authorable in #2970 and the packer only ever resamples DOWN, so
// without the clip an imported direction is permanently stuck at whatever
// frame count it arrived with: re-deriving it needs the clip re-extracted.
// Matched by exact shape rather than a blanket `.mp4` rule so unrelated videos
// in the source tree still stay behind. `raw/` frames deliberately remain
// excluded below — ~5× the bytes of the clip they were extracted from, and
// re-derivation re-extracts them on demand. (The name itself lives in paths.js —
// the importer, the postprocess, and the walk service all key on it.)
const SOURCE_CLIP_REF = new RegExp(`${RUN_DIR_MATCH.source}/generated/${SOURCE_CLIP_NAME.replace('.', '\\.')}$`);
const EXCLUDED_RUN_SEGMENTS = /(^|\/)(raw|frames|review)\//;
// Free-text log fields in run records can contain path-looking strings that
// were never assets — don't traverse them.
const NOISE_KEYS = /tail$|^stdout|^stderr|^log$/i;

function importableRel(value, characterId) {
  const rel = relToCharacterDir(value, characterId);
  if (!rel || EXCLUDED_RUN_SEGMENTS.test(rel)) return null;
  return ASSET_REF.test(rel) || SOURCE_CLIP_REF.test(rel) ? canonicalizeImportedRunPath(rel) : null;
}

/**
 * Recursively collect importable asset paths referenced anywhere in a
 * manifest's JSON — and, when a manifest hash-pins a referenced file (a
 * `{ path, sha256 }` record, or the `<base>Path`/`<base>Sha256` sibling-key
 * convention), the expectation pair so the copied bytes get verified too.
 */
function collectManifestRefs(node, characterId, refs, expectations) {
  if (typeof node === 'string') {
    const rel = importableRel(node, characterId);
    if (rel) refs.add(rel);
  } else if (Array.isArray(node)) {
    for (const v of node) collectManifestRefs(v, characterId, refs, expectations);
  } else if (node && typeof node === 'object') {
    for (const [k, v] of Object.entries(node)) {
      if (NOISE_KEYS.test(k)) continue;
      collectManifestRefs(v, characterId, refs, expectations);
    }
    if (typeof node.sha256 === 'string' && typeof node.path === 'string') {
      const rel = importableRel(node.path, characterId);
      if (rel) expectations.push({ relPath: rel, sha256: node.sha256 });
    }
    for (const [k, v] of Object.entries(node)) {
      if (!k.endsWith('Sha256') || typeof v !== 'string') continue;
      const base = k.slice(0, -'Sha256'.length);
      const pathVal = node[base] ?? node[`${base}Path`];
      if (typeof pathVal !== 'string') continue;
      const rel = importableRel(pathVal, characterId);
      if (rel) expectations.push({ relPath: rel, sha256: v });
    }
  }
}

// Read a character-dir JSON with the same drift tolerance, so a manifest
// naming `runs/<run-id>/…` still expands when the source stores that run under
// `grok/<run-id>/`. Read-then-fall-back rather than probe-then-read: readJson
// already collapses a missing/corrupt file to null, so the twin costs a
// syscall only when the declared path really isn't there.
async function readCharJson(sourceRoot, srcCharDir, rel) {
  const direct = await readSourceJson(sourceRoot, join(srcCharDir, rel));
  if (direct) return direct;
  const alt = altRunLayoutPath(rel);
  return alt ? readSourceJson(sourceRoot, join(srcCharDir, alt)) : null;
}

// Copy one character-dir file into PortOS's neutral imported-run layout. The
// source lookup remains drift-tolerant, while every local write maps a legacy
// `grok/` run prefix to `runs/` so a re-import cannot recreate the residue
// migration 203 healed on earlier installs.
async function copyCharFile(sourceRoot, srcCharDir, destDir, rel) {
  const src = await charSourcePath(sourceRoot, srcCharDir, rel);
  if (!src) return null;
  const destRel = canonicalizeImportedRunPath(rel);
  const dest = join(destDir, destRel);
  await ensureDir(dirname(dest));
  return await copySourceFile(sourceRoot, src, dest) ? destRel : null;
}

// Recursively collect JSON manifests in one imported tree. A missing optional
// subtree is an empty list; the importer only calls this after copying an
// approved direction, so a non-directory is not an error worth surfacing.
async function listJsonFiles(dir) {
  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
  const files = [];
  for (const entry of entries) {
    const abs = join(dir, entry.name);
    if (entry.isDirectory()) files.push(...await listJsonFiles(abs));
    else if (entry.isFile() && entry.name.endsWith('.json')) files.push(abs);
  }
  return files;
}

async function normalizeImportedJsonFile(characterId, abs) {
  const current = await readJson(abs);
  if (current === null) return null;
  const normalized = deepCanonicalizeImportedRunPaths(characterId, current);
  if (JSON.stringify(normalized) !== JSON.stringify(current)) await atomicWrite(abs, normalized);
  return normalized;
}

// A rewritten run record gets new bytes, so the selection and walk-set must
// repin it just as migration 203 does. The path helper accepts both imported
// source provenance and PortOS-relative forms, keeping the write-side layout
// rule in one place.
async function repinDirectionRunManifests(destDir, characterId, directions) {
  for (const entry of Object.values(directions || {})) {
    if (!entry || typeof entry !== 'object') continue;
    if (typeof entry.runPath === 'string') entry.runPath = canonicalizeImportedRunPathValue(characterId, entry.runPath);
    if (typeof entry.runManifest !== 'string') continue;
    entry.runManifest = canonicalizeImportedRunPathValue(characterId, entry.runManifest);
    const rel = toRecordRelativeAssetPath(characterId, entry.runManifest);
    // eslint-disable-next-line no-await-in-loop -- an imported set has at most eight directions.
    if (rel && await pathExists(join(destDir, rel))) {
      // eslint-disable-next-line no-await-in-loop -- each hash reads one small run record.
      entry.runManifestSha256 = await sha256File(join(destDir, rel));
    }
  }
}

async function normalizeImportedDirections({ destDir, characterId, abs, repin = false }) {
  const current = await readJson(abs);
  if (current === null) return null;
  const normalized = deepCanonicalizeImportedRunPaths(characterId, current);
  if (repin && normalized?.directions) await repinDirectionRunManifests(destDir, characterId, normalized.directions);
  if (JSON.stringify(normalized) !== JSON.stringify(current)) await atomicWrite(abs, normalized);
  return normalized;
}

// Imported manifests initially retain source bytes so all source sha256 pins
// can be verified. Normalize only after that verification completes, then
// rebuild the local run-manifest → selection → walk-set checksum cascade.
async function normalizeImportedRunLayout(destDir, characterId) {
  const runsDir = join(destDir, 'runs');
  for (const abs of await listJsonFiles(runsDir)) await normalizeImportedJsonFile(characterId, abs);

  const walkDir = join(destDir, 'walk');
  const selectionAbs = join(walkDir, `${characterId}-walk-selection-v1.json`);
  const walkSetAbs = join(walkDir, `${characterId}-walk-set-v1.json`);
  const selection = await normalizeImportedDirections({ destDir, characterId, abs: selectionAbs, repin: true });
  const selectionSha256 = selection && await pathExists(selectionAbs) ? await sha256File(selectionAbs) : null;
  const walkSet = await normalizeImportedDirections({ destDir, characterId, abs: walkSetAbs, repin: true });
  if (walkSet && selectionSha256 && typeof walkSet.selectionSha256 === 'string') {
    walkSet.selectionSha256 = selectionSha256;
    await atomicWrite(walkSetAbs, walkSet);
  }

  for (const abs of await listJsonFiles(join(destDir, 'runtime'))) await normalizeImportedJsonFile(characterId, abs);
  const walkSetSha256 = walkSet && await pathExists(walkSetAbs) ? await sha256File(walkSetAbs) : null;
  const pointerAbs = join(destDir, 'runtime', 'current.json');
  const pointer = await readJson(pointerAbs);
  if (!pointer) return;
  const normalizedPointer = deepCanonicalizeImportedRunPaths(characterId, pointer);
  if (walkSetSha256 && typeof normalizedPointer.walkSetSha256 === 'string') normalizedPointer.walkSetSha256 = walkSetSha256;
  const manifestRel = toRecordRelativeAssetPath(characterId, normalizedPointer.manifestPath) || normalizedPointer.manifestPath;
  if (typeof manifestRel === 'string' && await pathExists(join(destDir, manifestRel))) {
    normalizedPointer.manifestSha256 = await sha256File(join(destDir, manifestRel));
  }
  if (JSON.stringify(normalizedPointer) !== JSON.stringify(pointer)) await atomicWrite(pointerAbs, normalizedPointer);
}

/**
 * Import one APPROVED direction of a finalized walk set: the run manifest
 * (hash-verified), then only the assets the run's manifests declare — packed
 * strips, packaged manifests, previews, and the run's source clip. Works for
 * both grok runs (record at
 * run root, assets under generated/) and imagegen redraw runs (manifest +
 * strips at the version root, surrounded by unselected candidates that must
 * stay behind).
 */
async function importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun }) {
  const manifestRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runManifest, characterId));
  if (!manifestRel) {
    result.errors.push(`walk set ${direction}: unsafe run path rejected: ${dir.runManifest || dir.runPath}`);
    return;
  }
  const copiedManifestRel = await copyCharFile(sourceRoot, srcCharDir, destDir, manifestRel);
  if (!copiedManifestRel) {
    result.errors.push(`walk set ${direction}: run manifest missing: ${manifestRel}`);
    return;
  }
  result.files += 1;
  copiedThisRun.add(copiedManifestRel);
  if (dir.runManifestSha256) hashExpectations.push({ relPath: copiedManifestRel, sha256: dir.runManifestSha256 });

  const refs = new Set();
  // Read the copy just made rather than the source: same bytes, one fewer
  // resolution + read, and it matches how the reference/ and walk/ loops below
  // expand their manifests.
  collectManifestRefs(await readJson(join(destDir, copiedManifestRel)), characterId, refs, hashExpectations);
  const runRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runPath, characterId));
  if (runRel && (await charSourcePath(sourceRoot, srcCharDir, `${runRel}/generated/review-preview.json`))) {
    refs.add(`${runRel}/generated/review-preview.json`);
  }
  // One extra expansion level: packaged manifests / previews the run record
  // names in turn declare the strip files (and their hash pins).
  for (const rel of [...refs]) {
    if (rel.endsWith('.json')) collectManifestRefs(await readCharJson(sourceRoot, srcCharDir, rel), characterId, refs, hashExpectations);
  }
  // The source clip (#2984) — a fallback for a run whose manifests never name
  // it, since a direction with no clip can never be re-derived at a different
  // frame count. Deliberately AFTER the expansion above: the usual carrier of
  // `sourceVideoPath` is the packaged manifest, one level down, so probing
  // earlier would miss a clip already collected and copy the same bytes to two
  // destinations (once per run layout). The RUN_DIR_MATCH gate keeps this to
  // real run entries (an imagegen redraw entry has no run dir and no clip), and
  // the twin check ignores a manifest that named the clip under the OTHER
  // layout — that path is already in `refs` and denotes the same file.
  const clipRel = runRel && `${runRel}/generated/${SOURCE_CLIP_NAME}`;
  if (clipRel && RUN_DIR_MATCH.test(runRel) && !refs.has(clipRel) && !refs.has(altRunLayoutPath(clipRel))
      && (await charSourcePath(sourceRoot, srcCharDir, clipRel))) {
    refs.add(clipRel);
  }
  refs.delete(manifestRel);
  for (const rel of [...refs].sort()) {
    if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {
      result.files += 1;
      copiedThisRun.add(rel);
    } else {
      // A manifest-declared asset the source no longer has is a real gap —
      // an "imported" subject must not silently omit an approved artifact.
      result.errors.push(`walk set ${direction}: referenced asset missing: ${rel}`);
    }
  }
}

async function importCharacter({ sourceRoot, characterId, spec, specPath, selection }) {
  const result = { id: characterId, kind: 'character', files: 0, verified: 0, errors: [] };
  const srcCharDir = join(sourceRoot, 'art-source', 'sprites', characterId);
  const destDir = spriteDir(characterId);

  // A destination with a LOCKED reference set (#2896) is immutable evidence
  // — a same-id import would copy the source's reference tree straight over
  // the frozen identity root, bypassing every lock check. Refuse the whole
  // character; the user can delete the record or import under another id.
  const destManifest = await readJSONFile(join(destDir, 'reference', `${characterId}-reference-set-v1.json`), null);
  if (destManifest?.mainReference?.locked) {
    result.errors.push('destination has a locked reference set — refusing to overwrite immutable artifacts (delete the record or import under a different id)');
    return result;
  }
  await ensureDir(destDir);

  // Character spec — verbatim copy for provenance alongside the record.
  if (!await copySourceFile(sourceRoot, specPath, join(destDir, 'character-spec.json'))) {
    result.errors.push('character spec missing from source');
    return result;
  }
  result.files += 1;

  const hashExpectations = [];
  // Every file THIS run copied (dest-relative). Verification is scoped to
  // this set so a stale destination copy can't vouch for a file the current
  // source no longer provides.
  const copiedThisRun = new Set(['character-spec.json']);

  // reference/ — locked main + anchors; candidates/ (unapproved) stay behind.
  if (await sourcePath(sourceRoot, join(srcCharDir, 'reference'))) {
    const copied = await copyTree(
      sourceRoot,
      join(srcCharDir, 'reference'),
      join(destDir, 'reference'),
      (rel, isDir) => !(isDir && basename(rel) === 'candidates') && !rel.includes('candidates/'),
    );
    result.files += copied.length;
    for (const rel of copied) copiedThisRun.add(`reference/${rel}`);
    for (const rel of copied) {
      if (!rel.endsWith('.json')) continue;
      const manifest = await readJson(join(destDir, 'reference', rel));
      for (const anchor of manifest?.anchors || []) {
        const anchorRel = relToCharacterDir(anchor?.path, characterId);
        if (anchorRel && anchor?.sha256) hashExpectations.push({ relPath: anchorRel, sha256: anchor.sha256 });
      }
    }
  }

  // walk/ — the manifests themselves are small provenance and copy whole,
  // but ASSETS import only for APPROVED directions of a FINALIZED set. Draft
  // selections / publication records / pending directions contribute nothing.
  const walkDir = join(srcCharDir, 'walk');
  if (await sourcePath(sourceRoot, walkDir)) {
    const copied = await copyTree(sourceRoot, walkDir, join(destDir, 'walk'));
    result.files += copied.length;
    for (const rel of copied) copiedThisRun.add(`walk/${rel}`);
    for (const rel of copied) {
      if (!rel.endsWith('.json')) continue;
      const walkSet = await readJson(join(destDir, 'walk', rel));
      if (walkSet?.kind !== 'finalized-eight-direction-walk-set' || walkSet?.status !== 'final') continue;
      // The set's own top-level references (selectionPath/selectionSha256 …)
      // are hash-pinned provenance too. Harvest them from the metadata ONLY —
      // never from `directions`, or non-approved directions' run files would
      // ride in through the generic collector.
      const { directions, ...walkSetMeta } = walkSet;
      const metaRefs = new Set();
      collectManifestRefs(walkSetMeta, characterId, metaRefs, hashExpectations);
      for (const metaRel of [...metaRefs].sort()) {
        if (copiedThisRun.has(metaRel)) continue;
        if (await copyCharFile(sourceRoot, srcCharDir, destDir, metaRel)) {
          result.files += 1;
          copiedThisRun.add(metaRel);
        } else result.errors.push(`walk set: referenced file missing: ${metaRel}`);
      }
      for (const [direction, dir] of Object.entries(directions || {})) {
        if (dir?.status !== 'approved') {
          result.errors.push(`walk set ${direction}: not approved (${dir?.status ?? 'missing'}) — skipped`);
          continue;
        }
        await importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun });
      }
    }
  }

  // runtime/ — immutable published atlas archives. (No wholesale imagegen/
  // copy: approved redraw outputs arrive via the walk-set expansion above;
  // the rest of imagegen/ is unselected candidates and raw intermediates.)
  if (await sourcePath(sourceRoot, join(srcCharDir, 'runtime'))) {
    const copied = await copyTree(sourceRoot, join(srcCharDir, 'runtime'), join(destDir, 'runtime'));
    result.files += copied.length;
    for (const rel of copied) copiedThisRun.add(`runtime/${rel}`);
  }

  // Published-selection provenance: copy and hash-verify every character-dir
  // artifact the catalog's selection declares (immutable atlas archive, keyed
  // source, publication manifest, …) via the generic collector — pins ride in
  // through the {path,sha256} / <base>Path+<base>Sha256 conventions, and
  // repo-anchored references outside the character dir are ignored.
  if (selection?.characterId === characterId && selection.selected) {
    const selRefs = new Set();
    collectManifestRefs(selection.selected, characterId, selRefs, hashExpectations);
    for (const rel of [...selRefs].sort()) {
      if (copiedThisRun.has(rel)) continue;
      if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {
        result.files += 1;
        copiedThisRun.add(rel);
      } else result.errors.push(`published selection: referenced asset missing: ${rel}`);
    }
  }

  const { verified, dropped } = await verifyHashes(destDir, hashExpectations, result.errors, copiedThisRun);
  result.verified = verified;
  result.files -= dropped;
  await normalizeImportedRunLayout(destDir, characterId);

  const record = await upsertImportedRecord(characterId, {
    kind: 'character',
    name: spec?.displayName || characterId,
    status: 'imported',
    spec,
    chromaKey: LEGACY_CHROMA_KEY,
    importedFrom: { sourceRoot, importedAt: new Date().toISOString() },
  });
  result.record = record;
  return result;
}

async function importPropsFamily({ sourceRoot, familyId, srcFamilyDir }) {
  const result = { id: familyId, kind: 'props', files: 0, verified: 0, errors: [] };
  const destDir = join(spriteDir(familyId), 'atlas');
  const copied = await copyTree(sourceRoot, srcFamilyDir, destDir, (rel, isDir) => {
    if (isDir) return true;
    const lower = rel.toLowerCase();
    return PROP_FILE_EXTENSIONS.some((ext) => lower.endsWith(ext));
  });
  result.files += copied.length;
  const record = await upsertImportedRecord(familyId, {
    kind: 'props',
    name: familyId,
    status: 'imported',
    importedFrom: { sourceRoot, importedAt: new Date().toISOString() },
  });
  result.record = record;
  return result;
}

/**
 * Import approved sprite assets from `sourceRoot`. Each subject (a character or a
 * props family) holds one backup admission lease from its first copied byte
 * through the record row that marks it imported (#9982): a re-import copies over
 * files an existing row already describes, and a snapshot taken halfway would
 * pair them with a row that says the import finished. A multi-subject import
 * releases the lease between subjects so a cut waits for one subject, not all.
 * Options:
 *  - characters: limit to these character ids (default: all specs found)
 *  - includeProps: also import game/assets/sprites/ prop families (default true)
 *
 * Returns { results: [...perSubject], totals: { subjects, files, verified, errors } }.
 */
export async function importFromSource({ sourceRoot, characters, includeProps = true }) {
  // The UI placeholder suggests a ~/… path; resolve it before probing, or a
  // correct tilde path 400s with a misleading "not a sprite pipeline".
  sourceRoot = await realpath(expandHome(sourceRoot)).catch((err) => {
    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;
    throw err;
  });
  const specsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'art-pipeline', 'characters'));
  const propsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'game', 'assets', 'sprites'));
  if (!specsDir && !propsDir) {
    throw new ServerError(
      'Source root does not look like a sprite pipeline (expected art-pipeline/characters/ or game/assets/sprites/)',
      { status: 400, code: 'INVALID_SOURCE_ROOT' },
    );
  }

  await ensureDir(PATHS.sprites);
  // Read the published-selection pointer up front so the matching character's
  // import can verify the hash-pinned runtime artifacts it declares.
  const selection = await readSourceJson(sourceRoot, join(sourceRoot, 'art-pipeline', 'catalog', 'runtime-selection.json'));
  const results = [];
  // Ids imported THIS run (drives the runtime-selection provenance copy) vs
  // every character id the specs dir declares (drives the props-loop skip). A
  // filtered import must still exclude ALL characters' game dirs from the
  // props walk — otherwise `characters: ['a']` treats character b's published
  // atlas dir as a props family and overwrites b's record with kind:'props'.
  const characterIds = new Set();
  const allSpecCharacterIds = new Set();

  if (specsDir) {
    const wanted = Array.isArray(characters) && characters.length ? new Set(characters) : null;
    for (const entry of (await readdir(specsDir)).sort()) {
      if (!entry.endsWith('.json')) continue;
      const specPath = join(specsDir, entry);
      const spec = await readSourceJson(sourceRoot, specPath);
      // A real character spec carries characterId; sibling files in the specs
      // dir (JSON-schema definitions, templates) don't and are not subjects.
      if (!spec?.characterId) continue;
      const characterId = spec.characterId;
      if (!isValidSpriteId(characterId)) {
        results.push({ id: characterId, kind: 'character', files: 0, verified: 0, errors: ['invalid character id'] });
        continue;
      }
      allSpecCharacterIds.add(characterId);
      if (wanted && !wanted.has(characterId)) continue;
      characterIds.add(characterId);
      results.push(await withBackupAssetPublication(
        () => importCharacter({ sourceRoot, characterId, spec, specPath, selection }),
      ));
    }
  }

  if (propsDir && includeProps) {
    for (const entry of (await readdir(propsDir, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
      if (!entry.isDirectory()) continue;
      // A character's published atlas dir duplicates what runtime/ archives —
      // only non-character families import from the game tree. Excludes every
      // spec-declared character, imported this run or not.
      if (allSpecCharacterIds.has(entry.name)) continue;
      if (!isValidSpriteId(entry.name)) continue;
      results.push(await withBackupAssetPublication(
        () => importPropsFamily({ sourceRoot, familyId: entry.name, srcFamilyDir: join(propsDir, entry.name) }),
      ));
    }
  }

  // Published-atlas pointer — provenance for the current selection, kept with
  // the character it points at.
  if (selection?.characterId && characterIds.has(selection.characterId)) {
    const destDir = join(spriteDir(selection.characterId), 'catalog');
    await ensureDir(destDir);
    await atomicWrite(join(destDir, 'runtime-selection.json'), selection);
  }

  const totals = results.reduce(
    (acc, r) => ({
      subjects: acc.subjects + 1,
      files: acc.files + r.files,
      verified: acc.verified + r.verified,
      errors: acc.errors + r.errors.length,
    }),
    { subjects: 0, files: 0, verified: 0, errors: 0 },
  );
  console.log(`🎞️ Sprite import: ${totals.subjects} subjects, ${totals.files} files, ${totals.verified} hash-verified, ${totals.errors} errors`);
  return { results, totals };
}