Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Sprites — production-asset importer (issue #2895, phase 1).
*
* Ingests the approved/final artifacts of an ElsewhereAcres-layout sprite
* pipeline into data/sprites/, creating/refreshing a sprite record per
* subject. Expected source layout (all paths relative to `sourceRoot`):
*
* art-pipeline/characters/<id>.json character specs
* art-source/sprites/<id>/reference/ locked main + anchors (+ candidates/, skipped)
* art-source/sprites/<id>/walk/ finalized walk-set manifests
* art-source/sprites/<id>/grok|runs/<run-id>/ animation runs (only files the walk set references)
* art-source/sprites/<id>/runtime/vN/ immutable published atlas archives
* art-source/sprites/<id>/imagegen/vN/ approved redraw sources
* art-pipeline/catalog/runtime-selection.json current published-atlas pointer
* game/assets/sprites/<family>/ props atlas families (PNG + README)
*
* Only approved/final artifacts import: reference candidates and extracted
* frame intermediates stay behind, while a run's source clip does come across
* (#2984 — see SOURCE_CLIP_REF). Where a source manifest carries a
* sha256 for a file, the copied bytes are verified against it — a mismatch is
* recorded per-item (the import continues) so one corrupt file can't abort a
* whole tree. Runs on demand only (a user action) — never at boot.
*/
import { join, basename, dirname } from 'path';
import { readdir, realpath } from 'fs/promises';
import { PATHS, ensureDir, sha256File, atomicWrite, pathExists, readJSONFile, expandHome, copyFileGuarded, rmGuarded } from '../../lib/fileUtils.js';
import { isWithinRoot } from '../../lib/workspaceRoots.js';
import { ServerError } from '../../lib/errorHandler.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { upsertImportedRecord } from './records.js';
import {
spriteDir,
RUN_DIR_MATCH,
altRunLayoutPath,
canonicalizeImportedRunPath,
canonicalizeImportedRunPathValue,
deepCanonicalizeImportedRunPaths,
SOURCE_CLIP_NAME,
toRecordRelativeAssetPath,
} from './paths.js';
import { isValidSpriteId } from './recordsLogic.js';
// The source pipeline keys everything on magenta; imported characters carry it
// so later re-renders stay consistent with their existing anchors/strips.
const LEGACY_CHROMA_KEY = '#FF00FF';
const PROP_FILE_EXTENSIONS = ['.png', '.md'];
// Absent-or-corrupt manifest reads collapse to null — the importer treats both
// as "nothing to import from this manifest."
const readJson = (p) => readJSONFile(p, null, { logError: false });
// The selected repository is the read boundary, including metadata and assets
// reached through manifests. Check at use time and return the canonical path:
// later copies must not follow the original (possibly redirected) symlink.
// Missing optional artifacts retain their existing null/skip behavior.
async function sourcePath(sourceRoot, candidate) {
const resolved = await realpath(candidate).catch((err) => {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;
throw err;
});
// Both paths are realpath results. Do not fold case based on the OS: a
// case-sensitive volume may have a distinct sibling with the same spelling.
if (resolved && !isWithinRoot(resolved, sourceRoot)) {
throw new ServerError('Sprite import source path resolves outside the selected source root', {
status: 400, code: 'SOURCE_OUTSIDE_ROOT',
});
}
return resolved;
}
async function readSourceJson(sourceRoot, candidate) {
const checked = await sourcePath(sourceRoot, candidate);
return checked ? readJson(checked) : null;
}
async function copySourceFile(sourceRoot, candidate, dest) {
const checked = await sourcePath(sourceRoot, candidate);
if (!checked) return false;
await copyFileGuarded(checked, dest);
return true;
}
async function charSourcePath(sourceRoot, srcCharDir, rel) {
const direct = await sourcePath(sourceRoot, join(srcCharDir, rel));
if (direct) return direct;
const alt = altRunLayoutPath(rel);
return alt ? sourcePath(sourceRoot, join(srcCharDir, alt)) : null;
}
/**
* Recursively copy `srcDir` into `destDir`, filtering by `shouldCopy(relPath,
* isDir)`. Returns the list of copied file paths relative to destDir.
*/
async function copyTree(sourceRoot, srcDir, destDir, shouldCopy = () => true) {
const copied = [];
const ensuredDirs = new Set();
async function walk(current, rel) {
const checked = await sourcePath(sourceRoot, current);
if (!checked) return;
let entries;
try {
entries = await readdir(checked, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
if (entry.name.startsWith('.')) continue;
const entryRel = rel ? `${rel}/${entry.name}` : entry.name;
if (!shouldCopy(entryRel, entry.isDirectory())) continue;
const srcPath = join(current, entry.name);
if (entry.isDirectory()) {
await walk(srcPath, entryRel);
} else if (entry.isFile()) {
const destPath = join(destDir, entryRel);
const parent = dirname(destPath);
if (!ensuredDirs.has(parent)) {
await ensureDir(parent);
ensuredDirs.add(parent);
}
if (await copySourceFile(sourceRoot, srcPath, destPath)) copied.push(entryRel);
}
}
}
await walk(srcDir, '');
return copied;
}
/**
* Verify hash-pinned files against the sha256s the source manifests declare.
* Only files copied by THIS run count — a stale copy left in the destination
* by an earlier import must not vouch for a file the current source no
* longer provides. Duplicate expectations (the same file pinned by more than
* one manifest) collapse to one check per (path, hash).
*/
async function verifyHashes(destDir, expectations, errors, copiedThisRun) {
const seen = new Set();
const unique = expectations.filter(({ relPath, sha256 }) => {
if (typeof sha256 !== 'string' || !sha256) return false;
const key = `${relPath}\n${sha256}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
const outcomes = await Promise.all(
unique.map(async ({ relPath, sha256 }) => {
if (!copiedThisRun.has(relPath)) return { error: `missing from source: ${relPath}` };
if ((await sha256File(join(destDir, relPath))) === sha256) return { verified: true };
// A still that fails its pin stays on disk with an error recorded — the
// record is still browsable and the mismatch is visible. A source CLIP
// is different: it is the INPUT to re-deriving a cycle (#2984), so bytes
// the manifest disowns would let a later re-derive silently emit frames
// nobody approved. Drop it — that run just stays non-re-derivable until
// a clean re-import.
if (SOURCE_CLIP_REF.test(relPath)) {
await rmGuarded(join(destDir, relPath), { force: true });
return { dropped: relPath };
}
return { error: `sha256 mismatch: ${relPath}` };
}),
);
for (const o of outcomes) if (o.error) errors.push(o.error);
// Collapse drops by PATH, not by outcome: one clip pinned by two manifests
// that disagree on its hash yields two mismatching expectations for the same
// file, and counting both would decrement the caller's file total twice for a
// single deleted clip (and report the same run twice).
const dropped = [...new Set(outcomes.filter((o) => o.dropped).map((o) => o.dropped))];
for (const relPath of dropped) {
errors.push(`run ${relPath.split('/')[1]}: source clip failed sha256 verification — not imported`);
}
// The caller subtracts `dropped` from its file count: a clip removed here was
// counted when it was copied, and an import summary must not claim a file it
// deliberately deleted.
return { verified: outcomes.filter((o) => o.verified).length, dropped: dropped.length };
}
// Manifest paths reference files repo-relative (art-source/sprites/<id>/…) or
// occasionally bare. Normalize to a path relative to the character's source
// dir so it can be re-anchored under data/sprites/<id>/. Manifest content is
// request-influenced data feeding join() as a WRITE destination, so a `..`
// segment (or empty segment) rejects the whole path — a corrupt/crafted
// manifest must not be able to copy outside data/sprites/.
function relToCharacterDir(manifestPath, characterId) {
if (typeof manifestPath !== 'string' || !manifestPath) return null;
// Anchor to the SOURCE tree — a published game-tree path
// (game/assets/sprites/<id>/…) or engine resource URI also contains
// "sprites/<id>/" but is not a character-dir asset.
const marker = `art-source/sprites/${characterId}/`;
const idx = manifestPath.indexOf(marker);
const rel = idx >= 0 ? manifestPath.slice(idx + marker.length) : manifestPath.replace(/^\/+/, '');
// A repo-anchored path OUTSIDE the character dir (contract docs, pipeline
// catalogs) is provenance, not a character asset — don't misread it as
// character-relative and then report it "missing".
if (idx < 0 && /^(art-pipeline|art-source|game)\//.test(rel)) return null;
// Split on BOTH separators — on Windows a `..\` segment would otherwise ride
// through the check as one opaque segment and still traverse in join().
if (!rel || rel.split(/[\\/]/).some((seg) => seg === '..' || seg === '')) return null;
return rel;
}
// Asset references worth importing from a run's manifests: images, previews,
// and packaged manifests — never raw/extracted intermediates.
const ASSET_REF = /\.(png|gif|json)$/i;
// A run's i2v source clip — the ONE video that imports (#2984). Frame counts
// became authorable in #2970 and the packer only ever resamples DOWN, so
// without the clip an imported direction is permanently stuck at whatever
// frame count it arrived with: re-deriving it needs the clip re-extracted.
// Matched by exact shape rather than a blanket `.mp4` rule so unrelated videos
// in the source tree still stay behind. `raw/` frames deliberately remain
// excluded below — ~5× the bytes of the clip they were extracted from, and
// re-derivation re-extracts them on demand. (The name itself lives in paths.js —
// the importer, the postprocess, and the walk service all key on it.)
const SOURCE_CLIP_REF = new RegExp(`${RUN_DIR_MATCH.source}/generated/${SOURCE_CLIP_NAME.replace('.', '\\.')}$`);
const EXCLUDED_RUN_SEGMENTS = /(^|\/)(raw|frames|review)\//;
// Free-text log fields in run records can contain path-looking strings that
// were never assets — don't traverse them.
const NOISE_KEYS = /tail$|^stdout|^stderr|^log$/i;
function importableRel(value, characterId) {
const rel = relToCharacterDir(value, characterId);
if (!rel || EXCLUDED_RUN_SEGMENTS.test(rel)) return null;
return ASSET_REF.test(rel) || SOURCE_CLIP_REF.test(rel) ? canonicalizeImportedRunPath(rel) : null;
}
/**
* Recursively collect importable asset paths referenced anywhere in a
* manifest's JSON — and, when a manifest hash-pins a referenced file (a
* `{ path, sha256 }` record, or the `<base>Path`/`<base>Sha256` sibling-key
* convention), the expectation pair so the copied bytes get verified too.
*/
function collectManifestRefs(node, characterId, refs, expectations) {
if (typeof node === 'string') {
const rel = importableRel(node, characterId);
if (rel) refs.add(rel);
} else if (Array.isArray(node)) {
for (const v of node) collectManifestRefs(v, characterId, refs, expectations);
} else if (node && typeof node === 'object') {
for (const [k, v] of Object.entries(node)) {
if (NOISE_KEYS.test(k)) continue;
collectManifestRefs(v, characterId, refs, expectations);
}
if (typeof node.sha256 === 'string' && typeof node.path === 'string') {
const rel = importableRel(node.path, characterId);
if (rel) expectations.push({ relPath: rel, sha256: node.sha256 });
}
for (const [k, v] of Object.entries(node)) {
if (!k.endsWith('Sha256') || typeof v !== 'string') continue;
const base = k.slice(0, -'Sha256'.length);
const pathVal = node[base] ?? node[`${base}Path`];
if (typeof pathVal !== 'string') continue;
const rel = importableRel(pathVal, characterId);
if (rel) expectations.push({ relPath: rel, sha256: v });
}
}
}
// Read a character-dir JSON with the same drift tolerance, so a manifest
// naming `runs/<run-id>/…` still expands when the source stores that run under
// `grok/<run-id>/`. Read-then-fall-back rather than probe-then-read: readJson
// already collapses a missing/corrupt file to null, so the twin costs a
// syscall only when the declared path really isn't there.
async function readCharJson(sourceRoot, srcCharDir, rel) {
const direct = await readSourceJson(sourceRoot, join(srcCharDir, rel));
if (direct) return direct;
const alt = altRunLayoutPath(rel);
return alt ? readSourceJson(sourceRoot, join(srcCharDir, alt)) : null;
}
// Copy one character-dir file into PortOS's neutral imported-run layout. The
// source lookup remains drift-tolerant, while every local write maps a legacy
// `grok/` run prefix to `runs/` so a re-import cannot recreate the residue
// migration 203 healed on earlier installs.
async function copyCharFile(sourceRoot, srcCharDir, destDir, rel) {
const src = await charSourcePath(sourceRoot, srcCharDir, rel);
if (!src) return null;
const destRel = canonicalizeImportedRunPath(rel);
const dest = join(destDir, destRel);
await ensureDir(dirname(dest));
return await copySourceFile(sourceRoot, src, dest) ? destRel : null;
}
// Recursively collect JSON manifests in one imported tree. A missing optional
// subtree is an empty list; the importer only calls this after copying an
// approved direction, so a non-directory is not an error worth surfacing.
async function listJsonFiles(dir) {
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
const files = [];
for (const entry of entries) {
const abs = join(dir, entry.name);
if (entry.isDirectory()) files.push(...await listJsonFiles(abs));
else if (entry.isFile() && entry.name.endsWith('.json')) files.push(abs);
}
return files;
}
async function normalizeImportedJsonFile(characterId, abs) {
const current = await readJson(abs);
if (current === null) return null;
const normalized = deepCanonicalizeImportedRunPaths(characterId, current);
if (JSON.stringify(normalized) !== JSON.stringify(current)) await atomicWrite(abs, normalized);
return normalized;
}
// A rewritten run record gets new bytes, so the selection and walk-set must
// repin it just as migration 203 does. The path helper accepts both imported
// source provenance and PortOS-relative forms, keeping the write-side layout
// rule in one place.
async function repinDirectionRunManifests(destDir, characterId, directions) {
for (const entry of Object.values(directions || {})) {
if (!entry || typeof entry !== 'object') continue;
if (typeof entry.runPath === 'string') entry.runPath = canonicalizeImportedRunPathValue(characterId, entry.runPath);
if (typeof entry.runManifest !== 'string') continue;
entry.runManifest = canonicalizeImportedRunPathValue(characterId, entry.runManifest);
const rel = toRecordRelativeAssetPath(characterId, entry.runManifest);
// eslint-disable-next-line no-await-in-loop -- an imported set has at most eight directions.
if (rel && await pathExists(join(destDir, rel))) {
// eslint-disable-next-line no-await-in-loop -- each hash reads one small run record.
entry.runManifestSha256 = await sha256File(join(destDir, rel));
}
}
}
async function normalizeImportedDirections({ destDir, characterId, abs, repin = false }) {
const current = await readJson(abs);
if (current === null) return null;
const normalized = deepCanonicalizeImportedRunPaths(characterId, current);
if (repin && normalized?.directions) await repinDirectionRunManifests(destDir, characterId, normalized.directions);
if (JSON.stringify(normalized) !== JSON.stringify(current)) await atomicWrite(abs, normalized);
return normalized;
}
// Imported manifests initially retain source bytes so all source sha256 pins
// can be verified. Normalize only after that verification completes, then
// rebuild the local run-manifest → selection → walk-set checksum cascade.
async function normalizeImportedRunLayout(destDir, characterId) {
const runsDir = join(destDir, 'runs');
for (const abs of await listJsonFiles(runsDir)) await normalizeImportedJsonFile(characterId, abs);
const walkDir = join(destDir, 'walk');
const selectionAbs = join(walkDir, `${characterId}-walk-selection-v1.json`);
const walkSetAbs = join(walkDir, `${characterId}-walk-set-v1.json`);
const selection = await normalizeImportedDirections({ destDir, characterId, abs: selectionAbs, repin: true });
const selectionSha256 = selection && await pathExists(selectionAbs) ? await sha256File(selectionAbs) : null;
const walkSet = await normalizeImportedDirections({ destDir, characterId, abs: walkSetAbs, repin: true });
if (walkSet && selectionSha256 && typeof walkSet.selectionSha256 === 'string') {
walkSet.selectionSha256 = selectionSha256;
await atomicWrite(walkSetAbs, walkSet);
}
for (const abs of await listJsonFiles(join(destDir, 'runtime'))) await normalizeImportedJsonFile(characterId, abs);
const walkSetSha256 = walkSet && await pathExists(walkSetAbs) ? await sha256File(walkSetAbs) : null;
const pointerAbs = join(destDir, 'runtime', 'current.json');
const pointer = await readJson(pointerAbs);
if (!pointer) return;
const normalizedPointer = deepCanonicalizeImportedRunPaths(characterId, pointer);
if (walkSetSha256 && typeof normalizedPointer.walkSetSha256 === 'string') normalizedPointer.walkSetSha256 = walkSetSha256;
const manifestRel = toRecordRelativeAssetPath(characterId, normalizedPointer.manifestPath) || normalizedPointer.manifestPath;
if (typeof manifestRel === 'string' && await pathExists(join(destDir, manifestRel))) {
normalizedPointer.manifestSha256 = await sha256File(join(destDir, manifestRel));
}
if (JSON.stringify(normalizedPointer) !== JSON.stringify(pointer)) await atomicWrite(pointerAbs, normalizedPointer);
}
/**
* Import one APPROVED direction of a finalized walk set: the run manifest
* (hash-verified), then only the assets the run's manifests declare — packed
* strips, packaged manifests, previews, and the run's source clip. Works for
* both grok runs (record at
* run root, assets under generated/) and imagegen redraw runs (manifest +
* strips at the version root, surrounded by unselected candidates that must
* stay behind).
*/
async function importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun }) {
const manifestRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runManifest, characterId));
if (!manifestRel) {
result.errors.push(`walk set ${direction}: unsafe run path rejected: ${dir.runManifest || dir.runPath}`);
return;
}
const copiedManifestRel = await copyCharFile(sourceRoot, srcCharDir, destDir, manifestRel);
if (!copiedManifestRel) {
result.errors.push(`walk set ${direction}: run manifest missing: ${manifestRel}`);
return;
}
result.files += 1;
copiedThisRun.add(copiedManifestRel);
if (dir.runManifestSha256) hashExpectations.push({ relPath: copiedManifestRel, sha256: dir.runManifestSha256 });
const refs = new Set();
// Read the copy just made rather than the source: same bytes, one fewer
// resolution + read, and it matches how the reference/ and walk/ loops below
// expand their manifests.
collectManifestRefs(await readJson(join(destDir, copiedManifestRel)), characterId, refs, hashExpectations);
const runRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runPath, characterId));
if (runRel && (await charSourcePath(sourceRoot, srcCharDir, `${runRel}/generated/review-preview.json`))) {
refs.add(`${runRel}/generated/review-preview.json`);
}
// One extra expansion level: packaged manifests / previews the run record
// names in turn declare the strip files (and their hash pins).
for (const rel of [...refs]) {
if (rel.endsWith('.json')) collectManifestRefs(await readCharJson(sourceRoot, srcCharDir, rel), characterId, refs, hashExpectations);
}
// The source clip (#2984) — a fallback for a run whose manifests never name
// it, since a direction with no clip can never be re-derived at a different
// frame count. Deliberately AFTER the expansion above: the usual carrier of
// `sourceVideoPath` is the packaged manifest, one level down, so probing
// earlier would miss a clip already collected and copy the same bytes to two
// destinations (once per run layout). The RUN_DIR_MATCH gate keeps this to
// real run entries (an imagegen redraw entry has no run dir and no clip), and
// the twin check ignores a manifest that named the clip under the OTHER
// layout — that path is already in `refs` and denotes the same file.
const clipRel = runRel && `${runRel}/generated/${SOURCE_CLIP_NAME}`;
if (clipRel && RUN_DIR_MATCH.test(runRel) && !refs.has(clipRel) && !refs.has(altRunLayoutPath(clipRel))
&& (await charSourcePath(sourceRoot, srcCharDir, clipRel))) {
refs.add(clipRel);
}
refs.delete(manifestRel);
for (const rel of [...refs].sort()) {
if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {
result.files += 1;
copiedThisRun.add(rel);
} else {
// A manifest-declared asset the source no longer has is a real gap —
// an "imported" subject must not silently omit an approved artifact.
result.errors.push(`walk set ${direction}: referenced asset missing: ${rel}`);
}
}
}
async function importCharacter({ sourceRoot, characterId, spec, specPath, selection }) {
const result = { id: characterId, kind: 'character', files: 0, verified: 0, errors: [] };
const srcCharDir = join(sourceRoot, 'art-source', 'sprites', characterId);
const destDir = spriteDir(characterId);
// A destination with a LOCKED reference set (#2896) is immutable evidence
// — a same-id import would copy the source's reference tree straight over
// the frozen identity root, bypassing every lock check. Refuse the whole
// character; the user can delete the record or import under another id.
const destManifest = await readJSONFile(join(destDir, 'reference', `${characterId}-reference-set-v1.json`), null);
if (destManifest?.mainReference?.locked) {
result.errors.push('destination has a locked reference set — refusing to overwrite immutable artifacts (delete the record or import under a different id)');
return result;
}
await ensureDir(destDir);
// Character spec — verbatim copy for provenance alongside the record.
if (!await copySourceFile(sourceRoot, specPath, join(destDir, 'character-spec.json'))) {
result.errors.push('character spec missing from source');
return result;
}
result.files += 1;
const hashExpectations = [];
// Every file THIS run copied (dest-relative). Verification is scoped to
// this set so a stale destination copy can't vouch for a file the current
// source no longer provides.
const copiedThisRun = new Set(['character-spec.json']);
// reference/ — locked main + anchors; candidates/ (unapproved) stay behind.
if (await sourcePath(sourceRoot, join(srcCharDir, 'reference'))) {
const copied = await copyTree(
sourceRoot,
join(srcCharDir, 'reference'),
join(destDir, 'reference'),
(rel, isDir) => !(isDir && basename(rel) === 'candidates') && !rel.includes('candidates/'),
);
result.files += copied.length;
for (const rel of copied) copiedThisRun.add(`reference/${rel}`);
for (const rel of copied) {
if (!rel.endsWith('.json')) continue;
const manifest = await readJson(join(destDir, 'reference', rel));
for (const anchor of manifest?.anchors || []) {
const anchorRel = relToCharacterDir(anchor?.path, characterId);
if (anchorRel && anchor?.sha256) hashExpectations.push({ relPath: anchorRel, sha256: anchor.sha256 });
}
}
}
// walk/ — the manifests themselves are small provenance and copy whole,
// but ASSETS import only for APPROVED directions of a FINALIZED set. Draft
// selections / publication records / pending directions contribute nothing.
const walkDir = join(srcCharDir, 'walk');
if (await sourcePath(sourceRoot, walkDir)) {
const copied = await copyTree(sourceRoot, walkDir, join(destDir, 'walk'));
result.files += copied.length;
for (const rel of copied) copiedThisRun.add(`walk/${rel}`);
for (const rel of copied) {
if (!rel.endsWith('.json')) continue;
const walkSet = await readJson(join(destDir, 'walk', rel));
if (walkSet?.kind !== 'finalized-eight-direction-walk-set' || walkSet?.status !== 'final') continue;
// The set's own top-level references (selectionPath/selectionSha256 …)
// are hash-pinned provenance too. Harvest them from the metadata ONLY —
// never from `directions`, or non-approved directions' run files would
// ride in through the generic collector.
const { directions, ...walkSetMeta } = walkSet;
const metaRefs = new Set();
collectManifestRefs(walkSetMeta, characterId, metaRefs, hashExpectations);
for (const metaRel of [...metaRefs].sort()) {
if (copiedThisRun.has(metaRel)) continue;
if (await copyCharFile(sourceRoot, srcCharDir, destDir, metaRel)) {
result.files += 1;
copiedThisRun.add(metaRel);
} else result.errors.push(`walk set: referenced file missing: ${metaRel}`);
}
for (const [direction, dir] of Object.entries(directions || {})) {
if (dir?.status !== 'approved') {
result.errors.push(`walk set ${direction}: not approved (${dir?.status ?? 'missing'}) — skipped`);
continue;
}
await importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun });
}
}
}
// runtime/ — immutable published atlas archives. (No wholesale imagegen/
// copy: approved redraw outputs arrive via the walk-set expansion above;
// the rest of imagegen/ is unselected candidates and raw intermediates.)
if (await sourcePath(sourceRoot, join(srcCharDir, 'runtime'))) {
const copied = await copyTree(sourceRoot, join(srcCharDir, 'runtime'), join(destDir, 'runtime'));
result.files += copied.length;
for (const rel of copied) copiedThisRun.add(`runtime/${rel}`);
}
// Published-selection provenance: copy and hash-verify every character-dir
// artifact the catalog's selection declares (immutable atlas archive, keyed
// source, publication manifest, …) via the generic collector — pins ride in
// through the {path,sha256} / <base>Path+<base>Sha256 conventions, and
// repo-anchored references outside the character dir are ignored.
if (selection?.characterId === characterId && selection.selected) {
const selRefs = new Set();
collectManifestRefs(selection.selected, characterId, selRefs, hashExpectations);
for (const rel of [...selRefs].sort()) {
if (copiedThisRun.has(rel)) continue;
if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {
result.files += 1;
copiedThisRun.add(rel);
} else result.errors.push(`published selection: referenced asset missing: ${rel}`);
}
}
const { verified, dropped } = await verifyHashes(destDir, hashExpectations, result.errors, copiedThisRun);
result.verified = verified;
result.files -= dropped;
await normalizeImportedRunLayout(destDir, characterId);
const record = await upsertImportedRecord(characterId, {
kind: 'character',
name: spec?.displayName || characterId,
status: 'imported',
spec,
chromaKey: LEGACY_CHROMA_KEY,
importedFrom: { sourceRoot, importedAt: new Date().toISOString() },
});
result.record = record;
return result;
}
async function importPropsFamily({ sourceRoot, familyId, srcFamilyDir }) {
const result = { id: familyId, kind: 'props', files: 0, verified: 0, errors: [] };
const destDir = join(spriteDir(familyId), 'atlas');
const copied = await copyTree(sourceRoot, srcFamilyDir, destDir, (rel, isDir) => {
if (isDir) return true;
const lower = rel.toLowerCase();
return PROP_FILE_EXTENSIONS.some((ext) => lower.endsWith(ext));
});
result.files += copied.length;
const record = await upsertImportedRecord(familyId, {
kind: 'props',
name: familyId,
status: 'imported',
importedFrom: { sourceRoot, importedAt: new Date().toISOString() },
});
result.record = record;
return result;
}
/**
* Import approved sprite assets from `sourceRoot`. Each subject (a character or a
* props family) holds one backup admission lease from its first copied byte
* through the record row that marks it imported (#9982): a re-import copies over
* files an existing row already describes, and a snapshot taken halfway would
* pair them with a row that says the import finished. A multi-subject import
* releases the lease between subjects so a cut waits for one subject, not all.
* Options:
* - characters: limit to these character ids (default: all specs found)
* - includeProps: also import game/assets/sprites/ prop families (default true)
*
* Returns { results: [...perSubject], totals: { subjects, files, verified, errors } }.
*/
export async function importFromSource({ sourceRoot, characters, includeProps = true }) {
// The UI placeholder suggests a ~/… path; resolve it before probing, or a
// correct tilde path 400s with a misleading "not a sprite pipeline".
sourceRoot = await realpath(expandHome(sourceRoot)).catch((err) => {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;
throw err;
});
const specsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'art-pipeline', 'characters'));
const propsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'game', 'assets', 'sprites'));
if (!specsDir && !propsDir) {
throw new ServerError(
'Source root does not look like a sprite pipeline (expected art-pipeline/characters/ or game/assets/sprites/)',
{ status: 400, code: 'INVALID_SOURCE_ROOT' },
);
}
await ensureDir(PATHS.sprites);
// Read the published-selection pointer up front so the matching character's
// import can verify the hash-pinned runtime artifacts it declares.
const selection = await readSourceJson(sourceRoot, join(sourceRoot, 'art-pipeline', 'catalog', 'runtime-selection.json'));
const results = [];
// Ids imported THIS run (drives the runtime-selection provenance copy) vs
// every character id the specs dir declares (drives the props-loop skip). A
// filtered import must still exclude ALL characters' game dirs from the
// props walk — otherwise `characters: ['a']` treats character b's published
// atlas dir as a props family and overwrites b's record with kind:'props'.
const characterIds = new Set();
const allSpecCharacterIds = new Set();
if (specsDir) {
const wanted = Array.isArray(characters) && characters.length ? new Set(characters) : null;
for (const entry of (await readdir(specsDir)).sort()) {
if (!entry.endsWith('.json')) continue;
const specPath = join(specsDir, entry);
const spec = await readSourceJson(sourceRoot, specPath);
// A real character spec carries characterId; sibling files in the specs
// dir (JSON-schema definitions, templates) don't and are not subjects.
if (!spec?.characterId) continue;
const characterId = spec.characterId;
if (!isValidSpriteId(characterId)) {
results.push({ id: characterId, kind: 'character', files: 0, verified: 0, errors: ['invalid character id'] });
continue;
}
allSpecCharacterIds.add(characterId);
if (wanted && !wanted.has(characterId)) continue;
characterIds.add(characterId);
results.push(await withBackupAssetPublication(
() => importCharacter({ sourceRoot, characterId, spec, specPath, selection }),
));
}
}
if (propsDir && includeProps) {
for (const entry of (await readdir(propsDir, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
if (!entry.isDirectory()) continue;
// A character's published atlas dir duplicates what runtime/ archives —
// only non-character families import from the game tree. Excludes every
// spec-declared character, imported this run or not.
if (allSpecCharacterIds.has(entry.name)) continue;
if (!isValidSpriteId(entry.name)) continue;
results.push(await withBackupAssetPublication(
() => importPropsFamily({ sourceRoot, familyId: entry.name, srcFamilyDir: join(propsDir, entry.name) }),
));
}
}
// Published-atlas pointer — provenance for the current selection, kept with
// the character it points at.
if (selection?.characterId && characterIds.has(selection.characterId)) {
const destDir = join(spriteDir(selection.characterId), 'catalog');
await ensureDir(destDir);
await atomicWrite(join(destDir, 'runtime-selection.json'), selection);
}
const totals = results.reduce(
(acc, r) => ({
subjects: acc.subjects + 1,
files: acc.files + r.files,
verified: acc.verified + r.verified,
errors: acc.errors + r.errors.length,
}),
{ subjects: 0, files: 0, verified: 0, errors: 0 },
);
console.log(`🎞️ Sprite import: ${totals.subjects} subjects, ${totals.files} files, ${totals.verified} hash-verified, ${totals.errors} errors`);
return { results, totals };
}