Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: confine sprite imports to the selected source root
atomantic5cb7fe4a7be4
diff --git a/server/services/sprites/importer.js b/server/services/sprites/importer.jsindex 1682ad5d3c..925d06b87e 100644--- a/server/services/sprites/importer.js+++ b/server/services/sprites/importer.js@@ -23,8 +23,9 @@*/import { join, basename, dirname } from 'path';-import { readdir } from 'fs/promises';+import { readdir, realpath } from 'fs/promises';import { PATHS, ensureDir, sha256File, atomicWrite, pathExists, readJSONFile, expandHome, copyFileGuarded, rmGuarded } from '../../lib/fileUtils.js';+import { isWithinRoot } from '../../lib/workspaceRoots.js';import { ServerError } from '../../lib/errorHandler.js';import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';import { upsertImportedRecord } from './records.js';@@ -35,7 +36,6 @@ import {canonicalizeImportedRunPath,canonicalizeImportedRunPathValue,deepCanonicalizeImportedRunPaths,- resolveDriftTolerantRel,SOURCE_CLIP_NAME,toRecordRelativeAssetPath,} from './paths.js';@@ -51,17 +51,57 @@ const PROP_FILE_EXTENSIONS = ['.png', '.md'];// as "nothing to import from this manifest."const readJson = (p) => readJSONFile(p, null, { logError: false });+// The selected repository is the read boundary, including metadata and assets+// reached through manifests. Check at use time and return the canonical path:+// later copies must not follow the original (possibly redirected) symlink.+// Missing optional artifacts retain their existing null/skip behavior.+async function sourcePath(sourceRoot, candidate) {+ const resolved = await realpath(candidate).catch((err) => {+ if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;+ throw err;+ });+ // Both paths are realpath results. Do not fold case based on the OS: a+ // case-sensitive volume may have a distinct sibling with the same spelling.+ if (resolved && !isWithinRoot(resolved, sourceRoot)) {+ throw new ServerError('Sprite import source path resolves outside the selected source root', {+ status: 400, code: 'SOURCE_OUTSIDE_ROOT',+ });+ }+ return resolved;+}++async function readSourceJson(sourceRoot, candidate) {+ const checked = await sourcePath(sourceRoot, candidate);+ return checked ? readJson(checked) : null;+}++async function copySourceFile(sourceRoot, candidate, dest) {+ const checked = await sourcePath(sourceRoot, candidate);+ if (!checked) return false;+ await copyFileGuarded(checked, dest);+ return true;+}++async function charSourcePath(sourceRoot, srcCharDir, rel) {+ const direct = await sourcePath(sourceRoot, join(srcCharDir, rel));+ if (direct) return direct;+ const alt = altRunLayoutPath(rel);+ return alt ? sourcePath(sourceRoot, join(srcCharDir, alt)) : null;+}+/*** Recursively copy `srcDir` into `destDir`, filtering by `shouldCopy(relPath,* isDir)`. Returns the list of copied file paths relative to destDir.*/-async function copyTree(srcDir, destDir, shouldCopy = () => true) {+async function copyTree(sourceRoot, srcDir, destDir, shouldCopy = () => true) {const copied = [];const ensuredDirs = new Set();async function walk(current, rel) {+ const checked = await sourcePath(sourceRoot, current);+ if (!checked) return;let entries;try {- entries = await readdir(current, { withFileTypes: true });+ entries = await readdir(checked, { withFileTypes: true });} catch {return;}@@ -79,8 +119,7 @@ async function copyTree(srcDir, destDir, shouldCopy = () => true) {await ensureDir(parent);ensuredDirs.add(parent);}- await copyFileGuarded(srcPath, destPath);- copied.push(entryRel);+ if (await copySourceFile(sourceRoot, srcPath, destPath)) copied.push(entryRel);}}}@@ -221,25 +260,24 @@ function collectManifestRefs(node, characterId, refs, expectations) {// `grok/<run-id>/`. Read-then-fall-back rather than probe-then-read: readJson// already collapses a missing/corrupt file to null, so the twin costs a// syscall only when the declared path really isn't there.-async function readCharJson(srcCharDir, rel) {- const direct = await readJson(join(srcCharDir, rel));+async function readCharJson(sourceRoot, srcCharDir, rel) {+ const direct = await readSourceJson(sourceRoot, join(srcCharDir, rel));if (direct) return direct;const alt = altRunLayoutPath(rel);- return alt ? readJson(join(srcCharDir, alt)) : null;+ return alt ? readSourceJson(sourceRoot, join(srcCharDir, alt)) : null;}// Copy one character-dir file into PortOS's neutral imported-run layout. The// source lookup remains drift-tolerant, while every local write maps a legacy// `grok/` run prefix to `runs/` so a re-import cannot recreate the residue// migration 203 healed on earlier installs.-async function copyCharFile(srcCharDir, destDir, rel) {- const srcRel = await resolveDriftTolerantRel(srcCharDir, rel);- if (!srcRel) return null;+async function copyCharFile(sourceRoot, srcCharDir, destDir, rel) {+ const src = await charSourcePath(sourceRoot, srcCharDir, rel);+ if (!src) return null;const destRel = canonicalizeImportedRunPath(rel);const dest = join(destDir, destRel);await ensureDir(dirname(dest));- await copyFileGuarded(join(srcCharDir, srcRel), dest);- return destRel;+ return await copySourceFile(sourceRoot, src, dest) ? destRel : null;}// Recursively collect JSON manifests in one imported tree. A missing optional@@ -333,13 +371,13 @@ async function normalizeImportedRunLayout(destDir, characterId) {* strips at the version root, surrounded by unselected candidates that must* stay behind).*/-async function importApprovedDirection({ srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun }) {+async function importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun }) {const manifestRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runManifest, characterId));if (!manifestRel) {result.errors.push(`walk set ${direction}: unsafe run path rejected: ${dir.runManifest || dir.runPath}`);return;}- const copiedManifestRel = await copyCharFile(srcCharDir, destDir, manifestRel);+ const copiedManifestRel = await copyCharFile(sourceRoot, srcCharDir, destDir, manifestRel);if (!copiedManifestRel) {result.errors.push(`walk set ${direction}: run manifest missing: ${manifestRel}`);return;@@ -354,13 +392,13 @@ async function importApprovedDirection({ srcCharDir, destDir, characterId, direc// expand their manifests.collectManifestRefs(await readJson(join(destDir, copiedManifestRel)), characterId, refs, hashExpectations);const runRel = canonicalizeImportedRunPath(relToCharacterDir(dir.runPath, characterId));- if (runRel && (await resolveDriftTolerantRel(srcCharDir, `${runRel}/generated/review-preview.json`))) {+ if (runRel && (await charSourcePath(sourceRoot, srcCharDir, `${runRel}/generated/review-preview.json`))) {refs.add(`${runRel}/generated/review-preview.json`);}// One extra expansion level: packaged manifests / previews the run record// names in turn declare the strip files (and their hash pins).for (const rel of [...refs]) {- if (rel.endsWith('.json')) collectManifestRefs(await readCharJson(srcCharDir, rel), characterId, refs, hashExpectations);+ if (rel.endsWith('.json')) collectManifestRefs(await readCharJson(sourceRoot, srcCharDir, rel), characterId, refs, hashExpectations);}// The source clip (#2984) — a fallback for a run whose manifests never name// it, since a direction with no clip can never be re-derived at a different@@ -373,12 +411,12 @@ async function importApprovedDirection({ srcCharDir, destDir, characterId, direc// layout — that path is already in `refs` and denotes the same file.const clipRel = runRel && `${runRel}/generated/${SOURCE_CLIP_NAME}`;if (clipRel && RUN_DIR_MATCH.test(runRel) && !refs.has(clipRel) && !refs.has(altRunLayoutPath(clipRel))- && (await resolveDriftTolerantRel(srcCharDir, clipRel))) {+ && (await charSourcePath(sourceRoot, srcCharDir, clipRel))) {refs.add(clipRel);}refs.delete(manifestRel);for (const rel of [...refs].sort()) {- if (await copyCharFile(srcCharDir, destDir, rel)) {+ if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {result.files += 1;copiedThisRun.add(rel);} else {@@ -406,7 +444,10 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectawait ensureDir(destDir);// Character spec — verbatim copy for provenance alongside the record.- await copyFileGuarded(specPath, join(destDir, 'character-spec.json'));+ if (!await copySourceFile(sourceRoot, specPath, join(destDir, 'character-spec.json'))) {+ result.errors.push('character spec missing from source');+ return result;+ }result.files += 1;const hashExpectations = [];@@ -416,8 +457,9 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectconst copiedThisRun = new Set(['character-spec.json']);// reference/ — locked main + anchors; candidates/ (unapproved) stay behind.- if (await pathExists(join(srcCharDir, 'reference'))) {+ if (await sourcePath(sourceRoot, join(srcCharDir, 'reference'))) {const copied = await copyTree(+ sourceRoot,join(srcCharDir, 'reference'),join(destDir, 'reference'),(rel, isDir) => !(isDir && basename(rel) === 'candidates') && !rel.includes('candidates/'),@@ -438,8 +480,8 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, select// but ASSETS import only for APPROVED directions of a FINALIZED set. Draft// selections / publication records / pending directions contribute nothing.const walkDir = join(srcCharDir, 'walk');- if (await pathExists(walkDir)) {- const copied = await copyTree(walkDir, join(destDir, 'walk'));+ if (await sourcePath(sourceRoot, walkDir)) {+ const copied = await copyTree(sourceRoot, walkDir, join(destDir, 'walk'));result.files += copied.length;for (const rel of copied) copiedThisRun.add(`walk/${rel}`);for (const rel of copied) {@@ -455,7 +497,7 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectcollectManifestRefs(walkSetMeta, characterId, metaRefs, hashExpectations);for (const metaRel of [...metaRefs].sort()) {if (copiedThisRun.has(metaRel)) continue;- if (await copyCharFile(srcCharDir, destDir, metaRel)) {+ if (await copyCharFile(sourceRoot, srcCharDir, destDir, metaRel)) {result.files += 1;copiedThisRun.add(metaRel);} else result.errors.push(`walk set: referenced file missing: ${metaRel}`);@@ -465,7 +507,7 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectresult.errors.push(`walk set ${direction}: not approved (${dir?.status ?? 'missing'}) — skipped`);continue;}- await importApprovedDirection({ srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun });+ await importApprovedDirection({ sourceRoot, srcCharDir, destDir, characterId, direction, dir, result, hashExpectations, copiedThisRun });}}}@@ -473,8 +515,8 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, select// runtime/ — immutable published atlas archives. (No wholesale imagegen/// copy: approved redraw outputs arrive via the walk-set expansion above;// the rest of imagegen/ is unselected candidates and raw intermediates.)- if (await pathExists(join(srcCharDir, 'runtime'))) {- const copied = await copyTree(join(srcCharDir, 'runtime'), join(destDir, 'runtime'));+ if (await sourcePath(sourceRoot, join(srcCharDir, 'runtime'))) {+ const copied = await copyTree(sourceRoot, join(srcCharDir, 'runtime'), join(destDir, 'runtime'));result.files += copied.length;for (const rel of copied) copiedThisRun.add(`runtime/${rel}`);}@@ -489,7 +531,7 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectcollectManifestRefs(selection.selected, characterId, selRefs, hashExpectations);for (const rel of [...selRefs].sort()) {if (copiedThisRun.has(rel)) continue;- if (await copyCharFile(srcCharDir, destDir, rel)) {+ if (await copyCharFile(sourceRoot, srcCharDir, destDir, rel)) {result.files += 1;copiedThisRun.add(rel);} else result.errors.push(`published selection: referenced asset missing: ${rel}`);@@ -516,7 +558,7 @@ async function importCharacter({ sourceRoot, characterId, spec, specPath, selectasync function importPropsFamily({ sourceRoot, familyId, srcFamilyDir }) {const result = { id: familyId, kind: 'props', files: 0, verified: 0, errors: [] };const destDir = join(spriteDir(familyId), 'atlas');- const copied = await copyTree(srcFamilyDir, destDir, (rel, isDir) => {+ const copied = await copyTree(sourceRoot, srcFamilyDir, destDir, (rel, isDir) => {if (isDir) return true;const lower = rel.toLowerCase();return PROP_FILE_EXTENSIONS.some((ext) => lower.endsWith(ext));@@ -548,12 +590,13 @@ async function importPropsFamily({ sourceRoot, familyId, srcFamilyDir }) {export async function importFromSource({ sourceRoot, characters, includeProps = true }) {// The UI placeholder suggests a ~/… path; resolve it before probing, or a// correct tilde path 400s with a misleading "not a sprite pipeline".- sourceRoot = expandHome(sourceRoot);- const specsDir = join(sourceRoot, 'art-pipeline', 'characters');- const propsDir = join(sourceRoot, 'game', 'assets', 'sprites');- const hasSpecs = await pathExists(specsDir);- const hasProps = await pathExists(propsDir);- if (!hasSpecs && !hasProps) {+ sourceRoot = await realpath(expandHome(sourceRoot)).catch((err) => {+ if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return null;+ throw err;+ });+ const specsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'art-pipeline', 'characters'));+ const propsDir = sourceRoot && await sourcePath(sourceRoot, join(sourceRoot, 'game', 'assets', 'sprites'));+ if (!specsDir && !propsDir) {throw new ServerError('Source root does not look like a sprite pipeline (expected art-pipeline/characters/ or game/assets/sprites/)',{ status: 400, code: 'INVALID_SOURCE_ROOT' },@@ -563,7 +606,7 @@ export async function importFromSource({ sourceRoot, characters, includeProps =await ensureDir(PATHS.sprites);// Read the published-selection pointer up front so the matching character's// import can verify the hash-pinned runtime artifacts it declares.- const selection = await readJson(join(sourceRoot, 'art-pipeline', 'catalog', 'runtime-selection.json'));+ const selection = await readSourceJson(sourceRoot, join(sourceRoot, 'art-pipeline', 'catalog', 'runtime-selection.json'));const results = [];// Ids imported THIS run (drives the runtime-selection provenance copy) vs// every character id the specs dir declares (drives the props-loop skip). A@@ -573,12 +616,12 @@ export async function importFromSource({ sourceRoot, characters, includeProps =const characterIds = new Set();const allSpecCharacterIds = new Set();- if (hasSpecs) {+ if (specsDir) {const wanted = Array.isArray(characters) && characters.length ? new Set(characters) : null;for (const entry of (await readdir(specsDir)).sort()) {if (!entry.endsWith('.json')) continue;const specPath = join(specsDir, entry);- const spec = await readJson(specPath);+ const spec = await readSourceJson(sourceRoot, specPath);// A real character spec carries characterId; sibling files in the specs// dir (JSON-schema definitions, templates) don't and are not subjects.if (!spec?.characterId) continue;@@ -596,7 +639,7 @@ export async function importFromSource({ sourceRoot, characters, includeProps =}}- if (hasProps && includeProps) {+ if (propsDir && includeProps) {for (const entry of (await readdir(propsDir, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {if (!entry.isDirectory()) continue;// A character's published atlas dir duplicates what runtime/ archives —diff --git a/server/services/sprites/importerContainment.test.js b/server/services/sprites/importerContainment.test.jsnew file mode 100644index 0000000000..66d83333bb--- /dev/null+++ b/server/services/sprites/importerContainment.test.js@@ -0,0 +1,129 @@+import { afterAll, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'fs';+import { dirname, join } from 'path';+import { createTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';+import { request } from '../../lib/testHelper.js';++const fixtureRoot = createTempDataRoot('sprite-import-containment-');+const dataRoot = join(fixtureRoot, 'data');+vi.mock('../../lib/fileUtils.js', async (importOriginal) => (+ makePathsProxy(await importOriginal(), { dataRoot })+));++const { importFromSource } = await import('./importer.js');+const { getRecord } = await import('./records.js');+const { mountAssetRoutes } = await import('../assetMounts.js');+const app = express();+mountAssetRoutes(app);+afterAll(() => rmSync(fixtureRoot, { recursive: true, force: true }));++const CANARY = 'INVENTED-OUTSIDE-SOURCE-CANARY';+function write(path, value) {+ mkdirSync(dirname(path), { recursive: true });+ writeFileSync(path, typeof value === 'string' ? value : JSON.stringify(value));+}++function link(target, path, type = 'file') {+ mkdirSync(dirname(path), { recursive: true });+ symlinkSync(target, path, type);+}++function fixture(id) {+ const sourceRoot = join(fixtureRoot, id, 'source');+ const outside = join(fixtureRoot, id, 'outside');+ const charDir = join(sourceRoot, 'art-source/sprites', id);+ const specPath = join(sourceRoot, 'art-pipeline/characters', `${id}.json`);+ const selectionPath = join(sourceRoot, 'art-pipeline/catalog/runtime-selection.json');+ write(specPath, { characterId: id, displayName: 'Example character' });+ write(join(outside, 'canary.png'), CANARY);+ return { id, sourceRoot, outside, charDir, specPath, selectionPath };+}++describe('sprite import source containment at persistence and asset-serving boundaries', () => {+ it.each([+ ['reference-directory', (f) => {+ link(f.outside, join(f.charDir, 'reference'), 'junction');+ return 'reference/canary.png';+ }],+ ['selected-asset', (f) => {+ link(join(f.outside, 'canary.png'), join(f.charDir, 'selected.png'));+ write(f.selectionPath, { characterId: f.id, selected: { path: 'selected.png' } });+ return 'selected.png';+ }],+ ['expanded-run-manifest', (f) => {+ write(join(f.charDir, 'walk/set.json'), {+ kind: 'finalized-eight-direction-walk-set', status: 'final',+ directions: { south: { status: 'approved', runPath: 'runs/run-1', runManifest: 'runs/run-1/run.json' } },+ });+ // The walk set names runs/, while the source has the legacy grok/ twin.+ write(join(f.charDir, 'grok/run-1/run.json'), { manifestPath: 'runs/run-1/packaged.json' });+ write(join(f.outside, 'packaged.json'), { secret: CANARY, asset: 'selected.png' });+ link(join(f.outside, 'packaged.json'), join(f.charDir, 'grok/run-1/packaged.json'));+ return 'runs/run-1/packaged.json';+ }],+ ['character-spec', (f) => {+ rmSync(f.specPath);+ write(join(f.outside, 'spec.json'), { characterId: f.id, displayName: CANARY });+ link(join(f.outside, 'spec.json'), f.specPath);+ return 'character-spec.json';+ }],+ ['specs-directory', (f) => {+ rmSync(dirname(f.specPath), { recursive: true });+ write(join(f.outside, `${f.id}.json`), { characterId: f.id, displayName: CANARY });+ link(f.outside, dirname(f.specPath), 'junction');+ return 'character-spec.json';+ }],+ ['selection-metadata', (f) => {+ write(join(f.outside, 'selection.json'), { characterId: f.id, selected: {}, secret: CANARY });+ link(join(f.outside, 'selection.json'), f.selectionPath);+ return 'catalog/runtime-selection.json';+ }],+ ['props-directory', (f) => {+ write(join(f.outside, f.id, 'atlas.png'), CANARY);+ rmSync(dirname(f.specPath), { recursive: true });+ link(f.outside, join(f.sourceRoot, 'game/assets/sprites'), 'junction');+ return 'atlas/atlas.png';+ }],+ ])('rejects an escaping %s before persisting or serving outside bytes', async (id, prepare) => {+ const f = fixture(id);+ const asset = prepare(f);+ await expect(importFromSource({ sourceRoot: f.sourceRoot })).rejects.toMatchObject({ code: 'SOURCE_OUTSIDE_ROOT', status: 400 });+ expect(await getRecord(id)).toBeNull();+ const response = await request(app).get(`/data/sprites/${id}/${asset}`);+ expect(response.status).toBe(404);+ expect(response.text).not.toContain(CANARY);+ });++ it('rejects a distinct sibling whose name differs only by case', async (context) => {+ const f = fixture('case-sensitive');+ const sibling = join(dirname(f.sourceRoot), 'SOURCE');+ if (existsSync(sibling)) return context.skip('The fixture volume is case-insensitive; there is no distinct sibling.');+ write(join(sibling, 'canary.png'), CANARY);+ link(sibling, join(f.charDir, 'reference'), 'junction');+ await expect(importFromSource({ sourceRoot: f.sourceRoot })).rejects.toMatchObject({ code: 'SOURCE_OUTSIDE_ROOT' });+ expect(await getRecord(f.id)).toBeNull();+ const response = await request(app).get('/data/sprites/case-sensitive/reference/canary.png');+ expect(response.status).toBe(404);+ });++ it('preserves normal imports, a selected root alias, and contained selected-file links', async () => {+ const f = fixture('contained');+ write(join(f.charDir, 'reference/main.png'), 'APPROVED-REFERENCE');+ write(join(f.sourceRoot, 'shared/selected.png'), 'APPROVED-SELECTED');+ link(join(f.sourceRoot, 'shared/selected.png'), join(f.charDir, 'selected.png'));+ write(f.selectionPath, { characterId: f.id, selected: { path: 'selected.png' } });+ write(join(f.sourceRoot, 'game/assets/sprites/props/atlas.png'), 'APPROVED-PROPS');+ const alias = join(fixtureRoot, 'selected-root-alias');+ link(f.sourceRoot, alias, 'junction');++ const { totals } = await importFromSource({ sourceRoot: alias });+ expect(totals).toMatchObject({ subjects: 2, errors: 0 });+ expect(await getRecord(f.id)).toMatchObject({ kind: 'character', name: 'Example character' });+ expect(await getRecord('props')).toMatchObject({ kind: 'props' });+ expect(readFileSync(join(dataRoot, 'sprites/contained/selected.png'), 'utf8')).toBe('APPROVED-SELECTED');+ const response = await request(app).get('/data/sprites/contained/selected.png');+ expect(response.status).toBe(200);+ expect(response.text).toBe('APPROVED-SELECTED');+ });+});