Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep synced Digital Twin documents within their root (#11277)
Adam Eivya5e1a8dd491d
diff --git a/server/lib/README.md b/server/lib/README.mdindex f2e58df188..49f8dee11d 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -126,7 +126,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `creativeCommissionSpec.js` | `GENERATION_KEY_DEFS` / `ABILITY_GENERATION_SPEC` / `resolvedDefault` and the quality / aspect-ratio / image-mode / video-mode enums — the generation-param spec `creativeCommissionValidation.js` builds its Zod schema from and the ability adapters sanitize against. A pure leaf (like `creativeBriefLimits.js`) so `client/src/components/creative-commission/commissionForm.js` derives its fields/bounds/defaults from it directly instead of hand-copying them (#6816). || `creativeCommissionValidation.js` | Creative Commission (Autonomous Creation Engine) create/update + brief/schedule/generation schemas. The brief field caps it enforces come from `creativeBriefLimits.js`; the generation-param spec comes from `creativeCommissionSpec.js`. || `creativeDirectorValidation.js` | Creative Director project/treatment/scene, inert Video draft preferences, and Create-Suite importer schemas. The directive `goal` cap it enforces comes from `creativeBriefLimits.js`. |-| `digitalTwinValidation.js` | Digital twin document/category schemas. |+| `digitalTwinValidation.js` | Digital twin document/category schemas, including `documentFilenameSchema` for safe Markdown basenames across imports, federation and stored document access. || `eidoverseValidation.js` | Eidoverse world-projection schemas — the `EIDOVERSE_PROJECTION_SOURCE_KEYS` allowlist, the V1/V2 projection recipe union (includes/assets/terrain/scale/districts/environment, with the `..`-escape guard on every asset path), and the world augment/say/config-patch route bodies (incl. the 8KB augment-argument cap). Split out of `validation.js` (#5698), which re-exports it. || `filmStyleGrammarValidation.js` | Zod contract for a film style grammar record (`filmStyleGrammarSchema`), its field caps (`FILM_STYLE_LIMITS`), categories, renderable `FILM_STYLE_PARTS`, the `parts` selector, and the `/api/film-styles/:id` params and `?parts=` prompt-preview query schemas. || `fableLoomValidation.js` | FableLoom branching-narrative route schemas (loom/episode/node/transition CRUD, weave/branch/review, play turns). |diff --git a/server/lib/digitalTwinValidation.js b/server/lib/digitalTwinValidation.jsindex 064bc04db9..37f96d0cf8 100644--- a/server/lib/digitalTwinValidation.js+++ b/server/lib/digitalTwinValidation.js@@ -7,6 +7,7 @@ import {EMOJI_USAGE_VALUES} from './personaTraitBlend.js';import { partialWithoutDefaults } from './zodCompat.js';+import { isSafeFilename } from './pathSafety.js';// Document category enumexport const documentCategoryEnum = z.enum([@@ -44,6 +45,13 @@ export const enrichmentCategoryEnum = z.enum(['personality_assessments']);+// Imports and service callers may use existing peer document names (including+// spaces and uppercase extensions), but every document stays a Markdown basename.+export const documentFilenameSchema = z.string().min(1).refine(+ name => isSafeFilename(name, ['.md']) && !name.startsWith('.'),+ 'Filename must be a safe markdown basename'+);+// Document metadata schemaexport const documentMetaSchema = z.object({id: z.string().min(1),diff --git a/server/services/digital-twin-document-safety.test.js b/server/services/digital-twin-document-safety.test.jsnew file mode 100644index 0000000000..e62096583b--- /dev/null+++ b/server/services/digital-twin-document-safety.test.js@@ -0,0 +1,78 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';+import { join } from 'node:path';+import { tmpdir } from 'node:os';++const calls = vi.hoisted(() => []);+const parent = await mkdtemp(join(tmpdir(), 'digital-twin-safety-'));+const root = join(parent, 'digital-twin');+const outside = join(parent, 'outside.md');+vi.mock('../lib/fileUtils.js', async original => {+ const actual = await original();+ return { ...actual, PATHS: { ...actual.PATHS, digitalTwin: root } };+});+vi.mock('fs/promises', async original => {+ const actual = await original();+ return { ...actual, ...Object.fromEntries(['readFile', 'stat', 'unlink'].map(method => [method, (...args) => {+ calls.push({ method, path: args[0] });+ return actual[method](...args);+ }])) };+});+vi.mock('../lib/databaseMaintenanceJournal.js', () => ({ assertDatabaseAdmission() {} }));+const { applyDigitalTwinRemote, mergeMeta } = await import('./digital-twin-sync.js');+const { getDocuments, getDocumentById, deleteDocument, updateDocument } = await import('./digital-twin-documents.js');+const { DEFAULT_META, cache } = await import('./digital-twin-meta.js');+const doc = (filename, id = filename) => ({ id, filename, title: 'Synthetic', category: 'core', enabled: true,+ priority: 50, weight: 5, createdAt: '2026-01-01T00:00:00.000Z' });+const unsafeNames = () => ['../outside.md', outside, '..\\outside.md', 'C:\\outside.md', 'nested/doc.md',+ 'identity.json', '.hidden.md', 'bad\0.md', '', null, 42, {}];+async function seed(documents) {+ await writeFile(join(root, 'meta.json'), JSON.stringify({ ...structuredClone(DEFAULT_META), documents }));+ cache.meta.data = null; cache.meta.timestamp = 0;+}+beforeEach(async () => {+ await rm(root, { recursive: true, force: true }); await mkdir(root);+ await writeFile(outside, 'Synthetic outside sentinel');+ await seed([]); calls.length = 0;+});+afterAll(async () => { await rm(parent, { recursive: true, force: true }); });++describe('Digital Twin document filename boundary', () => {+ it('rejects unsafe peer metadata, bytes and tombstones while retaining valid Markdown names', async () => {+ const valid = doc('My Doc.MD');+ const invalid = unsafeNames().map((filename, i) => doc(filename, `unsafe-${i}`));+ const remote = { meta: { documents: [...invalid, valid], deletedDocuments: invalid.map(({ filename }) =>+ ({ filename, deletedAt: '2025-01-01T00:00:00.000Z' })) },+ documents: { '../outside.md': 'REPLACEMENT', '..\\outside.md': 'REPLACEMENT', 'My Doc.MD': '# Valid peer body' } };+ // Missing local metadata must not bypass the ingress policy either.+ expect(mergeMeta(null, remote.meta).merged.documents).toEqual([valid]);+ await applyDigitalTwinRemote(remote);+ const saved = JSON.parse(await readFile(join(root, 'meta.json'), 'utf8'));+ expect(saved.documents).toEqual([valid]);+ expect(saved.deletedDocuments).toEqual([]);+ expect(await getDocumentById(valid.id)).toMatchObject({ content: '# Valid peer body' });+ expect((await getDocuments()).map(doc => doc.filename)).toEqual(['My Doc.MD']);+ expect(calls.filter(call => call.path === outside)).toEqual([]);+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');+ });++ it('ignores malformed legacy rows before read/stat/update/delete and cleans them on the next peer merge', async () => {+ const valid = doc('Existing.MD');+ const invalid = unsafeNames().map((filename, i) => doc(filename, `legacy-${i}`));+ await writeFile(join(root, valid.filename), '# Existing body');+ await seed([null, ...invalid, valid]);+ expect((await getDocuments()).map(doc => doc.id)).toEqual([valid.id]);+ for (const entry of invalid) {+ expect(await getDocumentById(entry.id)).toBeNull();+ expect(await updateDocument(entry.id, { content: 'REPLACEMENT' })).toBeNull();+ expect(await deleteDocument(entry.id)).toBe(false);+ }+ expect(calls.filter(call => call.path === outside)).toEqual([]);+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');+ await applyDigitalTwinRemote({ meta: { documents: [] } });+ expect(JSON.parse(await readFile(join(root, 'meta.json'), 'utf8')).documents).toEqual([valid]);+ expect(await deleteDocument(valid.id)).toBe(true);+ expect(await getDocuments()).toEqual([]);+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');+ });+});diff --git a/server/services/digital-twin-documents.js b/server/services/digital-twin-documents.jsindex 91ef3c2cce..54b7629f1f 100644--- a/server/services/digital-twin-documents.js+++ b/server/services/digital-twin-documents.js@@ -4,13 +4,19 @@ import { join } from 'path';import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';import { atomicWrite } from '../lib/fileUtils.js';+import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';import { extractVersion } from './digital-twin-meta.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';+// Legacy metadata may bypass schema parsing; reject individual bad records+// before any filename reaches the filesystem, without hiding valid neighbors.+const usableDocument = doc => doc && documentFilenameSchema.safeParse(doc.filename).success;+const documentRecords = meta => Array.isArray(meta.documents) ? meta.documents : [];+export async function getDocuments() {const meta = await loadMeta();- const existing = meta.documents.filter(doc => existsSync(join(DIGITAL_TWIN_DIR, doc.filename)));+ const existing = documentRecords(meta).filter(usableDocument).filter(doc => existsSync(join(DIGITAL_TWIN_DIR, doc.filename)));const stats = await Promise.all(existing.map(doc => stat(join(DIGITAL_TWIN_DIR, doc.filename))));return existing.map((doc, i) => ({...doc,@@ -21,7 +27,7 @@ export async function getDocuments() {export async function getDocumentById(id) {const meta = await loadMeta();- const docMeta = meta.documents.find(d => d.id === id);+ const docMeta = documentRecords(meta).find(d => usableDocument(d) && d.id === id);if (!docMeta) return null;@@ -90,7 +96,7 @@ async function createDocumentLeased(data) {async function updateDocumentLeased(id, updates) {const meta = await loadMeta();- const docIndex = meta.documents.findIndex(d => d.id === id);+ const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);if (docIndex === -1) return null;@@ -126,7 +132,7 @@ async function updateDocumentLeased(id, updates) {async function deleteDocumentLeased(id) {const meta = await loadMeta();- const docIndex = meta.documents.findIndex(d => d.id === id);+ const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);if (docIndex === -1) return false;diff --git a/server/services/digital-twin-sync.js b/server/services/digital-twin-sync.jsindex 24c4d63406..a3c67f77f3 100644--- a/server/services/digital-twin-sync.js+++ b/server/services/digital-twin-sync.js@@ -57,7 +57,8 @@ import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';*/import { meatspaceEvents } from './meatspaceEvents.js';-import { join, basename } from 'path';+import { join } from 'path';+import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';import { readdir, readFile, unlink } from 'fs/promises';import { existsSync } from 'fs';import { atomicWrite, readJSONFile, ensureDir, PATHS } from '../lib/fileUtils.js';@@ -435,11 +436,16 @@ function documentLiveStamp(doc) {* Returns the surviving documents plus the merged tombstone list.*/export function mergeDocumentsWithTombstones(local, remote) {- const { merged: tombstonesUnion } = mergeTombstones(local?.deletedDocuments, remote?.deletedDocuments, DOC_TOMBSTONE_OPTS);+ const safeRecords = records => (Array.isArray(records) ? records : [])+ .filter(record => isPlainObject(record) && safeMdName(record.filename));+ const localDocuments = safeRecords(local?.documents);+ const remoteDocuments = safeRecords(remote?.documents);+ const { merged: tombstonesUnion } = mergeTombstones(+ safeRecords(local?.deletedDocuments), safeRecords(remote?.deletedDocuments), DOC_TOMBSTONE_OPTS+ );const remoteStamps = new Map();- for (const doc of Array.isArray(remote?.documents) ? remote.documents : []) {- if (!isPlainObject(doc) || typeof doc.filename !== 'string') continue;+ for (const doc of remoteDocuments) {const prev = remoteStamps.get(doc.filename);remoteStamps.set(doc.filename, {createdAt: newerStamp(doc.createdAt, prev?.createdAt),@@ -447,7 +453,7 @@ export function mergeDocumentsWithTombstones(local, remote) {});}- const { merged: unioned } = unionByKey(local?.documents, remote?.documents, 'filename');+ const { merged: unioned } = unionByKey(localDocuments, remoteDocuments, 'filename');const documents = unioned.map((doc) => {const peer = remoteStamps.get(doc.filename);@@ -525,7 +531,13 @@ export function mergePersonasWithTombstones(local, remote) {*/export function mergeMeta(local, remote) {if (!isPlainObject(remote)) return { merged: local, changed: false };- if (!isPlainObject(local)) return { merged: remote, changed: true };+ if (!isPlainObject(local)) {+ const safeDocuments = mergeDocumentsWithTombstones(null, remote);+ return { merged: { ...remote,+ ...(Object.hasOwn(remote, 'documents') ? { documents: safeDocuments.documents } : {}),+ ...(Object.hasOwn(remote, 'deletedDocuments') ? { deletedDocuments: safeDocuments.deletedDocuments } : {})+ }, changed: true };+ }let changed = false;const merged = { ...local };@@ -736,12 +748,7 @@ export function mergeAutobiographyStories(local, remote) {// Sanitize a peer-supplied document name down to a safe `*.md` basename so a// malformed/buggy payload can't write outside the digital-twin dir.export function safeMdName(name) {- if (typeof name !== 'string') return null;- const base = basename(name);- if (base !== name) return null;- if (!base.toLowerCase().endsWith('.md')) return null;- if (base.startsWith('.')) return null;- return base;+ return documentFilenameSchema.safeParse(name).success ? name : null;}// --- Snapshot ---diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.jsindex 50755ae094..cbe3d0924c 100644--- a/server/services/digital-twin.test.js+++ b/server/services/digital-twin.test.js@@ -31,7 +31,8 @@ vi.mock('./promptService.js', () => ({buildPrompt: vi.fn()}));-vi.mock('../lib/digitalTwinValidation.js', () => ({+vi.mock('../lib/digitalTwinValidation.js', async importOriginal => ({+ ...await importOriginal(),digitalTwinMetaSchema: {safeParse: vi.fn((data) => ({ success: true, data }))},