Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep synced Digital Twin documents within their root (#11277)

Adam Eivya5e1a8dd491d
diff --git a/server/lib/README.md b/server/lib/README.md
index f2e58df188..49f8dee11d 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -126,7 +126,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `creativeCommissionSpec.js` | `GENERATION_KEY_DEFS` / `ABILITY_GENERATION_SPEC` / `resolvedDefault` and the quality / aspect-ratio / image-mode / video-mode enums — the generation-param spec `creativeCommissionValidation.js` builds its Zod schema from and the ability adapters sanitize against. A pure leaf (like `creativeBriefLimits.js`) so `client/src/components/creative-commission/commissionForm.js` derives its fields/bounds/defaults from it directly instead of hand-copying them (#6816). |
| `creativeCommissionValidation.js` | Creative Commission (Autonomous Creation Engine) create/update + brief/schedule/generation schemas. The brief field caps it enforces come from `creativeBriefLimits.js`; the generation-param spec comes from `creativeCommissionSpec.js`. |
| `creativeDirectorValidation.js` | Creative Director project/treatment/scene, inert Video draft preferences, and Create-Suite importer schemas. The directive `goal` cap it enforces comes from `creativeBriefLimits.js`. |
-| `digitalTwinValidation.js` | Digital twin document/category schemas. |
+| `digitalTwinValidation.js` | Digital twin document/category schemas, including `documentFilenameSchema` for safe Markdown basenames across imports, federation and stored document access. |
| `eidoverseValidation.js` | Eidoverse world-projection schemas — the `EIDOVERSE_PROJECTION_SOURCE_KEYS` allowlist, the V1/V2 projection recipe union (includes/assets/terrain/scale/districts/environment, with the `..`-escape guard on every asset path), and the world augment/say/config-patch route bodies (incl. the 8KB augment-argument cap). Split out of `validation.js` (#5698), which re-exports it. |
| `filmStyleGrammarValidation.js` | Zod contract for a film style grammar record (`filmStyleGrammarSchema`), its field caps (`FILM_STYLE_LIMITS`), categories, renderable `FILM_STYLE_PARTS`, the `parts` selector, and the `/api/film-styles/:id` params and `?parts=` prompt-preview query schemas. |
| `fableLoomValidation.js` | FableLoom branching-narrative route schemas (loom/episode/node/transition CRUD, weave/branch/review, play turns). |
diff --git a/server/lib/digitalTwinValidation.js b/server/lib/digitalTwinValidation.js
index 064bc04db9..37f96d0cf8 100644
--- a/server/lib/digitalTwinValidation.js
+++ b/server/lib/digitalTwinValidation.js
@@ -7,6 +7,7 @@ import {
EMOJI_USAGE_VALUES
} from './personaTraitBlend.js';
import { partialWithoutDefaults } from './zodCompat.js';
+import { isSafeFilename } from './pathSafety.js';
// Document category enum
export const documentCategoryEnum = z.enum([
@@ -44,6 +45,13 @@ export const enrichmentCategoryEnum = z.enum([
'personality_assessments'
]);
+// Imports and service callers may use existing peer document names (including
+// spaces and uppercase extensions), but every document stays a Markdown basename.
+export const documentFilenameSchema = z.string().min(1).refine(
+ name => isSafeFilename(name, ['.md']) && !name.startsWith('.'),
+ 'Filename must be a safe markdown basename'
+);
+
// Document metadata schema
export const documentMetaSchema = z.object({
id: z.string().min(1),
diff --git a/server/services/digital-twin-document-safety.test.js b/server/services/digital-twin-document-safety.test.js
new file mode 100644
index 0000000000..e62096583b
--- /dev/null
+++ b/server/services/digital-twin-document-safety.test.js
@@ -0,0 +1,78 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const calls = vi.hoisted(() => []);
+const parent = await mkdtemp(join(tmpdir(), 'digital-twin-safety-'));
+const root = join(parent, 'digital-twin');
+const outside = join(parent, 'outside.md');
+vi.mock('../lib/fileUtils.js', async original => {
+ const actual = await original();
+ return { ...actual, PATHS: { ...actual.PATHS, digitalTwin: root } };
+});
+vi.mock('fs/promises', async original => {
+ const actual = await original();
+ return { ...actual, ...Object.fromEntries(['readFile', 'stat', 'unlink'].map(method => [method, (...args) => {
+ calls.push({ method, path: args[0] });
+ return actual[method](...args);
+ }])) };
+});
+vi.mock('../lib/databaseMaintenanceJournal.js', () => ({ assertDatabaseAdmission() {} }));
+const { applyDigitalTwinRemote, mergeMeta } = await import('./digital-twin-sync.js');
+const { getDocuments, getDocumentById, deleteDocument, updateDocument } = await import('./digital-twin-documents.js');
+const { DEFAULT_META, cache } = await import('./digital-twin-meta.js');
+const doc = (filename, id = filename) => ({ id, filename, title: 'Synthetic', category: 'core', enabled: true,
+ priority: 50, weight: 5, createdAt: '2026-01-01T00:00:00.000Z' });
+const unsafeNames = () => ['../outside.md', outside, '..\\outside.md', 'C:\\outside.md', 'nested/doc.md',
+ 'identity.json', '.hidden.md', 'bad\0.md', '', null, 42, {}];
+async function seed(documents) {
+ await writeFile(join(root, 'meta.json'), JSON.stringify({ ...structuredClone(DEFAULT_META), documents }));
+ cache.meta.data = null; cache.meta.timestamp = 0;
+}
+beforeEach(async () => {
+ await rm(root, { recursive: true, force: true }); await mkdir(root);
+ await writeFile(outside, 'Synthetic outside sentinel');
+ await seed([]); calls.length = 0;
+});
+afterAll(async () => { await rm(parent, { recursive: true, force: true }); });
+
+describe('Digital Twin document filename boundary', () => {
+ it('rejects unsafe peer metadata, bytes and tombstones while retaining valid Markdown names', async () => {
+ const valid = doc('My Doc.MD');
+ const invalid = unsafeNames().map((filename, i) => doc(filename, `unsafe-${i}`));
+ const remote = { meta: { documents: [...invalid, valid], deletedDocuments: invalid.map(({ filename }) =>
+ ({ filename, deletedAt: '2025-01-01T00:00:00.000Z' })) },
+ documents: { '../outside.md': 'REPLACEMENT', '..\\outside.md': 'REPLACEMENT', 'My Doc.MD': '# Valid peer body' } };
+ // Missing local metadata must not bypass the ingress policy either.
+ expect(mergeMeta(null, remote.meta).merged.documents).toEqual([valid]);
+ await applyDigitalTwinRemote(remote);
+ const saved = JSON.parse(await readFile(join(root, 'meta.json'), 'utf8'));
+ expect(saved.documents).toEqual([valid]);
+ expect(saved.deletedDocuments).toEqual([]);
+ expect(await getDocumentById(valid.id)).toMatchObject({ content: '# Valid peer body' });
+ expect((await getDocuments()).map(doc => doc.filename)).toEqual(['My Doc.MD']);
+ expect(calls.filter(call => call.path === outside)).toEqual([]);
+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');
+ });
+
+ it('ignores malformed legacy rows before read/stat/update/delete and cleans them on the next peer merge', async () => {
+ const valid = doc('Existing.MD');
+ const invalid = unsafeNames().map((filename, i) => doc(filename, `legacy-${i}`));
+ await writeFile(join(root, valid.filename), '# Existing body');
+ await seed([null, ...invalid, valid]);
+ expect((await getDocuments()).map(doc => doc.id)).toEqual([valid.id]);
+ for (const entry of invalid) {
+ expect(await getDocumentById(entry.id)).toBeNull();
+ expect(await updateDocument(entry.id, { content: 'REPLACEMENT' })).toBeNull();
+ expect(await deleteDocument(entry.id)).toBe(false);
+ }
+ expect(calls.filter(call => call.path === outside)).toEqual([]);
+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');
+ await applyDigitalTwinRemote({ meta: { documents: [] } });
+ expect(JSON.parse(await readFile(join(root, 'meta.json'), 'utf8')).documents).toEqual([valid]);
+ expect(await deleteDocument(valid.id)).toBe(true);
+ expect(await getDocuments()).toEqual([]);
+ expect(await readFile(outside, 'utf8')).toBe('Synthetic outside sentinel');
+ });
+});
diff --git a/server/services/digital-twin-documents.js b/server/services/digital-twin-documents.js
index 91ef3c2cce..54b7629f1f 100644
--- a/server/services/digital-twin-documents.js
+++ b/server/services/digital-twin-documents.js
@@ -4,13 +4,19 @@ import { join } from 'path';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';
import { atomicWrite } from '../lib/fileUtils.js';
+import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';
import { extractVersion } from './digital-twin-meta.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';
+// Legacy metadata may bypass schema parsing; reject individual bad records
+// before any filename reaches the filesystem, without hiding valid neighbors.
+const usableDocument = doc => doc && documentFilenameSchema.safeParse(doc.filename).success;
+const documentRecords = meta => Array.isArray(meta.documents) ? meta.documents : [];
+
export async function getDocuments() {
const meta = await loadMeta();
- const existing = meta.documents.filter(doc => existsSync(join(DIGITAL_TWIN_DIR, doc.filename)));
+ const existing = documentRecords(meta).filter(usableDocument).filter(doc => existsSync(join(DIGITAL_TWIN_DIR, doc.filename)));
const stats = await Promise.all(existing.map(doc => stat(join(DIGITAL_TWIN_DIR, doc.filename))));
return existing.map((doc, i) => ({
...doc,
@@ -21,7 +27,7 @@ export async function getDocuments() {
export async function getDocumentById(id) {
const meta = await loadMeta();
- const docMeta = meta.documents.find(d => d.id === id);
+ const docMeta = documentRecords(meta).find(d => usableDocument(d) && d.id === id);
if (!docMeta) return null;
@@ -90,7 +96,7 @@ async function createDocumentLeased(data) {
async function updateDocumentLeased(id, updates) {
const meta = await loadMeta();
- const docIndex = meta.documents.findIndex(d => d.id === id);
+ const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);
if (docIndex === -1) return null;
@@ -126,7 +132,7 @@ async function updateDocumentLeased(id, updates) {
async function deleteDocumentLeased(id) {
const meta = await loadMeta();
- const docIndex = meta.documents.findIndex(d => d.id === id);
+ const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);
if (docIndex === -1) return false;
diff --git a/server/services/digital-twin-sync.js b/server/services/digital-twin-sync.js
index 24c4d63406..a3c67f77f3 100644
--- a/server/services/digital-twin-sync.js
+++ b/server/services/digital-twin-sync.js
@@ -57,7 +57,8 @@ import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
*/
import { meatspaceEvents } from './meatspaceEvents.js';
-import { join, basename } from 'path';
+import { join } from 'path';
+import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';
import { readdir, readFile, unlink } from 'fs/promises';
import { existsSync } from 'fs';
import { atomicWrite, readJSONFile, ensureDir, PATHS } from '../lib/fileUtils.js';
@@ -435,11 +436,16 @@ function documentLiveStamp(doc) {
* Returns the surviving documents plus the merged tombstone list.
*/
export function mergeDocumentsWithTombstones(local, remote) {
- const { merged: tombstonesUnion } = mergeTombstones(local?.deletedDocuments, remote?.deletedDocuments, DOC_TOMBSTONE_OPTS);
+ const safeRecords = records => (Array.isArray(records) ? records : [])
+ .filter(record => isPlainObject(record) && safeMdName(record.filename));
+ const localDocuments = safeRecords(local?.documents);
+ const remoteDocuments = safeRecords(remote?.documents);
+ const { merged: tombstonesUnion } = mergeTombstones(
+ safeRecords(local?.deletedDocuments), safeRecords(remote?.deletedDocuments), DOC_TOMBSTONE_OPTS
+ );
const remoteStamps = new Map();
- for (const doc of Array.isArray(remote?.documents) ? remote.documents : []) {
- if (!isPlainObject(doc) || typeof doc.filename !== 'string') continue;
+ for (const doc of remoteDocuments) {
const prev = remoteStamps.get(doc.filename);
remoteStamps.set(doc.filename, {
createdAt: newerStamp(doc.createdAt, prev?.createdAt),
@@ -447,7 +453,7 @@ export function mergeDocumentsWithTombstones(local, remote) {
});
}
- const { merged: unioned } = unionByKey(local?.documents, remote?.documents, 'filename');
+ const { merged: unioned } = unionByKey(localDocuments, remoteDocuments, 'filename');
const documents = unioned
.map((doc) => {
const peer = remoteStamps.get(doc.filename);
@@ -525,7 +531,13 @@ export function mergePersonasWithTombstones(local, remote) {
*/
export function mergeMeta(local, remote) {
if (!isPlainObject(remote)) return { merged: local, changed: false };
- if (!isPlainObject(local)) return { merged: remote, changed: true };
+ if (!isPlainObject(local)) {
+ const safeDocuments = mergeDocumentsWithTombstones(null, remote);
+ return { merged: { ...remote,
+ ...(Object.hasOwn(remote, 'documents') ? { documents: safeDocuments.documents } : {}),
+ ...(Object.hasOwn(remote, 'deletedDocuments') ? { deletedDocuments: safeDocuments.deletedDocuments } : {})
+ }, changed: true };
+ }
let changed = false;
const merged = { ...local };
@@ -736,12 +748,7 @@ export function mergeAutobiographyStories(local, remote) {
// Sanitize a peer-supplied document name down to a safe `*.md` basename so a
// malformed/buggy payload can't write outside the digital-twin dir.
export function safeMdName(name) {
- if (typeof name !== 'string') return null;
- const base = basename(name);
- if (base !== name) return null;
- if (!base.toLowerCase().endsWith('.md')) return null;
- if (base.startsWith('.')) return null;
- return base;
+ return documentFilenameSchema.safeParse(name).success ? name : null;
}
// --- Snapshot ---
diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.js
index 50755ae094..cbe3d0924c 100644
--- a/server/services/digital-twin.test.js
+++ b/server/services/digital-twin.test.js
@@ -31,7 +31,8 @@ vi.mock('./promptService.js', () => ({
buildPrompt: vi.fn()
}));
-vi.mock('../lib/digitalTwinValidation.js', () => ({
+vi.mock('../lib/digitalTwinValidation.js', async importOriginal => ({
+ ...await importOriginal(),
digitalTwinMetaSchema: {
safeParse: vi.fn((data) => ({ success: true, data }))
},