Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: contain Digital Twin imports within Markdown documents (#11271)
Adam Eivy876a58a224ae
diff --git a/server/lib/README.md b/server/lib/README.mdindex 442c26da64..27240b51bc 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -128,6 +128,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `creativeCommissionValidation.js` | Creative Commission (Autonomous Creation Engine) create/update + brief/schedule/generation schemas. The brief field caps it enforces come from `creativeBriefLimits.js`; the generation-param spec comes from `creativeCommissionSpec.js`. || `creativeDirectorValidation.js` | Creative Director project/treatment/scene, inert Video draft preferences, and Create-Suite importer schemas. The directive `goal` cap it enforces comes from `creativeBriefLimits.js`. || `digitalTwinValidation.js` | Digital twin document/category schemas, including `documentFilenameSchema` for safe Markdown basenames across imports, federation and stored document access. |+| `digitalTwinValidation.js` | Digital twin document/category schemas, including the shared safe Markdown-basename `documentFilenameSchema` for imports and document writes. || `eidoverseValidation.js` | Eidoverse world-projection schemas — the `EIDOVERSE_PROJECTION_SOURCE_KEYS` allowlist, the V1/V2 projection recipe union (includes/assets/terrain/scale/districts/environment, with the `..`-escape guard on every asset path), and the world augment/say/config-patch route bodies (incl. the 8KB augment-argument cap). Split out of `validation.js` (#5698), which re-exports it. || `filmStyleGrammarValidation.js` | Zod contract for a film style grammar record (`filmStyleGrammarSchema`), its field caps (`FILM_STYLE_LIMITS`), categories, renderable `FILM_STYLE_PARTS`, the `parts` selector, and the `/api/film-styles/:id` params and `?parts=` prompt-preview query schemas. || `fableLoomValidation.js` | FableLoom branching-narrative route schemas (loom/episode/node/transition CRUD, weave/branch/review, play turns). |diff --git a/server/lib/digitalTwinValidation.js b/server/lib/digitalTwinValidation.jsindex 37f96d0cf8..05d63e66ce 100644--- a/server/lib/digitalTwinValidation.js+++ b/server/lib/digitalTwinValidation.js@@ -320,7 +320,7 @@ export const soulMetaSchema = digitalTwinMetaSchema; // Alias for backwards comp// Create document inputexport const createDocumentInputSchema = z.object({- filename: z.string().min(1).max(100).regex(/^[\w\-]+\.md$/, 'Filename must be a valid markdown filename'),+ filename: documentFilenameSchema.max(100).regex(/^[\w\-]+\.md$/, 'Filename must be a valid markdown filename'),title: z.string().min(1).max(200),category: documentCategoryEnum,content: z.string().min(1).max(1000000),diff --git a/server/routes/digital-twin/import.filenames.test.js b/server/routes/digital-twin/import.filenames.test.jsnew file mode 100644index 0000000000..2a97c4f468--- /dev/null+++ b/server/routes/digital-twin/import.filenames.test.js@@ -0,0 +1,92 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';+import { join } from 'node:path';+import { createTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';+import { request } from '../../lib/testHelper.js';++const tempRoot = createTempDataRoot('digital-twin-import-filenames-');+const twinDir = join(tempRoot, 'digital-twin');++vi.mock('../../lib/fileUtils.js', async original =>+ makePathsProxy(await original(), { dataRoot: tempRoot }));+vi.mock('../../services/promptRunner.js', () => ({ runPromptThroughProvider: vi.fn() }));+vi.mock('../../services/spotifyBrowserImport.js', () => ({}));+// Keep the import/save and document writers real; unrelated routes need no services.+vi.mock('../../services/digital-twin.js', async () => ({+ saveImportAsDocument: (await import('../../services/digital-twin-import.js')).saveImportAsDocument,+ createDocument: (await import('../../services/digital-twin-documents.js')).createDocument,+}));++const { default: importRouter } = await import('./import.js');+const { default: documentRouter } = await import('./documents.js');+const { createDocument, updateDocument } = await import('../../services/digital-twin-documents.js');+const { saveImportAsDocument } = await import('../../services/digital-twin-import.js');+const { cache, loadMeta, saveMeta } = await import('../../services/digital-twin-meta.js');++const app = express();+app.use(express.json());+app.use('/api/digital-twin', importRouter, documentRouter);++const content = '# Synthetic profile\n';+const suggestedDoc = filename => ({ filename, title: 'Synthetic profile', category: 'core', content });+const invalidNames = [+ '../escaped.md', join(tempRoot, 'absolute.md'), 'nested/profile.md',+ '..\\escaped.md', 'nested\\profile.md', 'C:\\profile.md',+ '..', '.hidden.md', 'profile.txt', 'profile.md\0suffix',+];++beforeEach(() => {+ for (const entry of readdirSync(tempRoot)) rmSync(join(tempRoot, entry), { recursive: true, force: true });+ cache.meta.data = null;+ cache.meta.timestamp = 0;+});+afterAll(() => rmSync(tempRoot, { recursive: true, force: true }));++describe('Digital Twin document filename boundary (#11271)', () => {+ it('refuses unsafe import HTTP filenames before any file or metadata is created', async () => {+ for (const filename of invalidNames) {+ const res = await request(app).post('/api/digital-twin/import/save')+ .send({ source: 'synthetic', suggestedDoc: suggestedDoc(filename) });+ expect(res.status, filename).toBe(400);+ expect(res.body.code).toBe('VALIDATION_ERROR');+ expect(readdirSync(tempRoot), filename).toEqual([]);+ }+ });++ it('protects direct document and import service callers before directory initialization', async () => {+ for (const filename of invalidNames) {+ await expect(createDocument(suggestedDoc(filename))).rejects.toThrow();+ await expect(saveImportAsDocument('synthetic', suggestedDoc(filename))).rejects.toThrow();+ expect(readdirSync(tempRoot), filename).toEqual([]);+ }+ });++ it('creates and updates safe Markdown imports and preserves ordinary document creation', async () => {+ const first = await request(app).post('/api/digital-twin/import/save')+ .send({ source: 'synthetic', suggestedDoc: suggestedDoc('Profile Notes.MD') });+ expect(first.status).toBe(200);+ expect(readFileSync(join(twinDir, 'Profile Notes.MD'), 'utf8')).toBe(content);++ const updatedContent = '# Revised synthetic profile\n';+ const updated = await request(app).post('/api/digital-twin/import/save').send({+ source: 'synthetic', suggestedDoc: { ...suggestedDoc('Profile Notes.MD'), content: updatedContent },+ });+ expect(updated.status).toBe(200);+ expect(updated.body.document.id).toBe(first.body.document.id);+ expect(readFileSync(join(twinDir, 'Profile Notes.MD'), 'utf8')).toBe(updatedContent);++ const normal = await request(app).post('/api/digital-twin/documents').send(suggestedDoc('ordinary.md'));+ expect(normal.status).toBe(201);+ expect(readFileSync(join(twinDir, 'ordinary.md'), 'utf8')).toBe(content);+ expect((await loadMeta()).documents).toHaveLength(2);+ });++ it('refuses an unsafe stored filename on update without changing file or metadata', async () => {+ await saveMeta({ version: '1.0.0', documents: [{ id: 'legacy', ...suggestedDoc('../escaped.md') }] });+ const metaBefore = readFileSync(join(twinDir, 'meta.json'), 'utf8');+ expect(await updateDocument('legacy', { content: '# Rejected update' })).toBeNull();+ expect(existsSync(join(tempRoot, 'escaped.md'))).toBe(false);+ expect(readFileSync(join(twinDir, 'meta.json'), 'utf8')).toBe(metaBefore);+ });+});diff --git a/server/routes/digital-twin/import.js b/server/routes/digital-twin/import.jsindex 87cb5594c5..46f4aeffe2 100644--- a/server/routes/digital-twin/import.js+++ b/server/routes/digital-twin/import.js@@ -8,13 +8,13 @@ import { z } from 'zod';import * as digitalTwinService from '../../services/digital-twin.js';import { asyncHandler, ServerError } from '../../lib/errorHandler.js';import { validateRequest } from '../../lib/validation.js';-import { importDataInputSchema } from '../../lib/digitalTwinValidation.js';+import { documentFilenameSchema, importDataInputSchema } from '../../lib/digitalTwinValidation.js';import * as spotifyBrowserImport from '../../services/spotifyBrowserImport.js';const importSaveSchema = z.object({source: z.string().min(1),suggestedDoc: z.object({- filename: z.string().min(1),+ filename: documentFilenameSchema,content: z.string().min(1),title: z.string().optional(),category: z.string().optional(),diff --git a/server/services/digital-twin-documents.js b/server/services/digital-twin-documents.jsindex 54b7629f1f..53d64c46d6 100644--- a/server/services/digital-twin-documents.js+++ b/server/services/digital-twin-documents.js@@ -1,10 +1,12 @@import { readFile, unlink, stat } from 'fs/promises';import { existsSync } from 'fs';-import { join } from 'path';+import { join, resolve } from 'path';import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';import { atomicWrite } from '../lib/fileUtils.js';import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';+import { isPathInsideDir } from '../lib/pathSafety.js';+import { ServerError } from '../lib/errorHandler.js';import { extractVersion } from './digital-twin-meta.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';@@ -52,11 +54,21 @@ export const createDocument = (data) => withBackupAssetPublication(() => withMetexport const updateDocument = (id, updates) => withBackupAssetPublication(() => withMetaLock(() => updateDocumentLeased(id, updates)));export const deleteDocument = (id) => withBackupAssetPublication(() => withMetaLock(() => deleteDocumentLeased(id)));+function documentWritePath(filename) {+ documentFilenameSchema.parse(filename);+ const filePath = resolve(DIGITAL_TWIN_DIR, filename);+ if (!isPathInsideDir(DIGITAL_TWIN_DIR, filePath)) {+ throw new ServerError('Invalid document destination', { status: 400, code: 'VALIDATION_ERROR' });+ }+ return filePath;+}+async function createDocumentLeased(data) {+ // Validate before even creating the document directory or initializing metadata.+ const filePath = documentWritePath(data.filename);await ensureSoulDir();const meta = await loadMeta();- const filePath = join(DIGITAL_TWIN_DIR, data.filename);// Check if file already existsif (existsSync(filePath)) {@@ -101,7 +113,7 @@ async function updateDocumentLeased(id, updates) {if (docIndex === -1) return null;const docMeta = meta.documents[docIndex];- const filePath = join(DIGITAL_TWIN_DIR, docMeta.filename);+ const filePath = documentWritePath(docMeta.filename);// Update file content if providedif (updates.content) {diff --git a/server/services/digital-twin-import.js b/server/services/digital-twin-import.jsindex 0ce8a591fc..8769557241 100644--- a/server/services/digital-twin-import.js+++ b/server/services/digital-twin-import.js@@ -13,6 +13,7 @@ import {parseTraitsResponse} from './digital-twin-analysis.js';import { digitalTwinEvents } from './digital-twin-meta.js';+import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';/*** Parse Goodreads CSV export@@ -429,7 +430,8 @@ export async function analyzeImportedData(source, rawData, providerId, model) {* Save imported analysis as a document*/export async function saveImportAsDocument(source, suggestedDoc) {- const { filename, title, category, content } = suggestedDoc;+ const { title, category, content } = suggestedDoc;+ const filename = documentFilenameSchema.parse(suggestedDoc.filename);// Check if document already existsconst meta = await loadMeta();