Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: develop still-image effects in document videos (#11319)
[._.]/ Adam Eivy2b76e204b714
diff --git a/server/services/musicVideo/compositionDocument.js b/server/services/musicVideo/compositionDocument.jsindex b1d03ad8df..f563ebf23c 100644--- a/server/services/musicVideo/compositionDocument.js+++ b/server/services/musicVideo/compositionDocument.js@@ -50,6 +50,11 @@ const SHARED_ROOT = join(TEMPLATE_ROOT, 'shared');const LYRIC_TYPE_REF = /\blyricType\.(?:js|css)\b/;const LYRIC_TYPE_SCAN_MAX = 2 * 1024 * 1024;const LAYERED_ENGINE_TAG = '<script src="engine.js"></script>';+const IMAGE_EFFECT_TAG = `<script type="module">+import { imageEffectState, IMAGE_EFFECT_FOG_COLOR } from './musicVideoImageEffects.js';+globalThis.PORTOS_MV_IMAGE_EFFECTS = { imageEffectState, IMAGE_EFFECT_FOG_COLOR };+</script><!-- portos-allow-hidden-content: shared still-image effect curves -->+`;const LYRIC_TYPE_TAGS = '<link rel="stylesheet" href="lyricType.css">\n'+ '<script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->\n';@@ -152,13 +157,15 @@ function storeVersion(projectId, files, source, options = {}) {}/**- * A layered page from before #10583 loads the engine through the template's own- * tag; it gains the lyric-type stylesheet and module in front of that tag. A page- * that already names lyricType, or loads its engine some other way, is unchanged.+ * On explicit engine adoption, a template-shaped page gains the shared module+ * tags the current engine needs. Existing module tags and authored entrypoints+ * are preserved.*/-function withLyricTypeTags(page) {- if (!page.includes(LAYERED_ENGINE_TAG) || LYRIC_TYPE_REF.test(page)) return page;- return page.replace(LAYERED_ENGINE_TAG, () => LYRIC_TYPE_TAGS + LAYERED_ENGINE_TAG);+function withLayeredModuleTags(page) {+ if (!page.includes(LAYERED_ENGINE_TAG)) return page;+ const tags = (LYRIC_TYPE_REF.test(page) ? '' : LYRIC_TYPE_TAGS)+ + (page.includes('PORTOS_MV_IMAGE_EFFECTS') ? '' : IMAGE_EFFECT_TAG);+ return page.replace(LAYERED_ENGINE_TAG, () => tags + LAYERED_ENGINE_TAG);}/**@@ -181,6 +188,19 @@ async function withSharedLyricType(files) {return missing.length ? [...files, ...missing] : files;}+// Ship the actual browser-safe effect definition, preserving an authored copy.+async function withSharedImageEffects(files) {+ if (files.some(file => file.rel === 'musicVideoImageEffects.js')) return files;+ for (const file of files) {+ if (!/\.(?:html?|m?js)$/i.test(file.rel) || (file.size ?? file.data?.length ?? 0) > LYRIC_TYPE_SCAN_MAX) continue;+ const text = file.data ? file.data.toString('utf8') : await readFile(file.abs, 'utf8');+ if (!/\bmusicVideoImageEffects\.js\b/.test(text)) continue;+ const abs = fileURLToPath(new URL('../../lib/musicVideoImageEffects.js', import.meta.url));+ return [...files, { rel: 'musicVideoImageEffects.js', abs, size: (await stat(abs)).size }];+ }+ return files;+}+// Shared toon kit remains a local static graph, including its vendored geometry// dependency. Authored copies win; unrelated documents gain no dependencies.async function withSharedToonWorld(files) {@@ -205,7 +225,7 @@ async function withSharedToonWorld(files) {async function storeVersionNow(projectId, inputFiles, source, { draft = false, verifyCurrent = () => {} } = {}) {assertDocumentShape(inputFiles.map((file) => file.rel));- const files = await withSharedToonWorld(await withSharedLyricType(inputFiles));+ const files = await withSharedToonWorld(await withSharedImageEffects(await withSharedLyricType(inputFiles)));const totalBytes = files.reduce((sum, file) => sum + (file.data?.length ?? file.size), 0);if (files.length > DOCUMENT_MAX_FILES || totalBytes > DOCUMENT_MAX_BYTES) {throw refuse('Shared document assets exceed the document file or byte limit', 'COMPOSITION_DOCUMENT_TOO_LARGE', 413);@@ -364,7 +384,7 @@ export async function importDocumentTemplate(projectId, templateId = 'layered')export async function readTemplateDocumentFiles(templateId = 'layered') {if (!MUSIC_VIDEO_DOCUMENT_TEMPLATES.includes(templateId)) throw refuse('Unknown composition template', 'VALIDATION_ERROR', 400);const { files } = await collectTree(join(TEMPLATE_ROOT, templateId), { excludeTests: true });- return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));+ return new Map((await withSharedImageEffects(await withSharedLyricType(files))).map((file) => [file.rel, { abs: file.abs, size: file.size }]));}/**@@ -374,7 +394,7 @@ export async function readTemplateDocumentFiles(templateId = 'layered') {*/export async function readLyricOverlayDocumentFiles() {const { files } = await collectTree(join(TEMPLATE_ROOT, 'lyricOverlay'), { excludeTests: true });- return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));+ return new Map((await withSharedImageEffects(await withSharedLyricType(files))).map((file) => [file.rel, { abs: file.abs, size: file.size }]));}/** Stage a host-assembled generated document for review before selection. */@@ -409,6 +429,8 @@ export async function stageGeneratedDocument(projectId, generatedFiles, { verify// Known compatible shipped layered engines. Recognition only;// customized engines belong to the author and are never silently replaced.const UPGRADABLE_LAYERED_ENGINES = new Set([+ // Last shipped engine before document still-image effects (#11319).+ '9581bb2d190e62ea2eb6a709bc989b1f913c501236554cd0301379877dc8a5fc','dc27611ebaa429c28bd467b003495d2a1c7dc0ede73be0cfc28db4b97c964dcf','2556a0905a85958b9107cfa75ab0c5bfde167a708e29b511d577065bb530ea58','e2267a068c92a3c7dee49db5f3e57caff1ac11aee1312c0d82e28352850aad3b',@@ -423,7 +445,7 @@ const engineDigest = bytes => createHash('sha256').update(bytes.toString('utf8')/*** Explicitly adopt the shipped layered engine in a new immutable document.* All authored scripts, HTML, fonts and assets retain their exact bytes, except- * that a template-shaped index.html gains the shared lyric-type tags the shipped+ * that a template-shaped index.html gains the shared module tags the shipped* engine needs (see below). The* new pointer invalidates revision-bound render/review evidence naturally.*/@@ -451,12 +473,11 @@ export function upgradeDocumentEngine(projectId, directory) {const loaded = await Promise.all(files.map(async file => ({rel: file.rel, data: file.rel === 'engine.js' ? shipped : await readFile(file.abs),})));- // The shipped engine draws its words with the shared lyricType.js (#10583). A page that- // loads the engine through the template's own tag gets the module's two tags in front- // of it (storeVersionNow adds the module and its faces); a page that doesn't is the- // author's and is left alone.+ // Add the shared lyric-type and still-effect modules only to a template-shaped+ // page. storeVersionNow supplies missing module files and fonts; authored+ // entrypoints keep their bytes.const index = loaded.find(file => file.rel === 'index.html');- if (index) index.data = Buffer.from(withLyricTypeTags(index.data.toString('utf8')));+ if (index) index.data = Buffer.from(withLayeredModuleTags(index.data.toString('utf8')));const result = await storeVersionNow(projectId, loaded, project.composition.document.source || { kind: 'directory', name: null }, { verifyCurrent });return { ...result, changed: true };});diff --git a/server/services/musicVideo/compositionDocument.lyricType.test.js b/server/services/musicVideo/compositionDocument.lyricType.test.jsindex 16a196cdb3..bc99a74761 100644--- a/server/services/musicVideo/compositionDocument.lyricType.test.js+++ b/server/services/musicVideo/compositionDocument.lyricType.test.js@@ -38,12 +38,13 @@ async function uploaded(name, files) {describe('shared lyric-type assets in composition documents', () => {it('ships lyricType.js, its stylesheet and OFL faces with the layered template', async () => {const { id } = await projects.createProject({ name: 'Example Song' });- const { files } = await listed((await importDocumentTemplate(id)).document);+ const { root, files } = await listed((await importDocumentTemplate(id)).document);expect(files).toEqual(expect.arrayContaining(['lyricType.js', 'lyricType.css', 'fonts/archivo-variable.woff2', 'fonts/ibm-plex-mono-500.woff2',- 'fonts/LICENSE-Archivo.txt', 'fonts/LICENSE-IBM-Plex.txt', 'engine.js', 'index.html',+ 'fonts/LICENSE-Archivo.txt', 'fonts/LICENSE-IBM-Plex.txt', 'engine.js', 'index.html', 'musicVideoImageEffects.js',]));expect(files.filter((file) => file.endsWith('.test.js'))).toEqual([]);+ expect(await readFile(join(root, 'musicVideoImageEffects.js'), 'utf8')).toBe(await readFile(new URL('../../lib/musicVideoImageEffects.js', import.meta.url), 'utf8'));});it('adds the module and faces to an uploaded document that imports it, and keeps a document\'s own copy', async () => {diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.jsindex 90fe23fc02..d78e200b9e 100644--- a/server/services/musicVideo/documentRender.browser.test.js+++ b/server/services/musicVideo/documentRender.browser.test.js@@ -112,6 +112,68 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {} finally { await _cleanupTestBrowser({ browser, proc, cleanup: () => {} }); }});+ // Uniquely catches dropped effect data, missing staged modules and preview/export curve drift.+ it('develops a still from white identically in the document render and live preview', async () => {+ await mkdir(PATHS.images, { recursive: true });+ const image = await sharp({ create: { width: 64, height: 36, channels: 3, background: '#ee6721' } }).png().toBuffer();+ await writeFile(join(PATHS.images, 'effect-still.png'), image);+ const created = await projects.createProject({ name: 'Synthetic image effect' });+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,+ audioAnalysis: { durationSec: 2, sections: [] },+ scenes: [{ sceneId: 'develop', startSec: 0.25, endSec: 2, visualLayer: 'still', stillMove: 'hold',+ referenceImageId: 'effect-still.png', imageEffect: 'flash-develop', imageEffectSec: 0.5 }],+ } }));+ await importDocumentTemplate(created.id);+ const project = await projects.getProject(created.id);+ const { openIsolatedDocumentPreview } = await import('./documentRender.js');+ const capture = await openIsolatedDocumentPreview({ project, jobId: 'still-effect', signal: testSignal });+ const page = await browser.newPage();+ try {+ const loadPreview = async current => {+ const preview = await buildDocumentPreview(current);+ await page.setContent(preview.html);+ await page.evaluate(({ key, encoded }) => {+ addEventListener('message', ({ data }) => {+ if (data?.type !== 'portos-mv:request') return;+ const blob = new Blob([Uint8Array.from(atob(encoded), c => c.charCodeAt(0))], { type: 'image/png' });+ window.postMessage({ type: 'portos-mv:asset', key, blob }, '*');+ });+ window.postMessage({ type: 'portos-mv:manifest', keys: [key] }, '*');+ }, { key: preview.assets[0].key, encoded: image.toString('base64') });+ await page.waitForFunction(() => globalThis.portosComposition);+ };+ await loadPreview(project);+ const pixels = [];+ for (const t of [0.25, 0.5, 0.75, 1.25]) {+ const sample = `(async () => {+ await portosComposition.seek(${t});+ const canvas = document.getElementById('stage');+ return [...canvas.getContext('2d').getImageData(canvas.width / 2, canvas.height / 2, 1, 1).data].slice(0, 3);+ })()`;+ const live = await page.evaluate(sample);+ const rendered = await capture.page.evaluate(sample);+ expect(rendered).toEqual(live);+ pixels.push(live);+ }+ expect(pixels[0]).toEqual([255, 255, 255]);+ expect(pixels[1][0] - pixels[1][2]).toBeLessThan(pixels[2][0] - pixels[2][2]);+ expect(pixels[2][0]).toBeGreaterThan(pixels[2][1] + 80);+ expect(pixels[2][1]).toBeGreaterThan(pixels[2][2] + 40);+ // A developed still is exactly the no-effect picture at the same song time.+ await loadPreview({ ...project, scenes: project.scenes.map(scene => ({ ...scene, imageEffect: null })) });+ const held = await page.evaluate(async () => {+ await globalThis.portosComposition.seek(1.25);+ const canvas = document.getElementById('stage');+ return [...canvas.getContext('2d').getImageData(canvas.width / 2, canvas.height / 2, 1, 1).data].slice(0, 3);+ });+ expect(held).toEqual(pixels[3]);+ } finally {+ await page.close();+ await capture.page.close();+ await rm(capture.scratchRoot, { recursive: true, force: true });+ }+ }, 60000);+it('keeps network refusal inside the owned capture browser and leaves the browsing session alive', async () => {const { openComposition } = await import('../htmlComposition/browser.js');const directory = 'compositions/synthetic-owned-refusal';diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.jsindex fad7600410..d2fd9cf777 100644--- a/server/services/musicVideo/documentRender.js+++ b/server/services/musicVideo/documentRender.js@@ -1,4 +1,5 @@import { assertMusicVideoMediaSelections, musicVideoMediaMode } from '../../lib/musicVideoMediaPolicy.js';+import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';import { musicVideoGradeFilter } from '../../lib/musicVideoGrade.js';import { filmLookRuntimeSource, isFilmLookNeutral } from '../../lib/filmLook.js';import { titleCardVisible } from '../../lib/musicVideoTitleCard.js';@@ -190,6 +191,7 @@ export function buildDocumentData(project, { media = new Map(), frame, clock, soendSec: start != null && end != null && end > start ? end : null,shotMode: scene.shotMode === 'performance' ? 'performance' : 'cutaway',visualLayer: documentSceneVisualLayer(project, scene, { generated }),+ imageEffect: sceneImageEffect(scene, { layered: true }),stillMove: typeof scene.stillMove === 'string' ? scene.stillMove : null,// #10589: the planned camera move the template's camera rig follows (only when planned)....(typeof scene.camera?.move === 'string' ? { camera: structuredClone(scene.camera) } : {}),diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.jsindex e6cdc76cc2..30b4acc870 100644--- a/server/services/musicVideo/documentRender.test.js+++ b/server/services/musicVideo/documentRender.test.js@@ -80,7 +80,7 @@ async function fixture({ shipped = {} } = {}) {composition: { mode: 'document', textCues: [{ id: 'c1', text: 'hero', startSec: 1, endSec: 2, emphasis: 'hero' }], overlay: { enabled: true, ticker: ['a'] } },scenes: [{ sceneId: 'a', order: 0, startSec: 0, endSec: 4, videoHistoryId: 'vh-a', referenceImageId: 'still-b.png' },- { sceneId: 'b', order: 1, startSec: 4, endSec: 8, videoHistoryId: 'vh-gone', referenceImageId: 'still-b.png', visualLayer: 'still', stillMove: 'push' },+ { sceneId: 'b', order: 1, startSec: 4, endSec: 8, videoHistoryId: 'vh-gone', referenceImageId: 'still-b.png', visualLayer: 'still', stillMove: 'push', imageEffect: 'flash-develop', imageEffectSec: 6 },{ sceneId: 'c', order: 2, startSec: 8, endSec: 9, visualLayer: 'card', cardText: 'Title' },{ sceneId: 'd', order: 3, startSec: 9, endSec: 12, shotMode: 'performance', videoHistoryId: 'vh-a', takes: [performanceTake('vh-a')] },],@@ -138,8 +138,9 @@ describe('composition document staging', () => {const byId = Object.fromEntries(data.scenes.map((s) => [s.sceneId, s]));expect(byId.a.media).toMatchObject({ kind: 'video', src: 'media/scene-a.mp4', inSec: 0, outSec: 4, fps: 24 });expect(byId.b.media).toMatchObject({ kind: 'image', src: 'media/scene-b.png' });- expect(byId.b).toMatchObject({ visualLayer: 'still', stillMove: 'push' });- expect(byId.c).toMatchObject({ media: null, visualLayer: 'card', cardText: 'Title' });+ expect(byId.b).toMatchObject({ visualLayer: 'still', stillMove: 'push', imageEffect: { effect: 'flash-develop', developSec: 6 } });+ expect(byId.a.imageEffect).toBeNull();+ expect(byId.c).toMatchObject({ imageEffect: null, media: null, visualLayer: 'card', cardText: 'Title' });expect(byId.d.media).toMatchObject({ kind: 'video', src: 'media/scene-d.mp4', inSec: 0.5, outSec: 3.5 });expect(data.render).toEqual({ width: 1920, height: 1080, fps: 24, frames: 720, durationSec: 30 });expect(data.song).toMatchObject({ durationSec: 30, beats: [0, 0.5], downbeats: [0] });diff --git a/server/services/musicVideo/documentTemplates/layered/engine.js b/server/services/musicVideo/documentTemplates/layered/engine.jsindex 37a3f92685..774c4355de 100644--- a/server/services/musicVideo/documentTemplates/layered/engine.js+++ b/server/services/musicVideo/documentTemplates/layered/engine.js@@ -294,14 +294,14 @@let lyricType = null;// lyricType.js is a module, so it runs after this classic script; it publishes// itself on globalThis before DOMContentLoaded.- const lyricTypeModule = () => new Promise((resolve, reject) => {- const check = () => (globalThis.PORTOS_LYRIC_TYPE ? resolve(globalThis.PORTOS_LYRIC_TYPE)- : reject(new Error('lyricType.js did not load — the layered template needs it beside index.html')));- if (globalThis.PORTOS_LYRIC_TYPE || document.readyState !== 'loading') check();+ const documentModule = (key, filename) => new Promise((resolve, reject) => {+ const check = () => (globalThis[key] ? resolve(globalThis[key])+ : reject(new Error(`${filename} did not load — the layered template needs it beside index.html`)));+ if (globalThis[key] || document.readyState !== 'loading') check();else document.addEventListener('DOMContentLoaded', check, { once: true });});async function buildLyricType() {- const { createLyricType } = await lyricTypeModule();+ const { createLyricType } = await documentModule('PORTOS_LYRIC_TYPE', 'lyricType.js');const words = { palette: { fill: C.paper, ink: C.ink, accent: C.accent, strike: C.alert } };// Subtitles: the director's subtitle cues when there are any, else the lyrics.const lines = SUBTITLE_CUES.length ? [...HERO_LINES, ...SUBTITLE_CUES] : null;@@ -500,9 +500,13 @@};let footageVisibility = null;+ let imageEffects = null;+function render(t, scene, source, state, reviewFootage = false) {ctx.fillStyle = C.ink; ctx.fillRect(0, 0, W, H);const authored = sectionFunction(t);+ const effect = source && scene?.visualLayer === 'still' && scene.media?.kind === 'image' && scene.imageEffect+ ? imageEffects?.imageEffectState(scene.imageEffect, t - scene.startSec) : null;if (scene) {const lt = t - scene.startSec; const d = scene.endSec - scene.startSec;const card = authored?.fn ? null : cardFor(scene);@@ -510,7 +514,11 @@else if (source) {const energetic = isHighEnergy(t) || scene.shotMode === 'performance';const punch = 1 + 0.018 * pulse(t, energetic ? beats : downs, 8) * state.reactiveGain;+ ctx.save();+ if (effect) ctx.filter = `contrast(${effect.contrast}) saturate(${effect.saturation})`;drawCover(source, cameraView(scene, seg(t, scene.startSec, scene.endSec)), punch);+ ctx.restore();+ if (effect) _flash(effect.fog, imageEffects.IMAGE_EFFECT_FOG_COLOR);}if (isHighEnergy(t)) glitch(t, pulse(t, downs, 8) * state.reactiveGain);}@@ -532,6 +540,7 @@}vignette(0.5);grain(t, 0.09);+ if (effect) _flash(effect.flash);drawHud(t);drawNarrativeEvents(state);if (lyricType) lyricType.draw(ctx, t, { width: W, height: H });@@ -543,6 +552,9 @@if (window.PORTOS_MV_ASSETS) assetUrls = await window.PORTOS_MV_ASSETS;await Promise.all([F.stencil(40), F.mono(20), F.mono(20, 600), F.cond(20), F.cond(20, 700)].map((font) => document.fonts.load(font)));await buildLyricType();+ if (SCENES.some(scene => scene.imageEffect && scene.visualLayer === 'still')) {+ imageEffects = await documentModule('PORTOS_MV_IMAGE_EFFECTS', 'musicVideoImageEffects.js');+ }buildGrain();})();globalThis.portosComposition = {diff --git a/server/services/musicVideo/documentTemplates/layered/index.html b/server/services/musicVideo/documentTemplates/layered/index.htmlindex 289ce11bd8..a37bc17ac7 100644--- a/server/services/musicVideo/documentTemplates/layered/index.html+++ b/server/services/musicVideo/documentTemplates/layered/index.html@@ -23,6 +23,10 @@ html, body { margin: 0; height: 100%; background: #000; overflow: hidden; }<script src="portos-mv.js"></script><!-- portos-allow-hidden-content: the render-time data file --><script src="cameraRig.js"></script><!-- portos-allow-hidden-content: the reviewed camera-move paths the engine reads --><script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->+<!-- portos-allow-hidden-content: shared still-image effect curves --><script type="module">+import { imageEffectState, IMAGE_EFFECT_FOG_COLOR } from './musicVideoImageEffects.js';+globalThis.PORTOS_MV_IMAGE_EFFECTS = { imageEffectState, IMAGE_EFFECT_FOG_COLOR };+</script><script src="engine.js"></script><!-- portos-allow-hidden-content: the reviewed template engine beside this page --></body></html>