Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: develop still-image effects in document videos (#11319)

[._.]/ Adam Eivy2b76e204b714
diff --git a/server/services/musicVideo/compositionDocument.js b/server/services/musicVideo/compositionDocument.js
index b1d03ad8df..f563ebf23c 100644
--- a/server/services/musicVideo/compositionDocument.js
+++ b/server/services/musicVideo/compositionDocument.js
@@ -50,6 +50,11 @@ const SHARED_ROOT = join(TEMPLATE_ROOT, 'shared');
const LYRIC_TYPE_REF = /\blyricType\.(?:js|css)\b/;
const LYRIC_TYPE_SCAN_MAX = 2 * 1024 * 1024;
const LAYERED_ENGINE_TAG = '<script src="engine.js"></script>';
+const IMAGE_EFFECT_TAG = `<script type="module">
+import { imageEffectState, IMAGE_EFFECT_FOG_COLOR } from './musicVideoImageEffects.js';
+globalThis.PORTOS_MV_IMAGE_EFFECTS = { imageEffectState, IMAGE_EFFECT_FOG_COLOR };
+</script><!-- portos-allow-hidden-content: shared still-image effect curves -->
+`;
const LYRIC_TYPE_TAGS = '<link rel="stylesheet" href="lyricType.css">\n'
+ '<script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->\n';
@@ -152,13 +157,15 @@ function storeVersion(projectId, files, source, options = {}) {
}
/**
- * A layered page from before #10583 loads the engine through the template's own
- * tag; it gains the lyric-type stylesheet and module in front of that tag. A page
- * that already names lyricType, or loads its engine some other way, is unchanged.
+ * On explicit engine adoption, a template-shaped page gains the shared module
+ * tags the current engine needs. Existing module tags and authored entrypoints
+ * are preserved.
*/
-function withLyricTypeTags(page) {
- if (!page.includes(LAYERED_ENGINE_TAG) || LYRIC_TYPE_REF.test(page)) return page;
- return page.replace(LAYERED_ENGINE_TAG, () => LYRIC_TYPE_TAGS + LAYERED_ENGINE_TAG);
+function withLayeredModuleTags(page) {
+ if (!page.includes(LAYERED_ENGINE_TAG)) return page;
+ const tags = (LYRIC_TYPE_REF.test(page) ? '' : LYRIC_TYPE_TAGS)
+ + (page.includes('PORTOS_MV_IMAGE_EFFECTS') ? '' : IMAGE_EFFECT_TAG);
+ return page.replace(LAYERED_ENGINE_TAG, () => tags + LAYERED_ENGINE_TAG);
}
/**
@@ -181,6 +188,19 @@ async function withSharedLyricType(files) {
return missing.length ? [...files, ...missing] : files;
}
+// Ship the actual browser-safe effect definition, preserving an authored copy.
+async function withSharedImageEffects(files) {
+ if (files.some(file => file.rel === 'musicVideoImageEffects.js')) return files;
+ for (const file of files) {
+ if (!/\.(?:html?|m?js)$/i.test(file.rel) || (file.size ?? file.data?.length ?? 0) > LYRIC_TYPE_SCAN_MAX) continue;
+ const text = file.data ? file.data.toString('utf8') : await readFile(file.abs, 'utf8');
+ if (!/\bmusicVideoImageEffects\.js\b/.test(text)) continue;
+ const abs = fileURLToPath(new URL('../../lib/musicVideoImageEffects.js', import.meta.url));
+ return [...files, { rel: 'musicVideoImageEffects.js', abs, size: (await stat(abs)).size }];
+ }
+ return files;
+}
+
// Shared toon kit remains a local static graph, including its vendored geometry
// dependency. Authored copies win; unrelated documents gain no dependencies.
async function withSharedToonWorld(files) {
@@ -205,7 +225,7 @@ async function withSharedToonWorld(files) {
async function storeVersionNow(projectId, inputFiles, source, { draft = false, verifyCurrent = () => {} } = {}) {
assertDocumentShape(inputFiles.map((file) => file.rel));
- const files = await withSharedToonWorld(await withSharedLyricType(inputFiles));
+ const files = await withSharedToonWorld(await withSharedImageEffects(await withSharedLyricType(inputFiles)));
const totalBytes = files.reduce((sum, file) => sum + (file.data?.length ?? file.size), 0);
if (files.length > DOCUMENT_MAX_FILES || totalBytes > DOCUMENT_MAX_BYTES) {
throw refuse('Shared document assets exceed the document file or byte limit', 'COMPOSITION_DOCUMENT_TOO_LARGE', 413);
@@ -364,7 +384,7 @@ export async function importDocumentTemplate(projectId, templateId = 'layered')
export async function readTemplateDocumentFiles(templateId = 'layered') {
if (!MUSIC_VIDEO_DOCUMENT_TEMPLATES.includes(templateId)) throw refuse('Unknown composition template', 'VALIDATION_ERROR', 400);
const { files } = await collectTree(join(TEMPLATE_ROOT, templateId), { excludeTests: true });
- return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
+ return new Map((await withSharedImageEffects(await withSharedLyricType(files))).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
}
/**
@@ -374,7 +394,7 @@ export async function readTemplateDocumentFiles(templateId = 'layered') {
*/
export async function readLyricOverlayDocumentFiles() {
const { files } = await collectTree(join(TEMPLATE_ROOT, 'lyricOverlay'), { excludeTests: true });
- return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
+ return new Map((await withSharedImageEffects(await withSharedLyricType(files))).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
}
/** Stage a host-assembled generated document for review before selection. */
@@ -409,6 +429,8 @@ export async function stageGeneratedDocument(projectId, generatedFiles, { verify
// Known compatible shipped layered engines. Recognition only;
// customized engines belong to the author and are never silently replaced.
const UPGRADABLE_LAYERED_ENGINES = new Set([
+ // Last shipped engine before document still-image effects (#11319).
+ '9581bb2d190e62ea2eb6a709bc989b1f913c501236554cd0301379877dc8a5fc',
'dc27611ebaa429c28bd467b003495d2a1c7dc0ede73be0cfc28db4b97c964dcf',
'2556a0905a85958b9107cfa75ab0c5bfde167a708e29b511d577065bb530ea58',
'e2267a068c92a3c7dee49db5f3e57caff1ac11aee1312c0d82e28352850aad3b',
@@ -423,7 +445,7 @@ const engineDigest = bytes => createHash('sha256').update(bytes.toString('utf8')
/**
* Explicitly adopt the shipped layered engine in a new immutable document.
* All authored scripts, HTML, fonts and assets retain their exact bytes, except
- * that a template-shaped index.html gains the shared lyric-type tags the shipped
+ * that a template-shaped index.html gains the shared module tags the shipped
* engine needs (see below). The
* new pointer invalidates revision-bound render/review evidence naturally.
*/
@@ -451,12 +473,11 @@ export function upgradeDocumentEngine(projectId, directory) {
const loaded = await Promise.all(files.map(async file => ({
rel: file.rel, data: file.rel === 'engine.js' ? shipped : await readFile(file.abs),
})));
- // The shipped engine draws its words with the shared lyricType.js (#10583). A page that
- // loads the engine through the template's own tag gets the module's two tags in front
- // of it (storeVersionNow adds the module and its faces); a page that doesn't is the
- // author's and is left alone.
+ // Add the shared lyric-type and still-effect modules only to a template-shaped
+ // page. storeVersionNow supplies missing module files and fonts; authored
+ // entrypoints keep their bytes.
const index = loaded.find(file => file.rel === 'index.html');
- if (index) index.data = Buffer.from(withLyricTypeTags(index.data.toString('utf8')));
+ if (index) index.data = Buffer.from(withLayeredModuleTags(index.data.toString('utf8')));
const result = await storeVersionNow(projectId, loaded, project.composition.document.source || { kind: 'directory', name: null }, { verifyCurrent });
return { ...result, changed: true };
});
diff --git a/server/services/musicVideo/compositionDocument.lyricType.test.js b/server/services/musicVideo/compositionDocument.lyricType.test.js
index 16a196cdb3..bc99a74761 100644
--- a/server/services/musicVideo/compositionDocument.lyricType.test.js
+++ b/server/services/musicVideo/compositionDocument.lyricType.test.js
@@ -38,12 +38,13 @@ async function uploaded(name, files) {
describe('shared lyric-type assets in composition documents', () => {
it('ships lyricType.js, its stylesheet and OFL faces with the layered template', async () => {
const { id } = await projects.createProject({ name: 'Example Song' });
- const { files } = await listed((await importDocumentTemplate(id)).document);
+ const { root, files } = await listed((await importDocumentTemplate(id)).document);
expect(files).toEqual(expect.arrayContaining([
'lyricType.js', 'lyricType.css', 'fonts/archivo-variable.woff2', 'fonts/ibm-plex-mono-500.woff2',
- 'fonts/LICENSE-Archivo.txt', 'fonts/LICENSE-IBM-Plex.txt', 'engine.js', 'index.html',
+ 'fonts/LICENSE-Archivo.txt', 'fonts/LICENSE-IBM-Plex.txt', 'engine.js', 'index.html', 'musicVideoImageEffects.js',
]));
expect(files.filter((file) => file.endsWith('.test.js'))).toEqual([]);
+ expect(await readFile(join(root, 'musicVideoImageEffects.js'), 'utf8')).toBe(await readFile(new URL('../../lib/musicVideoImageEffects.js', import.meta.url), 'utf8'));
});
it('adds the module and faces to an uploaded document that imports it, and keeps a document\'s own copy', async () => {
diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.js
index 90fe23fc02..d78e200b9e 100644
--- a/server/services/musicVideo/documentRender.browser.test.js
+++ b/server/services/musicVideo/documentRender.browser.test.js
@@ -112,6 +112,68 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {
} finally { await _cleanupTestBrowser({ browser, proc, cleanup: () => {} }); }
});
+ // Uniquely catches dropped effect data, missing staged modules and preview/export curve drift.
+ it('develops a still from white identically in the document render and live preview', async () => {
+ await mkdir(PATHS.images, { recursive: true });
+ const image = await sharp({ create: { width: 64, height: 36, channels: 3, background: '#ee6721' } }).png().toBuffer();
+ await writeFile(join(PATHS.images, 'effect-still.png'), image);
+ const created = await projects.createProject({ name: 'Synthetic image effect' });
+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,
+ audioAnalysis: { durationSec: 2, sections: [] },
+ scenes: [{ sceneId: 'develop', startSec: 0.25, endSec: 2, visualLayer: 'still', stillMove: 'hold',
+ referenceImageId: 'effect-still.png', imageEffect: 'flash-develop', imageEffectSec: 0.5 }],
+ } }));
+ await importDocumentTemplate(created.id);
+ const project = await projects.getProject(created.id);
+ const { openIsolatedDocumentPreview } = await import('./documentRender.js');
+ const capture = await openIsolatedDocumentPreview({ project, jobId: 'still-effect', signal: testSignal });
+ const page = await browser.newPage();
+ try {
+ const loadPreview = async current => {
+ const preview = await buildDocumentPreview(current);
+ await page.setContent(preview.html);
+ await page.evaluate(({ key, encoded }) => {
+ addEventListener('message', ({ data }) => {
+ if (data?.type !== 'portos-mv:request') return;
+ const blob = new Blob([Uint8Array.from(atob(encoded), c => c.charCodeAt(0))], { type: 'image/png' });
+ window.postMessage({ type: 'portos-mv:asset', key, blob }, '*');
+ });
+ window.postMessage({ type: 'portos-mv:manifest', keys: [key] }, '*');
+ }, { key: preview.assets[0].key, encoded: image.toString('base64') });
+ await page.waitForFunction(() => globalThis.portosComposition);
+ };
+ await loadPreview(project);
+ const pixels = [];
+ for (const t of [0.25, 0.5, 0.75, 1.25]) {
+ const sample = `(async () => {
+ await portosComposition.seek(${t});
+ const canvas = document.getElementById('stage');
+ return [...canvas.getContext('2d').getImageData(canvas.width / 2, canvas.height / 2, 1, 1).data].slice(0, 3);
+ })()`;
+ const live = await page.evaluate(sample);
+ const rendered = await capture.page.evaluate(sample);
+ expect(rendered).toEqual(live);
+ pixels.push(live);
+ }
+ expect(pixels[0]).toEqual([255, 255, 255]);
+ expect(pixels[1][0] - pixels[1][2]).toBeLessThan(pixels[2][0] - pixels[2][2]);
+ expect(pixels[2][0]).toBeGreaterThan(pixels[2][1] + 80);
+ expect(pixels[2][1]).toBeGreaterThan(pixels[2][2] + 40);
+ // A developed still is exactly the no-effect picture at the same song time.
+ await loadPreview({ ...project, scenes: project.scenes.map(scene => ({ ...scene, imageEffect: null })) });
+ const held = await page.evaluate(async () => {
+ await globalThis.portosComposition.seek(1.25);
+ const canvas = document.getElementById('stage');
+ return [...canvas.getContext('2d').getImageData(canvas.width / 2, canvas.height / 2, 1, 1).data].slice(0, 3);
+ });
+ expect(held).toEqual(pixels[3]);
+ } finally {
+ await page.close();
+ await capture.page.close();
+ await rm(capture.scratchRoot, { recursive: true, force: true });
+ }
+ }, 60000);
+
it('keeps network refusal inside the owned capture browser and leaves the browsing session alive', async () => {
const { openComposition } = await import('../htmlComposition/browser.js');
const directory = 'compositions/synthetic-owned-refusal';
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index fad7600410..d2fd9cf777 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -1,4 +1,5 @@
import { assertMusicVideoMediaSelections, musicVideoMediaMode } from '../../lib/musicVideoMediaPolicy.js';
+import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
import { musicVideoGradeFilter } from '../../lib/musicVideoGrade.js';
import { filmLookRuntimeSource, isFilmLookNeutral } from '../../lib/filmLook.js';
import { titleCardVisible } from '../../lib/musicVideoTitleCard.js';
@@ -190,6 +191,7 @@ export function buildDocumentData(project, { media = new Map(), frame, clock, so
endSec: start != null && end != null && end > start ? end : null,
shotMode: scene.shotMode === 'performance' ? 'performance' : 'cutaway',
visualLayer: documentSceneVisualLayer(project, scene, { generated }),
+ imageEffect: sceneImageEffect(scene, { layered: true }),
stillMove: typeof scene.stillMove === 'string' ? scene.stillMove : null,
// #10589: the planned camera move the template's camera rig follows (only when planned).
...(typeof scene.camera?.move === 'string' ? { camera: structuredClone(scene.camera) } : {}),
diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.js
index e6cdc76cc2..30b4acc870 100644
--- a/server/services/musicVideo/documentRender.test.js
+++ b/server/services/musicVideo/documentRender.test.js
@@ -80,7 +80,7 @@ async function fixture({ shipped = {} } = {}) {
composition: { mode: 'document', textCues: [{ id: 'c1', text: 'hero', startSec: 1, endSec: 2, emphasis: 'hero' }], overlay: { enabled: true, ticker: ['a'] } },
scenes: [
{ sceneId: 'a', order: 0, startSec: 0, endSec: 4, videoHistoryId: 'vh-a', referenceImageId: 'still-b.png' },
- { sceneId: 'b', order: 1, startSec: 4, endSec: 8, videoHistoryId: 'vh-gone', referenceImageId: 'still-b.png', visualLayer: 'still', stillMove: 'push' },
+ { sceneId: 'b', order: 1, startSec: 4, endSec: 8, videoHistoryId: 'vh-gone', referenceImageId: 'still-b.png', visualLayer: 'still', stillMove: 'push', imageEffect: 'flash-develop', imageEffectSec: 6 },
{ sceneId: 'c', order: 2, startSec: 8, endSec: 9, visualLayer: 'card', cardText: 'Title' },
{ sceneId: 'd', order: 3, startSec: 9, endSec: 12, shotMode: 'performance', videoHistoryId: 'vh-a', takes: [performanceTake('vh-a')] },
],
@@ -138,8 +138,9 @@ describe('composition document staging', () => {
const byId = Object.fromEntries(data.scenes.map((s) => [s.sceneId, s]));
expect(byId.a.media).toMatchObject({ kind: 'video', src: 'media/scene-a.mp4', inSec: 0, outSec: 4, fps: 24 });
expect(byId.b.media).toMatchObject({ kind: 'image', src: 'media/scene-b.png' });
- expect(byId.b).toMatchObject({ visualLayer: 'still', stillMove: 'push' });
- expect(byId.c).toMatchObject({ media: null, visualLayer: 'card', cardText: 'Title' });
+ expect(byId.b).toMatchObject({ visualLayer: 'still', stillMove: 'push', imageEffect: { effect: 'flash-develop', developSec: 6 } });
+ expect(byId.a.imageEffect).toBeNull();
+ expect(byId.c).toMatchObject({ imageEffect: null, media: null, visualLayer: 'card', cardText: 'Title' });
expect(byId.d.media).toMatchObject({ kind: 'video', src: 'media/scene-d.mp4', inSec: 0.5, outSec: 3.5 });
expect(data.render).toEqual({ width: 1920, height: 1080, fps: 24, frames: 720, durationSec: 30 });
expect(data.song).toMatchObject({ durationSec: 30, beats: [0, 0.5], downbeats: [0] });
diff --git a/server/services/musicVideo/documentTemplates/layered/engine.js b/server/services/musicVideo/documentTemplates/layered/engine.js
index 37a3f92685..774c4355de 100644
--- a/server/services/musicVideo/documentTemplates/layered/engine.js
+++ b/server/services/musicVideo/documentTemplates/layered/engine.js
@@ -294,14 +294,14 @@
let lyricType = null;
// lyricType.js is a module, so it runs after this classic script; it publishes
// itself on globalThis before DOMContentLoaded.
- const lyricTypeModule = () => new Promise((resolve, reject) => {
- const check = () => (globalThis.PORTOS_LYRIC_TYPE ? resolve(globalThis.PORTOS_LYRIC_TYPE)
- : reject(new Error('lyricType.js did not load — the layered template needs it beside index.html')));
- if (globalThis.PORTOS_LYRIC_TYPE || document.readyState !== 'loading') check();
+ const documentModule = (key, filename) => new Promise((resolve, reject) => {
+ const check = () => (globalThis[key] ? resolve(globalThis[key])
+ : reject(new Error(`${filename} did not load — the layered template needs it beside index.html`)));
+ if (globalThis[key] || document.readyState !== 'loading') check();
else document.addEventListener('DOMContentLoaded', check, { once: true });
});
async function buildLyricType() {
- const { createLyricType } = await lyricTypeModule();
+ const { createLyricType } = await documentModule('PORTOS_LYRIC_TYPE', 'lyricType.js');
const words = { palette: { fill: C.paper, ink: C.ink, accent: C.accent, strike: C.alert } };
// Subtitles: the director's subtitle cues when there are any, else the lyrics.
const lines = SUBTITLE_CUES.length ? [...HERO_LINES, ...SUBTITLE_CUES] : null;
@@ -500,9 +500,13 @@
};
let footageVisibility = null;
+ let imageEffects = null;
+
function render(t, scene, source, state, reviewFootage = false) {
ctx.fillStyle = C.ink; ctx.fillRect(0, 0, W, H);
const authored = sectionFunction(t);
+ const effect = source && scene?.visualLayer === 'still' && scene.media?.kind === 'image' && scene.imageEffect
+ ? imageEffects?.imageEffectState(scene.imageEffect, t - scene.startSec) : null;
if (scene) {
const lt = t - scene.startSec; const d = scene.endSec - scene.startSec;
const card = authored?.fn ? null : cardFor(scene);
@@ -510,7 +514,11 @@
else if (source) {
const energetic = isHighEnergy(t) || scene.shotMode === 'performance';
const punch = 1 + 0.018 * pulse(t, energetic ? beats : downs, 8) * state.reactiveGain;
+ ctx.save();
+ if (effect) ctx.filter = `contrast(${effect.contrast}) saturate(${effect.saturation})`;
drawCover(source, cameraView(scene, seg(t, scene.startSec, scene.endSec)), punch);
+ ctx.restore();
+ if (effect) _flash(effect.fog, imageEffects.IMAGE_EFFECT_FOG_COLOR);
}
if (isHighEnergy(t)) glitch(t, pulse(t, downs, 8) * state.reactiveGain);
}
@@ -532,6 +540,7 @@
}
vignette(0.5);
grain(t, 0.09);
+ if (effect) _flash(effect.flash);
drawHud(t);
drawNarrativeEvents(state);
if (lyricType) lyricType.draw(ctx, t, { width: W, height: H });
@@ -543,6 +552,9 @@
if (window.PORTOS_MV_ASSETS) assetUrls = await window.PORTOS_MV_ASSETS;
await Promise.all([F.stencil(40), F.mono(20), F.mono(20, 600), F.cond(20), F.cond(20, 700)].map((font) => document.fonts.load(font)));
await buildLyricType();
+ if (SCENES.some(scene => scene.imageEffect && scene.visualLayer === 'still')) {
+ imageEffects = await documentModule('PORTOS_MV_IMAGE_EFFECTS', 'musicVideoImageEffects.js');
+ }
buildGrain();
})();
globalThis.portosComposition = {
diff --git a/server/services/musicVideo/documentTemplates/layered/index.html b/server/services/musicVideo/documentTemplates/layered/index.html
index 289ce11bd8..a37bc17ac7 100644
--- a/server/services/musicVideo/documentTemplates/layered/index.html
+++ b/server/services/musicVideo/documentTemplates/layered/index.html
@@ -23,6 +23,10 @@ html, body { margin: 0; height: 100%; background: #000; overflow: hidden; }
<script src="portos-mv.js"></script><!-- portos-allow-hidden-content: the render-time data file -->
<script src="cameraRig.js"></script><!-- portos-allow-hidden-content: the reviewed camera-move paths the engine reads -->
<script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->
+<!-- portos-allow-hidden-content: shared still-image effect curves --><script type="module">
+import { imageEffectState, IMAGE_EFFECT_FOG_COLOR } from './musicVideoImageEffects.js';
+globalThis.PORTOS_MV_IMAGE_EFFECTS = { imageEffectState, IMAGE_EFFECT_FOG_COLOR };
+</script>
<script src="engine.js"></script><!-- portos-allow-hidden-content: the reviewed template engine beside this page -->
</body>
</html>