Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
import { mkdir, mkdtemp, readdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { execFileSync, spawn } from 'node:child_process';
import { chromium } from 'playwright-core';
import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
import { findFfmpeg, probeVideoDuration } from '../../lib/ffmpeg.js';
import { PATHS } from '../../lib/fileUtils.js';
import { _cleanupTestBrowser, _createTestArtifactsRoot, _waitForTestChrome } from '../htmlComposition/testBrowserCleanup.js';
import { beatSnapClips, buildMusicVideoFfmpegArgs, resolveSceneClips } from './render.js';
import { composedLyricWindows, projectTypographyPlan, renderableCues, sectionCardCues } from './composition.js';
import { COMPOSITION_SCRATCH_DIR, removeCompositionScratch, renderTypographyOverlays } from './compositionRender.js';
import { resolveTestChrome } from '../../lib/browserSuiteGate.js';
let endpoint;
vi.mock('../browserService.js', () => ({ cdpRequest: path => fetch(`${endpoint}${path}`) }));
vi.mock('../../lib/fileUtils.js', async importOriginal => makePathsProxy(await importOriginal(), {
dataRoot: () => lazyTempDataRoot('portos-mv-composition-'),
}));
const chrome = resolveTestChrome();
const ffmpeg = await findFfmpeg();
// Vitest runs no hooks for a file whose every test is skipped, so on a runner
// without Chrome or ffmpeg the afterAll below never fires — yet importing
// compositionRender.js above already minted the lazy root (#9045).
if (!chrome || !ffmpeg) cleanupTempDataRoots();
let proc;
let browser;
let artifacts;
const detachedChromeProcessGroupId = () => process.platform === 'win32' ? undefined : proc?.pid;
const FPS = 24;
const run = (args) => execFileSync(ffmpeg, ['-v', 'error', ...args], { maxBuffer: 256 * 1024 * 1024 });
// Decode a video to raw frames of `pix_fmt`.
const frames = (path, pixFmt) => run(['-i', path, '-f', 'rawvideo', '-pix_fmt', pixFmt, '-']);
// Every overlay pixel with visible alpha lies inside the 10% title-safe area.
function assertInsideSafeArea(path, width, height) {
const raw = frames(path, 'rgba');
const frameBytes = width * height * 4;
const x0 = width * 0.1; const x1 = width * 0.9; const y0 = height * 0.1; const y1 = height * 0.9;
let drawn = 0;
for (let f = 0; f < raw.length / frameBytes; f++) {
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
if (raw[f * frameBytes + (y * width + x) * 4 + 3] <= 8) continue;
drawn += 1;
if (x < x0 || x >= x1 || y < y0 || y >= y1) throw new Error(`frame ${f}: text pixel at (${x},${y}) outside the safe area`);
}
}
}
return drawn;
}
describe.skipIf(!chrome || !ffmpeg)('music-video typography overlay with real Chrome and ffmpeg (#8984)', () => {
beforeAll(async () => {
// A per-run profile: a fixed path can be locked by an earlier run, and Chrome's exit must not hide behind a timeout.
await mkdir(PATHS.data, { recursive: true });
const profile = await mkdtemp(join(PATHS.data, 'chrome-test-profile-'));
proc = spawn(chrome, ['--headless=new', '--mute-audio', '--no-sandbox', '--no-first-run', '--disable-background-networking', '--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank'], { stdio: ['ignore', 'ignore', 'pipe'], detached: process.platform !== 'win32' });
const ws = await _waitForTestChrome(proc);
endpoint = new URL(ws).origin.replace('ws:', 'http:');
artifacts = _createTestArtifactsRoot();
browser = await chromium.connectOverCDP(endpoint, { artifactsDir: artifacts.path });
}, 30000);
afterAll(async () => {
await _cleanupTestBrowser({ browser, proc, artifacts, cleanup: cleanupTempDataRoots, detachedProcessGroupId: detachedChromeProcessGroupId() });
}, 25000);
it('lays timed text over the cut on one continuous audio master, covering the edit within a frame', async () => {
const dir = join(PATHS.data, 'fixture');
await mkdir(dir, { recursive: true });
const [w, h] = [320, 180];
// Two 1.5s "scenes" (blue, green) cut over a 3s tone.
run(['-f', 'lavfi', '-i', `color=blue:s=${w}x${h}:r=${FPS}:d=1.5`, '-y', join(dir, 'a.mp4')]);
run(['-f', 'lavfi', '-i', `color=green:s=${w}x${h}:r=${FPS}:d=1.5`, '-y', join(dir, 'b.mp4')]);
run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-y', join(dir, 'song.wav')]);
const clip = (name) => ({ videoPath: join(dir, name), width: w, height: h, fps: FPS, duration: 1.5, sourceSec: 1.5, loop: false, inSec: 0, outSec: 1.5 });
const clips = [clip('a.mp4'), clip('b.mp4')];
// A hero line spanning the cut at 1.5s.
const cues = [{ id: 'c1', text: 'ACROSS THE CUT', startSec: 1, endSec: 2, template: 'fade', placement: 'center', emphasis: 'hero' }];
const overlays = await renderTypographyOverlays({ jobId: 'job-a', cues, style: { color: '#ffffff', font: 'sans' }, width: w, height: h, fps: FPS, durationSec: 3 });
expect(overlays).toEqual([{ path: expect.stringContaining('overlay-0.mov'), startSec: 1, durationSec: 1 }]);
const output = join(dir, 'out.mp4');
const { args } = buildMusicVideoFfmpegArgs(clips, join(dir, 'song.wav'), output, { audioDurationSec: 3, overlays });
run(args.filter((arg, i) => !(arg === '-progress' || args[i - 1] === '-progress')));
expect(Math.abs(await probeVideoDuration(output) - 3)).toBeLessThanOrEqual(1 / FPS);
// One continuous master: the tone never drops out across the cut or the text.
const pcm = run(['-i', output, '-map', '0:a', '-ac', '1', '-ar', '8000', '-f', 's16le', '-']);
const samples = new Int16Array(Uint8Array.from(pcm).buffer);
expect(Math.abs(samples.length / 8000 - 3)).toBeLessThanOrEqual(1 / FPS);
for (let start = 0; start + 80 <= samples.length - 800; start += 80) {
let peak = 0;
for (let i = start; i < start + 80; i++) peak = Math.max(peak, Math.abs(samples[i]));
expect(peak, `silent 10ms window at ${start / 8000}s`).toBeGreaterThan(1000);
}
const rgb = frames(output, 'rgb24');
const frameBytes = w * h * 3;
expect(rgb.length / frameBytes).toBe(3 * FPS);
const whiteIn = (f) => {
let n = 0;
for (let p = 0; p < w * h; p++) {
const o = f * frameBytes + p * 3;
if (rgb[o] > 200 && rgb[o + 1] > 200 && rgb[o + 2] > 200) n += 1;
}
return n;
};
// No text before the cue; text drawn over BOTH scenes on either side of the cut; gone after.
expect(whiteIn(12)).toBe(0);
expect(whiteIn(33)).toBeGreaterThan(100);
expect(whiteIn(39)).toBeGreaterThan(100);
expect(whiteIn(60)).toBe(0);
// Footage passes through untouched outside the text: the corner keeps its scene colour.
const corner = (f) => [...rgb.subarray(f * frameBytes, f * frameBytes + 3)];
expect(corner(33)[2]).toBeGreaterThan(200); // blue scene
expect(corner(39)[1]).toBeGreaterThan(100); // green scene
// Same time, same frame: a second capture of the same cues decodes identically.
const again = await renderTypographyOverlays({ jobId: 'job-b', cues, style: {}, width: w, height: h, fps: FPS, durationSec: 3 });
expect(frames(again[0].path, 'rgba').equals(frames(overlays[0].path, 'rgba'))).toBe(true);
await removeCompositionScratch('job-a');
await removeCompositionScratch('job-b');
expect(await readdir(join(PATHS.data, COMPOSITION_SCRATCH_DIR))).toEqual([]);
}, 60000);
it('draws a composed render\'s lyrics in the shared lyric type when it has no subtitle cues', async () => {
const [w, h] = [320, 180];
const words = (text, start) => text.split(' ').map((word, i) => ({ w: word, startSec: start + i * 0.3, endSec: start + i * 0.3 + 0.25 }));
const project = {
audioAnalysis: { durationSec: 4 }, lyricMarkers: [], scenes: [],
lyricCues: [{ text: 'found a shot', startSec: 1, endSec: 1.85, words: words('found a shot', 1) }],
composition: { mode: 'composed', textCues: [] },
};
const windows = composedLyricWindows(project, 4);
const overlays = await renderTypographyOverlays({ jobId: 'job-lyrics', cues: [], style: {}, width: w, height: h, fps: FPS, durationSec: 4,
lyrics: { project, windows } });
expect(overlays).toEqual([{ path: expect.stringContaining('lyrics-overlay-0.mov'), startSec: 1, durationSec: 69 / FPS }]); // 1s to 1.85s + the 2s tail, snapped out to whole frames
const raw = frames(overlays[0].path, 'rgba');
const frameBytes = w * h * 4;
const drawn = (f) => {
const box = { n: 0, minY: h };
for (let p = 0; p < w * h; p++) {
if (raw[f * frameBytes + p * 4 + 3] <= 8) continue;
box.n += 1; box.minY = Math.min(box.minY, Math.floor(p / w));
}
return box;
};
// The whole line is up after its last onset, in the lower-left zone, and it has left by the window's end.
const shown = drawn(Math.round(0.9 * FPS));
expect(shown.n).toBeGreaterThan(100);
expect(shown.minY).toBeGreaterThan(h / 2);
expect(drawn(raw.length / frameBytes - 1).n).toBe(0);
await removeCompositionScratch('job-lyrics');
}, 60000);
// #8985 acceptance: footage, a still with a push-in, a code-rendered title
// card and animated text on one 30s timebase under one continuous song.
it('renders footage, a still, a title card and text on one timebase without gaps, repeats or audio resets', async () => {
const [w, h] = [320, 180];
await mkdir(PATHS.videos, { recursive: true });
await mkdir(PATHS.images, { recursive: true });
const dir = join(PATHS.data, 'fixture-layers');
await mkdir(dir, { recursive: true });
run(['-f', 'lavfi', '-i', `color=blue:s=${w}x${h}:r=${FPS}:d=8`, '-y', join(PATHS.videos, 'a.mp4')]);
run(['-f', 'lavfi', '-i', `color=magenta:s=${w}x${h}:r=${FPS}:d=11`, '-y', join(PATHS.videos, 'b.mp4')]);
run(['-f', 'lavfi', '-i', 'color=red:s=640x360', '-frames:v', '1', '-y', join(PATHS.images, 'still.png')]);
run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=30', '-y', join(dir, 'song.wav')]);
const entry = (id, filename, sec) => ({ id, filename, width: w, height: h, fps: FPS, numFrames: sec * FPS });
await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([entry('h-a', 'a.mp4', 8), entry('h-b', 'b.mp4', 11)]));
const scenes = [
{ sceneId: 's1', order: 0, videoHistoryId: 'h-a', startSec: 0, endSec: 8, beatAligned: true, loop: false },
// An off-grid boundary (15.37s is not a whole frame) and no footage.
{ sceneId: 's2', order: 1, visualLayer: 'still', stillMove: 'push', referenceImageId: 'still.png', videoHistoryId: null, startSec: 8, endSec: 15.37 },
{ sceneId: 's3', order: 2, visualLayer: 'card', cardText: 'CHAPTER TWO', cardColor: '#202020', videoHistoryId: null, startSec: 15.37, endSec: 19 },
{ sceneId: 's4', order: 3, videoHistoryId: 'h-b', startSec: 19, endSec: 30, beatAligned: true, loop: false },
];
const composition = { mode: 'composed', style: { color: '#ffffff', font: 'sans' },
textCues: [{ id: 'c1', text: 'OVER THE CUT', startSec: 7, endSec: 9, template: 'rise', placement: 'center', emphasis: 'hero' }] };
const clips = beatSnapClips(await resolveSceneClips({ scenes, composition }, { layered: true }), null, { scenes });
const song = join(dir, 'song.wav');
const output = join(dir, 'out.mp4');
const plan = buildMusicVideoFfmpegArgs(clips, song, output, { audioDurationSec: 30, frameGrid: true });
// Every section starts where the previous one ends, on the frame grid.
expect(plan.sections.map((s) => [s.layer, Math.round(s.startSec * FPS), Math.round(s.endSec * FPS)])).toEqual([
['footage', 0, 192], ['still', 192, 369], ['card', 369, 456], ['footage', 456, 720],
]);
const cues = [...renderableCues(composition, plan.totalDuration), ...sectionCardCues(clips, plan.sections, plan.totalDuration)];
const overlays = await renderTypographyOverlays({ jobId: 'job-layers', cues, style: composition.style, width: w, height: h, fps: FPS, durationSec: plan.totalDuration });
const { args } = buildMusicVideoFfmpegArgs(clips, song, output, { audioDurationSec: 30, overlays, frameGrid: true });
run(args.filter((arg, i) => !(arg === '-progress' || args[i - 1] === '-progress')));
expect(Math.abs(await probeVideoDuration(output) - 30)).toBeLessThanOrEqual(1 / FPS);
const pcm = run(['-i', output, '-map', '0:a', '-ac', '1', '-ar', '8000', '-f', 's16le', '-']);
const samples = new Int16Array(Uint8Array.from(pcm).buffer);
expect(Math.abs(samples.length / 8000 - 30)).toBeLessThanOrEqual(1 / FPS);
for (let start = 0; start + 80 <= samples.length - 800; start += 80) {
let peak = 0;
for (let i = start; i < start + 80; i++) peak = Math.max(peak, Math.abs(samples[i]));
expect(peak, `silent 10ms window at ${start / 8000}s`).toBeGreaterThan(1000);
}
const rgb = frames(output, 'rgb24');
const frameBytes = w * h * 3;
expect(rgb.length / frameBytes).toBe(30 * FPS);
const corner = (f) => [...rgb.subarray(f * frameBytes, f * frameBytes + 3)];
const whiteIn = (f) => {
let n = 0;
for (let p = 0; p < w * h; p++) {
const o = f * frameBytes + p * 3;
if (rgb[o] > 200 && rgb[o + 1] > 200 && rgb[o + 2] > 200) n += 1;
}
return n;
};
const isBlue = ([r, g, b]) => b > 200 && r < 60 && g < 60;
const isRed = ([r, g, b]) => r > 200 && g < 60 && b < 60;
const isCard = ([r, g, b]) => Math.max(r, g, b) < 60;
const isMagenta = ([r, g, b]) => r > 200 && b > 200 && g < 60;
// Each section holds its own picture right up to its boundary frame — no
// gap, no black filler, and no footage repeated into the still or card.
expect(isBlue(corner(0)) && isBlue(corner(191))).toBe(true);
expect(isRed(corner(192)) && isRed(corner(368))).toBe(true);
expect(isCard(corner(369)) && isCard(corner(455))).toBe(true);
expect(isMagenta(corner(456)) && isMagenta(corner(719))).toBe(true);
// The lyric cue is drawn across the footage→still cut; the card carries its own title.
expect(whiteIn(100)).toBe(0);
expect(whiteIn(186)).toBeGreaterThan(100);
expect(whiteIn(200)).toBeGreaterThan(100);
expect(whiteIn(280)).toBe(0);
expect(whiteIn(420)).toBeGreaterThan(100);
expect(whiteIn(600)).toBe(0);
await removeCompositionScratch('job-layers');
}, 120000);
// #11185: a silence card planned with no text shows its spoken line (the
// " / " joiner becomes a line break) in the lyric type's cream and ink, with
// its own face shipped, inside the title-safe area and on no plate.
it('draws a blank card\'s lyric line in the lyric type, inside the safe area', async () => {
const [w, h] = [384, 256];
const scenes = [{ sceneId: 'quiet', order: 0, visualLayer: 'card', cardText: '', lyricText: 'Example spoken line / Then the answer', startSec: 0, endSec: 2 }];
const clips = await resolveSceneClips({ scenes }, { layered: true });
expect(clips[0].cardText).toBe('Example spoken line / Then the answer');
const plan = buildMusicVideoFfmpegArgs(clips, null, join(PATHS.data, 'unused.mp4'), { audioDurationSec: 2, frameGrid: true });
const cues = sectionCardCues(clips, plan.sections, 2);
expect(cues).toMatchObject([{ text: 'Example spoken line\nThen the answer', card: true }]);
const overlays = await renderTypographyOverlays({ jobId: 'job-card', cues, style: {}, width: w, height: h, fps: 12, durationSec: 2 });
expect(existsSync(join(PATHS.data, COMPOSITION_SCRATCH_DIR, 'job-card', 'doc', 'archivo-variable.woff2'))).toBe(true);
expect(assertInsideSafeArea(overlays[0].path, w, h)).toBeGreaterThan(1000);
// A mid-card frame shows the cream fill inside its ink outline.
const raw = frames(overlays[0].path, 'rgba');
const frame = raw.subarray(12 * w * h * 4, 13 * w * h * 4);
let cream = 0;
let ink = 0;
for (let p = 0; p < w * h; p++) {
const [r, g, b, a] = frame.subarray(p * 4, p * 4 + 4);
if (a < 250) continue;
if (r > 220 && g > 210 && b > 190 && b < r) cream += 1;
else if (r < 40 && g < 40 && b < 40) ink += 1;
}
expect(cream).toBeGreaterThan(300);
expect(ink).toBeGreaterThan(300);
await removeCompositionScratch('job-card');
}, 60000);
// The opening title card reaches composed renders: the overlay page draws it
// with the same runtime as the live preview, in its lower-left corner.
it('draws the opening title card over a composed render, lower left', async () => {
const [w, h] = [384, 216];
const project = { composition: { mode: 'composed' }, titleCard: {
enabled: true, title: 'Example Song', artist: 'Example Artist', year: '2026', style: 'mtv',
position: 'lower-left', startSec: 0, endSec: 2, fadeSec: 0.25 } };
const { cues, style } = projectTypographyPlan(project, [], [], 2);
const overlays = await renderTypographyOverlays({ jobId: 'job-title-card', cues, style, width: w, height: h, fps: 12, durationSec: 2 });
expect(overlays).toHaveLength(1);
const raw = frames(overlays[0].path, 'rgba');
const frame = raw.subarray(12 * w * h * 4, 13 * w * h * 4);
let lowerLeft = 0;
let elsewhere = 0;
for (let p = 0; p < w * h; p++) {
if (frame[p * 4 + 3] < 128) continue;
const [x, y] = [p % w, Math.floor(p / w)];
if (x < w / 2 && y > h / 2) lowerLeft += 1;
else elsewhere += 1;
}
expect(lowerLeft).toBeGreaterThan(1000);
expect(elsewhere).toBe(0);
await removeCompositionScratch('job-title-card');
}, 60000);
it.each([['16:9', 480, 270], ['9:16', 270, 480], ['a small 16:9 frame', 160, 90]])('keeps every motion template inside the title-safe area at %s', async (_label, width, height) => {
const long = 'A deliberately long lyric line that has to wrap and shrink to stay inside the frame';
const cues = [
{ id: 'hero', text: long, startSec: 0, endSec: 0.75, template: 'pop', placement: 'center', emphasis: 'hero' },
{ id: 'low', text: long, startSec: 1, endSec: 1.75, template: 'rise', placement: 'lower', emphasis: 'subtitle' },
{ id: 'up', text: long, startSec: 2, endSec: 2.75, template: 'typewriter', placement: 'upper', emphasis: 'subtitle' },
// Worst case: a maximum-length hero line that cannot fit even at the minimum size.
{ id: 'max', text: 'W'.repeat(500), startSec: 3, endSec: 3.75, template: 'rise', placement: 'lower', emphasis: 'hero' },
];
const overlays = await renderTypographyOverlays({ jobId: `safe-${width}`, cues, style: { font: 'serif' }, width, height, fps: 12, durationSec: 4 });
let drawn = 0;
for (const overlay of overlays) drawn += assertInsideSafeArea(overlay.path, width, height);
expect(drawn).toBeGreaterThan(1000);
}, 60000);
it('cancels a capture in progress and leaves no overlay behind', async () => {
const controller = new AbortController();
const cues = [{ id: 'c', text: 'Cancel me', startSec: 0, endSec: 20, template: 'fade', placement: 'lower', emphasis: 'subtitle' }];
const capture = renderTypographyOverlays({ jobId: 'job-cancel', cues, style: {}, width: 320, height: 180, fps: 24, durationSec: 20, signal: controller.signal,
onProgress: (fraction) => { if (fraction > 0.05) controller.abort(new Error('Render cancelled')); } });
await expect(capture).rejects.toThrow();
await removeCompositionScratch('job-cancel');
expect(existsSync(join(PATHS.data, COMPOSITION_SCRATCH_DIR, 'job-cancel'))).toBe(false);
}, 60000);
});