Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

[issue-11630] Align paused footage and saved frame colors (#11645)

[._.]/c53c07052dd3
diff --git a/client/src/components/musicVideo/PreviewVideo.jsx b/client/src/components/musicVideo/PreviewVideo.jsx
new file mode 100644
index 0000000000..249f672e96
--- /dev/null
+++ b/client/src/components/musicVideo/PreviewVideo.jsx
@@ -0,0 +1,64 @@
+import { useEffect, useRef } from 'react';
+
+/** Native playback, with the same sRGB decoded surface as Save frame when paused. */
+export default function PreviewVideo({ videoRef, audioRef, active = true, className, style, ...props }) {
+ const canvasRef = useRef(null);
+ useEffect(() => {
+ const video = videoRef.current;
+ const canvas = canvasRef.current;
+ const audio = audioRef.current;
+ if (!video || !canvas || !audio) return undefined;
+ let handle = 0;
+ const invalidate = () => {
+ cancelAnimationFrame(handle);
+ delete video.dataset.pausedFrame;
+ canvas.style.visibility = 'hidden';
+ video.style.visibility = audio.paused ? 'hidden' : '';
+ };
+ const paint = () => {
+ if (!active || !audio.paused || video.seeking || video.readyState < 2) return;
+ const scale = Math.min(1, Math.sqrt(16777216 / (video.videoWidth * video.videoHeight)));
+ const width = Math.max(1, Math.round(video.videoWidth * scale));
+ const height = Math.max(1, Math.round(video.videoHeight * scale));
+ if (!video.videoWidth || !video.videoHeight) return;
+ canvas.width = width; canvas.height = height;
+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });
+ if (!ctx) { video.dataset.previewError = 'Frame capture unavailable'; return; }
+ try {
+ ctx.drawImage(video, 0, 0, width, height);
+ delete video.dataset.previewError;
+ video.dataset.pausedFrame = String(video.currentTime);
+ canvas.style.visibility = '';
+ video.style.visibility = 'hidden';
+ } catch { invalidate(); video.dataset.previewError = 'Video frame unavailable'; }
+ };
+ const schedule = () => {
+ invalidate();
+ if (!active || !audio.paused) return;
+ // seeked precedes presentation in Chromium; let the decoded frame reach
+ // the compositor before drawing. A new seek cancels this pending paint.
+ handle = requestAnimationFrame(() => { handle = requestAnimationFrame(paint); });
+ };
+ const events = ['loadeddata', 'seeked', 'pause'];
+ for (const event of events) video.addEventListener(event, schedule);
+ video.addEventListener('seeking', invalidate);
+ video.addEventListener('emptied', invalidate);
+ audio.addEventListener('play', invalidate);
+ audio.addEventListener('pause', schedule);
+ schedule();
+ return () => {
+ invalidate();
+ for (const event of events) video.removeEventListener(event, schedule);
+ video.removeEventListener('seeking', invalidate);
+ video.removeEventListener('emptied', invalidate);
+ audio.removeEventListener('play', invalidate);
+ audio.removeEventListener('pause', schedule);
+ };
+ }, [videoRef, audioRef, active, props.src]);
+ const fit = className.includes('object-cover') ? 'object-cover' : 'object-contain';
+ return <div className={className} style={style}>
+ <video {...props} ref={videoRef} data-preview-video="true" className={`h-full w-full ${fit}`} />
+ <canvas ref={canvasRef} tabIndex={-1} aria-hidden="true" data-paused-video="true"
+ className={`pointer-events-none absolute inset-0 h-full w-full ${fit}`} style={{ visibility: 'hidden' }} />
+ </div>;
+}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 03c3e2ce11..339692db31 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -16,6 +16,7 @@ import StatsOutroCanvas from './StatsOutroCanvas.jsx';
import ShotStepper from './ShotStepper.jsx';
import FrameStepper from './FrameStepper.jsx';
import { capturePreviewFrame, lyricFrameCaptureBridge } from '../../lib/musicVideoFrameCapture.js';
+import PreviewVideo from './PreviewVideo.jsx';
import { sceneFrameFit } from './SceneFrameFit.jsx';
import TransitionGlitch from './TransitionGlitch.jsx';
import ImageEffectStill from './ImageEffectStill.jsx';
@@ -346,9 +347,11 @@ function ShotClip({ scene, audioRef, time, current, preload = 'auto' }) {
// The final render's frame fit: zoomed to fill, or whole over the still, the VHS edge or black bars.
const fit = sceneFrameFit(scene);
return (
- <video
- key={reload}
- ref={videoRef}
+ <PreviewVideo
+ key={`${src}:${reload}`}
+ videoRef={videoRef}
+ audioRef={audioRef}
+ active={current}
src={src}
muted
playsInline
@@ -436,7 +439,8 @@ function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {
const el = mediaRef.current;
const audio = audioRef.current;
if (!el) return;
- el.style.opacity = String(incomingOpacity(toggle, t, fps));
+ const surface = el.tagName === 'VIDEO' ? el.parentElement : el;
+ surface.style.opacity = String(incomingOpacity(toggle, t, fps));
if (el.tagName !== 'VIDEO' || el.readyState < 1) return;
const duration = Number.isFinite(el.duration) ? el.duration : null;
const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true, fromSec: toggle.windowStartSec });
@@ -455,7 +459,7 @@ function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {
const fit = sceneFrameFit(scene);
const className = `pointer-events-none absolute inset-0 h-full w-full ${src && fit !== 'fill' ? 'object-contain' : 'object-cover'} ${fit === 'black' ? 'bg-black' : ''}`;
if (src) {
- return <video ref={mediaRef} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}
+ return <PreviewVideo key={src} videoRef={mediaRef} audioRef={audioRef} active={time >= toggle.windowStartSec && time < toggle.cutSec} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}
data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;
}
if (!scene.referenceImageId) return null;
@@ -552,7 +556,18 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
await new Promise((resolve, reject) => {
let handle;
const timer = setTimeout(() => { cancelAnimationFrame(handle); reject(new Error('Preview did not settle')); }, 2000);
- handle = requestAnimationFrame(() => { handle = requestAnimationFrame(() => { clearTimeout(timer); resolve(); }); });
+ let paints = 0;
+ const settle = () => {
+ const clips = [...(figureRef.current?.querySelectorAll('video[data-preview-video]') || [])]
+ .filter((video) => Number(getComputedStyle(video.parentElement).opacity) > 0);
+ const failedClip = clips.find((video) => video.dataset.previewError);
+ if (failedClip) { clearTimeout(timer); reject(new Error(failedClip.dataset.previewError)); return; }
+ if (++paints >= 2 && clips.every((video) => !video.seeking && video.dataset.pausedFrame === String(video.currentTime))) {
+ clearTimeout(timer); resolve(); return;
+ }
+ handle = requestAnimationFrame(settle);
+ };
+ handle = requestAnimationFrame(settle);
});
const unchanged = () => {
const current = captureBasis.current;
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index aaabf2c7b2..0fb02aa10b 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -220,10 +220,10 @@ describe('StoryboardAnimatic clips', () => {
expect(clip.muted).toBe(true);
// The still stays underneath, and the clip stays transparent until it has a frame.
expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-a.png');
- expect(clip.className).toContain('opacity-0');
+ expect(clip.parentElement.className).toContain('opacity-0');
Object.defineProperty(clip, 'readyState', { value: 4, configurable: true });
fireEvent.loadedData(clip);
- expect(clip.className).toContain('opacity-100');
+ expect(clip.parentElement.className).toContain('opacity-100');
const audio = screen.getByLabelText('Song for the storyboard animatic');
Object.defineProperty(audio, 'currentTime', { value: 12, configurable: true });
fireEvent.timeUpdate(audio);
@@ -249,13 +249,13 @@ describe('StoryboardAnimatic clips', () => {
const toggle = await screen.findByTestId('beat-toggle-clip');
await waitFor(() => expect(toggle.getAttribute('src')).toBe('/data/videos/clip-b.mov'));
Object.defineProperty(toggle, 'readyState', { value: 4, configurable: true });
- expect(toggle.style.opacity).toBe('0');
+ expect(toggle.parentElement.style.opacity).toBe('0');
at(3.2);
- expect(toggle.style.opacity).toBe('1');
+ expect(toggle.parentElement.style.opacity).toBe('1');
// 0.8 s before the shot starts: 1.2 s of clip before its 1.5 s in-point.
expect(toggle.currentTime).toBeCloseTo(0.3, 2);
at(3.7);
- expect(toggle.style.opacity).toBe('0');
+ expect(toggle.parentElement.style.opacity).toBe('0');
// Past the cut the incoming shot's own clip starts where it would without the toggle.
const clipB = screen.getByLabelText('Clip for Two');
Object.defineProperty(clipB, 'readyState', { value: 4, configurable: true });
@@ -278,13 +278,13 @@ describe('StoryboardAnimatic clips', () => {
const at = (t) => { Object.defineProperty(audio, 'currentTime', { value: t, configurable: true }); fireEvent.timeUpdate(audio); };
at(2.9);
const incoming = await screen.findByTestId('beat-toggle-clip');
- expect(incoming.style.opacity).toBe('0');
+ expect(incoming.parentElement.style.opacity).toBe('0');
at(3.5);
- const half = Number(incoming.style.opacity);
+ const half = Number(incoming.parentElement.style.opacity);
expect(half).toBeGreaterThan(0.4);
expect(half).toBeLessThan(0.6);
at(3.95);
- expect(Number(incoming.style.opacity)).toBeGreaterThan(half);
+ expect(Number(incoming.parentElement.style.opacity)).toBeGreaterThan(half);
at(7.4);
const black = await screen.findByTestId('fade-black');
expect(black.style.opacity).toBe('0');
@@ -313,12 +313,12 @@ describe('StoryboardAnimatic clips', () => {
expect(screen.getByLabelText('Clip for Three').getAttribute('preload')).toBe('metadata');
Object.defineProperty(next, 'readyState', { value: 4, configurable: true });
fireEvent.seeked(next);
- expect(next.className).toContain('opacity-0');
+ expect(next.parentElement.className).toContain('opacity-0');
const audio = screen.getByLabelText('Song for the storyboard animatic');
Object.defineProperty(audio, 'currentTime', { value: 1.2, configurable: true });
fireEvent.timeUpdate(audio);
expect(screen.getByLabelText('Clip for Two')).toBe(next);
- expect(next.className).toContain('opacity-100');
+ expect(next.parentElement.className).toContain('opacity-100');
expect(await screen.findByLabelText('Clip for Four')).toBeTruthy();
});
diff --git a/client/src/lib/musicVideoFrameCapture.js b/client/src/lib/musicVideoFrameCapture.js
index 4107ea3ddf..80fab88a49 100644
--- a/client/src/lib/musicVideoFrameCapture.js
+++ b/client/src/lib/musicVideoFrameCapture.js
@@ -9,12 +9,18 @@ export async function capturePreviewFrame(root, { iframeImages = new Map() } = {
const width = Math.ceil(box.width);
const height = Math.ceil(box.height);
if (!width || !height || width * height > 16777216) throw new Error('Preview size unavailable');
+ for (const video of root.querySelectorAll('video[data-preview-video]')) {
+ const surface = getComputedStyle(video.parentElement);
+ if (surface.display === 'none' || surface.visibility === 'hidden' || Number(surface.opacity) === 0) continue;
+ if (video.dataset.previewError) throw new Error(video.dataset.previewError);
+ if (video.seeking || video.dataset.pausedFrame !== String(video.currentTime)) throw new Error('Video frame is still loading');
+ }
const raster = (source, w, h) => {
if (!w || !h) throw new Error('Preview media has not decoded');
const canvas = document.createElement('canvas');
const scale = Math.min(1, Math.sqrt(16777216 / (w * h)));
canvas.width = Math.max(1, Math.round(w * scale)); canvas.height = Math.max(1, Math.round(h * scale));
- const ctx = canvas.getContext('2d');
+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });
if (!ctx) throw new Error('Frame capture unavailable');
ctx.drawImage(source, 0, 0, canvas.width, canvas.height);
return canvas.toDataURL('image/png');
@@ -75,14 +81,14 @@ export async function capturePreviewFrame(root, { iframeImages = new Map() } = {
finally { clearTimeout(timer); }
const canvas = document.createElement('canvas');
canvas.width = width + 1; canvas.height = height;
- const ctx = canvas.getContext('2d');
+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });
if (!ctx) throw new Error('Frame capture unavailable');
ctx.drawImage(image, 0, 0);
const probe = ctx.getImageData(width, height - 1, 1, 1).data;
if (probe[0] !== 13 || probe[1] !== 231 || probe[2] !== 97 || probe[3] !== 255) throw new Error('This browser cannot capture the preview');
const output = document.createElement('canvas');
output.width = width; output.height = height;
- const paint = output.getContext('2d');
+ const paint = output.getContext('2d', { colorSpace: 'srgb' });
if (!paint) throw new Error('Frame capture unavailable');
// The figure's black ground sits outside the film-look filter.
paint.fillStyle = '#000'; paint.fillRect(0, 0, width, height);
@@ -102,7 +108,7 @@ export function lyricFrameCaptureBridge() {
const output = document.createElement('canvas');
output.width = innerWidth; output.height = innerHeight;
if (!output.width || !output.height || output.width * output.height > 16777216) throw new Error('Invalid lyric frame');
- const ctx = output.getContext('2d');
+ const ctx = output.getContext('2d', { colorSpace: 'srgb' });
const layers = [...document.querySelectorAll('canvas')];
if (!ctx || !layers.length) throw new Error('Lyric frame unavailable');
for (const canvas of layers) {
diff --git a/docs/README.md b/docs/README.md
index 279175df6d..af9958f446 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -33,6 +33,7 @@ Index of everything under `docs/`. Start with the [root README](../README.md) fo
| [MODEL-COMPARISON.md](./MODEL-COMPARISON.md) | Sourced provider/model/effort comparisons, cost estimates and CoS research refresh |
| [THREEJS_MODELS.md](./THREEJS_MODELS.md) | Three.js procedural 3D model generation and trust boundary |
| [features/music-video-autonomous.md](./features/music-video-autonomous.md) | One-prompt autonomous music videos: brief → lyrics → mood board → Suno song → production, optional checkpoints, and the `music-video-autopilot` scheduled task that turns Brain ideas into videos |
+| [features/music-video-frame-capture.md](./features/music-video-frame-capture.md) | Shared paused-video/capture color interpretation, synthetic evidence and browser limits |
| [features/music-video-temporal-evidence.md](./features/music-video-temporal-evidence.md) | Performance source provenance, synchronized playback and optional local temporal analyzer protocol |
| [features/music-renderer-benchmarks.md](./features/music-renderer-benchmarks.md) | Technical and full-length listening evidence for local music renderer profiles |
| [CONTRIBUTING.md](./CONTRIBUTING.md) | Dev setup (PostgreSQL required), code conventions |
diff --git a/docs/features/music-video-frame-capture.md b/docs/features/music-video-frame-capture.md
new file mode 100644
index 0000000000..35fc04a502
--- /dev/null
+++ b/docs/features/music-video-frame-capture.md
@@ -0,0 +1,103 @@
+# Storyboard paused-video color and frame capture
+
+The storyboard keeps continuous playback in a native muted video element. On
+pause, each visible footage layer presents its decoded frame in an explicitly
+sRGB Canvas2D surface. Save frame freezes that same canvas, then rasterizes the
+actual preview layout, transitions, exact paused film look, lyrics and outro.
+Native playback resumes without a JavaScript per-frame rasterization loop.
+Hidden preloaded clips do not allocate decoded canvas frames.
+
+`PreviewVideo.jsx` owns the shared surface for ordinary footage and incoming
+transition footage. The wrapper owns fitting/background and opacity; the video
+and canvas have the same object fitting. Seeks invalidate the previous surface,
+and presentation waits two animation frames after decoded data/seek completion.
+The existing song-to-source mapping still owns offsets, speed, reverse and loop.
+Save frame waits for the visible paused surfaces (bounded to two seconds) and
+rejects a changed, seeking or unavailable surface instead of exporting native
+video through a different color path. Canvas storage is capped at 16 megapixels,
+matching capture's limit. Unsupported Canvas2D or SVG foreignObject capture
+reports an error; it does not upload a blank image.
+
+## Interpretation and evidence
+
+This is a shared SDR preview/capture interpretation, not a color-specific
+correction. The browser decodes the source's matrix and range into sRGB Canvas2D.
+The composed renderer (`server/services/musicVideo/render.js`) processes source
+YUV through FFmpeg scale/format and encodes yuv420p. The regression independently
+checks encoded YUV against the source matrix and browser raw video planes.
+FFmpeg's matrix-to-RGB decode remains a diagnostic comparison. The layered document
+renderer also uses Canvas2D `drawImage`. This change does not add an HDR tone map,
+a wide-gamut mastering workflow, or a new primaries/transfer conversion to the
+final renderer; agreement with a calibrated native-video display is not claimed.
+
+The synthetic browser fixture generates the original 120×240 cyan VP8 clip and
+480×240 VP9 charts with eight saturated/neutral patches, a sixteen-step grayscale
+ramp and an alternating, frame-specific marker. RGB is actually converted into
+limited-range SMPTE170M (BT601) or BT709 YUV before encoding. `VideoFrame.colorSpace`
+assertions verify matrix, primaries, transfer and range at the decoder boundary.
+Metadata-only BT709 tags on the original VP8 clip still reported SMPTE170M in
+Chrome, so that clip cannot stand in for a correctly decoded BT709 test.
+
+macOS arm64, headless Google Chrome 155.0.8059.80, hardware-decoded NV12, forced
+sRGB display profile:
+
+| Source / patch | Native video screenshot RGB | Decoded canvas / paused preview / PNG RGB | FFmpeg matrix decode RGB |
+| --- | --- | --- | --- |
+| BT601 cyan | 70, 253, 255 | 1, 255, 255 | 0, 255, 255 |
+| BT601 red | 248, 36, 0 | 254, 0, 0 | 254, 0, 0 |
+| BT601 magenta | 249, 52, 255 | 255, 0, 254 | 255, 0, 254 |
+| BT709 cyan | 0, 254, 255 | 0, 254, 255 | 0, 254, 255 |
+| BT709 red | 255, 1, 0 | 255, 1, 0 | 255, 0, 0 |
+| Both grayscale ramps | 0, 17, …, 255 | 0, 17, …, 255 | 0, 17, …, 255 |
+
+The original standalone VP8 cyan reproduction gave native RGBA
+`[70,253,255,255]` versus Canvas2D `[1,255,255,255]`, before any PortOS compositor.
+Prior probes with ImageBitmap default/none conversion, VideoFrame, neutral CSS
+filters, extra animation frames, and a display-P3 canvas converted back to sRGB
+did not eliminate it. The VP9 matrix checks above distinguish that presentation
+boundary from an incorrectly tagged source.
+
+The hosted Linux x64 diagnostic (Chrome 155.0.8059.39, FFmpeg 6.1.1) exposed a
+separate CPU conversion limit: BT709 magenta was `[254,0,244]` in native video,
+paused canvas and PNG, while FFmpeg produced `[253,0,253]`. Yellow, green and blue
+also differed in the blue channel. On the same local Chrome 155, disabling GPU
+and hardware video decoding changed NV12 to I420 and reproduced those blue
+values while browser raw YUV still matched FFmpeg exactly. FFmpeg 6.1.1 arm64/x64
+and FFmpeg 9.0.2 synthetic probes independently verified the encoded matrix.
+
+This is consistent with libyuv's default BT709 blue coefficient being capped at
+128/64 rather than 135/64; see the [pinned conversion source](https://chromium.googlesource.com/libyuv/libyuv/+/eeb06a6451ba2e21f8ff272c7b42fc62b9faecc5/source/row_common.cc#1645).
+The source YUV `[78,214,229]` predicts blue 244 with that approximation. Thus a
+universal three-level bound between FFmpeg RGB and browser RGB is invalid even
+when source tagging and decoding are correct. PortOS does not compensate for
+individual colors or promise identical RGB across CPU/GPU decoder paths.
+
+`server/services/musicVideo/animaticCapture.browser.test.js` validates source and
+presentation separately. Each saturated patch, grayscale step and moving marker
+must encode within two YUV levels of the mathematical limited-range 601/709
+matrix (the lossy chart's quantization bound). The browser's raw I420/NV12 planes
+must exactly match FFmpeg's decoded YUV for the requested frame, and matrix,
+primaries, transfer and range metadata must match. The paused product canvas must
+exactly match a fresh, independent sRGB draw of the source video. This catches
+stale canvas frames without treating the product canvas as its own reference.
+
+The real paused preview screenshot is compared to the saved PNG without replacing
+DOM nodes: mean channel error remains below 1, and patch readback/export differences
+remain at most 1. Native video and FFmpeg RGB remain diagnostic evidence; failures
+include complete synthetic samples and versions even when CI silences console
+output. Forward/backward and adjacent-frame seeks check the alternating marker,
+and pause/resume checks native playback restoration. Fitting, incoming footage
+with film look, still motion/effects, lyrics, title and outro remain covered.
+
+Run the fixture with:
+
+```sh
+npm test --prefix server -- services/musicVideo/animaticCapture.browser.test.js
+```
+
+The CI planner selects it when the capture helper, Save frame control or paused
+video surface changes. Hosted Linux Chrome runs the same fixture; failing assertions retain its
+browser/version and numeric evidence. Local macOS results do not substitute
+for that CI result. Safari was present but could not create a WebDriver session
+because Allow remote automation was disabled; Safari pixels remain unverified.
+No browser settings were changed for this investigation.
diff --git a/scripts/ci-test-plan.test.js b/scripts/ci-test-plan.test.js
index e33ff1c1f1..d41249ca67 100644
--- a/scripts/ci-test-plan.test.js
+++ b/scripts/ci-test-plan.test.js
@@ -1539,7 +1539,7 @@ describe('cross-workspace browser suites (#10312)', () => {
it('selects animatic pixel coverage when only the compositor or Save frame control changes', () => {
const cwd = fileURLToPath(new URL('../', import.meta.url));
const suite = 'server/services/musicVideo/animaticCapture.browser.test.js';
- for (const source of ['client/src/lib/musicVideoFrameCapture.js', 'client/src/components/musicVideo/FrameStepper.jsx']) {
+ for (const source of ['client/src/lib/musicVideoFrameCapture.js', 'client/src/components/musicVideo/FrameStepper.jsx', 'client/src/components/musicVideo/PreviewVideo.jsx']) {
const inputs = collectPlanInputs({ baseSha: 'HEAD', changedFiles: [source], cwd });
const trackedFiles = inputs.trackedFiles.filter((path) => !/\.(?:test|spec)\.[cm]?[jt]sx?$/.test(path) || BROWSER_SUITES.includes(path));
const plan = buildCiTestPlan(inputs.changedFiles, { ...inputs, trackedFiles });
diff --git a/server/services/musicVideo/animaticCapture.browser.test.js b/server/services/musicVideo/animaticCapture.browser.test.js
index 48b7f411d2..d4209a027a 100644
--- a/server/services/musicVideo/animaticCapture.browser.test.js
+++ b/server/services/musicVideo/animaticCapture.browser.test.js
@@ -27,7 +27,15 @@ const canRun = browserSuiteCanRun('animatic frame capture', { Chrome: chrome, ff
const still = { sceneId: 'still', order: 0, label: 'Example still', startSec: 0, endSec: 4, visualLayer: 'still', referenceImageId: 'grid.png', stillMove: 'hold' };
const project = (scenes = [still], extra = {}) => ({ id: 'capture-fixture', name: 'Example capture', scenes, composition: { mode: 'composed' }, ...extra });
-let dir, server, fixture, page, lyricProject;
+let dir, server, fixture, page, lyricProject, ffmpegVersion;
+const colorReferences = new Map();
+const colorYuvReferences = new Map();
+const chartColors = [[0, 255, 255], [255, 0, 255], [255, 255, 0], [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 255], [0, 0, 0]];
+const chartRgb = (x, y, frame) => {
+ if (y < 120) return chartColors[Math.floor(x / 60)];
+ const gray = Math.floor(x / 30) * 17;
+ return y < 200 ? [gray, gray, gray] : frame % 2 ? [230, 40, 190] : [frame * 2, 255 - frame * 2, frame];
+};
const errors = [];
afterAll(async () => {
await fixture?.close();
@@ -37,6 +45,7 @@ afterAll(async () => {
describe.skipIf(!canRun)('saved storyboard pixels in a real browser', () => {
beforeAll(async () => {
+ ffmpegVersion = execFileSync(ffmpeg, ['-version'], { encoding: 'utf8' }).split('\n')[0];
dir = await mkdtemp(join(tmpdir(), 'portos-animatic-capture-'));
await symlink(join(client, 'node_modules'), join(dir, 'node_modules'), 'dir');
const entry = `import React,{useState} from 'react';import{createRoot}from'react-dom/client';import Animatic from ${JSON.stringify(join(client, 'src/components/musicVideo/StoryboardAnimatic.jsx'))};import toast,{Toaster}from ${JSON.stringify(join(client, 'src/components/ui/Toast.jsx'))};
@@ -53,20 +62,40 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wi
await sharp(Buffer.from(grid)).png().toFile(join(dir, 'grid.png'));
await sharp({ create: { width: 200, height: 400, channels: 3, background: '#ff00ff' } }).png().toFile(join(dir, 'end.png'));
execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'color=c=cyan:s=120x240:r=24:d=4', '-c:v', 'libvpx', join(dir, 'clip.webm')]);
+ // RGB chart -> explicit limited-range YUV matrix, never a metadata-only retag.
+ const frames = Buffer.alloc(480 * 240 * 3 * 96);
+ for (let f = 0; f < 96; f++) for (let y = 0; y < 240; y++) for (let x = 0; x < 480; x++) {
+ const rgb = chartRgb(x, y, f);
+ const offset = ((f * 240 + y) * 480 + x) * 3;
+ for (let c = 0; c < 3; c++) frames[offset + c] = rgb[c];
+ }
+ for (const matrix of ['smpte170m', 'bt709']) {
+ const path = join(dir, `${matrix}.webm`);
+ execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'rawvideo', '-pixel_format', 'rgb24', '-video_size', '480x240', '-framerate', '24', '-i', 'pipe:0',
+ '-vf', `scale=out_color_matrix=${matrix}:out_range=tv,format=yuv420p`, '-colorspace', matrix, '-color_primaries', matrix, '-color_trc', matrix, '-color_range', 'tv',
+ '-c:v', 'libvpx-vp9', '-crf', '4', '-b:v', '2M', path], { input: frames });
+ // Retain RGB as diagnostic evidence. CPU/GPU browser conversion need not
+ // match FFmpeg RGB; raw decoded YUV independently verifies the source.
+ const rgb = execFileSync(ffmpeg, ['-v', 'error', '-i', path, '-vf', `scale=in_color_matrix=${matrix}:in_range=tv:out_range=pc,format=rgb24`,
+ '-f', 'rawvideo', '-pix_fmt', 'rgb24', 'pipe:1'], { maxBuffer: frames.length + 1024 });
+ colorReferences.set(matrix, rgb);
+ colorYuvReferences.set(matrix, execFileSync(ffmpeg, ['-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', 'yuv420p', 'pipe:1'],
+ { maxBuffer: frames.length + 1024 }));
+ }
execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=220:duration=8', join(dir, 'song.wav')]);
const app = express();
app.get('/data/images/:id', (req, res) => res.sendFile(join(dir, req.params.id)));
- app.get('/api/video-gen/history/:id', (req, res) => res.json({ id: req.params.id, filename: 'clip.webm', fps: 24, numFrames: 96, width: 120, height: 240 }));
+ app.get('/api/video-gen/history/:id', (req, res) => res.json({ id: req.params.id, filename: colorReferences.has(req.params.id) ? `${req.params.id}.webm` : 'clip.webm', fps: 24, numFrames: 96, width: 120, height: 240 }));
app.get('/api/music-video/:id/stats-outro', (req, res) => res.json({ stats: { shots: 2, stills: { made: 3, kept: 1 }, clips: { made: 2, kept: 1 }, floorImages: [] } }));
app.get('/api/music-video/:id/lyrics/overlay/preview', async (req, res, next) => {
try { res.json(await buildLyricPlaythroughPreview(lyricProject, { overlay: true })); }
catch (error) { next(error); }
});
- app.get('/data/videos/clip.webm', (req, res) => res.sendFile(join(dir, 'clip.webm')));
+ app.get('/data/videos/:name', (req, res) => res.sendFile(join(dir, req.params.name)));
app.get('/song.wav', (req, res) => res.sendFile(join(dir, 'song.wav')));
app.use(express.static(join(dir, 'dist')));
server = await new Promise((done) => { const listener = app.listen(0, '127.0.0.1', () => done(listener)); });
- fixture = await startOwnedBrowserFixture({ chromium, launchOptions: { executablePath: chrome, headless: true } });
+ fixture = await startOwnedBrowserFixture({ chromium, launchOptions: { executablePath: chrome, headless: true, args: ['--force-color-profile=srgb'] } });
page = await fixture.browser.newPage({ viewport: { width: 1000, height: 800 } });
page.on('pageerror', (error) => errors.push(error.message));
await page.goto(`http://127.0.0.1:${server.address().port}`);
@@ -78,32 +107,16 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wi
await page.waitForFunction((t) => Math.abs(document.querySelector('audio')?.currentTime - t) < 0.00001, t);
await page.waitForFunction(() => [...document.querySelectorAll('figure img')].every((image) => image.complete && image.naturalWidth > 0));
};
- const capture = async (label = '', { decodedVideoReference = false } = {}) => {
+ const capture = async (label = '') => {
+ await page.evaluate(() => { window.uploads = []; window.captureErrors = []; });
+ await page.waitForFunction(() => [...document.querySelectorAll('video[data-preview-video]')].every(video => Number(getComputedStyle(video.parentElement).opacity) === 0 || (!video.seeking && video.dataset.pausedFrame === String(video.currentTime))));
await page.getByRole('button', { name: 'Save frame', exact: true }).click();
await page.waitForFunction(() => window.uploads.length > 0 || window.captureErrors.length > 0);
expect(await page.evaluate(() => window.captureErrors), label).toEqual([]);
const saved = Buffer.from(await page.evaluate(() => window.uploads.at(-1)));
const picture = page.getByLabel('Storyboard animatic', { exact: true }).locator(':scope > div').first();
- if (decodedVideoReference) await picture.evaluate((root) => {
- // Chrome's native VIDEO display and Canvas2D currently convert source colors
- // differently (tracked in #11630). Compare framing/effects to decoded pixels through the actual
- // CSS layout, retaining the native-display discrepancy as separate evidence.
- window.restoreVideoReference = [];
- for (const video of root.querySelectorAll('video[data-ready="true"]:not([aria-hidden="true"])')) {
- const canvas = document.createElement('canvas');
- canvas.width = video.videoWidth; canvas.height = video.videoHeight;
- canvas.getContext('2d').drawImage(video, 0, 0);
- canvas.className = video.className; canvas.style.cssText = video.style.cssText;
- video.replaceWith(canvas);
- window.restoreVideoReference.push(() => canvas.replaceWith(video));
- }
- });
// Global notification chrome is outside the video frame being saved.
- let displayed;
- try { displayed = await picture.screenshot({ style: '[data-toast-id]{visibility:hidden}' }); }
- finally {
- if (decodedVideoReference) await page.evaluate(() => window.restoreVideoReference.forEach((restore) => restore()));
- }
+ const displayed = await picture.screenshot({ style: '[data-toast-id]{visibility:hidden}' });
const a = await sharp(saved).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
const b = await sharp(displayed).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
expect([a.info.width, a.info.height]).toEqual([b.info.width, b.info.height]);
@@ -133,10 +146,111 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wi
await show(project([{ ...still, visualLayer: 'footage', videoHistoryId: 'clip', frameFit },
{ ...still, sceneId: 'next', order: 1, startSec: 4, endSec: 8, visualLayer: 'footage', videoHistoryId: 'preloaded' }]), 1);
await page.waitForFunction(() => document.querySelector('video[data-ready="true"]:not([aria-hidden="true"])')?.seeking === false);
- await capture(frameFit, { decodedVideoReference: true });
+ await capture(frameFit);
}
});
+ it('keeps BT601/709 saturated colors and grayscale identical across paused preview, decoded pixels and PNG', async () => {
+ const points = [...Array.from({ length: 8 }, (_, i) => [i * 60 + 30, 60]), ...Array.from({ length: 16 }, (_, i) => [i * 30 + 15, 160]), [240, 220]];
+ const samples = (data, width, channels, locations) => locations.map(([x, y]) => [...data.subarray((y * width + x) * channels, (y * width + x) * channels + 3)]);
+ for (const matrix of ['smpte170m', 'bt709']) {
+ const value = project([{ ...still, referenceImageId: null, visualLayer: 'footage', videoHistoryId: matrix, frameFit: 'black' }]);
+ for (const t of [0.02, 2.02, 1.02, 1.06, 1.02]) {
+ await show(value, t);
+ const saved = await capture(`${matrix} @ ${t}`);
+ const pixels = await page.evaluate(async (points) => {
+ const video = document.querySelector('video[data-ready="true"]');
+ const canvas = video.nextElementSibling;
+ const ctx = canvas.getContext('2d');
+ const frame = new VideoFrame(video, { timestamp: 0 });
+ try {
+ const metadata = frame.colorSpace.toJSON();
+ const format = frame.format;
+ if (!['I420', 'NV12'].includes(format)) throw new Error(`Unexpected synthetic VP9 pixel format: ${format}`);
+ const bytes = new Uint8Array(frame.allocationSize());
+ const layout = await frame.copyTo(bytes);
+ const yuv = points.map(([x, y]) => [bytes[layout[0].offset + y * layout[0].stride + x], ...[1, 2].map(i => {
+ const plane = layout[format === 'NV12' ? 1 : i];
+ return bytes[plane.offset + Math.floor(y / 2) * plane.stride + (format === 'NV12' ? 2 * Math.floor(x / 2) + i - 1 : Math.floor(x / 2))];
+ })]);
+ // Read a new draw of the source video, never the product canvas.
+ const fresh = document.createElement('canvas'); fresh.width = video.videoWidth; fresh.height = video.videoHeight;
+ const decoded = fresh.getContext('2d', { colorSpace: 'srgb' }); decoded.drawImage(video, 0, 0);
+ const read = context => points.map(([x, y]) => [...context.getImageData(x, y, 1, 1).data].slice(0, 3));
+ return { metadata, format, yuv, currentTime: video.currentTime, samples: read(ctx), fresh: read(decoded) };
+ } finally { frame.close(); }
+ }, points);
+ expect(pixels.currentTime).toBeCloseTo(t, 5);
+ expect(pixels.metadata).toEqual({ matrix, primaries: matrix, transfer: matrix, fullRange: false });
+ const raw = await sharp(saved).raw().toBuffer({ resolveWithObject: true });
+ const scale = raw.info.width / 480;
+ const pngPoints = points.map(([x, y]) => [Math.floor(x * scale), Math.floor((raw.info.height - 240 * scale) / 2 + y * scale)]);
+ const exported = samples(raw.data, raw.info.width, raw.info.channels, pngPoints);
+ const frameIndex = Math.floor(t * 24);
+ const offset = frameIndex * 480 * 240 * 3;
+ const yuv = colorYuvReferences.get(matrix).subarray(frameIndex * 480 * 240 * 3 / 2);
+ const sourceYuv = points.map(([x, y]) => [yuv[y * 480 + x], yuv[480 * 240 + Math.floor(y / 2) * 240 + Math.floor(x / 2)],
+ yuv[480 * 240 * 5 / 4 + Math.floor(y / 2) * 240 + Math.floor(x / 2)]]);
+ const [kr, kb] = matrix === 'bt709' ? [0.2126, 0.0722] : [0.299, 0.114];
+ const expectedYuv = points.map(([x, y]) => {
+ const [r, g, b] = chartRgb(x, y, frameIndex).map(channel => channel / 255);
+ const luma = kr * r + (1 - kr - kb) * g + kb * b;
+ return [16 + 219 * luma, 128 + 112 * (b - luma) / (1 - kb), 128 + 112 * (r - luma) / (1 - kr)].map(Math.round);
+ });
+ const reference = samples(colorReferences.get(matrix).subarray(offset), 480, 3, points);
+ let displayed;
+ if (t === 0.02) {
+ // Retain native-display negative evidence; it is diagnostic, not the
+ // reference used to relax the strict actual-preview capture assertion.
+ const video = page.locator('video[data-ready="true"]');
+ await video.evaluate(v => { v.style.visibility = ''; v.nextElementSibling.style.visibility = 'hidden'; });
+ let native;
+ try { native = await video.screenshot(); }
+ finally { await video.evaluate(v => { v.style.visibility = 'hidden'; v.nextElementSibling.style.visibility = ''; }); }
+ const nativePixels = await sharp(native).raw().toBuffer({ resolveWithObject: true });
+ displayed = samples(nativePixels.data, nativePixels.info.width, nativePixels.info.channels, pngPoints);
+ console.log(`🎨 ${process.platform}/${process.arch} ${fixture.browser.version()} ${matrix} limited YUV -> sRGB: ${JSON.stringify({ native: displayed, decoded: pixels.samples, png: exported, ffmpeg: reference })}`);
+ }
+ // CI intentionally silences console output. Preserve the complete
+ // independent measurements in the assertion failure, before bailing.
+ const evidence = JSON.stringify({ platform: process.platform, arch: process.arch, browser: fixture.browser.version(), ffmpegVersion,
+ matrix, t, metadata: pixels.metadata, format: pixels.format, native: displayed, decoded: pixels.samples, fresh: pixels.fresh,
+ png: exported, ffmpeg: reference, sourceYuv, browserYuv: pixels.yuv, expectedYuv });
+ expect(pixels.yuv, `Source frame ${matrix} @ ${t}: ${evidence}`).toEqual(sourceYuv);
+ expect(pixels.samples, `Paused frame ${matrix} @ ${t}: ${evidence}`).toEqual(pixels.fresh);
+ for (let i = 0; i < points.length; i++) for (let c = 0; c < 3; c++) {
+ expect(Math.abs(exported[i][c] - pixels.samples[i][c]), `PNG ${matrix} ${i}/${c}: ${evidence}`).toBeLessThanOrEqual(1);
+ // Quantization from the lossy VP9 chart is bounded independently of
+ // any browser or FFmpeg RGB conversion and checked for every frame.
+ expect(Math.abs(sourceYuv[i][c] - expectedYuv[i][c]), `Encoded YUV ${matrix} ${i}/${c}: ${evidence}`).toBeLessThanOrEqual(2);
+ }
+ }
+ }
+ }, 30000);
+
+ it('uses native continuous playback, then freezes the selected source frame on pause and resume', async () => {
+ await show(project([{ ...still, referenceImageId: null, visualLayer: 'footage', videoHistoryId: 'bt709' }]), 0.02);
+ await capture();
+ await page.evaluate(async () => { await document.querySelector('audio').play(); });
+ await page.waitForFunction(() => { const v = document.querySelector('video'); return !v.paused && v.currentTime > 0.3 && !v.dataset.pausedFrame && getComputedStyle(v).visibility === 'visible'; });
+ await page.evaluate(() => document.querySelector('audio').pause());
+ await capture();
+ await page.evaluate(async () => { await document.querySelector('audio').play(); });
+ await page.waitForFunction(() => !document.querySelector('video').paused && !document.querySelector('video').dataset.pausedFrame);
+ await page.evaluate(() => document.querySelector('audio').pause());
+ await capture();
+ });
+
+ it('shares both footage surfaces through an incoming dissolve and the selected exact film look', async () => {
+ await show(project([
+ { ...still, visualLayer: 'footage', videoHistoryId: 'smpte170m', cutEffect: { style: 'dissolve', enabled: true, overlapSec: 1 } },
+ { ...still, sceneId: 'incoming', order: 1, startSec: 4, endSec: 8, visualLayer: 'footage', videoHistoryId: 'bt709' },
+ ], { filmLook: filmLookPreset('vhs') }), 3.5);
+ await capture();
+ expect(await page.locator('canvas[data-paused-video]:visible').count()).toBe(2);
+ expect(await page.locator('[data-film-look="exact"]').count()).toBe(1);
+ });
+
it('saves end-frame effects, flash/develop, incoming dissolves and fade-black in their displayed order', async () => {
for (const [effect, t] of [['end-fade', 1.3], ['end-fade', 3], ['flash-develop', 0.1], ['flash-develop', 1]]) {
await show(project([{ ...still, imageEffect: effect, imageEffectSec: 1, endImageId: 'end.png', stillMove: 'pan' }]), t);
@@ -232,4 +346,22 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wi
expect(await page.evaluate(() => window.captureErrors.at(-1))).toContain('Preview changed during capture');
await page.unroute('**/api/music-video/*/stats-outro');
});
+ it('reports an unavailable paused color surface without uploading a different native-video image', async () => {
+ await page.evaluate(() => {
+ const original = HTMLCanvasElement.prototype.getContext;
+ window.restoreContext = () => { HTMLCanvasElement.prototype.getContext = original; };
+ HTMLCanvasElement.prototype.getContext = function(...args) { return this.dataset.pausedVideo ? null : original.apply(this, args); };
+ });
+ try {
+ await show(project([{ ...still, visualLayer: 'footage', videoHistoryId: 'smpte170m' }]), 1.02);
+ await page.waitForFunction(() => document.querySelector('video')?.dataset.previewError);
+ await page.getByRole('button', { name: 'Save frame', exact: true }).click();
+ await page.waitForFunction(() => window.captureErrors.length > 0);
+ expect(await page.evaluate(() => window.uploads.length)).toBe(0);
+ expect(await page.evaluate(() => window.captureErrors.at(-1))).toContain('Frame capture unavailable');
+ expect(await page.locator('video').evaluate(v => getComputedStyle(v).visibility)).toBe('hidden');
+ } finally { await page.evaluate(() => window.restoreContext()); }
+ expect(errors).toEqual([]);
+ });
+
});