Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
[issue-11630] Align paused footage and saved frame colors (#11645)
[._.]/c53c07052dd3
diff --git a/client/src/components/musicVideo/PreviewVideo.jsx b/client/src/components/musicVideo/PreviewVideo.jsxnew file mode 100644index 0000000000..249f672e96--- /dev/null+++ b/client/src/components/musicVideo/PreviewVideo.jsx@@ -0,0 +1,64 @@+import { useEffect, useRef } from 'react';++/** Native playback, with the same sRGB decoded surface as Save frame when paused. */+export default function PreviewVideo({ videoRef, audioRef, active = true, className, style, ...props }) {+ const canvasRef = useRef(null);+ useEffect(() => {+ const video = videoRef.current;+ const canvas = canvasRef.current;+ const audio = audioRef.current;+ if (!video || !canvas || !audio) return undefined;+ let handle = 0;+ const invalidate = () => {+ cancelAnimationFrame(handle);+ delete video.dataset.pausedFrame;+ canvas.style.visibility = 'hidden';+ video.style.visibility = audio.paused ? 'hidden' : '';+ };+ const paint = () => {+ if (!active || !audio.paused || video.seeking || video.readyState < 2) return;+ const scale = Math.min(1, Math.sqrt(16777216 / (video.videoWidth * video.videoHeight)));+ const width = Math.max(1, Math.round(video.videoWidth * scale));+ const height = Math.max(1, Math.round(video.videoHeight * scale));+ if (!video.videoWidth || !video.videoHeight) return;+ canvas.width = width; canvas.height = height;+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });+ if (!ctx) { video.dataset.previewError = 'Frame capture unavailable'; return; }+ try {+ ctx.drawImage(video, 0, 0, width, height);+ delete video.dataset.previewError;+ video.dataset.pausedFrame = String(video.currentTime);+ canvas.style.visibility = '';+ video.style.visibility = 'hidden';+ } catch { invalidate(); video.dataset.previewError = 'Video frame unavailable'; }+ };+ const schedule = () => {+ invalidate();+ if (!active || !audio.paused) return;+ // seeked precedes presentation in Chromium; let the decoded frame reach+ // the compositor before drawing. A new seek cancels this pending paint.+ handle = requestAnimationFrame(() => { handle = requestAnimationFrame(paint); });+ };+ const events = ['loadeddata', 'seeked', 'pause'];+ for (const event of events) video.addEventListener(event, schedule);+ video.addEventListener('seeking', invalidate);+ video.addEventListener('emptied', invalidate);+ audio.addEventListener('play', invalidate);+ audio.addEventListener('pause', schedule);+ schedule();+ return () => {+ invalidate();+ for (const event of events) video.removeEventListener(event, schedule);+ video.removeEventListener('seeking', invalidate);+ video.removeEventListener('emptied', invalidate);+ audio.removeEventListener('play', invalidate);+ audio.removeEventListener('pause', schedule);+ };+ }, [videoRef, audioRef, active, props.src]);+ const fit = className.includes('object-cover') ? 'object-cover' : 'object-contain';+ return <div className={className} style={style}>+ <video {...props} ref={videoRef} data-preview-video="true" className={`h-full w-full ${fit}`} />+ <canvas ref={canvasRef} tabIndex={-1} aria-hidden="true" data-paused-video="true"+ className={`pointer-events-none absolute inset-0 h-full w-full ${fit}`} style={{ visibility: 'hidden' }} />+ </div>;+}diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsxindex 03c3e2ce11..339692db31 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx@@ -16,6 +16,7 @@ import StatsOutroCanvas from './StatsOutroCanvas.jsx';import ShotStepper from './ShotStepper.jsx';import FrameStepper from './FrameStepper.jsx';import { capturePreviewFrame, lyricFrameCaptureBridge } from '../../lib/musicVideoFrameCapture.js';+import PreviewVideo from './PreviewVideo.jsx';import { sceneFrameFit } from './SceneFrameFit.jsx';import TransitionGlitch from './TransitionGlitch.jsx';import ImageEffectStill from './ImageEffectStill.jsx';@@ -346,9 +347,11 @@ function ShotClip({ scene, audioRef, time, current, preload = 'auto' }) {// The final render's frame fit: zoomed to fill, or whole over the still, the VHS edge or black bars.const fit = sceneFrameFit(scene);return (- <video- key={reload}- ref={videoRef}+ <PreviewVideo+ key={`${src}:${reload}`}+ videoRef={videoRef}+ audioRef={audioRef}+ active={current}src={src}mutedplaysInline@@ -436,7 +439,8 @@ function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {const el = mediaRef.current;const audio = audioRef.current;if (!el) return;- el.style.opacity = String(incomingOpacity(toggle, t, fps));+ const surface = el.tagName === 'VIDEO' ? el.parentElement : el;+ surface.style.opacity = String(incomingOpacity(toggle, t, fps));if (el.tagName !== 'VIDEO' || el.readyState < 1) return;const duration = Number.isFinite(el.duration) ? el.duration : null;const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true, fromSec: toggle.windowStartSec });@@ -455,7 +459,7 @@ function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {const fit = sceneFrameFit(scene);const className = `pointer-events-none absolute inset-0 h-full w-full ${src && fit !== 'fill' ? 'object-contain' : 'object-cover'} ${fit === 'black' ? 'bg-black' : ''}`;if (src) {- return <video ref={mediaRef} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}+ return <PreviewVideo key={src} videoRef={mediaRef} audioRef={audioRef} active={time >= toggle.windowStartSec && time < toggle.cutSec} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;}if (!scene.referenceImageId) return null;@@ -552,7 +556,18 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colawait new Promise((resolve, reject) => {let handle;const timer = setTimeout(() => { cancelAnimationFrame(handle); reject(new Error('Preview did not settle')); }, 2000);- handle = requestAnimationFrame(() => { handle = requestAnimationFrame(() => { clearTimeout(timer); resolve(); }); });+ let paints = 0;+ const settle = () => {+ const clips = [...(figureRef.current?.querySelectorAll('video[data-preview-video]') || [])]+ .filter((video) => Number(getComputedStyle(video.parentElement).opacity) > 0);+ const failedClip = clips.find((video) => video.dataset.previewError);+ if (failedClip) { clearTimeout(timer); reject(new Error(failedClip.dataset.previewError)); return; }+ if (++paints >= 2 && clips.every((video) => !video.seeking && video.dataset.pausedFrame === String(video.currentTime))) {+ clearTimeout(timer); resolve(); return;+ }+ handle = requestAnimationFrame(settle);+ };+ handle = requestAnimationFrame(settle);});const unchanged = () => {const current = captureBasis.current;diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsxindex aaabf2c7b2..0fb02aa10b 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx@@ -220,10 +220,10 @@ describe('StoryboardAnimatic clips', () => {expect(clip.muted).toBe(true);// The still stays underneath, and the clip stays transparent until it has a frame.expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-a.png');- expect(clip.className).toContain('opacity-0');+ expect(clip.parentElement.className).toContain('opacity-0');Object.defineProperty(clip, 'readyState', { value: 4, configurable: true });fireEvent.loadedData(clip);- expect(clip.className).toContain('opacity-100');+ expect(clip.parentElement.className).toContain('opacity-100');const audio = screen.getByLabelText('Song for the storyboard animatic');Object.defineProperty(audio, 'currentTime', { value: 12, configurable: true });fireEvent.timeUpdate(audio);@@ -249,13 +249,13 @@ describe('StoryboardAnimatic clips', () => {const toggle = await screen.findByTestId('beat-toggle-clip');await waitFor(() => expect(toggle.getAttribute('src')).toBe('/data/videos/clip-b.mov'));Object.defineProperty(toggle, 'readyState', { value: 4, configurable: true });- expect(toggle.style.opacity).toBe('0');+ expect(toggle.parentElement.style.opacity).toBe('0');at(3.2);- expect(toggle.style.opacity).toBe('1');+ expect(toggle.parentElement.style.opacity).toBe('1');// 0.8 s before the shot starts: 1.2 s of clip before its 1.5 s in-point.expect(toggle.currentTime).toBeCloseTo(0.3, 2);at(3.7);- expect(toggle.style.opacity).toBe('0');+ expect(toggle.parentElement.style.opacity).toBe('0');// Past the cut the incoming shot's own clip starts where it would without the toggle.const clipB = screen.getByLabelText('Clip for Two');Object.defineProperty(clipB, 'readyState', { value: 4, configurable: true });@@ -278,13 +278,13 @@ describe('StoryboardAnimatic clips', () => {const at = (t) => { Object.defineProperty(audio, 'currentTime', { value: t, configurable: true }); fireEvent.timeUpdate(audio); };at(2.9);const incoming = await screen.findByTestId('beat-toggle-clip');- expect(incoming.style.opacity).toBe('0');+ expect(incoming.parentElement.style.opacity).toBe('0');at(3.5);- const half = Number(incoming.style.opacity);+ const half = Number(incoming.parentElement.style.opacity);expect(half).toBeGreaterThan(0.4);expect(half).toBeLessThan(0.6);at(3.95);- expect(Number(incoming.style.opacity)).toBeGreaterThan(half);+ expect(Number(incoming.parentElement.style.opacity)).toBeGreaterThan(half);at(7.4);const black = await screen.findByTestId('fade-black');expect(black.style.opacity).toBe('0');@@ -313,12 +313,12 @@ describe('StoryboardAnimatic clips', () => {expect(screen.getByLabelText('Clip for Three').getAttribute('preload')).toBe('metadata');Object.defineProperty(next, 'readyState', { value: 4, configurable: true });fireEvent.seeked(next);- expect(next.className).toContain('opacity-0');+ expect(next.parentElement.className).toContain('opacity-0');const audio = screen.getByLabelText('Song for the storyboard animatic');Object.defineProperty(audio, 'currentTime', { value: 1.2, configurable: true });fireEvent.timeUpdate(audio);expect(screen.getByLabelText('Clip for Two')).toBe(next);- expect(next.className).toContain('opacity-100');+ expect(next.parentElement.className).toContain('opacity-100');expect(await screen.findByLabelText('Clip for Four')).toBeTruthy();});diff --git a/client/src/lib/musicVideoFrameCapture.js b/client/src/lib/musicVideoFrameCapture.jsindex 4107ea3ddf..80fab88a49 100644--- a/client/src/lib/musicVideoFrameCapture.js+++ b/client/src/lib/musicVideoFrameCapture.js@@ -9,12 +9,18 @@ export async function capturePreviewFrame(root, { iframeImages = new Map() } = {const width = Math.ceil(box.width);const height = Math.ceil(box.height);if (!width || !height || width * height > 16777216) throw new Error('Preview size unavailable');+ for (const video of root.querySelectorAll('video[data-preview-video]')) {+ const surface = getComputedStyle(video.parentElement);+ if (surface.display === 'none' || surface.visibility === 'hidden' || Number(surface.opacity) === 0) continue;+ if (video.dataset.previewError) throw new Error(video.dataset.previewError);+ if (video.seeking || video.dataset.pausedFrame !== String(video.currentTime)) throw new Error('Video frame is still loading');+ }const raster = (source, w, h) => {if (!w || !h) throw new Error('Preview media has not decoded');const canvas = document.createElement('canvas');const scale = Math.min(1, Math.sqrt(16777216 / (w * h)));canvas.width = Math.max(1, Math.round(w * scale)); canvas.height = Math.max(1, Math.round(h * scale));- const ctx = canvas.getContext('2d');+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });if (!ctx) throw new Error('Frame capture unavailable');ctx.drawImage(source, 0, 0, canvas.width, canvas.height);return canvas.toDataURL('image/png');@@ -75,14 +81,14 @@ export async function capturePreviewFrame(root, { iframeImages = new Map() } = {finally { clearTimeout(timer); }const canvas = document.createElement('canvas');canvas.width = width + 1; canvas.height = height;- const ctx = canvas.getContext('2d');+ const ctx = canvas.getContext('2d', { colorSpace: 'srgb' });if (!ctx) throw new Error('Frame capture unavailable');ctx.drawImage(image, 0, 0);const probe = ctx.getImageData(width, height - 1, 1, 1).data;if (probe[0] !== 13 || probe[1] !== 231 || probe[2] !== 97 || probe[3] !== 255) throw new Error('This browser cannot capture the preview');const output = document.createElement('canvas');output.width = width; output.height = height;- const paint = output.getContext('2d');+ const paint = output.getContext('2d', { colorSpace: 'srgb' });if (!paint) throw new Error('Frame capture unavailable');// The figure's black ground sits outside the film-look filter.paint.fillStyle = '#000'; paint.fillRect(0, 0, width, height);@@ -102,7 +108,7 @@ export function lyricFrameCaptureBridge() {const output = document.createElement('canvas');output.width = innerWidth; output.height = innerHeight;if (!output.width || !output.height || output.width * output.height > 16777216) throw new Error('Invalid lyric frame');- const ctx = output.getContext('2d');+ const ctx = output.getContext('2d', { colorSpace: 'srgb' });const layers = [...document.querySelectorAll('canvas')];if (!ctx || !layers.length) throw new Error('Lyric frame unavailable');for (const canvas of layers) {diff --git a/docs/README.md b/docs/README.mdindex 279175df6d..af9958f446 100644--- a/docs/README.md+++ b/docs/README.md@@ -33,6 +33,7 @@ Index of everything under `docs/`. Start with the [root README](../README.md) fo| [MODEL-COMPARISON.md](./MODEL-COMPARISON.md) | Sourced provider/model/effort comparisons, cost estimates and CoS research refresh || [THREEJS_MODELS.md](./THREEJS_MODELS.md) | Three.js procedural 3D model generation and trust boundary || [features/music-video-autonomous.md](./features/music-video-autonomous.md) | One-prompt autonomous music videos: brief → lyrics → mood board → Suno song → production, optional checkpoints, and the `music-video-autopilot` scheduled task that turns Brain ideas into videos |+| [features/music-video-frame-capture.md](./features/music-video-frame-capture.md) | Shared paused-video/capture color interpretation, synthetic evidence and browser limits || [features/music-video-temporal-evidence.md](./features/music-video-temporal-evidence.md) | Performance source provenance, synchronized playback and optional local temporal analyzer protocol || [features/music-renderer-benchmarks.md](./features/music-renderer-benchmarks.md) | Technical and full-length listening evidence for local music renderer profiles || [CONTRIBUTING.md](./CONTRIBUTING.md) | Dev setup (PostgreSQL required), code conventions |diff --git a/docs/features/music-video-frame-capture.md b/docs/features/music-video-frame-capture.mdnew file mode 100644index 0000000000..35fc04a502--- /dev/null+++ b/docs/features/music-video-frame-capture.md@@ -0,0 +1,103 @@+# Storyboard paused-video color and frame capture++The storyboard keeps continuous playback in a native muted video element. On+pause, each visible footage layer presents its decoded frame in an explicitly+sRGB Canvas2D surface. Save frame freezes that same canvas, then rasterizes the+actual preview layout, transitions, exact paused film look, lyrics and outro.+Native playback resumes without a JavaScript per-frame rasterization loop.+Hidden preloaded clips do not allocate decoded canvas frames.++`PreviewVideo.jsx` owns the shared surface for ordinary footage and incoming+transition footage. The wrapper owns fitting/background and opacity; the video+and canvas have the same object fitting. Seeks invalidate the previous surface,+and presentation waits two animation frames after decoded data/seek completion.+The existing song-to-source mapping still owns offsets, speed, reverse and loop.+Save frame waits for the visible paused surfaces (bounded to two seconds) and+rejects a changed, seeking or unavailable surface instead of exporting native+video through a different color path. Canvas storage is capped at 16 megapixels,+matching capture's limit. Unsupported Canvas2D or SVG foreignObject capture+reports an error; it does not upload a blank image.++## Interpretation and evidence++This is a shared SDR preview/capture interpretation, not a color-specific+correction. The browser decodes the source's matrix and range into sRGB Canvas2D.+The composed renderer (`server/services/musicVideo/render.js`) processes source+YUV through FFmpeg scale/format and encodes yuv420p. The regression independently+checks encoded YUV against the source matrix and browser raw video planes.+FFmpeg's matrix-to-RGB decode remains a diagnostic comparison. The layered document+renderer also uses Canvas2D `drawImage`. This change does not add an HDR tone map,+a wide-gamut mastering workflow, or a new primaries/transfer conversion to the+final renderer; agreement with a calibrated native-video display is not claimed.++The synthetic browser fixture generates the original 120×240 cyan VP8 clip and+480×240 VP9 charts with eight saturated/neutral patches, a sixteen-step grayscale+ramp and an alternating, frame-specific marker. RGB is actually converted into+limited-range SMPTE170M (BT601) or BT709 YUV before encoding. `VideoFrame.colorSpace`+assertions verify matrix, primaries, transfer and range at the decoder boundary.+Metadata-only BT709 tags on the original VP8 clip still reported SMPTE170M in+Chrome, so that clip cannot stand in for a correctly decoded BT709 test.++macOS arm64, headless Google Chrome 155.0.8059.80, hardware-decoded NV12, forced+sRGB display profile:++| Source / patch | Native video screenshot RGB | Decoded canvas / paused preview / PNG RGB | FFmpeg matrix decode RGB |+| --- | --- | --- | --- |+| BT601 cyan | 70, 253, 255 | 1, 255, 255 | 0, 255, 255 |+| BT601 red | 248, 36, 0 | 254, 0, 0 | 254, 0, 0 |+| BT601 magenta | 249, 52, 255 | 255, 0, 254 | 255, 0, 254 |+| BT709 cyan | 0, 254, 255 | 0, 254, 255 | 0, 254, 255 |+| BT709 red | 255, 1, 0 | 255, 1, 0 | 255, 0, 0 |+| Both grayscale ramps | 0, 17, …, 255 | 0, 17, …, 255 | 0, 17, …, 255 |++The original standalone VP8 cyan reproduction gave native RGBA+`[70,253,255,255]` versus Canvas2D `[1,255,255,255]`, before any PortOS compositor.+Prior probes with ImageBitmap default/none conversion, VideoFrame, neutral CSS+filters, extra animation frames, and a display-P3 canvas converted back to sRGB+did not eliminate it. The VP9 matrix checks above distinguish that presentation+boundary from an incorrectly tagged source.++The hosted Linux x64 diagnostic (Chrome 155.0.8059.39, FFmpeg 6.1.1) exposed a+separate CPU conversion limit: BT709 magenta was `[254,0,244]` in native video,+paused canvas and PNG, while FFmpeg produced `[253,0,253]`. Yellow, green and blue+also differed in the blue channel. On the same local Chrome 155, disabling GPU+and hardware video decoding changed NV12 to I420 and reproduced those blue+values while browser raw YUV still matched FFmpeg exactly. FFmpeg 6.1.1 arm64/x64+and FFmpeg 9.0.2 synthetic probes independently verified the encoded matrix.++This is consistent with libyuv's default BT709 blue coefficient being capped at+128/64 rather than 135/64; see the [pinned conversion source](https://chromium.googlesource.com/libyuv/libyuv/+/eeb06a6451ba2e21f8ff272c7b42fc62b9faecc5/source/row_common.cc#1645).+The source YUV `[78,214,229]` predicts blue 244 with that approximation. Thus a+universal three-level bound between FFmpeg RGB and browser RGB is invalid even+when source tagging and decoding are correct. PortOS does not compensate for+individual colors or promise identical RGB across CPU/GPU decoder paths.++`server/services/musicVideo/animaticCapture.browser.test.js` validates source and+presentation separately. Each saturated patch, grayscale step and moving marker+must encode within two YUV levels of the mathematical limited-range 601/709+matrix (the lossy chart's quantization bound). The browser's raw I420/NV12 planes+must exactly match FFmpeg's decoded YUV for the requested frame, and matrix,+primaries, transfer and range metadata must match. The paused product canvas must+exactly match a fresh, independent sRGB draw of the source video. This catches+stale canvas frames without treating the product canvas as its own reference.++The real paused preview screenshot is compared to the saved PNG without replacing+DOM nodes: mean channel error remains below 1, and patch readback/export differences+remain at most 1. Native video and FFmpeg RGB remain diagnostic evidence; failures+include complete synthetic samples and versions even when CI silences console+output. Forward/backward and adjacent-frame seeks check the alternating marker,+and pause/resume checks native playback restoration. Fitting, incoming footage+with film look, still motion/effects, lyrics, title and outro remain covered.++Run the fixture with:++```sh+npm test --prefix server -- services/musicVideo/animaticCapture.browser.test.js+```++The CI planner selects it when the capture helper, Save frame control or paused+video surface changes. Hosted Linux Chrome runs the same fixture; failing assertions retain its+browser/version and numeric evidence. Local macOS results do not substitute+for that CI result. Safari was present but could not create a WebDriver session+because Allow remote automation was disabled; Safari pixels remain unverified.+No browser settings were changed for this investigation.diff --git a/scripts/ci-test-plan.test.js b/scripts/ci-test-plan.test.jsindex e33ff1c1f1..d41249ca67 100644--- a/scripts/ci-test-plan.test.js+++ b/scripts/ci-test-plan.test.js@@ -1539,7 +1539,7 @@ describe('cross-workspace browser suites (#10312)', () => {it('selects animatic pixel coverage when only the compositor or Save frame control changes', () => {const cwd = fileURLToPath(new URL('../', import.meta.url));const suite = 'server/services/musicVideo/animaticCapture.browser.test.js';- for (const source of ['client/src/lib/musicVideoFrameCapture.js', 'client/src/components/musicVideo/FrameStepper.jsx']) {+ for (const source of ['client/src/lib/musicVideoFrameCapture.js', 'client/src/components/musicVideo/FrameStepper.jsx', 'client/src/components/musicVideo/PreviewVideo.jsx']) {const inputs = collectPlanInputs({ baseSha: 'HEAD', changedFiles: [source], cwd });const trackedFiles = inputs.trackedFiles.filter((path) => !/\.(?:test|spec)\.[cm]?[jt]sx?$/.test(path) || BROWSER_SUITES.includes(path));const plan = buildCiTestPlan(inputs.changedFiles, { ...inputs, trackedFiles });diff --git a/server/services/musicVideo/animaticCapture.browser.test.js b/server/services/musicVideo/animaticCapture.browser.test.jsindex 48b7f411d2..d4209a027a 100644--- a/server/services/musicVideo/animaticCapture.browser.test.js+++ b/server/services/musicVideo/animaticCapture.browser.test.js@@ -27,7 +27,15 @@ const canRun = browserSuiteCanRun('animatic frame capture', { Chrome: chrome, ffconst still = { sceneId: 'still', order: 0, label: 'Example still', startSec: 0, endSec: 4, visualLayer: 'still', referenceImageId: 'grid.png', stillMove: 'hold' };const project = (scenes = [still], extra = {}) => ({ id: 'capture-fixture', name: 'Example capture', scenes, composition: { mode: 'composed' }, ...extra });-let dir, server, fixture, page, lyricProject;+let dir, server, fixture, page, lyricProject, ffmpegVersion;+const colorReferences = new Map();+const colorYuvReferences = new Map();+const chartColors = [[0, 255, 255], [255, 0, 255], [255, 255, 0], [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 255], [0, 0, 0]];+const chartRgb = (x, y, frame) => {+ if (y < 120) return chartColors[Math.floor(x / 60)];+ const gray = Math.floor(x / 30) * 17;+ return y < 200 ? [gray, gray, gray] : frame % 2 ? [230, 40, 190] : [frame * 2, 255 - frame * 2, frame];+};const errors = [];afterAll(async () => {await fixture?.close();@@ -37,6 +45,7 @@ afterAll(async () => {describe.skipIf(!canRun)('saved storyboard pixels in a real browser', () => {beforeAll(async () => {+ ffmpegVersion = execFileSync(ffmpeg, ['-version'], { encoding: 'utf8' }).split('\n')[0];dir = await mkdtemp(join(tmpdir(), 'portos-animatic-capture-'));await symlink(join(client, 'node_modules'), join(dir, 'node_modules'), 'dir');const entry = `import React,{useState} from 'react';import{createRoot}from'react-dom/client';import Animatic from ${JSON.stringify(join(client, 'src/components/musicVideo/StoryboardAnimatic.jsx'))};import toast,{Toaster}from ${JSON.stringify(join(client, 'src/components/ui/Toast.jsx'))};@@ -53,20 +62,40 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wiawait sharp(Buffer.from(grid)).png().toFile(join(dir, 'grid.png'));await sharp({ create: { width: 200, height: 400, channels: 3, background: '#ff00ff' } }).png().toFile(join(dir, 'end.png'));execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'color=c=cyan:s=120x240:r=24:d=4', '-c:v', 'libvpx', join(dir, 'clip.webm')]);+ // RGB chart -> explicit limited-range YUV matrix, never a metadata-only retag.+ const frames = Buffer.alloc(480 * 240 * 3 * 96);+ for (let f = 0; f < 96; f++) for (let y = 0; y < 240; y++) for (let x = 0; x < 480; x++) {+ const rgb = chartRgb(x, y, f);+ const offset = ((f * 240 + y) * 480 + x) * 3;+ for (let c = 0; c < 3; c++) frames[offset + c] = rgb[c];+ }+ for (const matrix of ['smpte170m', 'bt709']) {+ const path = join(dir, `${matrix}.webm`);+ execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'rawvideo', '-pixel_format', 'rgb24', '-video_size', '480x240', '-framerate', '24', '-i', 'pipe:0',+ '-vf', `scale=out_color_matrix=${matrix}:out_range=tv,format=yuv420p`, '-colorspace', matrix, '-color_primaries', matrix, '-color_trc', matrix, '-color_range', 'tv',+ '-c:v', 'libvpx-vp9', '-crf', '4', '-b:v', '2M', path], { input: frames });+ // Retain RGB as diagnostic evidence. CPU/GPU browser conversion need not+ // match FFmpeg RGB; raw decoded YUV independently verifies the source.+ const rgb = execFileSync(ffmpeg, ['-v', 'error', '-i', path, '-vf', `scale=in_color_matrix=${matrix}:in_range=tv:out_range=pc,format=rgb24`,+ '-f', 'rawvideo', '-pix_fmt', 'rgb24', 'pipe:1'], { maxBuffer: frames.length + 1024 });+ colorReferences.set(matrix, rgb);+ colorYuvReferences.set(matrix, execFileSync(ffmpeg, ['-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', 'yuv420p', 'pipe:1'],+ { maxBuffer: frames.length + 1024 }));+ }execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=220:duration=8', join(dir, 'song.wav')]);const app = express();app.get('/data/images/:id', (req, res) => res.sendFile(join(dir, req.params.id)));- app.get('/api/video-gen/history/:id', (req, res) => res.json({ id: req.params.id, filename: 'clip.webm', fps: 24, numFrames: 96, width: 120, height: 240 }));+ app.get('/api/video-gen/history/:id', (req, res) => res.json({ id: req.params.id, filename: colorReferences.has(req.params.id) ? `${req.params.id}.webm` : 'clip.webm', fps: 24, numFrames: 96, width: 120, height: 240 }));app.get('/api/music-video/:id/stats-outro', (req, res) => res.json({ stats: { shots: 2, stills: { made: 3, kept: 1 }, clips: { made: 2, kept: 1 }, floorImages: [] } }));app.get('/api/music-video/:id/lyrics/overlay/preview', async (req, res, next) => {try { res.json(await buildLyricPlaythroughPreview(lyricProject, { overlay: true })); }catch (error) { next(error); }});- app.get('/data/videos/clip.webm', (req, res) => res.sendFile(join(dir, 'clip.webm')));+ app.get('/data/videos/:name', (req, res) => res.sendFile(join(dir, req.params.name)));app.get('/song.wav', (req, res) => res.sendFile(join(dir, 'song.wav')));app.use(express.static(join(dir, 'dist')));server = await new Promise((done) => { const listener = app.listen(0, '127.0.0.1', () => done(listener)); });- fixture = await startOwnedBrowserFixture({ chromium, launchOptions: { executablePath: chrome, headless: true } });+ fixture = await startOwnedBrowserFixture({ chromium, launchOptions: { executablePath: chrome, headless: true, args: ['--force-color-profile=srgb'] } });page = await fixture.browser.newPage({ viewport: { width: 1000, height: 800 } });page.on('pageerror', (error) => errors.push(error.message));await page.goto(`http://127.0.0.1:${server.address().port}`);@@ -78,32 +107,16 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wiawait page.waitForFunction((t) => Math.abs(document.querySelector('audio')?.currentTime - t) < 0.00001, t);await page.waitForFunction(() => [...document.querySelectorAll('figure img')].every((image) => image.complete && image.naturalWidth > 0));};- const capture = async (label = '', { decodedVideoReference = false } = {}) => {+ const capture = async (label = '') => {+ await page.evaluate(() => { window.uploads = []; window.captureErrors = []; });+ await page.waitForFunction(() => [...document.querySelectorAll('video[data-preview-video]')].every(video => Number(getComputedStyle(video.parentElement).opacity) === 0 || (!video.seeking && video.dataset.pausedFrame === String(video.currentTime))));await page.getByRole('button', { name: 'Save frame', exact: true }).click();await page.waitForFunction(() => window.uploads.length > 0 || window.captureErrors.length > 0);expect(await page.evaluate(() => window.captureErrors), label).toEqual([]);const saved = Buffer.from(await page.evaluate(() => window.uploads.at(-1)));const picture = page.getByLabel('Storyboard animatic', { exact: true }).locator(':scope > div').first();- if (decodedVideoReference) await picture.evaluate((root) => {- // Chrome's native VIDEO display and Canvas2D currently convert source colors- // differently (tracked in #11630). Compare framing/effects to decoded pixels through the actual- // CSS layout, retaining the native-display discrepancy as separate evidence.- window.restoreVideoReference = [];- for (const video of root.querySelectorAll('video[data-ready="true"]:not([aria-hidden="true"])')) {- const canvas = document.createElement('canvas');- canvas.width = video.videoWidth; canvas.height = video.videoHeight;- canvas.getContext('2d').drawImage(video, 0, 0);- canvas.className = video.className; canvas.style.cssText = video.style.cssText;- video.replaceWith(canvas);- window.restoreVideoReference.push(() => canvas.replaceWith(video));- }- });// Global notification chrome is outside the video frame being saved.- let displayed;- try { displayed = await picture.screenshot({ style: '[data-toast-id]{visibility:hidden}' }); }- finally {- if (decodedVideoReference) await page.evaluate(() => window.restoreVideoReference.forEach((restore) => restore()));- }+ const displayed = await picture.screenshot({ style: '[data-toast-id]{visibility:hidden}' });const a = await sharp(saved).ensureAlpha().raw().toBuffer({ resolveWithObject: true });const b = await sharp(displayed).ensureAlpha().raw().toBuffer({ resolveWithObject: true });expect([a.info.width, a.info.height]).toEqual([b.info.width, b.info.height]);@@ -133,10 +146,111 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wiawait show(project([{ ...still, visualLayer: 'footage', videoHistoryId: 'clip', frameFit },{ ...still, sceneId: 'next', order: 1, startSec: 4, endSec: 8, visualLayer: 'footage', videoHistoryId: 'preloaded' }]), 1);await page.waitForFunction(() => document.querySelector('video[data-ready="true"]:not([aria-hidden="true"])')?.seeking === false);- await capture(frameFit, { decodedVideoReference: true });+ await capture(frameFit);}});+ it('keeps BT601/709 saturated colors and grayscale identical across paused preview, decoded pixels and PNG', async () => {+ const points = [...Array.from({ length: 8 }, (_, i) => [i * 60 + 30, 60]), ...Array.from({ length: 16 }, (_, i) => [i * 30 + 15, 160]), [240, 220]];+ const samples = (data, width, channels, locations) => locations.map(([x, y]) => [...data.subarray((y * width + x) * channels, (y * width + x) * channels + 3)]);+ for (const matrix of ['smpte170m', 'bt709']) {+ const value = project([{ ...still, referenceImageId: null, visualLayer: 'footage', videoHistoryId: matrix, frameFit: 'black' }]);+ for (const t of [0.02, 2.02, 1.02, 1.06, 1.02]) {+ await show(value, t);+ const saved = await capture(`${matrix} @ ${t}`);+ const pixels = await page.evaluate(async (points) => {+ const video = document.querySelector('video[data-ready="true"]');+ const canvas = video.nextElementSibling;+ const ctx = canvas.getContext('2d');+ const frame = new VideoFrame(video, { timestamp: 0 });+ try {+ const metadata = frame.colorSpace.toJSON();+ const format = frame.format;+ if (!['I420', 'NV12'].includes(format)) throw new Error(`Unexpected synthetic VP9 pixel format: ${format}`);+ const bytes = new Uint8Array(frame.allocationSize());+ const layout = await frame.copyTo(bytes);+ const yuv = points.map(([x, y]) => [bytes[layout[0].offset + y * layout[0].stride + x], ...[1, 2].map(i => {+ const plane = layout[format === 'NV12' ? 1 : i];+ return bytes[plane.offset + Math.floor(y / 2) * plane.stride + (format === 'NV12' ? 2 * Math.floor(x / 2) + i - 1 : Math.floor(x / 2))];+ })]);+ // Read a new draw of the source video, never the product canvas.+ const fresh = document.createElement('canvas'); fresh.width = video.videoWidth; fresh.height = video.videoHeight;+ const decoded = fresh.getContext('2d', { colorSpace: 'srgb' }); decoded.drawImage(video, 0, 0);+ const read = context => points.map(([x, y]) => [...context.getImageData(x, y, 1, 1).data].slice(0, 3));+ return { metadata, format, yuv, currentTime: video.currentTime, samples: read(ctx), fresh: read(decoded) };+ } finally { frame.close(); }+ }, points);+ expect(pixels.currentTime).toBeCloseTo(t, 5);+ expect(pixels.metadata).toEqual({ matrix, primaries: matrix, transfer: matrix, fullRange: false });+ const raw = await sharp(saved).raw().toBuffer({ resolveWithObject: true });+ const scale = raw.info.width / 480;+ const pngPoints = points.map(([x, y]) => [Math.floor(x * scale), Math.floor((raw.info.height - 240 * scale) / 2 + y * scale)]);+ const exported = samples(raw.data, raw.info.width, raw.info.channels, pngPoints);+ const frameIndex = Math.floor(t * 24);+ const offset = frameIndex * 480 * 240 * 3;+ const yuv = colorYuvReferences.get(matrix).subarray(frameIndex * 480 * 240 * 3 / 2);+ const sourceYuv = points.map(([x, y]) => [yuv[y * 480 + x], yuv[480 * 240 + Math.floor(y / 2) * 240 + Math.floor(x / 2)],+ yuv[480 * 240 * 5 / 4 + Math.floor(y / 2) * 240 + Math.floor(x / 2)]]);+ const [kr, kb] = matrix === 'bt709' ? [0.2126, 0.0722] : [0.299, 0.114];+ const expectedYuv = points.map(([x, y]) => {+ const [r, g, b] = chartRgb(x, y, frameIndex).map(channel => channel / 255);+ const luma = kr * r + (1 - kr - kb) * g + kb * b;+ return [16 + 219 * luma, 128 + 112 * (b - luma) / (1 - kb), 128 + 112 * (r - luma) / (1 - kr)].map(Math.round);+ });+ const reference = samples(colorReferences.get(matrix).subarray(offset), 480, 3, points);+ let displayed;+ if (t === 0.02) {+ // Retain native-display negative evidence; it is diagnostic, not the+ // reference used to relax the strict actual-preview capture assertion.+ const video = page.locator('video[data-ready="true"]');+ await video.evaluate(v => { v.style.visibility = ''; v.nextElementSibling.style.visibility = 'hidden'; });+ let native;+ try { native = await video.screenshot(); }+ finally { await video.evaluate(v => { v.style.visibility = 'hidden'; v.nextElementSibling.style.visibility = ''; }); }+ const nativePixels = await sharp(native).raw().toBuffer({ resolveWithObject: true });+ displayed = samples(nativePixels.data, nativePixels.info.width, nativePixels.info.channels, pngPoints);+ console.log(`🎨 ${process.platform}/${process.arch} ${fixture.browser.version()} ${matrix} limited YUV -> sRGB: ${JSON.stringify({ native: displayed, decoded: pixels.samples, png: exported, ffmpeg: reference })}`);+ }+ // CI intentionally silences console output. Preserve the complete+ // independent measurements in the assertion failure, before bailing.+ const evidence = JSON.stringify({ platform: process.platform, arch: process.arch, browser: fixture.browser.version(), ffmpegVersion,+ matrix, t, metadata: pixels.metadata, format: pixels.format, native: displayed, decoded: pixels.samples, fresh: pixels.fresh,+ png: exported, ffmpeg: reference, sourceYuv, browserYuv: pixels.yuv, expectedYuv });+ expect(pixels.yuv, `Source frame ${matrix} @ ${t}: ${evidence}`).toEqual(sourceYuv);+ expect(pixels.samples, `Paused frame ${matrix} @ ${t}: ${evidence}`).toEqual(pixels.fresh);+ for (let i = 0; i < points.length; i++) for (let c = 0; c < 3; c++) {+ expect(Math.abs(exported[i][c] - pixels.samples[i][c]), `PNG ${matrix} ${i}/${c}: ${evidence}`).toBeLessThanOrEqual(1);+ // Quantization from the lossy VP9 chart is bounded independently of+ // any browser or FFmpeg RGB conversion and checked for every frame.+ expect(Math.abs(sourceYuv[i][c] - expectedYuv[i][c]), `Encoded YUV ${matrix} ${i}/${c}: ${evidence}`).toBeLessThanOrEqual(2);+ }+ }+ }+ }, 30000);++ it('uses native continuous playback, then freezes the selected source frame on pause and resume', async () => {+ await show(project([{ ...still, referenceImageId: null, visualLayer: 'footage', videoHistoryId: 'bt709' }]), 0.02);+ await capture();+ await page.evaluate(async () => { await document.querySelector('audio').play(); });+ await page.waitForFunction(() => { const v = document.querySelector('video'); return !v.paused && v.currentTime > 0.3 && !v.dataset.pausedFrame && getComputedStyle(v).visibility === 'visible'; });+ await page.evaluate(() => document.querySelector('audio').pause());+ await capture();+ await page.evaluate(async () => { await document.querySelector('audio').play(); });+ await page.waitForFunction(() => !document.querySelector('video').paused && !document.querySelector('video').dataset.pausedFrame);+ await page.evaluate(() => document.querySelector('audio').pause());+ await capture();+ });++ it('shares both footage surfaces through an incoming dissolve and the selected exact film look', async () => {+ await show(project([+ { ...still, visualLayer: 'footage', videoHistoryId: 'smpte170m', cutEffect: { style: 'dissolve', enabled: true, overlapSec: 1 } },+ { ...still, sceneId: 'incoming', order: 1, startSec: 4, endSec: 8, visualLayer: 'footage', videoHistoryId: 'bt709' },+ ], { filmLook: filmLookPreset('vhs') }), 3.5);+ await capture();+ expect(await page.locator('canvas[data-paused-video]:visible').count()).toBe(2);+ expect(await page.locator('[data-film-look="exact"]').count()).toBe(1);+ });+it('saves end-frame effects, flash/develop, incoming dissolves and fade-black in their displayed order', async () => {for (const [effect, t] of [['end-fade', 1.3], ['end-fade', 3], ['flash-develop', 0.1], ['flash-develop', 1]]) {await show(project([{ ...still, imageEffect: effect, imageEffectSec: 1, endImageId: 'end.png', stillMove: 'pan' }]), t);@@ -232,4 +346,22 @@ function App(){const[state,setState]=useState(null);window.show=(project,t)=>{wiexpect(await page.evaluate(() => window.captureErrors.at(-1))).toContain('Preview changed during capture');await page.unroute('**/api/music-video/*/stats-outro');});+ it('reports an unavailable paused color surface without uploading a different native-video image', async () => {+ await page.evaluate(() => {+ const original = HTMLCanvasElement.prototype.getContext;+ window.restoreContext = () => { HTMLCanvasElement.prototype.getContext = original; };+ HTMLCanvasElement.prototype.getContext = function(...args) { return this.dataset.pausedVideo ? null : original.apply(this, args); };+ });+ try {+ await show(project([{ ...still, visualLayer: 'footage', videoHistoryId: 'smpte170m' }]), 1.02);+ await page.waitForFunction(() => document.querySelector('video')?.dataset.previewError);+ await page.getByRole('button', { name: 'Save frame', exact: true }).click();+ await page.waitForFunction(() => window.captureErrors.length > 0);+ expect(await page.evaluate(() => window.uploads.length)).toBe(0);+ expect(await page.evaluate(() => window.captureErrors.at(-1))).toContain('Frame capture unavailable');+ expect(await page.locator('video').evaluate(v => getComputedStyle(v).visibility)).toBe('hidden');+ } finally { await page.evaluate(() => window.restoreContext()); }+ expect(errors).toEqual([]);+ });+});