Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Fix GPU staging colors for native SMPTE-170M video (#11729)
[._.]/4722e2c0e3d9
diff --git a/server/lib/filmLookGpu.browser.test.js b/server/lib/filmLookGpu.browser.test.jsindex 65cd34d531..86195d4b7e 100644--- a/server/lib/filmLookGpu.browser.test.js+++ b/server/lib/filmLookGpu.browser.test.js@@ -87,6 +87,90 @@ describe.skipIf(!canRun).each([expect(psnr(svg, gpu)).toBeGreaterThanOrEqual(40);}, 60000);+ // Independent video pixels, not two consumers of one canvas. The untagged+ // SD clip decodes as NV12/SMPTE-C on macOS hardware and I420 in software;+ // only the former omits the primary conversion during canvas staging.+ it.for(['untagged SD', 'converted BT709'])('preserves native source colors and Neon rain for %s', async (kind, { skip }) => {+ if (!available) skip();+ const w = 640, h = 360, tagged = kind === 'converted BT709';+ const dir = mkdtempSync(join(tmpdir(), 'film-look-primary-'));+ const path = join(dir, 'clip.mp4');+ const pages = [];+ try {+ // testsrc is RGB: this performs a real matrix conversion, not a relabel+ // of the SD YUV bytes. Both frame and container tags mirror the encoder.+ execFileSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i',+ `${tagged ? 'testsrc' : 'testsrc2'}=s=${w}x${h}:r=24`, '-t', '0.25',+ ...(tagged ? ['-vf', 'scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709',+ '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709'] : []),+ '-c:v', 'libx264', '-threads', '1', '-pix_fmt', 'yuv420p', '-y', path], { timeout: 10000 });+ const clip = readFileSync(path);+ const yuv = execFileSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-threads', '1', '-i', path,+ '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'yuv420p', 'pipe:1'], { timeout: 10000, maxBuffer: w * h * 2 });+ // Uniform bar interiors avoid chroma-siting differences. Decode the+ // independent YUV planes with the declared limited-range 601/709 matrix.+ const xs = [40, 140, 340, 440, 540];+ const rawSamples = xs.map(x => {+ const p = w * h, uv = 100 * (w / 2) + x / 2;+ const y = (yuv[200 * w + x] - 16) / 219, u = (yuv[p + uv] - 128) / 224, v = (yuv[p * 1.25 + uv] - 128) / 224;+ const kr = tagged ? 0.2126 : 0.299, kb = tagged ? 0.0722 : 0.114, kg = 1 - kr - kb;+ return [y + 2 * (1 - kr) * v, y - 2 * kb * (1 - kb) / kg * u - 2 * kr * (1 - kr) / kg * v, y + 2 * (1 - kb) * u]+ .map(c => Math.round(Math.max(0, Math.min(1, c)) * 255));+ });+ const expectedRaw = tagged ? [[0, 0, 0], [0, 254, 255], [254, 255, 0], [0, 255, 1], [0, 0, 0]]+ : [[254, 0, 0], [0, 255, 1], [0, 0, 255], [255, 0, 254], [1, 255, 255]];+ rawSamples.forEach((rgb, i) => rgb.forEach((c, j) => expect(Math.abs(c - expectedRaw[i][j])).toBeLessThanOrEqual(2)));+ const origin = 'http://film-look-primary.test';+ const open = async gpu => {+ const p = await fixture.browser.newPage({ viewport: { width: w, height: h } });+ pages.push(p);+ await p.route(`${origin}/**`, route => route.fulfill(route.request().url().endsWith('.mp4')+ ? { body: clip, contentType: 'video/mp4' }+ : { contentType: 'text/html', body: `<!doctype html><style>*{margin:0}video,canvas{display:block;width:${w}px;height:${h}px}#out{position:fixed;left:0;top:0}</style>+ <video id="v" src="/clip.mp4" muted preload="auto"></video><svg id="fx" width="0" height="0" style="position:absolute"></svg>+ ${gpu ? `<canvas id="out" width="${w}" height="${h}"></canvas><script>${filmLookGpuRuntimeSource()}</script>` : ''}` }));+ await p.goto(`${origin}/`);+ await p.waitForFunction(() => document.getElementById('v').readyState >= 2, null, { timeout: 10000 });+ // The paused first frame is the frame independently decoded above.+ return p;+ };+ const svg = await open(false), gpu = await open(true);+ const metadata = await svg.evaluate(() => {+ const snapshot = new VideoFrame(document.getElementById('v'));+ try { return { format: snapshot.format, ...snapshot.colorSpace.toJSON() }; }+ finally { snapshot.close(); }+ });+ expect(metadata).toMatchObject({ fullRange: false, matrix: tagged ? 'bt709' : 'smpte170m', primaries: tagged ? 'bt709' : 'smpte170m' });+ expect(['NV12', 'I420']).toContain(metadata.format);+ expect(metadata.transfer).toBe(tagged || metadata.format === 'NV12' ? 'bt709' : 'smpte170m');+ const native = await sharp(await svg.screenshot()).removeAlpha().raw().toBuffer();+ if (!tagged) {+ const expectedNative = metadata.format === 'NV12'+ ? [[248, 36, 0], [63, 251, 0], [26, 35, 255], [249, 52, 255], [69, 253, 255]] : rawSamples;+ xs.forEach((x, i) => expectedNative[i].forEach((c, j) => expect(Math.abs(native[(200 * w + x) * 3 + j] - c)).toBeLessThanOrEqual(3)));+ }+ const look = filmLookPreset('neon-rain');+ await gpu.evaluate(({ look, w, h }) => {+ globalThis.primaryRenderer = globalThis.__portosFilmLookGpu.create(document.getElementById('out'));+ globalThis.primaryRenderer.render(document.getElementById('v'), look, { frame: 0, width: w, height: h, until: 0 });+ }, { look, w, h });+ const source = await sharp(await gpu.screenshot()).removeAlpha().raw().toBuffer();+ expect(psnr(native, source)).toBeGreaterThanOrEqual(40);+ const { filter, css } = filmLookFilterMarkup(look, { frame: 0, width: w, height: h });+ await svg.evaluate(({ filter, css }) => {+ document.getElementById('fx').innerHTML = `<defs>${filter}</defs>`;+ document.documentElement.style.filter = css;+ }, { filter, css });+ await gpu.evaluate(({ look, w, h }) => globalThis.primaryRenderer.render(document.getElementById('v'), look, { frame: 0, width: w, height: h }), { look, w, h });+ const reference = await sharp(await svg.screenshot()).removeAlpha().raw().toBuffer();+ const finished = await sharp(await gpu.screenshot()).removeAlpha().raw().toBuffer();+ expect(psnr(reference, finished)).toBeGreaterThanOrEqual(40);+ } finally {+ await Promise.all(pages.map(p => p.close()));+ rmSync(dir, { recursive: true, force: true });+ }+ }, 60000);+// A video is read the way the page's compositor reads it, scaled to fill the// frame. WebGL's own video upload converted colour differently (about 30 dB// off on the soft-analog look) and copied another size unscaled.diff --git a/server/lib/filmLookGpu.js b/server/lib/filmLookGpu.jsindex 8896fc9b92..0473e071b7 100644--- a/server/lib/filmLookGpu.js+++ b/server/lib/filmLookGpu.js@@ -183,6 +183,15 @@ function parseFilmLookFilter(filter) {return primitives;}+// Only this decoded-video contract was measured to omit the primary transform+// in Canvas2D. Software I420 (including SMPTE170M transfer) already matches the+// native compositor. Unknown contracts retain existing staging and admission.+function needsSmpte170mConversion(frame) {+ const space = frame?.colorSpace;+ return frame?.format === 'NV12' && space?.primaries === 'smpte170m'+ && space.matrix === 'smpte170m' && space.transfer === 'bt709' && space.fullRange === false;+}+/*** The browser source for the renderer. Defines `globalThis.__portosFilmLookGpu`:*@@ -200,6 +209,7 @@ export function filmLookGpuRuntimeSource() {const turbulenceTables = ${turbulenceTables.toString()};const blurKernel = ${blurKernel.toString()};const parseFilmLookFilter = ${parseFilmLookFilter.toString()};+ const needsSmpte170mConversion = ${needsSmpte170mConversion.toString()};const create = ${createFilmLookGpuRenderer.toString()};const available = () => {try {@@ -208,7 +218,7 @@ export function filmLookGpuRuntimeSource() {} catch { return false; }};Object.defineProperty(globalThis, '__portosFilmLookGpu', { configurable: false, writable: false,- value: Object.freeze({ available, create: (canvas) => create(canvas, { markup, turbulenceTables, blurKernel, parseFilmLookFilter }) }) });+ value: Object.freeze({ available, create: (canvas) => create(canvas, { markup, turbulenceTables, blurKernel, parseFilmLookFilter, needsSmpte170mConversion }) }) });})();`;}@@ -217,7 +227,7 @@ export function filmLookGpuRuntimeSource() {// premultiplied texture in image orientation (row 0 = top), as Skia's raster// filters keep them; the last pass flips into the canvas.function createFilmLookGpuRenderer(canvas, deps) {- const { markup, turbulenceTables, blurKernel: makeBlurKernel, parseFilmLookFilter } = deps;+ const { markup, turbulenceTables, blurKernel: makeBlurKernel, parseFilmLookFilter, needsSmpte170mConversion } = deps;const gl = canvas.getContext('webgl2', { alpha: false, antialias: false, depth: false, stencil: false, premultipliedAlpha: true, preserveDrawingBuffer: true });if (!gl) throw new Error('WebGL2 is unavailable');if (!gl.getExtension('EXT_color_buffer_float')) throw new Error('WebGL2 float render targets are unavailable');@@ -271,8 +281,28 @@ bool clipped() { vec2 p = gl_FragCoord.xy; return p.x < clip.x || p.y < clip.y |`;const PROGRAMS = {// The source picture, flipped from texture upload order into image order.- source: `uniform sampler2D a;-void main() { color = texelFetch(a, pixel(), 0); }`,+ // inv(RGB709→XYZ) × RGBSMPTE-C→XYZ, using D65 white (.3127,.3290),+ // 709 primaries (.64,.33),(.30,.60),(.15,.06) and SMPTE-C primaries+ // (.63,.34),(.31,.595),(.155,.07). Chromium renders BT709 transfer with+ // the sRGB curve (ui/gfx/color_space.cc), so transform in linear sRGB.+ source: `uniform sampler2D a; uniform int convertPrimaries;+vec3 linearRGB(vec3 c) {+ return mix(c / 12.92, pow((c + 0.055) / 1.055, vec3(2.4)), step(vec3(0.04045), c));+}+vec3 encodedRGB(vec3 c) {+ c = clamp(c, 0.0, 1.0);+ return mix(c * 12.92, 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055, step(vec3(0.0031308), c));+}+void main() {+ color = texelFetch(a, pixel(), 0);+ if (convertPrimaries == 1) {+ vec3 c = linearRGB(unpremul(color).rgb);+ c = mat3(0.939542064, 0.017772223, -0.001621600,+ 0.050181357, 0.965792862, -0.004369750,+ 0.010276579, 0.016434914, 1.005991350) * c;+ color.rgb = encodedRGB(c) * color.a;+ }+}`,flood: `uniform vec4 floodColor;void main() { color = clipped() ? vec4(0.0) : floodColor; }`,offset: `uniform sampler2D a; uniform vec2 shift; uniform int sampled;@@ -629,8 +659,21 @@ void main() {// texImage2D copies a video of another size unscaled instead of filling the// frame the way the page's object-fit does.let staging = null;+ let convertPrimaries = false;const stage = (source) => {+ convertPrimaries = false;if (typeof HTMLVideoElement === 'undefined' || !(source instanceof HTMLVideoElement)) return source;+ if (typeof VideoFrame !== 'undefined') {+ let snapshot;+ try {+ snapshot = new VideoFrame(source);+ convertPrimaries = needsSmpte170mConversion(snapshot);+ } catch { /* Metadata unavailable: preserve existing staging. */ }+ finally { snapshot?.close(); }+ }+ // This tuple narrows applicability; it is not a universal browser promise.+ // If a future canvas already converts it, the unchanged native-oracle+ // fidelity gate rejects double conversion and safely retains SVG.staging ||= document.createElement('canvas');if (staging.width !== width || staging.height !== height) { staging.width = width; staging.height = height; }staging.getContext('2d', { alpha: false }).drawImage(source, 0, 0, width, height);@@ -662,7 +705,10 @@ void main() {const frameRegion = [0, 0, width, height];const results = new Map();const sourceTarget = acquire();- run('source', sourceTarget, (u, bind) => bind('a', uploadTexture));+ run('source', sourceTarget, (u, bind) => {+ bind('a', uploadTexture);+ gl.uniform1i(u.convertPrimaries, convertPrimaries ? 1 : 0);+ });results.set('SourceGraphic', { target: sourceTarget, region: frameRegion });let previous = 'SourceGraphic';const release = (name) => {@@ -791,4 +837,4 @@ void main() {}// The pieces the renderer inlines, for unit tests against what Chrome draws.-export const _filmLookGpuTestHooks = Object.freeze({ turbulenceTables, turbulencePixel, blurKernel, parseFilmLookFilter });+export const _filmLookGpuTestHooks = Object.freeze({ turbulenceTables, turbulencePixel, blurKernel, parseFilmLookFilter, needsSmpte170mConversion });diff --git a/server/lib/filmLookGpu.test.js b/server/lib/filmLookGpu.test.jsindex 872657b0b6..e8dc995c02 100644--- a/server/lib/filmLookGpu.test.js+++ b/server/lib/filmLookGpu.test.js@@ -3,7 +3,7 @@ import { runInNewContext } from 'node:vm';import { FILM_LOOK_PRESETS, filmLookFilterMarkup, filmLookPreset } from './filmLook.js';import { _filmLookGpuTestHooks, filmLookGpuRuntimeSource } from './filmLookGpu.js';-const { blurKernel, parseFilmLookFilter, turbulencePixel, turbulenceTables } = _filmLookGpuTestHooks;+const { blurKernel, parseFilmLookFilter, turbulencePixel, turbulenceTables, needsSmpte170mConversion } = _filmLookGpuTestHooks;// Bytes headless Chromium 141 drew for a lone <feTurbulence> (64×48, sRGB,// userSpaceOnUse), read back from an unpremultiplied PNG screenshot over a@@ -20,6 +20,16 @@ const CHROME_TURBULENCE = [];describe('film look on the GPU', () => {+ it('corrects only the measured NV12 SMPTE170M primary contract', () => {+ const colorSpace = { primaries: 'smpte170m', matrix: 'smpte170m', transfer: 'bt709', fullRange: false };+ expect(needsSmpte170mConversion({ format: 'NV12', colorSpace })).toBe(true);+ for (const frame of [undefined, {}, { format: 'NV12' },+ ...['I420', 'RGBA', null].map(format => ({ format, colorSpace })),+ ...[{ primaries: 'bt709' }, { matrix: 'bt709' }, { transfer: 'smpte170m' }, { fullRange: true }, { fullRange: null }]+ .map(patch => ({ format: 'NV12', colorSpace: { ...colorSpace, ...patch } })),+ ]) expect(needsSmpte170mConversion(frame)).toBe(false);+ });+it('reproduces the feTurbulence pixels Chrome draws for a seed, not just their statistics', () => {for (const { seed, samples, ...options } of CHROME_TURBULENCE) {const tables = turbulenceTables(seed);