Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Fix GPU staging colors for native SMPTE-170M video (#11729)

[._.]/4722e2c0e3d9
diff --git a/server/lib/filmLookGpu.browser.test.js b/server/lib/filmLookGpu.browser.test.js
index 65cd34d531..86195d4b7e 100644
--- a/server/lib/filmLookGpu.browser.test.js
+++ b/server/lib/filmLookGpu.browser.test.js
@@ -87,6 +87,90 @@ describe.skipIf(!canRun).each([
expect(psnr(svg, gpu)).toBeGreaterThanOrEqual(40);
}, 60000);
+ // Independent video pixels, not two consumers of one canvas. The untagged
+ // SD clip decodes as NV12/SMPTE-C on macOS hardware and I420 in software;
+ // only the former omits the primary conversion during canvas staging.
+ it.for(['untagged SD', 'converted BT709'])('preserves native source colors and Neon rain for %s', async (kind, { skip }) => {
+ if (!available) skip();
+ const w = 640, h = 360, tagged = kind === 'converted BT709';
+ const dir = mkdtempSync(join(tmpdir(), 'film-look-primary-'));
+ const path = join(dir, 'clip.mp4');
+ const pages = [];
+ try {
+ // testsrc is RGB: this performs a real matrix conversion, not a relabel
+ // of the SD YUV bytes. Both frame and container tags mirror the encoder.
+ execFileSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i',
+ `${tagged ? 'testsrc' : 'testsrc2'}=s=${w}x${h}:r=24`, '-t', '0.25',
+ ...(tagged ? ['-vf', 'scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709',
+ '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709'] : []),
+ '-c:v', 'libx264', '-threads', '1', '-pix_fmt', 'yuv420p', '-y', path], { timeout: 10000 });
+ const clip = readFileSync(path);
+ const yuv = execFileSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-threads', '1', '-i', path,
+ '-frames:v', '1', '-f', 'rawvideo', '-pix_fmt', 'yuv420p', 'pipe:1'], { timeout: 10000, maxBuffer: w * h * 2 });
+ // Uniform bar interiors avoid chroma-siting differences. Decode the
+ // independent YUV planes with the declared limited-range 601/709 matrix.
+ const xs = [40, 140, 340, 440, 540];
+ const rawSamples = xs.map(x => {
+ const p = w * h, uv = 100 * (w / 2) + x / 2;
+ const y = (yuv[200 * w + x] - 16) / 219, u = (yuv[p + uv] - 128) / 224, v = (yuv[p * 1.25 + uv] - 128) / 224;
+ const kr = tagged ? 0.2126 : 0.299, kb = tagged ? 0.0722 : 0.114, kg = 1 - kr - kb;
+ return [y + 2 * (1 - kr) * v, y - 2 * kb * (1 - kb) / kg * u - 2 * kr * (1 - kr) / kg * v, y + 2 * (1 - kb) * u]
+ .map(c => Math.round(Math.max(0, Math.min(1, c)) * 255));
+ });
+ const expectedRaw = tagged ? [[0, 0, 0], [0, 254, 255], [254, 255, 0], [0, 255, 1], [0, 0, 0]]
+ : [[254, 0, 0], [0, 255, 1], [0, 0, 255], [255, 0, 254], [1, 255, 255]];
+ rawSamples.forEach((rgb, i) => rgb.forEach((c, j) => expect(Math.abs(c - expectedRaw[i][j])).toBeLessThanOrEqual(2)));
+ const origin = 'http://film-look-primary.test';
+ const open = async gpu => {
+ const p = await fixture.browser.newPage({ viewport: { width: w, height: h } });
+ pages.push(p);
+ await p.route(`${origin}/**`, route => route.fulfill(route.request().url().endsWith('.mp4')
+ ? { body: clip, contentType: 'video/mp4' }
+ : { contentType: 'text/html', body: `<!doctype html><style>*{margin:0}video,canvas{display:block;width:${w}px;height:${h}px}#out{position:fixed;left:0;top:0}</style>
+ <video id="v" src="/clip.mp4" muted preload="auto"></video><svg id="fx" width="0" height="0" style="position:absolute"></svg>
+ ${gpu ? `<canvas id="out" width="${w}" height="${h}"></canvas><script>${filmLookGpuRuntimeSource()}</script>` : ''}` }));
+ await p.goto(`${origin}/`);
+ await p.waitForFunction(() => document.getElementById('v').readyState >= 2, null, { timeout: 10000 });
+ // The paused first frame is the frame independently decoded above.
+ return p;
+ };
+ const svg = await open(false), gpu = await open(true);
+ const metadata = await svg.evaluate(() => {
+ const snapshot = new VideoFrame(document.getElementById('v'));
+ try { return { format: snapshot.format, ...snapshot.colorSpace.toJSON() }; }
+ finally { snapshot.close(); }
+ });
+ expect(metadata).toMatchObject({ fullRange: false, matrix: tagged ? 'bt709' : 'smpte170m', primaries: tagged ? 'bt709' : 'smpte170m' });
+ expect(['NV12', 'I420']).toContain(metadata.format);
+ expect(metadata.transfer).toBe(tagged || metadata.format === 'NV12' ? 'bt709' : 'smpte170m');
+ const native = await sharp(await svg.screenshot()).removeAlpha().raw().toBuffer();
+ if (!tagged) {
+ const expectedNative = metadata.format === 'NV12'
+ ? [[248, 36, 0], [63, 251, 0], [26, 35, 255], [249, 52, 255], [69, 253, 255]] : rawSamples;
+ xs.forEach((x, i) => expectedNative[i].forEach((c, j) => expect(Math.abs(native[(200 * w + x) * 3 + j] - c)).toBeLessThanOrEqual(3)));
+ }
+ const look = filmLookPreset('neon-rain');
+ await gpu.evaluate(({ look, w, h }) => {
+ globalThis.primaryRenderer = globalThis.__portosFilmLookGpu.create(document.getElementById('out'));
+ globalThis.primaryRenderer.render(document.getElementById('v'), look, { frame: 0, width: w, height: h, until: 0 });
+ }, { look, w, h });
+ const source = await sharp(await gpu.screenshot()).removeAlpha().raw().toBuffer();
+ expect(psnr(native, source)).toBeGreaterThanOrEqual(40);
+ const { filter, css } = filmLookFilterMarkup(look, { frame: 0, width: w, height: h });
+ await svg.evaluate(({ filter, css }) => {
+ document.getElementById('fx').innerHTML = `<defs>${filter}</defs>`;
+ document.documentElement.style.filter = css;
+ }, { filter, css });
+ await gpu.evaluate(({ look, w, h }) => globalThis.primaryRenderer.render(document.getElementById('v'), look, { frame: 0, width: w, height: h }), { look, w, h });
+ const reference = await sharp(await svg.screenshot()).removeAlpha().raw().toBuffer();
+ const finished = await sharp(await gpu.screenshot()).removeAlpha().raw().toBuffer();
+ expect(psnr(reference, finished)).toBeGreaterThanOrEqual(40);
+ } finally {
+ await Promise.all(pages.map(p => p.close()));
+ rmSync(dir, { recursive: true, force: true });
+ }
+ }, 60000);
+
// A video is read the way the page's compositor reads it, scaled to fill the
// frame. WebGL's own video upload converted colour differently (about 30 dB
// off on the soft-analog look) and copied another size unscaled.
diff --git a/server/lib/filmLookGpu.js b/server/lib/filmLookGpu.js
index 8896fc9b92..0473e071b7 100644
--- a/server/lib/filmLookGpu.js
+++ b/server/lib/filmLookGpu.js
@@ -183,6 +183,15 @@ function parseFilmLookFilter(filter) {
return primitives;
}
+// Only this decoded-video contract was measured to omit the primary transform
+// in Canvas2D. Software I420 (including SMPTE170M transfer) already matches the
+// native compositor. Unknown contracts retain existing staging and admission.
+function needsSmpte170mConversion(frame) {
+ const space = frame?.colorSpace;
+ return frame?.format === 'NV12' && space?.primaries === 'smpte170m'
+ && space.matrix === 'smpte170m' && space.transfer === 'bt709' && space.fullRange === false;
+}
+
/**
* The browser source for the renderer. Defines `globalThis.__portosFilmLookGpu`:
*
@@ -200,6 +209,7 @@ export function filmLookGpuRuntimeSource() {
const turbulenceTables = ${turbulenceTables.toString()};
const blurKernel = ${blurKernel.toString()};
const parseFilmLookFilter = ${parseFilmLookFilter.toString()};
+ const needsSmpte170mConversion = ${needsSmpte170mConversion.toString()};
const create = ${createFilmLookGpuRenderer.toString()};
const available = () => {
try {
@@ -208,7 +218,7 @@ export function filmLookGpuRuntimeSource() {
} catch { return false; }
};
Object.defineProperty(globalThis, '__portosFilmLookGpu', { configurable: false, writable: false,
- value: Object.freeze({ available, create: (canvas) => create(canvas, { markup, turbulenceTables, blurKernel, parseFilmLookFilter }) }) });
+ value: Object.freeze({ available, create: (canvas) => create(canvas, { markup, turbulenceTables, blurKernel, parseFilmLookFilter, needsSmpte170mConversion }) }) });
})();`;
}
@@ -217,7 +227,7 @@ export function filmLookGpuRuntimeSource() {
// premultiplied texture in image orientation (row 0 = top), as Skia's raster
// filters keep them; the last pass flips into the canvas.
function createFilmLookGpuRenderer(canvas, deps) {
- const { markup, turbulenceTables, blurKernel: makeBlurKernel, parseFilmLookFilter } = deps;
+ const { markup, turbulenceTables, blurKernel: makeBlurKernel, parseFilmLookFilter, needsSmpte170mConversion } = deps;
const gl = canvas.getContext('webgl2', { alpha: false, antialias: false, depth: false, stencil: false, premultipliedAlpha: true, preserveDrawingBuffer: true });
if (!gl) throw new Error('WebGL2 is unavailable');
if (!gl.getExtension('EXT_color_buffer_float')) throw new Error('WebGL2 float render targets are unavailable');
@@ -271,8 +281,28 @@ bool clipped() { vec2 p = gl_FragCoord.xy; return p.x < clip.x || p.y < clip.y |
`;
const PROGRAMS = {
// The source picture, flipped from texture upload order into image order.
- source: `uniform sampler2D a;
-void main() { color = texelFetch(a, pixel(), 0); }`,
+ // inv(RGB709→XYZ) × RGBSMPTE-C→XYZ, using D65 white (.3127,.3290),
+ // 709 primaries (.64,.33),(.30,.60),(.15,.06) and SMPTE-C primaries
+ // (.63,.34),(.31,.595),(.155,.07). Chromium renders BT709 transfer with
+ // the sRGB curve (ui/gfx/color_space.cc), so transform in linear sRGB.
+ source: `uniform sampler2D a; uniform int convertPrimaries;
+vec3 linearRGB(vec3 c) {
+ return mix(c / 12.92, pow((c + 0.055) / 1.055, vec3(2.4)), step(vec3(0.04045), c));
+}
+vec3 encodedRGB(vec3 c) {
+ c = clamp(c, 0.0, 1.0);
+ return mix(c * 12.92, 1.055 * pow(c, vec3(1.0 / 2.4)) - 0.055, step(vec3(0.0031308), c));
+}
+void main() {
+ color = texelFetch(a, pixel(), 0);
+ if (convertPrimaries == 1) {
+ vec3 c = linearRGB(unpremul(color).rgb);
+ c = mat3(0.939542064, 0.017772223, -0.001621600,
+ 0.050181357, 0.965792862, -0.004369750,
+ 0.010276579, 0.016434914, 1.005991350) * c;
+ color.rgb = encodedRGB(c) * color.a;
+ }
+}`,
flood: `uniform vec4 floodColor;
void main() { color = clipped() ? vec4(0.0) : floodColor; }`,
offset: `uniform sampler2D a; uniform vec2 shift; uniform int sampled;
@@ -629,8 +659,21 @@ void main() {
// texImage2D copies a video of another size unscaled instead of filling the
// frame the way the page's object-fit does.
let staging = null;
+ let convertPrimaries = false;
const stage = (source) => {
+ convertPrimaries = false;
if (typeof HTMLVideoElement === 'undefined' || !(source instanceof HTMLVideoElement)) return source;
+ if (typeof VideoFrame !== 'undefined') {
+ let snapshot;
+ try {
+ snapshot = new VideoFrame(source);
+ convertPrimaries = needsSmpte170mConversion(snapshot);
+ } catch { /* Metadata unavailable: preserve existing staging. */ }
+ finally { snapshot?.close(); }
+ }
+ // This tuple narrows applicability; it is not a universal browser promise.
+ // If a future canvas already converts it, the unchanged native-oracle
+ // fidelity gate rejects double conversion and safely retains SVG.
staging ||= document.createElement('canvas');
if (staging.width !== width || staging.height !== height) { staging.width = width; staging.height = height; }
staging.getContext('2d', { alpha: false }).drawImage(source, 0, 0, width, height);
@@ -662,7 +705,10 @@ void main() {
const frameRegion = [0, 0, width, height];
const results = new Map();
const sourceTarget = acquire();
- run('source', sourceTarget, (u, bind) => bind('a', uploadTexture));
+ run('source', sourceTarget, (u, bind) => {
+ bind('a', uploadTexture);
+ gl.uniform1i(u.convertPrimaries, convertPrimaries ? 1 : 0);
+ });
results.set('SourceGraphic', { target: sourceTarget, region: frameRegion });
let previous = 'SourceGraphic';
const release = (name) => {
@@ -791,4 +837,4 @@ void main() {
}
// The pieces the renderer inlines, for unit tests against what Chrome draws.
-export const _filmLookGpuTestHooks = Object.freeze({ turbulenceTables, turbulencePixel, blurKernel, parseFilmLookFilter });
+export const _filmLookGpuTestHooks = Object.freeze({ turbulenceTables, turbulencePixel, blurKernel, parseFilmLookFilter, needsSmpte170mConversion });
diff --git a/server/lib/filmLookGpu.test.js b/server/lib/filmLookGpu.test.js
index 872657b0b6..e8dc995c02 100644
--- a/server/lib/filmLookGpu.test.js
+++ b/server/lib/filmLookGpu.test.js
@@ -3,7 +3,7 @@ import { runInNewContext } from 'node:vm';
import { FILM_LOOK_PRESETS, filmLookFilterMarkup, filmLookPreset } from './filmLook.js';
import { _filmLookGpuTestHooks, filmLookGpuRuntimeSource } from './filmLookGpu.js';
-const { blurKernel, parseFilmLookFilter, turbulencePixel, turbulenceTables } = _filmLookGpuTestHooks;
+const { blurKernel, parseFilmLookFilter, turbulencePixel, turbulenceTables, needsSmpte170mConversion } = _filmLookGpuTestHooks;
// Bytes headless Chromium 141 drew for a lone <feTurbulence> (64×48, sRGB,
// userSpaceOnUse), read back from an unpremultiplied PNG screenshot over a
@@ -20,6 +20,16 @@ const CHROME_TURBULENCE = [
];
describe('film look on the GPU', () => {
+ it('corrects only the measured NV12 SMPTE170M primary contract', () => {
+ const colorSpace = { primaries: 'smpte170m', matrix: 'smpte170m', transfer: 'bt709', fullRange: false };
+ expect(needsSmpte170mConversion({ format: 'NV12', colorSpace })).toBe(true);
+ for (const frame of [undefined, {}, { format: 'NV12' },
+ ...['I420', 'RGBA', null].map(format => ({ format, colorSpace })),
+ ...[{ primaries: 'bt709' }, { matrix: 'bt709' }, { transfer: 'smpte170m' }, { fullRange: true }, { fullRange: null }]
+ .map(patch => ({ format: 'NV12', colorSpace: { ...colorSpace, ...patch } })),
+ ]) expect(needsSmpte170mConversion(frame)).toBe(false);
+ });
+
it('reproduces the feTurbulence pixels Chrome draws for a seed, not just their statistics', () => {
for (const { seed, samples, ...options } of CHROME_TURBULENCE) {
const tables = turbulenceTables(seed);