Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

perf: skip inactive cinematic work for toon ink-only frames (#10713)

Adam Eivy2c991bc96ebe
diff --git a/server/services/musicVideo/documentTemplates/shared/toonWorld.browser.test.js b/server/services/musicVideo/documentTemplates/shared/toonWorld.browser.test.js
index 790c7cc741..2f465a86b6 100644
--- a/server/services/musicVideo/documentTemplates/shared/toonWorld.browser.test.js
+++ b/server/services/musicVideo/documentTemplates/shared/toonWorld.browser.test.js
@@ -60,7 +60,8 @@ describe.skipIf(!canRun)('toon world headless dish proof', () => {
it('supports a manual/layered post stack and restores the caller target', async () => {
const result = await page.evaluate(() => window.manualProof());
- expect(result.changed).toBeGreaterThan(300);
+ expect(result.changed).toHaveLength(2);
+ for (const count of result.changed) expect(count).toBeGreaterThan(300);
expect(result.restored).toBe(true);
expect(result.feedbackRefused).toBe(true);
expect(errors).toEqual([]);
@@ -98,14 +99,19 @@ window.manualProof=()=>{
const input=new THREE.WebGLRenderTarget(640,360,{type:THREE.HalfFloatType,depthTexture:new THREE.DepthTexture(640,360,THREE.FloatType)});
const output=new THREE.WebGLRenderTarget(640,360,{depthBuffer:false});
const ink=kit.createInkPass(THREE,renderer);
- renderer.setRenderTarget(input);renderer.render(scene,camera);
+ const ortho=new THREE.OrthographicCamera(-12,12,6.75,-6.75,.1,100);
+ ortho.position.copy(camera.position);ortho.lookAt(0,1,0);
const read=()=>{const p=new Uint8Array(640*360*4);renderer.readRenderTargetPixels(output,0,0,640,360,p);return p;};
- ink.render(input,camera,output,{enabled:false});const baseline=read();
- ink.render(input,camera,output);const outlined=read();
- const restored=renderer.getRenderTarget()===input;
+ const changes=[];let restored=true;
+ for(const view of [camera,ortho]){
+ renderer.setRenderTarget(input);renderer.render(scene,view);
+ ink.render(input,view,output,{enabled:false});const baseline=read();
+ ink.render(input,view,output);const outlined=read();
+ changes.push(changed(baseline,outlined));restored&&=renderer.getRenderTarget()===input;
+ }
let feedbackRefused=false;try{ink.render(input,camera,input);}catch{feedbackRefused=true;}
renderer.setRenderTarget(null);ink.dispose();input.dispose();output.dispose();
- return {changed:changed(baseline,outlined),restored,feedbackRefused};
+ return {changed:changes,restored,feedbackRefused};
};
window.proof=()=>{
const baseline=draw(), ink=draw({ink:true});
diff --git a/server/services/musicVideo/documentTemplates/shared/toonWorld.js b/server/services/musicVideo/documentTemplates/shared/toonWorld.js
index a64ee2be08..873617af57 100644
--- a/server/services/musicVideo/documentTemplates/shared/toonWorld.js
+++ b/server/services/musicVideo/documentTemplates/shared/toonWorld.js
@@ -181,14 +181,21 @@ uniform vec2 uInkRes; uniform vec3 uInkColor;
uniform float uInkWidth, uInkDepthThreshold, uInkNormalThreshold, uInkEnabled;
vec3 inkPosition(vec2 uv){
float d = texture2D(tInkDepth, uv).x;
- vec4 p = uInkInverseProjection * vec4(uv * 2. - 1., d * 2. - 1., 1.);
- return p.xyz / p.w;
+ float z = d * 2. - 1.;
+ // Standard Three perspective/off-axis and orthographic projections have
+ // this sparse inverse. Avoid five full mat4 multiplies per output pixel.
+ vec2 xy = (uv * 2. - 1.) * vec2(uInkInverseProjection[0][0], uInkInverseProjection[1][1])
+ + vec2(uInkInverseProjection[3][0], uInkInverseProjection[3][1]);
+ return vec3(xy, z * uInkInverseProjection[2][2] + uInkInverseProjection[3][2])
+ / (z * uInkInverseProjection[2][3] + uInkInverseProjection[3][3]);
}
-float inkEdge(vec3 delta, vec3 nn, vec3 n, float planeThreshold2, float alignmentThreshold){
- float distanceToPlane = dot(delta, n), alignment = dot(n, nn);
- float discontinuity = step(planeThreshold2, distanceToPlane * distanceToPlane);
- float crease = step(alignment * alignment, alignmentThreshold * dot(nn, nn));
- return step(delta.z, 1e-5) * max(discontinuity, crease);
+float inkPlaneEdge(vec3 delta, vec3 n, float threshold){
+ float distanceToPlane = dot(delta, n);
+ return step(delta.z, 1e-5) * step(threshold, distanceToPlane * distanceToPlane);
+}
+float inkCrease(vec3 a, vec3 b, float alignment2){
+ float alignment = dot(a, b);
+ return step(alignment * alignment, alignment2 * dot(a, a) * dot(b, b));
}
vec3 applyInk(vec3 color, vec2 uv){
if (uInkEnabled < .5 || uInkWidth <= 0.) return color;
@@ -208,12 +215,16 @@ vec3 applyInk(vec3 color, vec2 uv){
float normalLength2 = max(dot(n, n), 1e-20);
float planeThreshold2 = uInkDepthThreshold * uInkDepthThreshold * max(dot(p, p), 1e-6) * normalLength2;
float alignment2 = (1. - uInkNormalThreshold) * (1. - uInkNormalThreshold);
- float alignmentThreshold = alignment2 * normalLength2;
- float edge = max(
- max(inkEdge(r, cross(r, ny), n, planeThreshold2, alignmentThreshold),
- inkEdge(-l, cross(l, ny), n, planeThreshold2, alignmentThreshold)),
- max(inkEdge(u, cross(nx, u), n, planeThreshold2, alignmentThreshold),
- inkEdge(-d, cross(nx, d), n, planeThreshold2, alignmentThreshold)));
+ float planeEdge = max(
+ max(inkPlaneEdge(r, n, planeThreshold2), inkPlaneEdge(-l, n, planeThreshold2)),
+ max(inkPlaneEdge(u, n, planeThreshold2), inkPlaneEdge(-d, n, planeThreshold2)));
+ // One of each pair is exactly the nearest normal selected above, so a
+ // single comparison per axis replaces four comparisons to that normal.
+ float creaseX = inkCrease(cross(r, ny), cross(l, ny), alignment2)
+ * step(abs(r.z) < abs(l.z) ? -l.z : r.z, 1e-5);
+ float creaseY = inkCrease(cross(nx, u), cross(nx, d), alignment2)
+ * step(abs(u.z) < abs(d.z) ? -d.z : u.z, 1e-5);
+ float edge = max(planeEdge, max(creaseX, creaseY));
return mix(color, uInkColor, edge);
}`;
diff --git a/server/services/musicVideo/documentTemplates/spatial/engine.js b/server/services/musicVideo/documentTemplates/spatial/engine.js
index 3088bf75dc..0e1b7eab23 100644
--- a/server/services/musicVideo/documentTemplates/spatial/engine.js
+++ b/server/services/musicVideo/documentTemplates/spatial/engine.js
@@ -172,8 +172,16 @@ export function createPost(THREE, renderer) {
c += (hash(vUv * uRes + mod(uFrame, 251.) * 7.13) - .5) * uGrain;
gl_FragColor = vec4(clamp(c, 0., 1.), 1.);
}`, { tColor: { value: null }, tBloom: { value: null }, uRes: { value: new THREE.Vector2() }, uBloom: { value: 0 }, uExposure: { value: 1 }, uVignette: { value: 0 }, uGrain: { value: 0 }, uFrame: { value: 0 }, ...ink.uniforms });
+ // Ink alone must preserve the direct path's palette and avoid paying for
+ // inactive bloom, highlight grading and grain in the cinematic composite.
+ const inkOnly = pass(/* glsl */`
+ ${toonWorld.inkShader}
+ varying vec2 vUv; uniform sampler2D tColor;
+ vec3 srgb(vec3 c){ return mix(c * 12.92, 1.055 * pow(c, vec3(1. / 2.4)) - .055, step(.0031308, c)); }
+ void main(){ gl_FragColor = vec4(srgb(clamp(applyInk(texture2D(tColor, vUv).rgb, vUv), 0., 1.)), 1.); }
+ `, { tColor: { value: null }, ...ink.uniforms });
let targets = [];
- let hdr, hdrDepth, colour, small, smallB, w = 1, h = 1;
+ let hdr, hdrDepth, inkTarget, colour, small, smallB, w = 1, h = 1;
// A software rasterizer (CI, GPU-less hosts) multiplies every MSAA sample, so
// it renders single-sampled; real GPUs keep 4x.
const gl = renderer.getContext();
@@ -190,10 +198,11 @@ export function createPost(THREE, renderer) {
// depth texture (resolving one is a full extra copy every frame).
hdr = new THREE.WebGLRenderTarget(w, h, { type: THREE.HalfFloatType, samples });
hdrDepth = new THREE.WebGLRenderTarget(w, h, { type: THREE.HalfFloatType, samples, depthTexture: new THREE.DepthTexture(w, h, THREE.FloatType) });
+ inkTarget = new THREE.WebGLRenderTarget(w, h, { samples, depthTexture: new THREE.DepthTexture(w, h, THREE.FloatType) });
colour = new THREE.WebGLRenderTarget(w, h, linear);
small = new THREE.WebGLRenderTarget(Math.ceil(w / 4), Math.ceil(h / 4), linear);
smallB = new THREE.WebGLRenderTarget(Math.ceil(w / 4), Math.ceil(h / 4), linear);
- targets = [hdr, hdrDepth, colour, small, smallB];
+ targets = [hdr, hdrDepth, inkTarget, colour, small, smallB];
// Allocate every target now: passes a lens skips must not create GPU
// textures on a later seek.
for (const target of targets) renderer.initRenderTarget(target);
@@ -219,6 +228,13 @@ export function createPost(THREE, renderer) {
renderer.setRenderTarget(null); renderer.render(world, view);
return;
}
+ if (inking && !focusing && !bloom && !vignette && !grain && exposure === 1) {
+ ink.configure(inkTarget.depthTexture, view, w, h, typeof lens.ink === 'object' ? lens.ink : {});
+ renderer.setRenderTarget(inkTarget); renderer.clear(); renderer.render(world, view);
+ inkOnly.uniforms.tColor.value = inkTarget.texture;
+ draw(inkOnly, null);
+ return;
+ }
const sceneTarget = focusing || inking ? hdrDepth : hdr;
ink.configure(sceneTarget.depthTexture, view, w, h, { ...(typeof lens.ink === 'object' ? lens.ink : {}), enabled: inking });
renderer.setRenderTarget(sceneTarget); renderer.clear(); renderer.render(world, view);