Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(effects): the recording's surface in the effect contract

EtienneLescot0392b3b960be
diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs
index 541840ae2..59b2a3b7d 100644
--- a/crates/compositor/src/compositor_linux.rs
+++ b/crates/compositor/src/compositor_linux.rs
@@ -3749,7 +3749,8 @@ impl Compositor {
let fx_inputs = crate::shader_fx::FxFrameInputs {
cursor: fx_cursor,
clicks: fx_clicks,
- footage: crate::shader_fx::bounds(&g.footage_quad([rw, rh]).corners),
+ footage: g.footage_quad([rw, rh]),
+ footage_aspect: g.footage_aspect([rw, rh]),
camera: lp.has_webcam.then_some(g.w_dst),
};
let effects = scene_ref
diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs
index 6014ec98c..8de571408 100644
--- a/crates/compositor/src/compositor_windows.rs
+++ b/crates/compositor/src/compositor_windows.rs
@@ -2811,7 +2811,8 @@ impl Compositor {
let fx_inputs = crate::shader_fx::FxFrameInputs {
cursor: fx_cursor,
clicks: fx_clicks,
- footage: crate::shader_fx::bounds(&g.footage_quad([self.rw(), self.rh()]).corners),
+ footage: g.footage_quad([self.rw(), self.rh()]),
+ footage_aspect: g.footage_aspect([self.rw(), self.rh()]),
camera: lp.has_webcam.then_some(w_dst),
};
let effects = scene_ref
@@ -3872,7 +3873,11 @@ mod tests {
/// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond.
fn shader_fx_scene(effects: serde_json::Value) -> Scene {
- let json = serde_json::json!({
+ Scene::from_json(&shader_fx_json(effects).to_string()).expect("scène")
+ }
+
+ fn shader_fx_json(effects: serde_json::Value) -> serde_json::Value {
+ serde_json::json!({
"clips": [],
"layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},
"effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0},
@@ -3882,8 +3887,7 @@ mod tests {
"cropByClip": [],
"output": {"width":320,"height":180,"fps":null},
"shaderEffects": effects,
- });
- Scene::from_json(&json.to_string()).expect("scène")
+ })
}
/// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie.
@@ -3982,6 +3986,59 @@ mod tests {
assert_eq!(corner, [0, 255, 0]);
}
+ /// `fx_to_footage` retrouve le point du métrage que le compositeur a dessiné sous un pixel,
+ /// à plat, sous un angle fixe (warp bilinéaire) et sous la caméra réelle (homographie), et
+ /// `fx_from_footage` le ramène sur ce pixel. En 1080p, l'arrondi 8 bits et le demi-pixel
+ /// tiennent sous 0,005 ; la pose orbit testée écarte l'homographie du bilinéaire de ~0,01.
+ #[test]
+ fn the_footage_plane_maps_back_to_what_the_compositor_drew() {
+ let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else {
+ eprintln!("pas de device D3D11 — test sauté");
+ return;
+ };
+ const PLANE: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n let p = fx_to_footage(uv);\n return vec4<f32>(p, distance(fx_from_footage(p), uv) * 100.0, 1.0);\n}\n";
+ for (rotation, projective) in [(None, false), (Some("left"), false), (Some("orbit"), true)] {
+ let mut json = shader_fx_json(serde_json::json!([fx(PLANE, "image", 0)]));
+ json["clips"] = serde_json::json!([{"screenPath": "/s.mp4", "webcamPath": "", "sourceStartSec": 0,
+ "sourceEndSec": 10, "webcamOffsetSec": 0, "hasAudio": false}]);
+ json["layout"]["screenRect"] = serde_json::json!({"x": 0.1, "y": 0.1, "width": 0.8, "height": 0.8});
+ json["cropByClip"] = serde_json::json!([null]);
+ if let Some(rotation) = rotation {
+ json["zoomRegions"] = serde_json::json!([{"clipIndex": 0, "startSec": 0, "endSec": 10,
+ "scale": 1.5, "focusX": 0.5, "focusY": 0.5, "rotation": rotation}]);
+ }
+ let comp = Compositor::new_sized(&gpu, 1920, 1080).expect("compositeur");
+ let scene = Scene::from_json(&json.to_string()).expect("scène");
+ comp.set_live_params(live_params_from_scene(&scene));
+ comp.set_scene(Some(scene));
+ let (screen, _tex) = nv12_frame(&gpu, 64, 64);
+ comp.set_timeline_time(Some(2.0));
+ comp.set_programme_time(Some(2.0));
+ unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose");
+ let quad = comp.footage_quad().expect("le métrage de la frame");
+ assert_eq!(quad.projective, projective, "{rotation:?}");
+ let (w, h, rgba) = unsafe { comp.readback_direct() }.expect("readback");
+ let [tl, tr, br, bl] = quad.corners.map(|[x, y]| (x, y));
+ for (s, t) in [(0.25, 0.3), (0.7, 0.4), (0.4, 0.75)] {
+ let (x, y) = if projective {
+ crate::regions::square_to_quad(&[tl, tr, br, bl], s, t)
+ } else {
+ let top = (tl.0 + (tr.0 - tl.0) * s, tl.1 + (tr.1 - tl.1) * s);
+ let bottom = (bl.0 + (br.0 - bl.0) * s, bl.1 + (br.1 - bl.1) * s);
+ (top.0 + (bottom.0 - top.0) * t, top.1 + (bottom.1 - top.1) * t)
+ };
+ let i = ((y * h as f32) as usize * w as usize + (x * w as f32) as usize) * 4;
+ let got = (rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0);
+ assert!(
+ (got.0 - s).abs() < 0.005 && (got.1 - t).abs() < 0.005,
+ "{rotation:?} ({s}, {t}) : lu {got:?}, coins {:?}",
+ quad.corners
+ );
+ assert!(rgba[i + 2] < 3, "{rotation:?} ({s}, {t}) : aller-retour à {}", rgba[i + 2]);
+ }
+ }
+ }
+
/// Les lanes s'appliquent dans l'ordre : bleu puis inversion donne jaune, l'inverse donne bleu.
#[test]
fn lanes_apply_in_order() {
diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs
index 24de39759..c68321efe 100644
--- a/crates/compositor/src/frame_geometry.rs
+++ b/crates/compositor/src/frame_geometry.rs
@@ -2857,12 +2857,17 @@ impl FrameGeometry {
let [x, y, w, h] = self.s_dst;
FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false }
}
+
+ /// Largeur sur hauteur du métrage dans son propre plan : celles du rect qu'incline le tilt.
+ pub fn footage_aspect(&self, render_px: [f32; 2]) -> f32 {
+ self.s_dst[2] * render_px[0] / (self.s_dst[3] * render_px[1]).max(1e-6)
+ }
}
/// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la
/// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la
/// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`).
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Default)]
pub struct FootageQuad {
pub corners: [[f32; 2]; 4],
pub projective: bool,
diff --git a/crates/compositor/src/shader_fx.rs b/crates/compositor/src/shader_fx.rs
index 87e4ad54e..b43f9e6ad 100644
--- a/crates/compositor/src/shader_fx.rs
+++ b/crates/compositor/src/shader_fx.rs
@@ -11,6 +11,7 @@
//! pas d'état d'une frame à l'autre. C'est ce qui garde la preview (qui saute d'une frame à
//! l'autre au seek) et l'export (qui les rend dans l'ordre) d'accord.
+use crate::frame_geometry::FootageQuad;
use crate::scene::{SceneShaderEffect, ShaderEffectTarget, ShaderParamDecl, ShaderParamType, ShaderParamValue};
use std::hash::{Hash, Hasher};
@@ -31,6 +32,9 @@ struct FxContext {
clicks: array<vec4<f32>, 4>,
footage: vec4<f32>,
camera: vec4<f32>,
+ footage_corners: array<vec4<f32>, 2>,
+ footage_aspect: f32,
+ footage_warp: f32,
};
@group(0) @binding(0) var fx_frame: texture_2d<f32>;
@group(0) @binding(1) var fx_sampler: sampler;
@@ -44,6 +48,98 @@ fn fx_px(n: f32) -> f32 {
return n / 1080.0;
}
+// The recording's surface, wherever the zoom, the layout and the 3D camera put it, through the
+// same warp the compositor draws it with. A point of the surface is (0,0) at the recording's
+// top-left corner and (1,1) at its bottom-right, extended past its edges. Mirror of
+// `quad_forward` in shaders.hlsl.
+fn fx_from_footage(p: vec2<f32>) -> vec2<f32> {
+ let r = ctx.resolution;
+ let c00 = ctx.footage_corners[0].xy * r;
+ let c10 = ctx.footage_corners[0].zw * r;
+ let c11 = ctx.footage_corners[1].xy * r;
+ let c01 = ctx.footage_corners[1].zw * r;
+ if (ctx.footage_warp > 0.5) {
+ let p1 = c10 - c00;
+ let p2 = c11 - c00;
+ let p3 = c01 - c00;
+ let d1 = p1 - p2;
+ let d2 = p3 - p2;
+ let d3 = p2 - p1 - p3;
+ let den = d1.x * d2.y - d2.x * d1.y;
+ let g = (d3.x * d2.y - d2.x * d3.y) / den;
+ let h = (d1.x * d3.y - d3.x * d1.y) / den;
+ return (c00 + (p1 * (1.0 + g) * p.x + p3 * (1.0 + h) * p.y) / (g * p.x + h * p.y + 1.0)) / r;
+ }
+ return (c00 + p.x * (c10 - c00) + p.y * (c01 - c00) + p.x * p.y * (c00 - c10 - c01 + c11)) / r;
+}
+
+fn fx_footage_root(t: f32, e: vec2<f32>, f: vec2<f32>, g: vec2<f32>, h: vec2<f32>) -> vec3<f32> {
+ let dx = e.x + g.x * t;
+ let dy = e.y + g.y * t;
+ var s = (h.y - f.y * t) / dy;
+ if (abs(dx) > abs(dy)) {
+ s = (h.x - f.x * t) / dx;
+ }
+ let inside = s >= -0.02 && s <= 1.02 && t >= -0.02 && t <= 1.02;
+ return vec3<f32>(s, t, select(0.0, 1.0, inside));
+}
+
+// The point of the recording's surface under `uv`: fx_from_footage backwards. Where no point of
+// the surface lands (past its horizon), a point far outside 0..1. Mirror of `quad_inverse` in
+// shaders.hlsl.
+fn fx_to_footage(uv: vec2<f32>) -> vec2<f32> {
+ let none = vec2<f32>(-1.0e4);
+ let r = ctx.resolution;
+ let c00 = ctx.footage_corners[0].xy * r;
+ let c10 = ctx.footage_corners[0].zw * r;
+ let c11 = ctx.footage_corners[1].xy * r;
+ let c01 = ctx.footage_corners[1].zw * r;
+ if (ctx.footage_warp > 0.5) {
+ let p1 = c10 - c00;
+ let p2 = c11 - c00;
+ let p3 = c01 - c00;
+ let d1 = p1 - p2;
+ let d2 = p3 - p2;
+ let d3 = p2 - p1 - p3;
+ let den = d1.x * d2.y - d2.x * d1.y;
+ let g = (d3.x * d2.y - d2.x * d3.y) / den;
+ let h = (d1.x * d3.y - d3.x * d1.y) / den;
+ let q = uv * r - c00;
+ let m00 = p1.x * (1.0 + g) - g * q.x;
+ let m01 = p3.x * (1.0 + h) - h * q.x;
+ let m10 = p1.y * (1.0 + g) - g * q.y;
+ let m11 = p3.y * (1.0 + h) - h * q.y;
+ let det = m00 * m11 - m01 * m10;
+ let st = vec2<f32>(q.x * m11 - m01 * q.y, m00 * q.y - q.x * m10) / det;
+ // Behind the camera: the homography reaches the point, the picture never shows it.
+ if (g * st.x + h * st.y + 1.0 <= 0.0) {
+ return none;
+ }
+ return st;
+ }
+ let e = c10 - c00;
+ let f = c01 - c00;
+ let g = c00 - c10 - c01 + c11;
+ let h = uv * r - c00;
+ let k2 = g.x * f.y - g.y * f.x;
+ let k1 = e.x * f.y - e.y * f.x + h.x * g.y - h.y * g.x;
+ let k0 = h.x * e.y - h.y * e.x;
+ if (abs(k2) < 1e-5 * abs(k1)) {
+ if (abs(k1) < 1e-6) {
+ return none;
+ }
+ return fx_footage_root(-k0 / k1, e, f, g, h).xy;
+ }
+ let disc = k1 * k1 - 4.0 * k2 * k0;
+ if (disc < 0.0) {
+ return none;
+ }
+ let q = -0.5 * (k1 + select(-1.0, 1.0, k1 >= 0.0) * sqrt(disc));
+ let r0 = fx_footage_root(q / k2, e, f, g, h);
+ let r1 = fx_footage_root(select(q / k2, k0 / q, abs(q) > 0.0), e, f, g, h);
+ return select(r1.xy, r0.xy, r0.z > 0.5);
+}
+
fn fx_pcg(v: u32) -> u32 {
let state = v * 747796405u + 2891336453u;
let word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u;
@@ -84,7 +180,7 @@ pub const MAX_ITERATIONS_PER_PIXEL: u64 = 4096;
/// Taille du constant buffer des paramètres : le plafond technique (32) de `vec4`.
pub const PARAMS_BUFFER_BYTES: usize = 32 * 16;
-/// Le constant buffer `ctx`, au layout de `FxContext` (144 octets).
+/// Le constant buffer `ctx`, au layout de `FxContext` (192 octets).
#[repr(C)]
#[derive(Debug, Clone, Copy, PartialEq, Default)]
pub struct FxContext {
@@ -96,6 +192,13 @@ pub struct FxContext {
pub clicks: [[f32; 4]; MAX_CLICKS],
pub footage: [f32; 4],
pub camera: [f32; 4],
+ /// Les coins du métrage, TL et TR puis BR et BL, en uv de sortie : `fx_to_footage`.
+ pub footage_corners: [[f32; 4]; 2],
+ /// Largeur sur hauteur du métrage dans son propre plan, avant toute inclinaison.
+ pub footage_aspect: f32,
+ /// 1 : l'homographie des coins (la caméra réelle), 0 : leur warp bilinéaire (`TiltedQuad`).
+ pub footage_warp: f32,
+ pub _pad: [f32; 2],
}
/// Ce que le plan du curseur et la géométrie de la frame donnent aux effets, en uv de sortie.
@@ -104,7 +207,8 @@ pub struct FxFrameInputs {
pub cursor: Option<[f32; 2]>,
/// Les clics récents, du plus récent au plus ancien : point et âge en secondes.
pub clicks: Vec<([f32; 2], f32)>,
- pub footage: [f32; 4],
+ pub footage: FootageQuad,
+ pub footage_aspect: f32,
pub camera: Option<[f32; 4]>,
}
@@ -558,6 +662,7 @@ pub fn context(
resolution: [f32; 2],
inputs: &FxFrameInputs,
) -> FxContext {
+ let [tl, tr, br, bl] = inputs.footage.corners;
let mut clicks = [[0.0; 4]; MAX_CLICKS];
for (slot, ([x, y], age)) in clicks.iter_mut().zip(&inputs.clicks) {
*slot = [*x, *y, *age, 1.0];
@@ -569,8 +674,12 @@ pub fn context(
region: region(effect, source_t, programme_t),
cursor: inputs.cursor.map_or([0.0; 4], |[x, y]| [x, y, 1.0, 0.0]),
clicks,
- footage: inputs.footage,
+ footage: bounds(&inputs.footage.corners),
camera: inputs.camera.unwrap_or([0.0; 4]),
+ footage_corners: [[tl[0], tl[1], tr[0], tr[1]], [br[0], br[1], bl[0], bl[1]]],
+ footage_aspect: inputs.footage_aspect,
+ footage_warp: if inputs.footage.projective { 1.0 } else { 0.0 },
+ _pad: [0.0; 2],
}
}
@@ -820,6 +929,8 @@ mod tests {
assert_eq!(offset("region"), &c.region as *const _ as usize - base);
assert_eq!(offset("clicks"), &c.clicks as *const _ as usize - base);
assert_eq!(offset("camera"), &c.camera as *const _ as usize - base);
+ assert_eq!(offset("footage_corners"), &c.footage_corners as *const _ as usize - base);
+ assert_eq!(offset("footage_warp"), &c.footage_warp as *const _ as usize - base);
}
#[test]
@@ -937,13 +1048,19 @@ mod tests {
let inputs = FxFrameInputs {
cursor: Some([0.25, 0.5]),
clicks: vec![([0.3, 0.6], 0.2), ([0.1, 0.1], 1.5)],
- footage: [0.1, 0.1, 0.8, 0.8],
+ footage: FootageQuad {
+ corners: [[0.25, 0.5], [0.75, 0.25], [0.625, 0.75], [0.25, 0.625]],
+ projective: true,
+ },
+ footage_aspect: 16.0 / 9.0,
camera: Some([0.7, 0.7, 0.2, 0.2]),
};
let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &inputs);
assert_eq!(ctx.cursor, [0.25, 0.5, 1.0, 0.0]);
assert_eq!(ctx.clicks[..3], [[0.3, 0.6, 0.2, 1.0], [0.1, 0.1, 1.5, 1.0], [0.0; 4]]);
- assert_eq!((ctx.footage, ctx.camera), ([0.1, 0.1, 0.8, 0.8], [0.7, 0.7, 0.2, 0.2]));
+ assert_eq!((ctx.footage, ctx.camera), ([0.25, 0.25, 0.5, 0.5], [0.7, 0.7, 0.2, 0.2]));
+ assert_eq!(ctx.footage_corners, [[0.25, 0.5, 0.75, 0.25], [0.625, 0.75, 0.25, 0.625]]);
+ assert_eq!((ctx.footage_aspect, ctx.footage_warp), (16.0 / 9.0, 1.0));
}
#[test]
diff --git a/docs/shader-effects.md b/docs/shader-effects.md
index b4d1a6ed2..039445a18 100644
--- a/docs/shader-effects.md
+++ b/docs/shader-effects.md
@@ -217,6 +217,10 @@ libre, et les erreurs pointent ainsi les lignes que l'agent a écrites.
| `ctx.clicks[i]` | `vec4<f32>` | Les 4 derniers clics des 2 dernières secondes, du plus récent au plus ancien. xy : le point du clic en uv de sortie. z : secondes écoulées depuis. w : 1 si l'entrée est occupée. |
| `ctx.footage` | `vec4<f32>` | Le rect de l'enregistrement en uv de sortie (x, y, w, h), après zoom et layout. Sous un tilt 3D, c'est sa boîte englobante. |
| `ctx.camera` | `vec4<f32>` | Le rect de la webcam en uv de sortie. Taille nulle sans caméra. |
+| `ctx.footage_aspect` | `f32` | Largeur sur hauteur de l'enregistrement dans son propre plan, pour les distances circulaires sur lui. |
+| `ctx.footage_corners`, `ctx.footage_warp` | | Ce que lisent les deux fonctions suivantes : les coins TL, TR, BR, BL du métrage en uv de sortie, et son warp (1 homographie, 0 bilinéaire). |
+| `fx_to_footage(uv)` | `vec2<f32>` | Le point du plan de l'enregistrement sous `uv` : (0, 0) son coin haut-gauche, (1, 1) le bas-droit, prolongé au-delà de ses bords. Loin hors de 0..1 là où aucun point du plan ne tombe. |
+| `fx_from_footage(p)` | `vec2<f32>` | L'inverse : où tombe en uv de sortie le point `p` du plan. |
| `params.<nom>` | selon le type | Les paramètres déclarés par l'effet (C.3). |
| `fx_px(n)` | `f32` | `n` pixels d'une sortie 1080p, en fraction de la hauteur. |
| `fx_hash(p)` | `f32` | Hasard déterministe dans [0, 1) pour un `vec2<u32>`, identique sur tous les GPU (PCG). |
@@ -224,6 +228,10 @@ libre, et les erreurs pointent ainsi les lignes que l'agent a écrites.
- **Tout ce qui est en uv de sortie** est déjà passé par le zoom, le layout, le cadre et le
tilt. Ces valeurs viennent de `plan_frame` et `plan_cursor` (`frame_geometry.rs`), que les
trois backends partagent.
+- **Le plan de l'enregistrement** suit le warp que le compositeur dessine : bilinéaire sous un
+ angle fixe, homographie sous la caméra réelle (`FootageQuad`, `quad_inverse` des shaders).
+ L'effet choisit son repère : l'image de sortie, ou ce plan. Le contrat ne décide pas lequel
+ convient à quel effet.
- **Le point d'un clic** est la pointe à l'instant du clic, au même endroit que l'impact intégré
(mode 16).
- **Le curseur dessiné** passe par le lissage et peut traîner un instant derrière ce point quand
diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts
index f97fa9cf6..b3e2f978b 100644
--- a/electron/ai-edition/deep-agent/service.ts
+++ b/electron/ai-edition/deep-agent/service.ts
@@ -263,7 +263,7 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
addCameraFullscreen:
"Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.",
addEffect:
- "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2<f32>) -> vec4<f32>`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `params.<name>` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2<u32>)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. Effects of one `lane` (0–7) cannot overlap; lanes apply in order, lane 0 first. The result gives the cost tier (frame() reads per pixel: light ≤ 8, about 0.06 ms per 1080p frame on a fast GPU; medium ≤ 32, 0.14 ms; heavy ≤ 128, 0.54 ms; very heavy beyond) and a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.",
+ "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2<f32>) -> vec4<f32>`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `params.<name>` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2<u32>)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. Effects of one `lane` (0–7) cannot overlap; lanes apply in order, lane 0 first. The result gives the cost tier (frame() reads per pixel: light ≤ 8, about 0.06 ms per 1080p frame on a fast GPU; medium ≤ 32, 0.14 ms; heavy ≤ 128, 0.54 ms; very heavy beyond) and a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.",
setEffect:
"Move, resize or re-tune an effect region by id (virtual seconds): its span, its parameter `values`, its `target` or its `lane`. Only the fields you pass change; a value equal to the default is dropped. Refused if the new span runs into another effect of the same lane.",
setEffectDefinition:
diff --git a/electron/mcp/effects-contract.test.ts b/electron/mcp/effects-contract.test.ts
index fce3e8b65..b8d54cbc8 100644
--- a/electron/mcp/effects-contract.test.ts
+++ b/electron/mcp/effects-contract.test.ts
@@ -32,7 +32,13 @@ describe("the effect contract resource", () => {
});
it("names every helper the prelude gives", () => {
- for (const helper of ["frame(uv)", "fx_px(n)", "fx_hash(p)"]) {
+ for (const helper of [
+ "frame(uv)",
+ "fx_px(n)",
+ "fx_hash(p)",
+ "fx_to_footage(uv)",
+ "fx_from_footage(p)",
+ ]) {
expect(EFFECTS_CONTRACT).toContain(helper);
}
});
diff --git a/electron/mcp/effects-contract.ts b/electron/mcp/effects-contract.ts
index 929de027e..47c28e37d 100644
--- a/electron/mcp/effects-contract.ts
+++ b/electron/mcp/effects-contract.ts
@@ -33,6 +33,10 @@ added after your code, so compiler errors point at your lines.
| ctx.clicks[i] | vec4<f32> | The last 4 clicks of the last 2 s, newest first. xy: where it happened, in uv; z: seconds since; w: 1 when the slot holds one. |
| ctx.footage | vec4<f32> | The recording's rect in uv (x, y, w, h), after zoom and layout; under a 3D tilt, its bounding box. |
| ctx.camera | vec4<f32> | The webcam's rect in uv; zero-sized without a camera. |
+| ctx.footage_aspect | f32 | The recording's width over height on its own surface, for round distances on it. |
+| ctx.footage_corners, ctx.footage_warp | | What the two functions below read: the recording's corners and its warp. Use the functions. |
+| fx_to_footage(uv) | vec2<f32> | The point of the recording's surface under uv: (0, 0) its top-left corner, (1, 1) its bottom-right, extended past its edges. Far outside 0..1 where no point of the surface lands. |
+| fx_from_footage(p) | vec2<f32> | Back: where point p of the recording's surface lands, in uv. |
| params.<name> | per type | Your declared parameters. |
| fx_px(n) | f32 | n pixels of a 1080p output, as a fraction of the height. |
| fx_hash(p) | f32 | A random number in [0, 1) for a vec2<u32>, identical on every GPU. |
@@ -40,6 +44,12 @@ added after your code, so compiler errors point at your lines.
Everything in uv is already placed through zoom, layout, frame and tilt: never recompute where
the screen, the camera or the pointer sit.
+There are two spaces to work in, and the effect picks the one that fits it. Output uv is the
+picture as it is framed. The recording's surface is the recording itself, wherever the zoom, the
+layout and the 3D camera put it, through the same warp the compositor draws it with: something
+computed there moves, scales and tilts with the recording. Map into it with fx_to_footage, compute,
+and map back with fx_from_footage before calling frame().
+
## Parameters
Declare each one as \`{ type, name, label, default }\`; a number also takes \`min\` and \`max\`.
@@ -92,27 +102,31 @@ multiplied by their bound), and its tier. Measured on a fast GPU (RTX 4070 Ti) a
An integrated GPU pays several times more. The tiers are recommendations, never refusals.
-## Example: a ripple on every click
+## Example: a ripple on the recording, on every click
Parameters: strength (number, 0–2, default 1), radius (number, 0.05–0.5, default 0.2),
-duration (number, 0.3–2, default 0.9).
+duration (number, 0.3–2, default 0.9). The wave is computed on the recording's surface, so it
+follows the zoom and lies flat on the recording under the 3D camera.
- fn ripple(uv: vec2<f32>, click: vec4<f32>) -> vec2<f32> {
+ fn ripple(p: vec2<f32>, click: vec4<f32>) -> vec2<f32> {
if (click.w < 0.5 || click.z > params.duration) { return vec2<f32>(0.0); }
let t = click.z / params.duration;
- let d = (uv - click.xy) * vec2<f32>(ctx.aspect, 1.0);
+ let metric = vec2<f32>(ctx.footage_aspect, 1.0);
+ let d = (p - fx_to_footage(click.xy)) * metric;
let r = length(d);
let x = params.radius * (1.0 - pow(1.0 - t, 3.0)) - r;
let env = smoothstep(-0.008, 0.0, x) * exp(-max(x, 0.0) / 0.07);
let wave = sin(x / 0.03 * 6.2831853) * env * (1.0 - t) * (1.0 - t) * params.strength;
let dir = select(vec2<f32>(0.0), d / r, r > 1e-5);
- return dir * wave * 0.01 / vec2<f32>(ctx.aspect, 1.0);
+ return dir * wave * 0.01 / metric;
}
fn effect(uv: vec2<f32>) -> vec4<f32> {
+ let p = fx_to_footage(uv);
var offset = vec2<f32>(0.0);
- for (var i = 0; i < 4; i++) { offset += ripple(uv, ctx.clicks[i]); }
- return frame(uv - offset);
+ for (var i = 0; i < 4; i++) { offset += ripple(p, ctx.clicks[i]); }
+ if (dot(offset, offset) == 0.0) { return frame(uv); }
+ return frame(fx_from_footage(p - offset));
}
## Example: a vignette that breathes across the region