Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(compositor): shader effects contract v1 and the Windows pass

EtienneLescot3e0500824dd8
diff --git a/crates/Cargo.lock b/crates/Cargo.lock
index da41d7070..ad32f5dc2 100644
--- a/crates/Cargo.lock
+++ b/crates/Cargo.lock
@@ -994,6 +994,7 @@ dependencies = [
"cosmic-text",
"image",
"metal 0.29.0",
+ "naga",
"ndarray 0.16.1",
"objc",
"ort",
diff --git a/crates/Cargo.toml b/crates/Cargo.toml
index a65f9aaef..48a77d775 100644
--- a/crates/Cargo.toml
+++ b/crates/Cargo.toml
@@ -60,6 +60,7 @@ version = "0.58"
features = [
"Win32_Foundation",
"Win32_Graphics_Direct3D",
+ "Win32_Graphics_Direct3D_Fxc", # D3DCompile à l'exécution des effets shader (shader_fx.rs)
"Win32_Graphics_Direct3D11",
"Win32_Graphics_Dxgi",
"Win32_Graphics_Dxgi_Common",
diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs
index ed98a7104..447df805c 100644
--- a/crates/compositor-view-napi/src/lib.rs
+++ b/crates/compositor-view-napi/src/lib.rs
@@ -388,6 +388,35 @@ pub fn set_active_clip(
}
}
+/// Ce que l'outil de l'agent apprend d'un effet avant de l'accepter (`docs/shader-effects.md`, E.3).
+#[napi(object)]
+pub struct ShaderEffectCheck {
+ /// L'erreur telle que l'agent doit la lire (ligne, colonne, extrait), ou `None` s'il est accepté.
+ pub error: Option<String>,
+ /// Le palier de coût : "light", "medium", "heavy" ou "very-heavy". Indicatif, jamais un refus.
+ pub tier: Option<String>,
+ /// Lectures de `frame()` par pixel, boucles multipliées par leur borne.
+ pub frame_reads: Option<f64>,
+ /// Itérations de boucle par pixel.
+ pub iterations: Option<f64>,
+}
+
+/// Vérifie le WGSL d'un effet shader : validation, bornes des boucles, compilation réelle pour le
+/// backend de la machine quand elle existe (FXC sur Windows), et palier de coût. `params_json` :
+/// les paramètres déclarés, `[{ "name": "strength", "type": "number" }, …]`.
+#[napi]
+pub fn check_shader_effect(code: String, params_json: String) -> ShaderEffectCheck {
+ match openscreen_compositor::shader_fx::check_for_agent_json(&code, &params_json) {
+ Ok(report) => ShaderEffectCheck {
+ error: None,
+ tier: Some(report.tier.name().to_string()),
+ frame_reads: Some(report.frame_reads as f64),
+ iterations: Some(report.iterations as f64),
+ },
+ Err(error) => ShaderEffectCheck { error: Some(error), tier: None, frame_reads: None, iterations: None },
+ }
+}
+
/// Installe la scène de l'app (JSON `SceneDescription`) sur la vue : layout preset piloté par
/// l'app au lieu de la fixture. JSON invalide → ignoré côté natif.
#[napi]
diff --git a/crates/compositor/Cargo.toml b/crates/compositor/Cargo.toml
index cc9ef6459..8f84f117b 100644
--- a/crates/compositor/Cargo.toml
+++ b/crates/compositor/Cargo.toml
@@ -37,6 +37,9 @@ ndarray = { workspace = true, optional = true }
serde.workspace = true
serde_json.workspace = true
image.workspace = true
+# Effets shader écrits à l'exécution (`shader_fx`) : la version que wgpu 24 tire déjà. Pas au-delà
+# de 24 : dès la 25, la sortie HLSL ne passe plus FXC en SM 5.0 (voir `shader_fx::to_hlsl`).
+naga = { version = "24", features = ["wgsl-in", "hlsl-out"] }
# Acces Vulkan brut, UNIQUEMENT pour ouvrir le device avec les extensions de
# memoire externe (cf. `d3d_linux::open_device_with_dmabuf_export`). Les versions
diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs
index 26dc8f67b..6014ec98c 100644
--- a/crates/compositor/src/compositor_windows.rs
+++ b/crates/compositor/src/compositor_windows.rs
@@ -16,7 +16,10 @@ use crate::frame_geometry::{
WEBCAM_SHADOW_OPACITY, WEBCAM_SHADOW_SPREAD_FRAC,
};
use crate::cursor::CursorTrack;
-use crate::scene::{Scene, SceneBackground, SceneCrop, SceneCursorSprite, WallpaperMotion};
+use crate::scene::{
+ Scene, SceneBackground, SceneCrop, SceneCursorSprite, SceneShaderEffect, ShaderEffectTarget,
+ WallpaperMotion,
+};
use crate::d3d::Gpu;
use crate::ffi::AVFrame;
use anyhow::{bail, Result};
@@ -26,7 +29,8 @@ use std::ffi::c_void;
use windows::core::Interface;
use windows::Win32::Foundation::RECT;
use windows::Win32::Graphics::Direct3D::{
- D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP,
+ D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST,
+ D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP,
};
use windows::Win32::Graphics::Direct3D11::*;
use windows::Win32::Graphics::Dxgi::Common::*;
@@ -85,6 +89,31 @@ struct BgCache {
tex: ID3D11Texture2D,
}
+/// Un effet shader de la scène (`shader_fx`), par hash de sa source.
+enum ShaderFx {
+ /// naga + FXC tournent sur un thread de fond.
+ Pending(std::sync::mpsc::Receiver<Result<crate::shader_fx::CompiledFx, String>>),
+ Ready(ID3D11VertexShader, ID3D11PixelShader),
+ /// Erreur déjà écrite ; l'effet ne se dessine pas.
+ Failed,
+}
+
+struct ShaderFxEntry {
+ state: ShaderFx,
+ /// Dernier usage, pour l'éviction (`SHADER_FX_CACHE`).
+ used: u64,
+}
+
+/// Combien d'effets compilés le compositeur garde. Un agent qui itère en crée un par version.
+const SHADER_FX_CACHE: usize = 64;
+
+/// Le calque transparent des textes, pour les effets qui ne visent qu'eux.
+struct TextLayer {
+ tex: ID3D11Texture2D,
+ rtv: ID3D11RenderTargetView,
+ srv: ID3D11ShaderResourceView,
+}
+
struct DofPyramid {
rtv: ID3D11RenderTargetView,
srv: ID3D11ShaderResourceView,
@@ -264,6 +293,19 @@ pub struct Compositor {
/// Pyramide de profondeur de champ, allouée au premier écran incliné et recréée quand la
/// texture décodeur change de taille. Rien n'est alloué tant qu'aucun tilt n'est rendu.
dof_pyramid: RefCell<Option<DofPyramid>>,
+ /// Effets shader de la scène, compilés une fois par source, les moins récents évincés au-delà
+ /// de `SHADER_FX_CACHE`.
+ shader_fx: RefCell<HashMap<u64, ShaderFxEntry>>,
+ shader_fx_tick: Cell<u64>,
+ /// `ctx` (b2) et `params` (b3) de la passe d'effet.
+ shader_fx_cb: ID3D11Buffer,
+ shader_fx_params_cb: ID3D11Buffer,
+ /// Alloué à la première frame qui porte un effet `text` ; à la taille de rendu.
+ text_layer: RefCell<Option<TextLayer>>,
+ /// Vrai par défaut : une frame dont l'effet compile encore attend son binaire, parce qu'un
+ /// export ne doit jamais rendre de frame sans l'effet. La preview le passe à faux et saute
+ /// l'effet le temps de la compilation (~10 ms), puis recompose (`shader_fx_landed`).
+ shader_fx_blocking: Cell<bool>,
/// Device WARP : la profondeur de champ y obéit à `DOF_ON_CPU_BACKEND`.
cpu_backend: bool,
}
@@ -426,6 +468,7 @@ impl Compositor {
bg_cache: RefCell::new(None),
live_readback_staging: RefCell::new(None),
nv12_readback_staging: RefCell::new(None),
+ text_layer: RefCell::new(None),
..self
})
}
@@ -654,6 +697,18 @@ impl Compositor {
};
let mut cbuf: Option<ID3D11Buffer> = None;
dev.CreateBuffer(&bd, None, Some(&mut cbuf))?;
+ let fx_bd = D3D11_BUFFER_DESC {
+ ByteWidth: std::mem::size_of::<crate::shader_fx::FxContext>() as u32,
+ ..bd
+ };
+ let mut shader_fx_cb: Option<ID3D11Buffer> = None;
+ dev.CreateBuffer(&fx_bd, None, Some(&mut shader_fx_cb))?;
+ let fx_params_bd = D3D11_BUFFER_DESC {
+ ByteWidth: crate::shader_fx::PARAMS_BUFFER_BYTES as u32,
+ ..bd
+ };
+ let mut shader_fx_params_cb: Option<ID3D11Buffer> = None;
+ dev.CreateBuffer(&fx_params_bd, None, Some(&mut shader_fx_params_cb))?;
// --- blend alpha prémultiplié ---
let mut bl = D3D11_BLEND_DESC::default();
@@ -791,6 +846,12 @@ impl Compositor {
seg_failed: RefCell::new(false),
nv12_readback_staging: RefCell::new(None),
dof_pyramid: RefCell::new(None),
+ shader_fx: RefCell::new(HashMap::new()),
+ shader_fx_tick: Cell::new(0),
+ shader_fx_cb: shader_fx_cb.unwrap(),
+ shader_fx_params_cb: shader_fx_params_cb.unwrap(),
+ shader_fx_blocking: Cell::new(true),
+ text_layer: RefCell::new(None),
cpu_backend: gpu.backend == crate::d3d::Backend::Cpu,
})
}
@@ -839,9 +900,207 @@ impl Compositor {
/// Installe (ou retire) la scène de l'app. Présente → `compose_frame` prend ses placements
/// depuis le layout preset au lieu du planning fixture.
pub fn set_scene(&self, s: Option<Scene>) {
+ // Les effets nouveaux partent compiler tout de suite, hors du thread de rendu : à la
+ // première frame qui les montre, ils sont en général déjà prêts.
+ if let Some(scene) = &s {
+ for effect in &scene.shader_effects {
+ self.shader_fx_entry(effect);
+ }
+ }
*self.scene.borrow_mut() = s;
}
+ /// Faux pour la preview : voir `shader_fx_blocking`.
+ pub fn set_shader_fx_blocking(&self, on: bool) {
+ self.shader_fx_blocking.set(on);
+ }
+
+ /// Fait aboutir les compilations terminées. Vrai si un effet vient d'être prêt : la preview
+ /// en pause doit alors recomposer, sinon elle resterait sur la frame sans l'effet.
+ pub fn shader_fx_landed(&self) -> bool {
+ let mut landed = false;
+ for entry in self.shader_fx.borrow_mut().values_mut() {
+ let got = match &entry.state {
+ ShaderFx::Pending(rx) => rx.try_recv(),
+ _ => continue,
+ };
+ match got {
+ Ok(result) => {
+ entry.state = self.finish_shader_fx(result);
+ landed = true;
+ }
+ Err(std::sync::mpsc::TryRecvError::Empty) => {}
+ Err(std::sync::mpsc::TryRecvError::Disconnected) => entry.state = ShaderFx::Failed,
+ }
+ }
+ landed
+ }
+
+ /// La clé de cache d'`effect`, sa compilation lancée s'il est neuf, et son dernier usage noté.
+ /// `None` pour un contrat que ce compositeur ne connaît pas : la région ne se dessine pas.
+ fn shader_fx_entry(&self, effect: &SceneShaderEffect) -> Option<u64> {
+ if effect.contract != crate::shader_fx::CONTRACT {
+ return None;
+ }
+ let key = crate::shader_fx::source_hash(effect);
+ let tick = self.shader_fx_tick.get() + 1;
+ self.shader_fx_tick.set(tick);
+ let mut fx = self.shader_fx.borrow_mut();
+ fx.entry(key)
+ .or_insert_with(|| ShaderFxEntry {
+ state: ShaderFx::Pending(crate::shader_fx::spawn_compile(effect)),
+ used: tick,
+ })
+ .used = tick;
+ // Un agent qui itère crée une entrée par version : on garde les plus récentes.
+ if fx.len() > SHADER_FX_CACHE {
+ if let Some(oldest) = fx.iter().min_by_key(|(_, e)| e.used).map(|(k, _)| *k) {
+ fx.remove(&oldest);
+ }
+ }
+ Some(key)
+ }
+
+ fn finish_shader_fx(&self, result: Result<crate::shader_fx::CompiledFx, String>) -> ShaderFx {
+ let created = result.and_then(|c| unsafe {
+ let (mut vs, mut ps) = (None, None);
+ self.dev
+ .CreateVertexShader(&c.vs, None, Some(&mut vs))
+ .and_then(|()| self.dev.CreatePixelShader(&c.ps, None, Some(&mut ps)))
+ .map_err(|e| e.to_string())?;
+ Ok((vs.unwrap(), ps.unwrap()))
+ });
+ match created {
+ Ok((vs, ps)) => ShaderFx::Ready(vs, ps),
+ Err(e) => {
+ eprintln!("[shader-fx] {e}");
+ ShaderFx::Failed
+ }
+ }
+ }
+
+ /// Les shaders d'`effect`. Le compositeur d'export attend leur compilation ; la preview
+ /// (`shader_fx_blocking` faux) rend `None` tant qu'ils compilent. `None` aussi s'ils ont échoué.
+ fn shader_fx_ready(&self, effect: &SceneShaderEffect) -> Option<(ID3D11VertexShader, ID3D11PixelShader)> {
+ let key = self.shader_fx_entry(effect)?;
+ let mut fx = self.shader_fx.borrow_mut();
+ let entry = fx.get_mut(&key)?;
+ if let ShaderFx::Pending(rx) = &entry.state {
+ let got = if self.shader_fx_blocking.get() {
+ rx.recv().map_err(|e| e.to_string())
+ } else {
+ match rx.try_recv() {
+ Ok(result) => Ok(result),
+ Err(std::sync::mpsc::TryRecvError::Empty) => return None,
+ Err(e) => Err(e.to_string()),
+ }
+ };
+ entry.state = match got {
+ Ok(result) => self.finish_shader_fx(result),
+ Err(e) => {
+ eprintln!("[shader-fx] {e}");
+ ShaderFx::Failed
+ }
+ };
+ }
+ match &entry.state {
+ ShaderFx::Ready(vs, ps) => Some((vs.clone(), ps.clone())),
+ _ => None,
+ }
+ }
+
+ unsafe fn upload_bytes(&self, buf: &ID3D11Buffer, bytes: &[u8]) -> bool {
+ let mut m = D3D11_MAPPED_SUBRESOURCE::default();
+ if self.ctx.Map(buf, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).is_err() {
+ return false;
+ }
+ std::ptr::copy_nonoverlapping(bytes.as_ptr(), m.pData as *mut u8, bytes.len());
+ self.ctx.Unmap(buf, 0);
+ true
+ }
+
+ /// Une passe d'effet sur `dst` (le RT principal, ou le calque des textes) : `dst` est copié
+ /// dans `ann_copy`, libre à ce stade, puis l'effet le repeint en entier.
+ unsafe fn apply_shader_effect(
+ &self,
+ effect: &SceneShaderEffect,
+ dst: (&ID3D11Texture2D, &ID3D11RenderTargetView),
+ cb: &crate::shader_fx::FxContext,
+ ) {
+ let Some((vs, ps)) = self.shader_fx_ready(effect) else { return };
+ let ctx_bytes = std::slice::from_raw_parts(
+ cb as *const crate::shader_fx::FxContext as *const u8,
+ std::mem::size_of::<crate::shader_fx::FxContext>(),
+ );
+ let params = crate::shader_fx::pack_params(effect);
+ let params_bytes = std::slice::from_raw_parts(
+ params.as_ptr() as *const u8,
+ (params.len() * 4).min(crate::shader_fx::PARAMS_BUFFER_BYTES),
+ );
+ if !self.upload_bytes(&self.shader_fx_cb, ctx_bytes)
+ || !self.upload_bytes(&self.shader_fx_params_cb, params_bytes)
+ {
+ return;
+ }
+ // On ne lit pas la cible sur laquelle on dessine : copie dans `ann_copy` (les flous
+ // d'annotation et le cristal de Prism Glow l'ont déjà lue).
+ self.ctx.CopySubresourceRegion(&self.ann_copy, 0, 0, 0, 0, dst.0, 0, None);
+ self.ctx.OMSetRenderTargets(Some(&[Some(dst.1.clone())]), None);
+ let vp = D3D11_VIEWPORT {
+ TopLeftX: 0.0,
+ TopLeftY: 0.0,
+ Width: self.rw(),
+ Height: self.rh(),
+ MinDepth: 0.0,
+ MaxDepth: 1.0,
+ };
+ self.ctx.RSSetViewports(Some(&[vp]));
+ self.ctx.IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
+ self.ctx.VSSetShader(&vs, None);
+ self.ctx.PSSetShader(&ps, None);
+ // Les registres du prélude tels que naga les sort : t0, s1, b2 (ctx), b3 (params).
+ self.ctx.PSSetShaderResources(0, Some(&[Some(self.ann_copy_srv.clone())]));
+ self.ctx.PSSetSamplers(1, Some(&[Some(self.sampler.clone())]));
+ self.ctx.PSSetConstantBuffers(
+ 2,
+ Some(&[Some(self.shader_fx_cb.clone()), Some(self.shader_fx_params_cb.clone())]),
+ );
+ self.ctx.OMSetBlendState(&self.blend_none, None, 0xffffffff);
+ self.ctx.Draw(3, 0);
+ self.ctx.PSSetShaderResources(0, Some(&[None]));
+ self.ctx.PSSetConstantBuffers(2, Some(&[None, None]));
+ self.bind_compose_state();
+ }
+
+ /// Le calque transparent des textes, à la taille de rendu, alloué à la première frame qui en a
+ /// besoin (un effet `ShaderEffectTarget::Text`).
+ unsafe fn text_layer(&self) -> Result<(ID3D11Texture2D, ID3D11RenderTargetView, ID3D11ShaderResourceView)> {
+ if let Some(l) = self.text_layer.borrow().as_ref() {
+ return Ok((l.tex.clone(), l.rtv.clone(), l.srv.clone()));
+ }
+ let mut d = D3D11_TEXTURE2D_DESC::default();
+ self.rt.GetDesc(&mut d);
+ d.BindFlags = (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32;
+ d.MiscFlags = 0;
+ d.CPUAccessFlags = 0;
+ d.Usage = D3D11_USAGE_DEFAULT;
+ let mut tex: Option<ID3D11Texture2D> = None;
+ self.dev.CreateTexture2D(&d, None, Some(&mut tex))?;
+ let tex = tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D (calque des textes)"))?;
+ let mut rtv: Option<ID3D11RenderTargetView> = None;
+ self.dev.CreateRenderTargetView(&tex, None, Some(&mut rtv))?;
+ let mut srv: Option<ID3D11ShaderResourceView> = None;
+ self.dev.CreateShaderResourceView(&tex, None, Some(&mut srv))?;
+ let layer = TextLayer {
+ tex,
+ rtv: rtv.ok_or_else(|| anyhow::anyhow!("RTV (calque des textes)"))?,
+ srv: srv.ok_or_else(|| anyhow::anyhow!("SRV (calque des textes)"))?,
+ };
+ let out = (layer.tex.clone(), layer.rtv.clone(), layer.srv.clone());
+ *self.text_layer.borrow_mut() = Some(layer);
+ Ok(out)
+ }
+
/// The Y (R8) and UV (R8G8) views of the decoder frame — over a private COPY, never
/// over the decoder surface itself.
///
@@ -2315,6 +2574,10 @@ impl Compositor {
// pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas).
self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true);
+ // La pointe et les clics récents, tels que les effets shader les lisent.
+ let mut fx_cursor = None;
+ let mut fx_clicks = Vec::new();
+
// --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce,
// et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) ---
if let Some(track) = cursor_ref.as_ref() {
@@ -2333,6 +2596,8 @@ impl Compositor {
)
.map(|p| p.for_backend(self.cpu_backend));
if let Some(plan) = plan {
+ fx_cursor = Some(plan.pointer);
+ fx_clicks = plan.clicks.clone();
let cursor_sprites: HashMap<String, SceneCursorSprite> = scene_ref
.as_ref()
.map(|s| s.cursor.cursor_sprites.clone())
@@ -2540,6 +2805,29 @@ impl Compositor {
}
}
+ // --- effets shader de l'agent, autour des textes selon leur cible (`docs/shader-effects.md`,
+ // D.1) : `image` avant les annotations et les sous-titres, `text` sur un calque des textes
+ // à part, `both` sur l'image finale.
+ let fx_inputs = crate::shader_fx::FxFrameInputs {
+ cursor: fx_cursor,
+ clicks: fx_clicks,
+ footage: crate::shader_fx::bounds(&g.footage_quad([self.rw(), self.rh()]).corners),
+ camera: lp.has_webcam.then_some(w_dst),
+ };
+ let effects = scene_ref
+ .as_ref()
+ .map(|s| crate::shader_fx::active_effects(&s.shader_effects, source_t))
+ .unwrap_or_default();
+ let fx_context = |e: &SceneShaderEffect| {
+ crate::shader_fx::context(e, source_t, g.programme_t, [self.rw(), self.rh()], &fx_inputs)
+ };
+ let with_target = |target: ShaderEffectTarget| -> Vec<&SceneShaderEffect> {
+ effects.iter().copied().filter(|e| e.target == target).collect()
+ };
+ for e in with_target(ShaderEffectTarget::Image) {
+ self.apply_shader_effect(e, (&self.rt, &self.rtv), &fx_context(e));
+ }
+
// --- annotations : calque le plus haut, comme dans le DOM de la preview (le calque y est
// monté après la vidéo). Ancrées sur `s_ann`, le rect ÉCRAN SANS ZOOM — c'est le conteneur
// que reçoit l'overlay web (`layout.screenRect`) — et volontairement pas sur le rect de
@@ -2553,7 +2841,34 @@ impl Compositor {
// `source_t`, la même base de temps que les zoom/speed regions : le temps SOURCE du clip,
// pas le compteur de frames. C'est ce qui garde une annotation alignée sur l'image quand
// une speed region répète ou saute des frames.
- self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false);
+ let text_effects = with_target(ShaderEffectTarget::Text);
+ if text_effects.is_empty() {
+ self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false);
+ } else {
+ // Les textes sur un calque transparent, l'effet sur ce calque seul, puis le calque
+ // posé sur l'image (« over » prémultiplié) : l'image dessous reste intacte.
+ let (tex, rtv, srv) = self.text_layer()?;
+ self.bind_compose_state();
+ self.ctx.OMSetRenderTargets(Some(&[Some(rtv.clone())]), None);
+ self.ctx.ClearRenderTargetView(&rtv, &[0.0, 0.0, 0.0, 0.0]);
+ self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false);
+ for e in text_effects {
+ self.apply_shader_effect(e, (&tex, &rtv), &fx_context(e));
+ }
+ self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None);
+ self.ctx.PSSetShaderResources(0, Some(&[Some(srv)]));
+ self.ctx.VSSetShader(&self.vs_fs, None);
+ self.ctx.PSSetShader(&self.ps_tex, None);
+ self.ctx.PSSetSamplers(0, Some(&[Some(self.sampler.clone())]));
+ self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff);
+ self.ctx.Draw(3, 0);
+ self.ctx.PSSetShaderResources(0, Some(&[None]));
+ self.bind_compose_state();
+ }
+
+ for e in with_target(ShaderEffectTarget::Both) {
+ self.apply_shader_effect(e, (&self.rt, &self.rtv), &fx_context(e));
+ }
Ok(())
}
@@ -3554,4 +3869,199 @@ mod tests {
let (rw, rh, rgba) = unsafe { comp.readback_direct() }.expect("readback");
assert_eq!((rw, rh, rgba.len()), (182, 322, 182 * 322 * 4));
}
+
+ /// Une scène à fond rouge plein, l'écran rétréci par le padding : les coins sont du fond.
+ fn shader_fx_scene(effects: serde_json::Value) -> Scene {
+ let json = serde_json::json!({
+ "clips": [],
+ "layout": {"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},
+ "effects": {"padding":0.5,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0},
+ "background": {"kind":"color","color":"#ff0000"},
+ "zoomRegions": [],
+ "cursor": {"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},
+ "cropByClip": [],
+ "output": {"width":320,"height":180,"fps":null},
+ "shaderEffects": effects,
+ });
+ Scene::from_json(&json.to_string()).expect("scène")
+ }
+
+ /// Un fragment d'effet de 1 s à 3 s, tel que l'app l'envoie.
+ fn fx(code: &str, target: &str, lane: u32) -> serde_json::Value {
+ serde_json::json!({
+ "clipIndex": 0, "startSec": 1.0, "endSec": 3.0, "target": target, "lane": lane,
+ "contract": 1, "code": code,
+ "params": [{"name": "level", "type": "number"}, {"name": "tint", "type": "color"}],
+ "values": {"level": 1.0, "tint": [0.0, 1.0, 0.0, 1.0]},
+ })
+ }
+
+ const INVERT: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n let c = frame(uv);\n return vec4<f32>(vec3<f32>(params.level) - c.rgb, 1.0);\n}\n";
+ const BLUE: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> { return vec4<f32>(0.0, 0.0, 1.0, 1.0); }";
+ const TINT: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> { return params.tint; }";
+ const SAME: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> { return frame(uv); }";
+
+ fn fx_cfg() -> crate::config::Cfg {
+ let mut cfg = crate::config::Cfg::c8();
+ cfg.bg_blur = 0.0;
+ cfg.mblur_n = 1;
+ cfg
+ }
+
+ /// Le pixel du coin haut-gauche (du fond) après une frame composée au temps source `t`.
+ fn corner_at(comp: &Compositor, gpu: &crate::d3d::Gpu, t: f32) -> [u8; 3] {
+ let (screen, _tex) = nv12_frame(gpu, 64, 64);
+ comp.set_timeline_time(Some(t));
+ comp.set_programme_time(Some(t));
+ unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose");
+ let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback");
+ [rgba[0], rgba[1], rgba[2]]
+ }
+
+ /// Le coin à 2 s, dans la région, sous les effets donnés.
+ fn corner_under(effects: serde_json::Value) -> Option<[u8; 3]> {
+ let gpu = crate::d3d::Gpu::create_auto(false).ok()?;
+ let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur");
+ comp.set_scene(Some(shader_fx_scene(effects)));
+ Some(corner_at(&comp, &gpu, 2.0))
+ }
+
+ /// La passe de l'agent repeint l'image composée, dans sa région et nulle part ailleurs. Le
+ /// compositeur par défaut est celui de l'export : il attend le shader, jamais de frame sans.
+ #[test]
+ fn a_shader_effect_repaints_the_composed_frame_inside_its_region() {
+ let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else {
+ eprintln!("pas de device D3D11 — test sauté");
+ return;
+ };
+ let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur");
+ comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image", 0)]))));
+ assert_eq!(corner_at(&comp, &gpu, 0.5), [255, 0, 0], "avant la région : le fond rouge");
+ assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255], "dans la région : rouge inversé");
+ assert_eq!(corner_at(&comp, &gpu, 3.5), [255, 0, 0], "après la région : le fond rouge");
+ }
+
+ /// La preview ne bloque pas : elle saute l'effet tant qu'il compile, et `shader_fx_landed`
+ /// lui dit quand recomposer. Ensuite la frame est celle de l'export, au pixel.
+ #[test]
+ fn the_preview_shows_the_effect_once_it_landed() {
+ let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else {
+ eprintln!("pas de device D3D11 — test sauté");
+ return;
+ };
+ let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur");
+ comp.set_shader_fx_blocking(false);
+ comp.set_scene(Some(shader_fx_scene(serde_json::json!([fx(INVERT, "image", 0)]))));
+ let start = std::time::Instant::now();
+ while !comp.shader_fx_landed() {
+ assert!(start.elapsed().as_secs() < 10, "la compilation n'a jamais abouti");
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ eprintln!("effet prêt en {:.1} ms", start.elapsed().as_secs_f64() * 1000.0);
+ assert_eq!(corner_at(&comp, &gpu, 2.0), [0, 255, 255]);
+ }
+
+ /// Un WGSL invalide, ou un contrat inconnu, ne casse pas le rendu : l'effet ne se dessine
+ /// pas, la frame reste celle de la composition.
+ #[test]
+ fn a_broken_or_unknown_effect_leaves_the_frame_untouched() {
+ let broken = fx("fn effect(uv: vec2<f32>) -> vec4<f32> { nope }", "image", 0);
+ let mut future = fx(BLUE, "image", 0);
+ future["contract"] = serde_json::json!(2);
+ for effect in [broken, future] {
+ let Some(corner) = corner_under(serde_json::json!([effect])) else { return };
+ assert_eq!(corner, [255, 0, 0]);
+ }
+ }
+
+ /// Les paramètres arrivent dans `params` au layout de la struct générée : la couleur `tint`
+ /// suit le nombre `level` dans le buffer, et se lit telle que l'app l'a écrite.
+ #[test]
+ fn a_color_parameter_reaches_the_shader() {
+ let Some(corner) = corner_under(serde_json::json!([fx(TINT, "image", 0)])) else { return };
+ assert_eq!(corner, [0, 255, 0]);
+ }
+
+ /// Les lanes s'appliquent dans l'ordre : bleu puis inversion donne jaune, l'inverse donne bleu.
+ #[test]
+ fn lanes_apply_in_order() {
+ let Some(blue_then_invert) =
+ corner_under(serde_json::json!([fx(INVERT, "image", 1), fx(BLUE, "image", 0)]))
+ else {
+ return;
+ };
+ assert_eq!(blue_then_invert, [255, 255, 0]);
+ let invert_then_blue =
+ corner_under(serde_json::json!([fx(INVERT, "image", 0), fx(BLUE, "image", 1)])).unwrap();
+ assert_eq!(invert_then_blue, [0, 0, 255]);
+ }
+
+ /// `text` ne touche que le calque des textes, posé ensuite sur l'image ; `both` voit tout.
+ /// Sans annotation, le calque est transparent : un effet `text` qui le rend tel quel laisse
+ /// l'image intacte, un effet `text` opaque la recouvre.
+ #[test]
+ fn the_three_targets_keep_to_their_layer() {
+ let Some(image_only) = corner_under(serde_json::json!([fx(BLUE, "image", 0)])) else { return };
+ assert_eq!(image_only, [0, 0, 255]);
+ let text_untouched =
+ corner_under(serde_json::json!([fx(BLUE, "image", 0), fx(SAME, "text", 0)])).unwrap();
+ assert_eq!(text_untouched, [0, 0, 255], "un calque transparent ne change pas l'image");
+ let text_opaque =
+ corner_under(serde_json::json!([fx(BLUE, "image", 0), fx(TINT, "text", 0)])).unwrap();
+ assert_eq!(text_opaque, [0, 255, 0], "le calque des textes, opaque, posé sur l'image");
+ let both = corner_under(serde_json::json!([fx(BLUE, "image", 0), fx(INVERT, "both", 0)])).unwrap();
+ assert_eq!(both, [255, 255, 0], "`both` voit l'image finale");
+ }
+
+ /// Calibre les paliers de coût : le surcoût par frame d'un effet qui lit `frame()` N fois,
+ /// à 1080p et en 4K, sur le GPU de la machine. Opt-in, comme les autres mesures :
+ ///
+ /// set OPENSCREEN_FX_BENCH=1 && cargo test -p openscreen-compositor --release
+ /// shader_fx_tier_calibration -- --nocapture
+ #[test]
+ fn shader_fx_tier_calibration() {
+ if std::env::var_os("OPENSCREEN_FX_BENCH").is_none() {
+ eprintln!("OPENSCREEN_FX_BENCH absent — saute");
+ return;
+ }
+ let gpu = crate::d3d::Gpu::create_backend(crate::d3d::Backend::Hardware, false)
+ .expect("device D3D11");
+ let (screen, _tex) = nv12_frame(&gpu, 1920, 1080);
+ let cfg = fx_cfg();
+ let reads_effect = |n: u32| {
+ format!(
+ "fn effect(uv: vec2<f32>) -> vec4<f32> {{\n var acc = vec4<f32>(0.0);\n for (var i = 0; i < {n}; i++) {{\n let a = f32(i) * 2.399;\n acc += frame(uv + vec2<f32>(cos(a), sin(a)) * fx_px(f32(i)));\n }}\n return acc / {n}.0;\n}}\n"
+ )
+ };
+ let per_frame = |w: u32, h: u32, effects: serde_json::Value| -> f64 {
+ let comp = Compositor::new_sized(&gpu, w, h).expect("compositeur");
+ comp.set_scene(Some(shader_fx_scene(effects)));
+ comp.set_timeline_time(Some(2.0));
+ comp.set_programme_time(Some(2.0));
+ unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &cfg) }.unwrap();
+ unsafe { comp.readback_direct() }.unwrap();
+ let n = 200;
+ let t0 = std::time::Instant::now();
+ for f in 0..n {
+ unsafe { comp.compose_frame(&*screen, &*screen, f as f32, &cfg) }.unwrap();
+ }
+ unsafe { comp.readback_direct() }.unwrap();
+ t0.elapsed().as_secs_f64() * 1000.0 / n as f64
+ };
+ for (w, h) in [(1920u32, 1080u32), (3840, 2160)] {
+ let base = (0..5).map(|_| per_frame(w, h, serde_json::json!([]))).fold(f64::MAX, f64::min);
+ for reads in [1u32, 8, 32, 128, 512] {
+ let code = reads_effect(reads);
+ let report = crate::shader_fx::analyze(&code, &[]).expect("analyse").report;
+ let with = (0..5)
+ .map(|_| per_frame(w, h, serde_json::json!([fx(&code, "image", 0)])))
+ .fold(f64::MAX, f64::min);
+ eprintln!(
+ "{w}x{h} : {reads:>3} lectures ({:>10}) : +{:.3} ms/frame",
+ report.tier.name(),
+ with - base
+ );
+ }
+ }
+ }
}
diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs
index 0decdb21f..24de39759 100644
--- a/crates/compositor/src/frame_geometry.rs
+++ b/crates/compositor/src/frame_geometry.rs
@@ -3599,6 +3599,13 @@ pub struct CursorPlan {
/// composée, le métrage et ses flous de confidentialité, que le backend lui copie juste avant
/// de le dessiner. Seuls des pixels floutés passent ainsi à travers le verre.
pub glass: bool,
+ /// Pour les effets shader (`shader_fx`) : la pointe en 0..1 de SORTIE, zoom, layout et tilt
+ /// appliqués — ce qu'un effet ne peut pas recalculer lui-même.
+ pub pointer: [f32; 2],
+ /// Les clics des `shader_fx::CLICK_WINDOW_S` dernières secondes, du plus récent au plus ancien,
+ /// au plus `shader_fx::MAX_CLICKS` : le point de la pointe AU MOMENT du clic, en 0..1 de sortie,
+ /// et l'âge en secondes.
+ pub clicks: Vec<([f32; 2], f32)>,
}
impl CursorPlan {
@@ -3887,7 +3894,19 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option<CursorP
Vec::new()
};
+ let clicks = input
+ .track
+ .clicks_with_points(input.t - crate::shader_fx::CLICK_WINDOW_S, input.t)
+ .collect::<Vec<_>>()
+ .into_iter()
+ .rev()
+ .take(crate::shader_fx::MAX_CLICKS)
+ .filter_map(|(tc, p)| Some((place(Some(p), g.s_dst)?.upright_center(), input.t - tc)))
+ .collect();
+
Some(CursorPlan {
+ pointer: placement.upright_center(),
+ clicks,
placement,
prev_placement,
size_px,
@@ -8849,6 +8868,46 @@ mod tests {
assert!(!plan(true, 1.08).expect("visible").impacts.is_empty(), "garde : l'anneau du clic");
}
+ /// Les effets shader lisent la pointe et les clics récents en uv de SORTIE : un clic garde le
+ /// point où la pointe était quand il a eu lieu, le plus récent vient en premier, et un clic
+ /// sort de la liste après la fenêtre.
+ #[test]
+ fn the_plan_hands_effects_the_pointer_and_the_recent_clicks() {
+ let scene = model_scene();
+ let cfg = crate::config::all().pop().expect("cfg");
+ let fg = full_frame_geometry();
+ let track = crate::cursor::CursorTrack::new_with_visibility(
+ vec![(0.0, 0.3, 0.6), (1.0, 0.3, 0.6), (1.2, 0.5, 0.4), (2.0, 0.8, 0.2), (6.0, 0.8, 0.2)],
+ vec![1.0, 1.2],
+ vec![],
+ vec![(0.0, true)],
+ );
+ let plan = |t: f32| {
+ plan_cursor(
+ &fg,
+ &CursorPlanInput {
+ render_px: [1920.0, 1080.0],
+ u_max: 1.0,
+ v_max: 1.0,
+ cfg: &cfg,
+ live: LiveParams::default(),
+ scene: Some(&scene),
+ track: &track,
+ t,
+ },
+ )
+ .expect("visible")
+ };
+ assert!(plan(0.5).clicks.is_empty(), "pas encore de clic");
+ let (first, second) = (plan(1.0).pointer, plan(1.2).pointer);
+ let clicks = plan(1.5).clicks;
+ assert_eq!(clicks.len(), 2);
+ assert_eq!((clicks[0].0, clicks[1].0), (second, first), "le plus récent d'abord, au point du clic");
+ assert!((clicks[0].1 - 0.3).abs() < 1e-4 && (clicks[1].1 - 0.5).abs() < 1e-4, "{clicks:?}");
+ assert_ne!(plan(2.5).pointer, second, "la pointe a bougé depuis");
+ assert_eq!(plan(1.1 + crate::shader_fx::CLICK_WINDOW_S).clicks.len(), 1, "le premier est sorti");
+ }
+
/// Sous un angle fixe, l'écran tourne autour du centre de sa boîte ZOOMÉE, réduit de son
/// containment : le curseur y perdait ~18 % au point de focus même, pendant que le zoom
/// grossissait l'image. Il y garde sa taille à plat, à la perspective de biais près ; ailleurs,
diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs
index b33679705..4aed79293 100644
--- a/crates/compositor/src/lib.rs
+++ b/crates/compositor/src/lib.rs
@@ -47,6 +47,8 @@ pub mod prism_mesh;
pub mod remux;
pub mod scene;
pub mod sculpt;
+// Effets shader écrits par un agent (docs/shader-effects.md) : WGSL -> naga -> backend.
+pub mod shader_fx;
// Segmentation du sujet webcam (masque -> `t3`). Le module compile toujours ; sans la feature
// `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate
// indépendant du choix de packaging d'ONNX Runtime.
diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs
index 16c75bd9e..2af211e20 100644
--- a/crates/compositor/src/live.rs
+++ b/crates/compositor/src/live.rs
@@ -1491,6 +1491,10 @@ unsafe fn render_thread(
// Chemin de PRODUCTION : matériel si possible, backend CPU sinon (voir `create_auto`).
let gpu = Gpu::create_auto(false)?;
let mut comp = Compositor::new(&gpu)?;
+ // La preview ne bloque jamais sur la compilation d'un effet shader : elle le saute le temps
+ // qu'il compile (voir `shader_fx_landed` plus bas). L'export, lui, attend.
+ #[cfg(windows)]
+ comp.set_shader_fx_blocking(false);
// Vue live = le VRAI enregistrement, pas la fenêtre fixture (100s@6s, taillée pour l'ancien
// fixture POC). On charge toute la piste depuis t=0 ; 24h couvre large toute recording réelle.
// Gardée à part (raw_cursor) pour pouvoir régénérer une variante lissée sans relire le
@@ -1781,6 +1785,12 @@ unsafe fn render_thread(
continue;
}
+ // Un effet shader vient de finir de compiler : la frame en pause doit le montrer.
+ #[cfg(windows)]
+ let fx_landed = comp.shader_fx_landed();
+ #[cfg(not(windows))]
+ let fx_landed = false;
+
// avance : seek app-piloté (presentTime) prioritaire, sinon lecture libre (60 fps)
let requested = shared.requested_frame.lock().unwrap().take();
let now = Instant::now();
@@ -1915,7 +1925,7 @@ unsafe fn render_thread(
break;
}
}
- } else if first || ip_changed || scene_changed || clip_changed || resized {
+ } else if first || ip_changed || scene_changed || clip_changed || resized || fx_landed {
// pause : recompose la frame courante (param / scène / clip / résolution changés).
(settle_until, last_settle) = open_settle_window(now);
let _ = player.recompose(&comp, &cfg);
diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs
index 895fa6b2a..7fea6bd7f 100644
--- a/crates/compositor/src/scene.rs
+++ b/crates/compositor/src/scene.rs
@@ -547,6 +547,76 @@ pub struct SceneCameraFullscreenRegion {
pub continues_after: bool,
}
+/// Un fragment de région d'effet shader (`docs/shader-effects.md`, B.5). Temps source du clip,
+/// comme les autres régions ; la définition (code, paramètres) voyage avec chaque fragment.
+#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SceneShaderEffect {
+ #[serde(default)]
+ pub clip_index: Option<usize>,
+ pub start_sec: f64,
+ pub end_sec: f64,
+ /// L'étendue de la pilule ENTIÈRE en temps programme, d'où `ctx.region`. Absente : la
+ /// progression court sur le fragment, en temps source.
+ #[serde(default)]
+ pub programme_start_sec: Option<f64>,
+ #[serde(default)]
+ pub programme_end_sec: Option<f64>,
+ #[serde(default)]
+ pub target: ShaderEffectTarget,
+ /// Ordre d'application : la lane 0 d'abord, au plus près du contenu.
+ #[serde(default)]
+ pub lane: u32,
+ /// Version du contrat. Une version que ce compositeur ne connaît pas ne se dessine pas.
+ #[serde(default = "default_contract")]
+ pub contract: u32,
+ /// Le WGSL de l'agent : une `fn effect(uv: vec2<f32>) -> vec4<f32>`.
+ pub code: String,
+ /// Les paramètres déclarés, dans l'ordre de la struct `FxParams` générée.
+ #[serde(default)]
+ pub params: Vec<ShaderParamDecl>,
+ /// Les valeurs, déjà normalisées par l'app : un nombre, ou 2 ou 4 floats.
+ #[serde(default)]
+ pub values: std::collections::HashMap<String, ShaderParamValue>,
+}
+
+fn default_contract() -> u32 {
+ crate::shader_fx::CONTRACT
+}
+
+/// Ce qu'un effet transforme : l'image sans les textes, les textes seuls, ou les deux.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ShaderEffectTarget {
+ #[default]
+ Image,
+ Text,
+ Both,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
+pub struct ShaderParamDecl {
+ pub name: String,
+ #[serde(rename = "type")]
+ pub ty: ShaderParamType,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ShaderParamType {
+ Number,
+ Toggle,
+ Color,
+ Point,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(untagged)]
+pub enum ShaderParamValue {
+ Scalar(f32),
+ Vector(Vec<f32>),
+}
+
/// Rendu du curseur.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -798,6 +868,9 @@ pub struct Scene {
/// Effet d'arrière-plan de la webcam. Absent = aucun effet.
#[serde(default)]
pub webcam_effect: Option<SceneWebcamEffect>,
+ /// Effets shader écrits par un agent (`shader_fx`). Absent = aucun.
+ #[serde(default)]
+ pub shader_effects: Vec<SceneShaderEffect>,
/// État de rendu interne, positionné par `for_clip_window` (jamais envoyé par l'app).
#[serde(skip)]
pub(crate) active_clip_index: usize,
@@ -845,6 +918,9 @@ impl Scene {
scene.annotations.retain(|annotation| {
belongs(annotation.clip_index, annotation.start_sec, annotation.end_sec)
});
+ scene.shader_effects.retain(|effect| {
+ belongs(effect.clip_index, effect.start_sec, effect.end_sec)
+ });
// Le layout dépend de la FORME de la source du clip (dimensions natives × crop), qui
// varie d'un clip à l'autre. On installe donc celui du clip composé dans les champs
// scalaires : `compose_frame` continue de lire un seul `layout`, sans jamais avoir à
diff --git a/crates/compositor/src/shader_fx.rs b/crates/compositor/src/shader_fx.rs
new file mode 100644
index 000000000..cda3ff6bf
--- /dev/null
+++ b/crates/compositor/src/shader_fx.rs
@@ -0,0 +1,902 @@
+//! Effets shader écrits par un agent (`docs/shader-effects.md`) : du WGSL qui définit
+//! `fn effect(uv: vec2<f32>) -> vec4<f32>`, appliqué en une passe plein cadre par `compose_frame`,
+//! donc à l'identique en preview et à l'export.
+//!
+//! Ce module porte ce qui ne dépend d'aucun backend : le contrat (le prélude et la struct des
+//! paramètres, générée), la validation, la borne des boucles, l'estimation du coût en paliers, la
+//! sélection des régions actives et le remplissage des constant buffers. Chaque backend traduit le
+//! module par naga et le compile une fois, hors de la boucle de rendu.
+//!
+//! L'effet est une fonction PURE de l'image composée, du temps, du curseur et de ses paramètres :
+//! pas d'état d'une frame à l'autre. C'est ce qui garde la preview (qui saute d'une frame à
+//! l'autre au seek) et l'export (qui les rend dans l'ordre) d'accord.
+
+use crate::scene::{SceneShaderEffect, ShaderEffectTarget, ShaderParamDecl, ShaderParamType, ShaderParamValue};
+use std::hash::{Hash, Hasher};
+
+/// La version du contrat que ce prélude implémente. Un prélude livré ne change plus : une entrée
+/// nouvelle s'ajoute à la fin de `FxContext`, un changement de sens crée la version suivante.
+pub const CONTRACT: u32 = 1;
+
+/// Les entrées que l'effet lit, en plus de ses paramètres. Placées APRÈS le code de l'agent (et
+/// après `FxParams`) : en WGSL l'ordre des déclarations de module est libre, et les erreurs
+/// pointent ainsi les lignes que l'agent a écrites. Voir la section C de la spec.
+pub const PRELUDE_V1: &str = r#"
+struct FxContext {
+ resolution: vec2<f32>,
+ time: f32,
+ aspect: f32,
+ region: vec4<f32>,
+ cursor: vec4<f32>,
+ clicks: array<vec4<f32>, 4>,
+ footage: vec4<f32>,
+ camera: vec4<f32>,
+};
+@group(0) @binding(0) var fx_frame: texture_2d<f32>;
+@group(0) @binding(1) var fx_sampler: sampler;
+@group(0) @binding(2) var<uniform> ctx: FxContext;
+
+fn frame(uv: vec2<f32>) -> vec4<f32> {
+ return textureSampleLevel(fx_frame, fx_sampler, uv, 0.0);
+}
+
+fn fx_px(n: f32) -> f32 {
+ return n / 1080.0;
+}
+
+fn fx_pcg(v: u32) -> u32 {
+ let state = v * 747796405u + 2891336453u;
+ let word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u;
+ return (word >> 22u) ^ word;
+}
+
+fn fx_hash(p: vec2<u32>) -> f32 {
+ return f32(fx_pcg(p.x ^ fx_pcg(p.y)) >> 8u) / 16777216.0;
+}
+
+struct FxVsOut {
+ @builtin(position) pos: vec4<f32>,
+ @location(0) uv: vec2<f32>,
+};
+
+@vertex
+fn fx_vs(@builtin(vertex_index) i: u32) -> FxVsOut {
+ let p = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));
+ var o: FxVsOut;
+ o.pos = vec4<f32>(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0, 0.0, 1.0);
+ o.uv = p;
+ return o;
+}
+
+@fragment
+fn fx_fs(v: FxVsOut) -> @location(0) vec4<f32> {
+ return effect(v.uv);
+}
+"#;
+
+/// Combien de clics récents l'effet voit (`ctx.clicks`), du plus récent au plus ancien.
+pub const MAX_CLICKS: usize = 4;
+/// Jusqu'où un clic reste visible des effets : au-delà, son entrée est vide (`w` = 0).
+pub const CLICK_WINDOW_S: f32 = 2.0;
+/// Le plafond de sécurité du travail par pixel, toutes boucles imbriquées comprises. Ce n'est pas
+/// un budget : c'est l'ordre de grandeur où un GPU intégré atteint le TDR en 1080p.
+pub const MAX_ITERATIONS_PER_PIXEL: u64 = 4096;
+/// Taille du constant buffer des paramètres : le plafond technique (32) de `vec4`.
+pub const PARAMS_BUFFER_BYTES: usize = 32 * 16;
+
+/// Le constant buffer `ctx`, au layout de `FxContext` (144 octets).
+#[repr(C)]
+#[derive(Debug, Clone, Copy, PartialEq, Default)]
+pub struct FxContext {
+ pub resolution: [f32; 2],
+ pub time: f32,
+ pub aspect: f32,
+ pub region: [f32; 4],
+ pub cursor: [f32; 4],
+ pub clicks: [[f32; 4]; MAX_CLICKS],
+ pub footage: [f32; 4],
+ pub camera: [f32; 4],
+}
+
+/// Ce que le plan du curseur et la géométrie de la frame donnent aux effets, en uv de sortie.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct FxFrameInputs {
+ pub cursor: Option<[f32; 2]>,
+ /// Les clics récents, du plus récent au plus ancien : point et âge en secondes.
+ pub clicks: Vec<([f32; 2], f32)>,
+ pub footage: [f32; 4],
+ pub camera: Option<[f32; 4]>,
+}
+
+/// Le palier de coût d'un effet (section C.7 de la spec) : indicatif, jamais un refus.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Tier {
+ Light,
+ Medium,
+ Heavy,
+ VeryHeavy,
+}
+
+impl Tier {
+ /// Les seuils, en lectures de `frame()` par pixel. Sur la RTX 4070 Ti, une lecture coûte
+ /// ~4 µs par frame en 1080p : léger ≤ 0,06 ms, moyen ≤ 0,14 ms, lourd ≤ 0,54 ms
+ /// (`shader_fx_tier_calibration`, rendering-performance.md).
+ pub fn of(frame_reads: u64) -> Tier {
+ match frame_reads {
+ 0..=8 => Tier::Light,
+ 9..=32 => Tier::Medium,
+ 33..=128 => Tier::Heavy,
+ _ => Tier::VeryHeavy,
+ }
+ }
+
+ pub fn name(self) -> &'static str {
+ match self {
+ Tier::Light => "light",
+ Tier::Medium => "medium",
+ Tier::Heavy => "heavy",
+ Tier::VeryHeavy => "very-heavy",
+ }
+ }
+}
+
+/// Ce que la validation sait d'un effet accepté.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EffectReport {
+ /// Lectures de texture par pixel, boucles multipliées par leur borne (le pire des branches).
+ pub frame_reads: u64,
+ /// Itérations de boucle par pixel, toutes boucles imbriquées comprises.
+ pub iterations: u64,
+ pub tier: Tier,
+}
+
+impl ShaderParamType {
+ fn wgsl(self) -> &'static str {
+ match self {
+ ShaderParamType::Number | ShaderParamType::Toggle => "f32",
+ ShaderParamType::Color => "vec4<f32>",
+ ShaderParamType::Point => "vec2<f32>",
+ }
+ }
+
+ /// (taille, alignement) en octets, règles WGSL de l'espace `uniform`.
+ fn size_align(self) -> (usize, usize) {
+ match self {
+ ShaderParamType::Number | ShaderParamType::Toggle => (4, 4),
+ ShaderParamType::Color => (16, 16),
+ ShaderParamType::Point => (8, 8),
+ }
+ }
+}
+
+const RESERVED_PARAM_NAMES: &[&str] = &["unused"];
+
+/// La struct `FxParams` et son binding, générés depuis les déclarations. WGSL refuse une struct
+/// vide : sans paramètre, un champ `unused` en tient lieu.
+fn params_source(params: &[ShaderParamDecl]) -> String {
+ let mut out = String::from("struct FxParams {\n");
+ if params.is_empty() {
+ out.push_str(" unused: f32,\n");
+ }
+ for p in params {
+ out.push_str(&format!(" {}: {},\n", p.name, p.ty.wgsl()));
+ }
+ out.push_str("};\n@group(0) @binding(3) var<uniform> params: FxParams;\n");
+ out
+}
+
+/// Le module complet que naga compile : le code de l'agent d'abord, pour que ses lignes gardent
+/// leur numéro dans les erreurs.
+pub fn module_source(user: &str, params: &[ShaderParamDecl]) -> String {
+ format!("{user}\n{}{PRELUDE_V1}", params_source(params))
+}
+
+fn check_params(params: &[ShaderParamDecl]) -> Result<(), String> {
+ let mut seen = std::collections::HashSet::new();
+ for p in params {
+ let valid = p.name.len() <= 32
+ && p.name.starts_with(|c: char| c.is_ascii_lowercase())
+ && p.name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_');
+ if !valid {
+ return Err(format!("parameter `{}`: names are snake_case, a-z first, at most 32 characters", p.name));
+ }
+ if RESERVED_PARAM_NAMES.contains(&p.name.as_str()) || !seen.insert(p.name.as_str()) {
+ return Err(format!("parameter `{}` is declared twice or reserved", p.name));
+ }
+ }
+ Ok(())
+}
+
+/// Le module parsé, validé, et sa première fonction d'entrée de fragment.
+pub struct Parsed {
+ pub module: naga::Module,
+ pub info: naga::valid::ModuleInfo,
+ pub report: EffectReport,
+}
+
+/// Valide un effet comme l'agent doit le lire : paramètres, syntaxe et types (naga), bornes des
+/// boucles, puis estimation du coût. L'erreur porte la ligne et la colonne du code de l'agent.
+pub fn analyze(user: &str, params: &[ShaderParamDecl]) -> Result<Parsed, String> {
+ check_params(params)?;
+ let src = module_source(user, params);
+ let module = naga::front::wgsl::parse_str(&src).map_err(|e| e.emit_to_string(&src))?;
+ let info = naga::valid::Validator::new(
+ naga::valid::ValidationFlags::all(),
+ naga::valid::Capabilities::empty(),
+ )
+ .validate(&module)
+ .map_err(|e| e.emit_to_string(&src))?;
+ let entry = module
+ .entry_points
+ .iter()
+ .find(|e| e.name == "fx_fs")
+ .ok_or("the module has no fx_fs entry point")?;
+ let mut costs = CostWalk { module: &module, src: &src, by_function: Default::default() };
+ let cost = costs.block(&entry.function, &entry.function.body)?;
+ if cost.iterations > MAX_ITERATIONS_PER_PIXEL {
+ return Err(format!(
+ "the loops run {} iterations per pixel, over the safety ceiling of {MAX_ITERATIONS_PER_PIXEL}: \
+ past it a GPU can stall long enough for the system to reset it",
+ cost.iterations
+ ));
+ }
+ let report = EffectReport { frame_reads: cost.reads, iterations: cost.iterations, tier: Tier::of(cost.reads) };
+ Ok(Parsed { module, info, report })
+}
+
+/// Ce que coûte un bloc, par pixel.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+struct Cost {
+ reads: u64,
+ iterations: u64,
+}
+
+impl Cost {
+ fn add(self, o: Cost) -> Cost {
+ Cost { reads: self.reads.saturating_add(o.reads), iterations: self.iterations.saturating_add(o.iterations) }
+ }
+ fn max(self, o: Cost) -> Cost {
+ Cost { reads: self.reads.max(o.reads), iterations: self.iterations.max(o.iterations) }
+ }
+ fn times(self, n: u64) -> Cost {
+ Cost { reads: self.reads.saturating_mul(n), iterations: self.iterations.saturating_mul(n) }
+ }
+}
+
+/// Parcourt l'IR de naga depuis `fx_fs` : les fonctions appelées sont dépliées, une branche
+/// compte pour la plus chère, une boucle multiplie son corps par sa borne.
+struct CostWalk<'a> {
+ module: &'a naga::Module,
+ src: &'a str,
+ by_function: std::collections::HashMap<naga::Handle<naga::Function>, Cost>,
+}
+
+impl CostWalk<'_> {
+ fn block(&mut self, func: &naga::Function, block: &naga::Block) -> Result<Cost, String> {
+ let mut total = Cost::default();
+ for (stmt, span) in block.span_iter() {
+ let c = match stmt {
+ naga::Statement::Emit(range) => {
+ let reads = range
+ .clone()
+ .filter(|&h| {
+ matches!(
+ func.expressions[h],
+ naga::Expression::ImageSample { .. } | naga::Expression::ImageLoad { .. }
+ )
+ })
+ .count() as u64;
+ Cost { reads, iterations: 0 }
+ }
+ naga::Statement::Block(b) => self.block(func, b)?,
+ naga::Statement::If { accept, reject, .. } => {
+ self.block(func, accept)?.max(self.block(func, reject)?)
+ }
+ naga::Statement::Switch { cases, .. } => {
+ let mut worst = Cost::default();
+ for case in cases {
+ worst = worst.max(self.block(func, &case.body)?);
+ }
+ worst
+ }
+ naga::Statement::Loop { body, continuing, break_if } => {
+ let trips = self.trip_count(func, block, body, continuing, *break_if, *span)?;
+ let inner = self.block(func, body)?.add(self.block(func, continuing)?);
+ Cost { reads: inner.reads, iterations: inner.iterations.saturating_add(1) }.times(trips)
+ }
+ naga::Statement::Call { function, .. } => self.function(*function)?,
+ _ => Cost::default(),
+ };
+ total = total.add(c);
+ }
+ Ok(total)
+ }
+
+ fn function(&mut self, h: naga::Handle<naga::Function>) -> Result<Cost, String> {
+ if let Some(c) = self.by_function.get(&h) {
+ return Ok(*c);
+ }
+ let module = self.module;
+ let f = &module.functions[h];
+ let c = self.block(f, &f.body)?;
+ self.by_function.insert(h, c);
+ Ok(c)
+ }
+
+ fn refuse(&self, span: naga::Span, why: &str) -> String {
+ let line = span.location(self.src).line_number;
+ format!(
+ "the loop at line {line} {why}. Write it as `for (var i = 0; i < 16; i++)`: a counter \
+ initialised with a literal, compared to a literal, stepped by a literal in the loop header"
+ )
+ }
+
+ /// Le nombre de tours d'une boucle `for` à bornes littérales. Toute autre forme est refusée :
+ /// une boucle qui ne finit pas bloque le GPU jusqu'au TDR.
+ fn trip_count(
+ &self,
+ func: &naga::Function,
+ parent: &naga::Block,
+ body: &naga::Block,
+ continuing: &naga::Block,
+ break_if: Option<naga::Handle<naga::Expression>>,
+ span: naga::Span,
+ ) -> Result<u64, String> {
+ use naga::{BinaryOperator as B, Expression as E, Statement as S};
+ if break_if.is_some() {
+ return Err(self.refuse(span, "ends on `break if`"));
+ }
+ // Le `for` de WGSL devient `loop { if (cond) {} else { break; } … continuing { pas } }`.
+ let guard = body.iter().find(|s| !matches!(s, S::Emit(_)));
+ let Some(S::If { condition, accept, reject }) = guard else {
+ return Err(self.refuse(span, "has no bound in its header"));
+ };
+ if !accept.is_empty() || reject.len() != 1 || !matches!(reject.iter().next(), Some(S::Break)) {
+ return Err(self.refuse(span, "has no bound in its header"));
+ }
+ let E::Binary { op, left, right } = func.expressions[*condition] else {
+ return Err(self.refuse(span, "is not bounded by a comparison"));
+ };
+ let (var, bound, op) = match (self.counter(func, left), self.literal(func, right)) {
+ (Some(v), Some(k)) => (v, k, op),
+ _ => match (self.literal(func, left), self.counter(func, right)) {
+ // `N > i` se lit `i < N`.
+ (Some(k), Some(v)) => (v, k, match op {
+ B::Greater => B::Less,
+ B::GreaterEqual => B::LessEqual,
+ B::Less => B::Greater,
+ B::LessEqual => B::GreaterEqual,
+ other => other,
+ }),
+ _ => return Err(self.refuse(span, "is not bounded by a literal")),
+ },
+ };
+ let init = func.local_variables[var]
+ .init
+ .and_then(|h| self.literal(func, h))
+ .or_else(|| self.store_before(func, parent, var))
+ .ok_or_else(|| self.refuse(span, "starts its counter from a value that is not a literal"))?;
+ let step = self
+ .step(func, continuing, var)
+ .ok_or_else(|| self.refuse(span, "does not step its counter by a literal in its header"))?;
+ if Self::writes(func, body, var) {
+ return Err(self.refuse(span, "changes its counter inside its body"));
+ }
+ let trips = match op {
+ B::Less if step > 0.0 => ((bound - init) / step).ceil(),
+ B::LessEqual if step > 0.0 => ((bound - init) / step).floor() + 1.0,
+ B::Greater if step < 0.0 => ((init - bound) / -step).ceil(),
+ B::GreaterEqual if step < 0.0 => ((init - bound) / -step).floor() + 1.0,
+ _ => return Err(self.refuse(span, "never reaches its bound")),
+ };
+ Ok(trips.max(0.0) as u64)
+ }
+
+ /// `Load(LocalVariable(v))` : le compteur d'une condition.
+ fn counter(&self, func: &naga::Function, h: naga::Handle<naga::Expression>) -> Option<naga::Handle<naga::LocalVariable>> {
+ match func.expressions[h] {
+ naga::Expression::Load { pointer } => match func.expressions[pointer] {
+ naga::Expression::LocalVariable(v) => Some(v),
+ _ => None,
+ },
+ _ => None,
+ }
+ }
+
+ /// Un littéral, directement ou par une constante de module.
+ fn literal(&self, func: &naga::Function, h: naga::Handle<naga::Expression>) -> Option<f64> {
+ let lit = |e: &naga::Expression| match *e {
+ naga::Expression::Literal(l) => match l {
+ naga::Literal::I32(v) => Some(v as f64),
+ naga::Literal::U32(v) => Some(v as f64),
+ naga::Literal::F32(v) => Some(v as f64),
+ naga::Literal::I64(v) => Some(v as f64),
+ naga::Literal::U64(v) => Some(v as f64),
+ naga::Literal::F64(v) => Some(v),
+ naga::Literal::AbstractInt(v) => Some(v as f64),
+ naga::Literal::AbstractFloat(v) => Some(v),
+ _ => None,
+ },
+ _ => None,
+ };
+ match &func.expressions[h] {
+ naga::Expression::Constant(c) => lit(&self.module.global_expressions[self.module.constants[*c].init]),
+ e => lit(e),
+ }
+ }
+
+ /// Le dernier `Store` d'un littéral dans `var` qui précède une boucle dans son bloc.
+ fn store_before(&self, func: &naga::Function, parent: &naga::Block, var: naga::Handle<naga::LocalVariable>) -> Option<f64> {
+ let mut found = None;
+ for s in parent.iter() {
+ match s {
+ naga::Statement::Store { pointer, value }
+ if matches!(func.expressions[*pointer], naga::Expression::LocalVariable(v) if v == var) =>
+ {
+ found = self.literal(func, *value);
+ }
+ naga::Statement::Loop { .. } => {}
+ _ => {}
+ }
+ }
+ found
+ }
+
+ /// Le pas du compteur dans `continuing` : `i = i + k` ou `i = i - k`, `k` littéral.
+ fn step(&self, func: &naga::Function, continuing: &naga::Block, var: naga::Handle<naga::LocalVariable>) -> Option<f64> {
+ continuing.iter().find_map(|s| match s {
+ naga::Statement::Store { pointer, value }
+ if matches!(func.expressions[*pointer], naga::Expression::LocalVariable(v) if v == var) =>
+ {
+ match func.expressions[*value] {
+ naga::Expression::Binary { op, left, right } if self.counter(func, left) == Some(var) => {
+ let k = self.literal(func, right)?;
+ match op {
+ naga::BinaryOperator::Add => Some(k),
+ naga::BinaryOperator::Subtract => Some(-k),
+ _ => None,
+ }
+ }
+ _ => None,
+ }
+ }
+ _ => None,
+ })
+ }
+
+ /// Le corps écrit-il le compteur (directement, ou en passant son pointeur à une fonction) ?
+ fn writes(func: &naga::Function, block: &naga::Block, var: naga::Handle<naga::LocalVariable>) -> bool {
+ let is_var = |h: naga::Handle<naga::Expression>| {
+ matches!(func.expressions[h], naga::Expression::LocalVariable(v) if v == var)
+ };
+ block.iter().any(|s| match s {
+ naga::Statement::Store { pointer, .. } => is_var(*pointer),
+ naga::Statement::Call { arguments, .. } => arguments.iter().any(|&a| is_var(a)),
+ naga::Statement::Block(b) => Self::writes(func, b, var),
+ naga::Statement::If { accept, reject, .. } => Self::writes(func, accept, var) || Self::writes(func, reject, var),
+ naga::Statement::Switch { cases, .. } => cases.iter().any(|c| Self::writes(func, &c.body, var)),
+ naga::Statement::Loop { body, continuing, .. } => {
+ Self::writes(func, body, var) || Self::writes(func, continuing, var)
+ }
+ _ => false,
+ })
+ }
+}
+
+/// HLSL SM 5.0 pour D3D11/FXC. naga 24 est la dernière version dont la sortie HLSL passe FXC :
+/// à partir de la 25 elle déclare les samplers dans un tas D3D12 (`register(s0, space0)`), que
+/// `ps_5_0` refuse (X3721). Ne pas monter naga sans faire passer ce module.
+pub fn to_hlsl(parsed: &Parsed) -> Result<String, String> {
+ let options = naga::back::hlsl::Options {
+ shader_model: naga::back::hlsl::ShaderModel::V5_0,
+ ..Default::default()
+ };
+ let mut out = String::new();
+ naga::back::hlsl::Writer::new(&mut out, &options)
+ .write(&parsed.module, &parsed.info, None)
+ .map_err(|e| e.to_string())?;
+ Ok(out)
+}
+
+/// Clé du cache des shaders compilés : ce qui change le binaire, et rien d'autre.
+pub fn source_hash(effect: &SceneShaderEffect) -> u64 {
+ let mut h = std::collections::hash_map::DefaultHasher::new();
+ effect.contract.hash(&mut h);
+ effect.code.hash(&mut h);
+ for p in &effect.params {
+ p.name.hash(&mut h);
+ p.ty.hash(&mut h);
+ }
+ h.finish()
+}
+
+/// Les régions actives au temps source `t`, dans l'ordre où elles s'appliquent : par lane, puis
+/// dans l'ordre de la scène.
+pub fn active_effects(effects: &[SceneShaderEffect], t: f32) -> Vec<&SceneShaderEffect> {
+ let t = t as f64;
+ let mut active: Vec<_> = effects.iter().filter(|e| t >= e.start_sec && t < e.end_sec).collect();
+ active.sort_by_key(|e| e.lane);
+ active
+}
+
+/// La boîte englobante (x, y, w, h) d'un quad, en uv de sortie : `ctx.footage` sous un tilt 3D.
+pub fn bounds(corners: &[[f32; 2]; 4]) -> [f32; 4] {
+ let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN);
+ for [x, y] in corners {
+ (x0, y0, x1, y1) = (x0.min(*x), y0.min(*y), x1.max(*x), y1.max(*y));
+ }
+ [x0, y0, x1 - x0, y1 - y0]
+}
+
+/// `ctx.region` : progression sur la pilule entière en temps programme quand l'app la donne, sur
+/// le fragment en temps source sinon.
+fn region(effect: &SceneShaderEffect, source_t: f32, programme_t: f32) -> [f32; 4] {
+ let (start, end, now) = match (effect.programme_start_sec, effect.programme_end_sec) {
+ (Some(s), Some(e)) => (s, e, programme_t as f64),
+ _ => (effect.start_sec, effect.end_sec, source_t as f64),
+ };
+ let span = (end - start).max(1e-6);
+ let since = (now - start).max(0.0);
+ [(since / span).clamp(0.0, 1.0) as f32, span as f32, since as f32, 0.0]
+}
+
+pub fn context(
+ effect: &SceneShaderEffect,
+ source_t: f32,
+ programme_t: f32,
+ resolution: [f32; 2],
+ inputs: &FxFrameInputs,
+) -> FxContext {
+ let mut clicks = [[0.0; 4]; MAX_CLICKS];
+ for (slot, ([x, y], age)) in clicks.iter_mut().zip(&inputs.clicks) {
+ *slot = [*x, *y, *age, 1.0];
+ }
+ FxContext {
+ resolution,
+ time: programme_t,
+ aspect: resolution[0] / resolution[1].max(1.0),
+ region: region(effect, source_t, programme_t),
+ cursor: inputs.cursor.map_or([0.0; 4], |[x, y]| [x, y, 1.0, 0.0]),
+ clicks,
+ footage: inputs.footage,
+ camera: inputs.camera.unwrap_or([0.0; 4]),
+ }
+}
+
+/// Les paramètres de la région, rangés comme la struct `FxParams` générée (règles d'alignement de
+/// WGSL), en mots de 4 octets. Une valeur absente vaut 0 : l'app envoie déjà les défauts.
+pub fn pack_params(effect: &SceneShaderEffect) -> Vec<f32> {
+ let mut words: Vec<f32> = Vec::new();
+ let mut offset = 0usize;
+ for p in &effect.params {
+ let (size, align) = p.ty.size_align();
+ offset = offset.div_ceil(align) * align;
+ let end = offset + size;
+ words.resize(end / 4, 0.0);
+ let value: Vec<f32> = match effect.values.get(&p.name) {
+ Some(ShaderParamValue::Scalar(v)) => vec![*v],
+ Some(ShaderParamValue::Vector(v)) => v.clone(),
+ None => Vec::new(),
+ };
+ for (i, v) in value.into_iter().take(size / 4).enumerate() {
+ words[offset / 4 + i] = v;
+ }
+ offset = end;
+ }
+ words.resize(offset.div_ceil(16).max(1) * 4, 0.0);
+ words
+}
+
+/// Le bytecode DXBC des deux étages, prêt pour `CreateVertexShader` / `CreatePixelShader`.
+#[cfg(windows)]
+pub struct CompiledFx {
+ pub vs: Vec<u8>,
+ pub ps: Vec<u8>,
+}
+
+#[cfg(windows)]
+pub fn compile_dxbc(code: &str, params: &[ShaderParamDecl]) -> Result<CompiledFx, String> {
+ let parsed = analyze(code, params)?;
+ let hlsl = to_hlsl(&parsed)?;
+ Ok(CompiledFx {
+ vs: d3d_compile(&hlsl, "fx_vs", "vs_5_0")?,
+ ps: d3d_compile(&hlsl, "fx_fs", "ps_5_0")?,
+ })
+}
+
+#[cfg(windows)]
+fn d3d_compile(hlsl: &str, entry: &str, profile: &str) -> Result<Vec<u8>, String> {
+ use windows::core::PCSTR;
+ use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3};
+ use windows::Win32::Graphics::Direct3D::ID3DBlob;
+
+ fn bytes(blob: &ID3DBlob) -> Vec<u8> {
+ unsafe {
+ std::slice::from_raw_parts(blob.GetBufferPointer() as *const u8, blob.GetBufferSize())
+ .to_vec()
+ }
+ }
+ let (entry_z, profile_z) = (format!("{entry}\0"), format!("{profile}\0"));
+ let (mut code, mut errors): (Option<ID3DBlob>, Option<ID3DBlob>) = (None, None);
+ let compiled = unsafe {
+ D3DCompile(
+ hlsl.as_ptr().cast(),
+ hlsl.len(),
+ PCSTR(b"effect.hlsl\0".as_ptr()),
+ None,
+ None,
+ PCSTR(entry_z.as_ptr()),
+ PCSTR(profile_z.as_ptr()),
+ D3DCOMPILE_OPTIMIZATION_LEVEL3,
+ 0,
+ &mut code,
+ Some(&mut errors),
+ )
+ };
+ match (compiled, code) {
+ (Ok(()), Some(code)) => Ok(bytes(&code)),
+ (result, _) => {
+ let log = errors.as_ref().map(bytes).unwrap_or_default();
+ Err(format!("{entry}: {result:?}\n{}", String::from_utf8_lossy(&log)))
+ }
+ }
+}
+
+/// Compile sur un thread à part ; le résultat arrive par le canal. naga + FXC prennent ~10 ms
+/// pour un effet de 60 lignes, soit une frame de preview : jamais sur le thread de rendu.
+#[cfg(windows)]
+pub fn spawn_compile(effect: &SceneShaderEffect) -> std::sync::mpsc::Receiver<Result<CompiledFx, String>> {
+ let (tx, rx) = std::sync::mpsc::channel();
+ let (code, params) = (effect.code.clone(), effect.params.clone());
+ std::thread::Builder::new()
+ .name("shader-fx".into())
+ .spawn(move || {
+ let _ = tx.send(compile_dxbc(&code, &params));
+ })
+ .expect("thread shader-fx");
+ rx
+}
+
+/// Ce que l'outil de l'agent demande avant d'accepter un code : la validation, la compilation
+/// réelle pour le backend de la machine quand elle existe, et le palier.
+pub fn check_for_agent(code: &str, params: &[ShaderParamDecl]) -> Result<EffectReport, String> {
+ let report = analyze(code, params)?.report;
+ #[cfg(windows)]
+ compile_dxbc(code, params)?;
+ Ok(report)
+}
+
+/// `check_for_agent`, les paramètres reçus en JSON (`[{ "name", "type" }, …]`) depuis l'addon.
+pub fn check_for_agent_json(code: &str, params_json: &str) -> Result<EffectReport, String> {
+ let params: Vec<ShaderParamDecl> =
+ serde_json::from_str(params_json).map_err(|e| format!("params: {e}"))?;
+ check_for_agent(code, &params)
+}
+
+/// Un effet tel que les tests et les bancs le construisent.
+#[cfg(test)]
+pub(crate) fn test_effect(code: &str, params: Vec<ShaderParamDecl>, values: &[(&str, ShaderParamValue)]) -> SceneShaderEffect {
+ SceneShaderEffect {
+ clip_index: Some(0),
+ start_sec: 1.0,
+ end_sec: 3.0,
+ programme_start_sec: None,
+ programme_end_sec: None,
+ target: ShaderEffectTarget::Image,
+ lane: 0,
+ contract: CONTRACT,
+ code: code.into(),
+ params,
+ values: values.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn decl(name: &str, ty: ShaderParamType) -> ShaderParamDecl {
+ ShaderParamDecl { name: name.into(), ty }
+ }
+
+ const VIGNETTE: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n let c = frame(uv);\n let d = distance(uv, vec2<f32>(0.5));\n return vec4<f32>(c.rgb * (1.0 - d * params.strength * ctx.region.x), c.a);\n}\n";
+
+ #[test]
+ fn a_valid_effect_becomes_hlsl_that_d3d11_accepts() {
+ let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("analyze");
+ let hlsl = to_hlsl(&parsed).expect("translate");
+ assert!(hlsl.contains("fx_vs") && hlsl.contains("fx_fs"));
+ // Les registres que les backends lient : t0, s1, b2 (ctx), b3 (params), sans `space`.
+ for reg in ["register(t0)", "register(s1)", "register(b2)", "register(b3)"] {
+ assert!(hlsl.contains(reg), "{reg} missing:\n{hlsl}");
+ }
+ assert!(!hlsl.contains("space"), "FXC ps_5_0 refuse les espaces de registres");
+ }
+
+ #[test]
+ fn an_error_points_at_the_line_the_agent_wrote() {
+ let broken = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n return frame(uv) * nope;\n}\n";
+ let err = analyze(broken, &[]).err().expect("must fail");
+ assert!(err.contains("nope") && err.contains(":2:"), "{err}");
+ }
+
+ #[test]
+ fn an_effect_without_the_entry_function_is_refused() {
+ assert!(analyze("fn other() {}", &[]).is_err());
+ }
+
+ #[test]
+ fn parameter_names_are_checked() {
+ for bad in ["Strength", "2x", "unused", "a-b"] {
+ assert!(analyze(VIGNETTE, &[decl(bad, ShaderParamType::Number)]).is_err(), "{bad}");
+ }
+ let twice = [decl("a", ShaderParamType::Number), decl("a", ShaderParamType::Color)];
+ assert!(analyze(VIGNETTE, &twice).is_err());
+ }
+
+ /// Le packer et la struct générée doivent placer chaque paramètre au même octet : un écart lirait
+ /// une couleur là où l'app a écrit une force.
+ #[test]
+ fn the_packer_matches_the_offsets_naga_gives_the_generated_struct() {
+ let params = vec![
+ decl("a", ShaderParamType::Number),
+ decl("tint", ShaderParamType::Color),
+ decl("on", ShaderParamType::Toggle),
+ decl("at", ShaderParamType::Point),
+ decl("b", ShaderParamType::Number),
+ ];
+ let code = "fn effect(uv: vec2<f32>) -> vec4<f32> { return frame(uv) * params.a + params.tint * params.on + vec4<f32>(params.at, params.b, 0.0); }";
+ let parsed = analyze(code, &params).expect("analyze");
+ let ty = parsed
+ .module
+ .types
+ .iter()
+ .find(|(_, t)| t.name.as_deref() == Some("FxParams"))
+ .expect("FxParams")
+ .1;
+ let naga::TypeInner::Struct { members, span } = &ty.inner else { panic!("struct") };
+ let values = [
+ ("a", ShaderParamValue::Scalar(1.0)),
+ ("tint", ShaderParamValue::Vector(vec![2.0, 3.0, 4.0, 5.0])),
+ ("on", ShaderParamValue::Scalar(6.0)),
+ ("at", ShaderParamValue::Vector(vec![7.0, 8.0])),
+ ("b", ShaderParamValue::Scalar(9.0)),
+ ];
+ let words = pack_params(&test_effect(code, params, &values));
+ assert_eq!(words.len() * 4, (*span as usize).div_ceil(16) * 16);
+ let first = |name: &str| members.iter().find(|m| m.name.as_deref() == Some(name)).unwrap().offset as usize / 4;
+ assert_eq!(words[first("a")], 1.0);
+ assert_eq!(&words[first("tint")..first("tint") + 4], &[2.0, 3.0, 4.0, 5.0]);
+ assert_eq!(words[first("on")], 6.0);
+ assert_eq!(&words[first("at")..first("at") + 2], &[7.0, 8.0]);
+ assert_eq!(words[first("b")], 9.0);
+ }
+
+ #[test]
+ fn the_context_matches_its_wgsl_struct() {
+ let parsed = analyze(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).unwrap();
+ let ty = parsed.module.types.iter().find(|(_, t)| t.name.as_deref() == Some("FxContext")).unwrap().1;
+ let naga::TypeInner::Struct { members, span } = &ty.inner else { panic!("struct") };
+ assert_eq!(std::mem::size_of::<FxContext>(), *span as usize);
+ let offset = |name: &str| members.iter().find(|m| m.name.as_deref() == Some(name)).unwrap().offset as usize;
+ let c = FxContext::default();
+ let base = &c as *const FxContext as usize;
+ assert_eq!(offset("region"), &c.region as *const _ as usize - base);
+ assert_eq!(offset("clicks"), &c.clicks as *const _ as usize - base);
+ assert_eq!(offset("camera"), &c.camera as *const _ as usize - base);
+ }
+
+ #[test]
+ fn a_for_loop_with_literal_bounds_is_counted() {
+ let code = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n var acc = vec4<f32>(0.0);\n for (var i = 0; i < 16; i++) {\n acc += frame(uv + vec2<f32>(f32(i) * 0.001, 0.0));\n }\n return acc / 16.0;\n}\n";
+ let r = analyze(code, &[]).expect("analyze").report;
+ assert_eq!((r.frame_reads, r.iterations, r.tier), (16, 16, Tier::Medium));
+ }
+
+ #[test]
+ fn nested_loops_multiply_and_called_functions_are_unfolded() {
+ let code = "fn tap(uv: vec2<f32>) -> vec4<f32> { return frame(uv) + frame(uv * 0.5); }\nfn effect(uv: vec2<f32>) -> vec4<f32> {\n var acc = vec4<f32>(0.0);\n for (var y = 0u; y <= 7u; y += 1u) {\n for (var x = 8; x > 0; x--) {\n acc += tap(uv);\n }\n }\n return acc;\n}\n";
+ let r = analyze(code, &[]).expect("analyze").report;
+ assert_eq!(r.frame_reads, 8 * 8 * 2);
+ assert_eq!(r.iterations, 8 * (8 + 1));
+ assert_eq!(r.tier, Tier::Heavy);
+ }
+
+ #[test]
+ fn a_branch_counts_for_the_dearer_side() {
+ let code = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n if (uv.x > 0.5) { return frame(uv) + frame(uv) + frame(uv); }\n return frame(uv);\n}\n";
+ assert_eq!(analyze(code, &[]).unwrap().report.frame_reads, 4, "branche chère + suite");
+ }
+
+ #[test]
+ fn loops_without_a_literal_bound_are_refused_with_their_line() {
+ let refused = [
+ ("fn effect(uv: vec2<f32>) -> vec4<f32> {\n var i = 0;\n loop {\n i++;\n if (i > 4) { break; }\n }\n return frame(uv);\n}\n", 3),
+ ("fn effect(uv: vec2<f32>) -> vec4<f32> {\n var i = 0;\n while (i < 4) {\n i++;\n }\n return frame(uv);\n}\n", 3),
+ ("fn effect(uv: vec2<f32>) -> vec4<f32> {\n let n = i32(uv.x * 100.0);\n for (var i = 0; i < n; i++) {}\n return frame(uv);\n}\n", 3),
+ ("fn effect(uv: vec2<f32>) -> vec4<f32> {\n for (var i = 0; i < 4; i++) { i = 0; }\n return frame(uv);\n}\n", 2),
+ ("fn effect(uv: vec2<f32>) -> vec4<f32> {\n for (var i = 0; i < 4; i--) {}\n return frame(uv);\n}\n", 2),
+ ];
+ for (code, line) in refused {
+ let err = analyze(code, &[]).err().unwrap_or_else(|| panic!("accepted: {code}"));
+ assert!(err.contains(&format!("line {line}")) && err.contains("for (var i = 0"), "{err}");
+ }
+ }
+
+ #[test]
+ fn a_module_constant_is_a_literal_bound() {
+ let code = "const TAPS = 12;\nfn effect(uv: vec2<f32>) -> vec4<f32> {\n var acc = vec4<f32>(0.0);\n for (var i = 0; i < TAPS; i += 3) { acc += frame(uv); }\n return acc;\n}\n";
+ assert_eq!(analyze(code, &[]).expect("analyze").report.iterations, 4);
+ }
+
+ #[test]
+ fn the_safety_ceiling_refuses_runaway_work() {
+ let code = "fn effect(uv: vec2<f32>) -> vec4<f32> {\n var a = 0.0;\n for (var i = 0; i < 100; i++) { for (var j = 0; j < 100; j++) { a += 1.0; } }\n return frame(uv) * a;\n}\n";
+ let err = analyze(code, &[]).err().expect("must fail");
+ assert!(err.contains("10100") && err.contains("4096"), "{err}");
+ }
+
+ #[test]
+ fn the_ripple_of_the_spec_is_light() {
+ let code = include_str!("../../../docs/shader-effects.md")
+ .split("```wgsl\n")
+ .nth(2)
+ .and_then(|s| s.split("```").next())
+ .expect("l'exemple C.6");
+ let params = vec![
+ decl("strength", ShaderParamType::Number),
+ decl("radius", ShaderParamType::Number),
+ decl("duration", ShaderParamType::Number),
+ ];
+ let r = analyze(code, &params).expect("l'exemple de la spec compile").report;
+ assert_eq!((r.frame_reads, r.tier), (1, Tier::Light));
+ }
+
+ #[test]
+ fn the_active_regions_and_their_progress() {
+ let mut a = test_effect(VIGNETTE, vec![], &[]);
+ a.lane = 1;
+ let b = test_effect(VIGNETTE, vec![], &[]);
+ let effects = [a, b];
+ assert!(active_effects(&effects, 0.5).is_empty());
+ let active = active_effects(&effects, 2.0);
+ assert_eq!(active.iter().map(|e| e.lane).collect::<Vec<_>>(), [0, 1], "par lane");
+ assert!(active_effects(&effects, 3.0).is_empty(), "fin exclusive");
+
+ let ctx = context(&effects[1], 2.0, 10.0, [1920.0, 1080.0], &FxFrameInputs::default());
+ assert_eq!(ctx.region, [0.5, 2.0, 1.0, 0.0], "fragment, temps source");
+ let mut whole = effects[1].clone();
+ (whole.programme_start_sec, whole.programme_end_sec) = (Some(8.0), Some(12.0));
+ let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &FxFrameInputs::default());
+ assert_eq!(ctx.region, [0.5, 4.0, 2.0, 0.0], "pilule entière, temps programme");
+ assert_eq!((ctx.cursor[2], ctx.clicks[0][3], ctx.camera), (0.0, 0.0, [0.0; 4]));
+
+ let inputs = FxFrameInputs {
+ cursor: Some([0.25, 0.5]),
+ clicks: vec![([0.3, 0.6], 0.2), ([0.1, 0.1], 1.5)],
+ footage: [0.1, 0.1, 0.8, 0.8],
+ camera: Some([0.7, 0.7, 0.2, 0.2]),
+ };
+ let ctx = context(&whole, 2.0, 10.0, [1920.0, 1080.0], &inputs);
+ assert_eq!(ctx.cursor, [0.25, 0.5, 1.0, 0.0]);
+ assert_eq!(ctx.clicks[..3], [[0.3, 0.6, 0.2, 1.0], [0.1, 0.1, 1.5, 1.0], [0.0; 4]]);
+ assert_eq!((ctx.footage, ctx.camera), ([0.1, 0.1, 0.8, 0.8], [0.7, 0.7, 0.2, 0.2]));
+ }
+
+ #[test]
+ fn the_cache_key_follows_code_and_layout_not_values() {
+ let p = vec![decl("strength", ShaderParamType::Number)];
+ let a = test_effect(VIGNETTE, p.clone(), &[("strength", ShaderParamValue::Scalar(1.0))]);
+ let mut b = test_effect(VIGNETTE, p, &[("strength", ShaderParamValue::Scalar(2.0))]);
+ assert_eq!(source_hash(&a), source_hash(&b), "une valeur ne recompile rien");
+ b.params[0].ty = ShaderParamType::Color;
+ assert_ne!(source_hash(&a), source_hash(&b), "un layout différent est un autre binaire");
+ }
+
+ #[cfg(windows)]
+ #[test]
+ fn fxc_compiles_both_stages() {
+ let c = compile_dxbc(VIGNETTE, &[decl("strength", ShaderParamType::Number)]).expect("compile");
+ assert!(!c.vs.is_empty() && !c.ps.is_empty());
+ }
+}
diff --git a/docs/shader-effects.md b/docs/shader-effects.md
index 5eafc05af..c9611976c 100644
--- a/docs/shader-effects.md
+++ b/docs/shader-effects.md
@@ -205,6 +205,7 @@ libre, et les erreurs pointent ainsi les lignes que l'agent a écrites.
|---|---|---|
| `frame(uv)` | `vec4<f32>` | L'image composée en ce point (D.1 dit ce qu'elle contient). Échantillonnage bilinéaire, bords répétés. |
| `ctx.resolution` | `vec2<f32>` | Taille de la sortie, en px. |
+| `ctx.aspect` | `f32` | Largeur sur hauteur de la sortie, pour les distances circulaires en uv. |
| `ctx.time` | `f32` | Secondes de programme, continues à travers coupes, clips et vitesses. |
| `ctx.region` | `vec4<f32>` | x : progression de 0 à 1 sur la pilule entière. y : sa durée en s. z : secondes depuis son début. |
| `ctx.cursor` | `vec4<f32>` | xy : la pointe en uv de sortie. z : 1 si elle est à l'écran. |
@@ -328,9 +329,13 @@ refusé parce que la machine qui l'écrit est modeste.
| Lourd | ≤ 128 | gros flou, bokeh | Sur des régions courtes : une transition, un moment fort. |
| Très lourd | au-delà | raymarching, simulations | Quelques secondes. L'export reste juste, mais plus lent ; la preview peut ralentir sur un petit GPU. |
-- **Les seuils sont à calibrer à la PR 1,** en ms mesurées sur le GPU intégré de référence (le
- Radeon 610M des mesures Linux) et sur la RTX du spike. La description de l'outil les donne
- ensuite en ms, pour que l'agent raisonne en temps et non en lectures.
+- **Mesuré sur la RTX 4070 Ti** (`shader_fx_tier_calibration`) : environ 4 µs par lecture en
+ 1080p et 17 µs en 4K, au-dessus d'un fixe de 0,03 ms. Un effet léger coûte au plus 0,06 ms par
+ frame en 1080p, un moyen 0,14 ms, un lourd 0,54 ms, 512 lectures 2,1 ms. Le détail est dans
+ [rendering-performance.md](../technical-documentation/engineering/rendering-performance.md).
+- **Le GPU intégré de référence** (le Radeon 610M des mesures Linux) reste à mesurer : on y attend
+ plusieurs fois ces coûts. La description de l'outil donnera les paliers en ms sur les deux
+ machines, pour que l'agent raisonne en temps et non en lectures.
- **Plusieurs effets au même instant s'additionnent.** Le retour d'un outil donne le palier de
l'effet, et le palier cumulé du moment le plus chargé de sa région.
- **Rien n'est refusé au coût.** Le palier et sa recommandation reviennent à l'agent (E.3), qui
diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts
index 5a285bb4e..de1bb08d7 100644
--- a/electron/native/compositor-view/addon.d.ts
+++ b/electron/native/compositor-view/addon.d.ts
@@ -63,6 +63,18 @@ export interface NativeSharedFramePacket {
sourceTimeSec?: number;
}
+/** What `checkShaderEffect` learns of an agent-written effect (`docs/shader-effects.md`, E.3). */
+export interface ShaderEffectCheck {
+ /** The error as the agent must read it (line, column, excerpt); absent when accepted. */
+ error?: string | null;
+ /** The indicative cost tier: never a refusal. */
+ tier?: "light" | "medium" | "heavy" | "very-heavy" | null;
+ /** `frame()` reads per pixel, loops multiplied by their bound. */
+ frameReads?: number | null;
+ /** Loop iterations per pixel. */
+ iterations?: number | null;
+}
+
export interface ExportStats {
frames: number;
wallS: number;
@@ -143,6 +155,10 @@ export interface CompositorViewAddon {
/** What this machine offers, asked without allocating a view — the export dialog
* needs the answer before any preview exists. Cached native-side. */
probeBackend(): CompositorBackend;
+ /** Checks an effect's WGSL: validation, literal loop bounds, the real compile for this machine's
+ * backend where there is one (FXC on Windows), and the cost tier. `paramsJson` is the declared
+ * parameters, `[{ "name": "strength", "type": "number" }, …]`. Absent from older builds. */
+ checkShaderEffect?(code: string, paramsJson: string): ShaderEffectCheck;
/** Whether the ONNX Runtime library is where the app staged it, and therefore whether a
* segmentation mask can be produced at all. Asked of the machine rather than guessed from
diff --git a/technical-documentation/architecture/decisions.md b/technical-documentation/architecture/decisions.md
index ac249fe6c..f0f16385d 100644
--- a/technical-documentation/architecture/decisions.md
+++ b/technical-documentation/architecture/decisions.md
@@ -26,6 +26,7 @@ A decision leaves this list only when the code stops honouring it.
| **Migrations are forward-only.** A document is migrated up to the current `schemaVersion` on open and never written back down. | Round-tripping through an older schema loses fields silently. |
| **Captions are derived from the transcript, not injected as annotations.** | The earlier design generated annotation objects from captions, which then drifted from the transcript the moment either was edited. The transcript is the SSOT for spoken words. |
| **One package, one repository.** No sidecar process, no local HTTP server, no monorepo. | The editing engine was adopted from a project that had a Python worker and a Fastify server; both were replaced by in-process TypeScript and Electron IPC. Adding a second runtime back is a large, permanent cost. |
+| **Agent-written shader effects are WGSL, translated by naga 24 and compiled off the render thread; an effect is a pure function of the frame, time, pointer and its parameters.** One source for the three backends; the document keeps only that source, never a binary. Cost is an indicative tier, never a refusal; the one hard rule is literal loop bounds. | The Windows spike measured +0.04 ms per 1080p frame for a heavy effect and an unchanged export time, and preview and export agreeing within codec noise. naga 25+ emits D3D12 sampler heaps that FXC `ps_5_0` rejects. A budget measured on the authoring machine was rejected: it would refuse high-end effects on a modest machine although the export renders them fine. See [docs/shader-effects.md](../../docs/shader-effects.md). |
## Rejected, with the reason
diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md
index 6ce4e305c..1b8a7e6c2 100644
--- a/technical-documentation/architecture/native-compositor.md
+++ b/technical-documentation/architecture/native-compositor.md
@@ -29,6 +29,7 @@ and without that flag the runtime corruption is silent.
| [`crates/compositor/src/ffi.rs`](../../crates/compositor/src/ffi.rs) | bindgen-generated `libav*` bindings (ffmpeg 8.x headers) |
| [`crates/compositor/src/compositor.rs`](../../crates/compositor/src/compositor_windows.rs) | HLSL compositor — every per-frame pass (`compose_frame`, background, screen, webcam, cursor, annotations, shadows, blur) |
| [`crates/compositor/src/scene.rs`](../../crates/compositor/src/scene.rs) | the `Scene` struct parsed from the app's `SceneDescription` JSON |
+| [`crates/compositor/src/shader_fx.rs`](../../crates/compositor/src/shader_fx.rs) | agent-written shader effects: the WGSL contract, validation and loop rule, cost tiers, parameter packing, background compilation (FXC on Windows) |
| [`crates/compositor/src/regions.rs`](../../crates/compositor/src/regions.rs) | zoom / speed / Full Camera regions — envelope shapes and per-frame state sampling |
| [`crates/compositor/src/pipeline.rs`](../../crates/compositor/src/pipeline_windows.rs) | demux + `D3D11VA` decode + composite + AMF encode + mux (`run_c0` for decode/encode only, `run_composited` for the full path) |
| [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, libavfilter `atempo` speed stretch (WSOLA fallback), multi-track mix, AAC encoder |
@@ -173,6 +174,39 @@ flowchart TB
BG --> SS --> SV --> CU --> WS --> WV --> ANN
```
+### Shader effects
+
+An agent-written effect is WGSL that defines `fn effect(uv: vec2<f32>) -> vec4<f32>`, carried by the
+scene (`Scene::shader_effects`) and applied by `compose_frame` as a full-frame pass. The design and
+the contract are in [docs/shader-effects.md](../../docs/shader-effects.md);
+[`shader_fx.rs`](../../crates/compositor/src/shader_fx.rs) holds everything that is not
+backend-specific.
+
+- **The module.** The agent's code comes first, then the generated `FxParams` struct (its declared
+ parameters, bound at `b3`), then the contract-v1 prelude: `frame()`, `ctx` (`b2`), `fx_px`,
+ `fx_hash` and the `fx_vs`/`fx_fs` entry points. Errors therefore point at the agent's lines.
+- **Validation** (`analyze`) is naga's, plus the loop rule: every loop is a `for` with literal
+ bounds, at most 4 096 iterations per pixel. It also estimates the cost, in `frame()` reads per
+ pixel, as an indicative tier (light, medium, heavy, very heavy). Nothing is refused on cost.
+- **Compilation never happens on the render thread.** `set_scene` starts one background compile per
+ new source (naga to HLSL SM 5.0, then FXC). The cache is keyed by contract, code and parameter
+ layout, and keeps the 64 most recently used.
+- **Blocking.** The export compositor waits for a pending compile and never draws a frame without
+ its effect. The live view (`set_shader_fx_blocking(false)`) skips the effect until it lands, then
+ recomposes the paused frame (`shader_fx_landed`).
+- **Targets.** `image` effects run before the annotations and captions. `text` effects run on a
+ transparent text layer, which is then drawn over the image. `both` effects run last. Within a
+ target, effects apply by lane, then in scene order. Each pass copies its destination into
+ `ann_copy` and repaints it with blending off.
+- **`ctx` comes from the frame geometry** and is in output uv: the pointer and the last four clicks
+ (`plan_cursor`), the footage rect (`FrameGeometry::footage_quad`, bounding box under a tilt), the
+ camera rect, the region's progress (over the whole pill in programme time when the app sends
+ `programmeStartSec`/`programmeEndSec`), and programme time.
+
+naga stays at 24. From 25 on, its HLSL output declares samplers in a D3D12 heap
+(`register(s0, space0)`), which FXC rejects for `ps_5_0` (X3721).
+`shader_fx::tests::a_valid_effect_becomes_hlsl_that_d3d11_accepts` guards that boundary.
+
## Scene contract
The app hands the native compositor a flat JSON description once per document
@@ -402,6 +436,11 @@ licence. D3D11VA + AMF survive the LGPL-shared build (verified:
rasteriser: one that needs no D3D device at all. Giving the helper a WARP
device would add a slow path nobody needs alongside a working one.
+### Known gap: shader effects draw on Windows only
+
+The Metal and wgpu compositors parse `shader_effects` but do not draw them: an effect is visible on
+Windows only. [docs/shader-effects.md](../../docs/shader-effects.md) (D.5) specifies both passes.
+
### Known gap: the capture fallback is unreachable on a host that fails D3D
That fallback is only reachable through the pre-flight probe, and the probe
diff --git a/technical-documentation/engineering/rendering-performance.md b/technical-documentation/engineering/rendering-performance.md
index 93c2387c4..20702e5c4 100644
--- a/technical-documentation/engineering/rendering-performance.md
+++ b/technical-documentation/engineering/rendering-performance.md
@@ -434,6 +434,22 @@ The editor preview was driven over CDP on the same two builds (not computer-use:
**The output did not change.** Every row above exports a byte-identical GIF, checked by hash. That needed one fix first: the median cut sorted `HashMap` entries with a stable sort, so ties kept the map's random per-process order and no two exports of the same project were byte-identical.
+## Shader effects — 2026-10-14
+
+An agent-written effect (`crates/compositor/src/shader_fx.rs`, [docs/shader-effects.md](../../docs/shader-effects.md)) adds one full-frame pass per active region: a copy of the render target into `ann_copy`, then the effect. Measured with `compositor_windows::tests::shader_fx_tier_calibration` (`OPENSCREEN_FX_BENCH=1`, release) on the RTX 4070 Ti: the extra time per composed frame of an effect that reads `frame()` N times, best of five runs of 200 frames.
+
+| `frame()` reads | Tier | 1080p | 4K |
+|---:|---|---:|---:|
+| 1 | light | +0.030 ms | +0.138 ms |
+| 8 | light | +0.061 ms | +0.211 ms |
+| 32 | medium | +0.137 ms | +0.606 ms |
+| 128 | heavy | +0.539 ms | +2.365 ms |
+| 512 | very heavy | +2.133 ms | +8.805 ms |
+
+- **The cost is linear in the reads,** about 4 µs per read at 1080p and 17 µs at 4K, over a fixed 0.03 ms (1080p) for the copy and the pass. It is the texture bandwidth that pays, which is why the tiers count reads.
+- **The export does not notice a light effect:** 180 frames at 1752×1080 took 520 ms without, 523 ms with the spike's effect, and 521 ms with a heavier one. The preview held the source's 30 fps either way.
+- **Not measured yet:** an integrated GPU, where the same reads cost several times more, and the Metal and wgpu passes.
+
## How we got here — the WebCodecs trail
> **This section is history.** It records the measurements that killed the browser-based export pipeline and motivated the native one. The code it describes is **gone**: `src/lib/exporter/videoExporter.ts`, `src/bench/runBench.ts` and the `npm run bench:export` script were deleted with the web MP4 pipeline. It is kept because it is the evidence for [why the compositor, not the encoder, was the wall](#the-wall-is-the-compositor) — which is the entire reason `crates/compositor/` exists — and because the [measurement hazards](#measurement-hazards) it uncovered still apply to any new benchmark here.