A virtual camera films an AI agent's screen while a tracker locks onto every glyph. Two fframes passes, one SkSL shader, all sound synthesized.
//! Pass 2: a virtual camera films the flat pass through `shaders/lens.sksl`.
//!
//! The shader does the physical part: perspective, depth of field focused on the tracker's
//! target, palette, bloom, chromatic aberration, LED grid and the tear on a lock. Every
//! per-frame value comes from `hud::shot`.
use fframes::{
AudioMap, AudioTimestamp::*, AudioTrack, Color, Duration, FFramesContext,
FFramesSyncedVideoFrame, Frame, Shader, ShaderUniforms, Svgr, SyncVideoFrameInput, Video,
};
pub const WIDTH: usize = 1920;
pub const HEIGHT: usize = 1080;
pub struct LensVideo {
shader: Shader,
}
impl LensVideo {
pub fn new() -> Self {
Self {
shader: Shader::sksl(include_str!("../shaders/lens.sksl")),
}
}
}
impl Default for LensVideo {
fn default() -> Self {
Self::new()
}
}
impl Video for LensVideo {
const FPS: usize = hud::FPS;
const WIDTH: usize = WIDTH;
const HEIGHT: usize = HEIGHT;
const BACKGROUND_COLOR: Color = Color::BLACK;
fn duration(&self) -> Duration<'_> {
Duration::Frames(hud::TOTAL_FRAMES)
}
fn audio(&self) -> AudioMap<'_> {
AudioMap::from([AudioTrack::new("sfx.wav", Second(0.)..Eof)])
}
fn render_frame<'a>(&'a self, frame: Frame, ctx: &FFramesContext<'a, '_>) -> Svgr<'a> {
let (si, f) = hud::split(frame.index);
let s = hud::shot(frame.index);
let input = SyncVideoFrameInput {
start_from: 0.0,
looping: false,
editor_fallback_image: None,
};
// A root without <svg> makes a full render hang instead of failing (fframes 1.2.0),
// so a missing frame draws black.
let Some(flat) = frame.get_synced_video_frame(ctx, "flat.mp4", &input) else {
return fframes::svgr!(
<svg xmlns="http://www.w3.org/2000/svg" width={WIDTH} height={HEIGHT}>
<rect width={WIDTH} height={HEIGHT} fill="#000000" />
</svg>
);
};
let flat = flat.into_image();
let p = s.palette.stops();
let uniforms = ShaderUniforms::new()
.image("iChannel0", &flat)
.float2("uAim", s.aim.0, s.aim.1)
.float("uZoom", s.zoom)
.float3("uRot", s.tilt_x, s.tilt_y, s.roll)
.float2("uFocus", s.focus.0, s.focus.1)
.float("uBlur", s.blur)
.float3("uP0", p[0][0], p[0][1], p[0][2])
.float3("uP1", p[1][0], p[1][1], p[1][2])
.float3("uP2", p[2][0], p[2][1], p[2][2])
.float3("uP3", p[3][0], p[3][1], p[3][2])
.float3("uP4", p[4][0], p[4][1], p[4][2])
.float("uLock", if s.lock { 1.0 } else { 0.0 })
.float("uCut", s.cut)
.float("uOut", s.out)
.float("uSeed", (si * 31 + f) as f32)
.float("uStage", stage());
let layer = self.shader.draw(&frame, uniforms);
fframes::svgr!(
<svg xmlns="http://www.w3.org/2000/svg" width={WIDTH} height={HEIGHT}>
<image href={layer.href()} x="0" y="0" width={WIDTH} height={HEIGHT} />
{if stage() >= 4.0 { camera_tracker(&s, si, f) } else { Svgr::empty() }}
</svg>
)
}
}
/// `LENS_STAGE=0..3` renders the shader up to one step (see lens.sksl); unset = all.
fn stage() -> f32 {
std::env::var("LENS_STAGE").ok().and_then(|v| v.parse().ok()).unwrap_or(4.0)
}
const MONO: &str = "JetBrains Mono";
const LOCK_RED: &str = "#ff3b30";
const RW: f32 = WIDTH as f32;
const RH: f32 = HEIGHT as f32;
/// The camera's own tracker, drawn sharp over the filmed screen: links between the boxes,
/// a dot running along each link, coordinate labels and brackets on the target. The
/// numbers are the real pixel positions of the boxes in this frame.
fn camera_tracker<'a>(s: &hud::Shot, si: usize, f: usize) -> Svgr<'a> {
let t = hud::track(si, f);
let ink = if s.palette == hud::Palette::Paper {
"#141414"
} else {
"#ffffff"
};
let proj = |q: (f32, f32)| hud::project(s, q, RW, RH);
let centers: Vec<Option<(f32, f32)>> = t.boxes.iter().map(|b| proj(b.center())).collect();
let mut items: Vec<Svgr> = vec![];
// Tails run off the frame: aim at a point on the screen plane, then extend on screen.
for (a, b) in &t.tails {
let mid = (a.0 + (b.0 - a.0) * 0.12, a.1 + (b.1 - a.1) * 0.12);
if let (Some(pa), Some(pm)) = (proj(*a), proj(mid)) {
let (dx, dy) = (pm.0 - pa.0, pm.1 - pa.1);
let len = (dx * dx + dy * dy).sqrt().max(1e-3);
let (ex, ey) = (pa.0 + dx / len * 3000.0, pa.1 + dy / len * 3000.0);
items.push(fframes::svgr!(<line x1={pa.0} y1={pa.1} x2={ex} y2={ey} stroke={ink} stroke-width="1.1" opacity="0.7" />));
}
}
for k in 1..centers.len() {
if let (Some(a), Some(b)) = (centers[k - 1], centers[k]) {
let p = (f as f32 * 0.17 + k as f32 * 0.31).fract();
let (mx, my) = (a.0 + (b.0 - a.0) * p, a.1 + (b.1 - a.1) * p);
items.push(fframes::svgr!(<g>
<line x1={a.0} y1={a.1} x2={b.0} y2={b.1} stroke={ink} stroke-width="1.3" />
<circle cx={mx} cy={my} r="3" fill={ink} />
</g>));
}
}
// Labels alternate above and below their boxes and skip a slot that is already taken.
let mut taken: Vec<(f32, f32)> = vec![];
for (k, (b, c)) in t.boxes.iter().zip(¢ers).enumerate() {
let below = k % 2 == 1 && !b.locked;
let anchor = if below { (b.x, b.y + b.h) } else { (b.x, b.y) };
let (Some(c), Some(corner)) = (*c, proj(anchor)) else {
continue;
};
let y = if below { corner.1 + 24.0 } else { corner.1 - 9.0 };
let free = b.locked
|| taken.iter().all(|(tx, ty)| (tx - corner.0).abs() > 190.0 || (ty - y).abs() > 26.0);
let color = if b.locked { LOCK_RED } else { ink };
items.push(fframes::svgr!(<circle cx={c.0} cy={c.1} r="3.2" fill={color} />));
if !free {
continue;
}
taken.push((corner.0, y));
let label = if b.locked {
format!("LOCK x: {:.0} y: {:.0}", c.0, c.1)
} else {
format!("x: {:.0} y: {:.0}", c.0, c.1)
};
items.push(fframes::svgr!(<text x={corner.0} y={y} font-family={MONO} font-weight="400" font-size="19" fill={color}>{label}</text>));
}
// Brackets around the target box, in screen space.
if let Some(b) = t.boxes.last() {
let corners = [
(b.x, b.y),
(b.x + b.w, b.y),
(b.x + b.w, b.y + b.h),
(b.x, b.y + b.h),
];
let pts: Vec<(f32, f32)> = corners.iter().filter_map(|q| proj(*q)).collect();
if pts.len() == 4 {
let color = if t.lock { LOCK_RED } else { ink };
let grow = if t.lock { 10.0 } else { 6.0 };
let (cx, cy) = (
pts.iter().map(|p| p.0).sum::<f32>() / 4.0,
pts.iter().map(|p| p.1).sum::<f32>() / 4.0,
);
for (i, p) in pts.iter().enumerate() {
let (ox, oy) = (p.0 - cx, p.1 - cy);
let l = (ox * ox + oy * oy).sqrt().max(1e-3);
let q = (p.0 + ox / l * grow, p.1 + oy / l * grow);
let n1 = pts[(i + 1) % 4];
let n0 = pts[(i + 3) % 4];
let arm = |n: (f32, f32)| {
let (dx, dy) = (n.0 - p.0, n.1 - p.1);
let l = (dx * dx + dy * dy).sqrt().max(1e-3);
let a = 18.0_f32.min(l * 0.4);
(q.0 + dx / l * a, q.1 + dy / l * a)
};
let (a1, a0) = (arm(n1), arm(n0));
items.push(fframes::svgr!(<path d={format!("M {:.1} {:.1} L {:.1} {:.1} L {:.1} {:.1}", a0.0, a0.1, q.0, q.1, a1.0, a1.1)}
fill="none" stroke={color} stroke-width="2.4" />));
}
}
}
fframes::svgr!(<g opacity={0.92 * (1.0 - s.out)}>{items}</g>)
}