A virtual camera films an AI agent's screen while a tracker locks onto every glyph. Two fframes passes, one SkSL shader, all sound synthesized.
Render on Linux with Vulkan, and fall back to the CPU
Kevin Rajanad650545de25
diff --git a/README.md b/README.mdindex c7d5ed6..85f0a51 100644--- a/README.md+++ b/README.md@@ -99,11 +99,40 @@ red lock frames tear the bass. See [Slab score](#slab-score).## Quick start-macOS uses Metal; Windows uses Vulkan. For Windows requirements and the PowerShell-render pipeline, see [Windows setup](docs/WINDOWS.md). The macOS instructions follow.+macOS uses Metal; Linux and Windows use Vulkan. For Windows requirements and the+PowerShell render pipeline, see [Windows setup](docs/WINDOWS.md).-Requirements: macOS with Apple silicon (Skia on Metal), Rust, ffmpeg, Python 3 with `numpy`-and `scipy`. The README images also need Pillow and `img2webp`.+### Requirements++- Rust (2024 edition), ffmpeg and ffprobe on `PATH`.+- Python 3 with `numpy` and `scipy`, or [uv](https://docs.astral.sh/uv/)+ (`render.sh` uses `uv run --with numpy --with scipy` if the packages are missing).+- macOS: Apple silicon, Skia on Metal (the existing path).+- Linux: a Vulkan loader and GPU driver, plus a C/C++ toolchain, pkg-config,+ libclang, fontconfig and OpenGL development libraries for Skia. On Debian/Ubuntu:+ `sudo apt install build-essential pkg-config libclang-dev libfontconfig1-dev libgl1-mesa-dev libvulkan1 ffmpeg`.+ For NVIDIA, install your distribution's proprietary Vulkan-capable driver.+ Rust Skia normally downloads a prebuilt; a source build also needs Python and Ninja.+- README images additionally need Pillow and `img2webp`.++Linux uses Vulkan, not Metal. The default `FF_BACKEND=auto` tries the platform GPU+and reports a CPU fallback if context creation fails. To choose explicitly:++```bash+FF_BACKEND=gpu tools/render.sh # Vulkan on Linux/Windows, Metal on macOS; no silent fallback+FF_BACKEND=cpu tools/render.sh # no GPU context is created (also works with GPUs hidden)+```++The CPU flat pass uses fframes' tiny-skia SVG renderer. The CPU lens pass uses+fframes' `SkiaCpuCtx` raster compatibility backend, **not** tiny-skia, because+tiny-skia cannot run SkSL. The original lens shader still runs: perspective,+depth of field, palette, aberration, bloom, LED grid, tear and grain are retained.+This is slower, and SVG antialiasing/filter kernels can differ from the GPU pass.+Backend selection is printed to stderr; invalid values fail rather than choosing+a different backend. `FF_BACKEND` also works when invoking either binary directly.+`preview` still needs a GPU/window; the fallback is for offline rendering.++### Render```bashgit clone https://github.com/mrsarac/ff-tracking && cd ff-trackingdiff --git a/flat/Cargo.toml b/flat/Cargo.tomlindex c239f00..2dba2ea 100644--- a/flat/Cargo.toml+++ b/flat/Cargo.toml@@ -8,15 +8,18 @@ publish = falsehud = { path = "../hud" }fframes = { workspace = true }-[target.'cfg(windows)'.dependencies]+[target.'cfg(not(target_os = "macos"))'.dependencies]fframes_skia_renderer = { workspace = true, features = ["vulkan"] }fframes_native_player = { workspace = true, features = ["vulkan"] }-[target.'cfg(not(windows))'.dependencies]+[target.'cfg(target_os = "macos")'.dependencies]fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }fframes_skia_renderer = { workspace = true, features = ["metal"] }fframes_native_player = { workspace = true, features = ["metal"] }+[target.'cfg(target_os = "linux")'.dependencies]+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }+[lib]name = "flat"path = "src/lib.rs"diff --git a/flat/src/main.rs b/flat/src/main.rsindex 24e1bc1..895c150 100644--- a/flat/src/main.rs+++ b/flat/src/main.rs@@ -1,20 +1,14 @@-use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider, cli};-#[cfg(not(windows))]-use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;-#[cfg(windows)]-use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;-use fframes_skia_renderer::{- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,-};-use flat::{FlatMedia, FlatVideo, HEIGHT, WIDTH};+use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider};+#[path = "../../tools/backend.rs"]+mod backend;+use flat::{FlatMedia, FlatVideo};use std::process::ExitCode;fn main() -> ExitCode {let media = FlatMedia::prepare().expect("media");let video = FlatVideo::new();- let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");- cli::new(+ backend::run(&video,RenderOptions {media: Some(&media),@@ -27,25 +21,7 @@ fn main() -> ExitCode {},..Default::default()},+ "out/pass/flat.mp4",+ false,)- .backend(SkiaFFramesRenderer::new(- if cfg!(windows) {- // Bound GPU contexts and frame buffers to limit memory use on Windows.- SkiaPipelineConfig {- concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),- encoder_threads: 2,- buffer_queue_size: 2,- ..Default::default()- }- } else {- SkiaPipelineConfig {- concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,- ..Default::default()- }- },- &gpu,- ))- .preview(fframes_native_player::cli_preview)- .default_output("out/pass/flat.mp4")- .run()}diff --git a/lens/Cargo.toml b/lens/Cargo.tomlindex fc3cf36..2d4d492 100644--- a/lens/Cargo.toml+++ b/lens/Cargo.toml@@ -8,15 +8,18 @@ publish = falsehud = { path = "../hud" }fframes = { workspace = true }-[target.'cfg(windows)'.dependencies]+[target.'cfg(not(target_os = "macos"))'.dependencies]fframes_skia_renderer = { workspace = true, features = ["vulkan"] }fframes_native_player = { workspace = true, features = ["vulkan"] }-[target.'cfg(not(windows))'.dependencies]+[target.'cfg(target_os = "macos")'.dependencies]fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }fframes_skia_renderer = { workspace = true, features = ["metal"] }fframes_native_player = { workspace = true, features = ["metal"] }+[target.'cfg(target_os = "linux")'.dependencies]+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }+[lib]name = "lens"path = "src/lib.rs"diff --git a/lens/src/main.rs b/lens/src/main.rsindex c5f6270..93c3271 100644--- a/lens/src/main.rs+++ b/lens/src/main.rs@@ -1,14 +1,9 @@use fframes::{- CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions, cli,+ CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions,};-#[cfg(not(windows))]-use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;-#[cfg(windows)]-use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;-use fframes_skia_renderer::{- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,-};-use lens::{HEIGHT, LensVideo, WIDTH};+#[path = "../../tools/backend.rs"]+mod backend;+use lens::LensVideo;use std::process::ExitCode;fn main() -> ExitCode {@@ -21,9 +16,8 @@ fn main() -> ExitCode {let fonts = fonts_dir.process_media_source().expect("fonts");let media = CombinedMediaProvider::from([&pass as &dyn MediaProvider, &fonts]);let video = LensVideo::new();- let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");- cli::new(+ backend::run(&video,RenderOptions {media: Some(&media),@@ -35,25 +29,16 @@ fn main() -> ExitCode {},..Default::default()},+ "out/tracking.mp4",+ true,)- .backend(SkiaFFramesRenderer::new(- if cfg!(windows) {- // Bound GPU contexts and frame buffers to limit memory use on Windows.- SkiaPipelineConfig {- concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),- encoder_threads: 2,- buffer_queue_size: 2,- ..Default::default()- }- } else {- SkiaPipelineConfig {- concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,- ..Default::default()- }- },- &gpu,- ))- .preview(fframes_native_player::cli_preview)- .default_output("out/tracking.mp4")- .run()+}++#[cfg(test)]+mod tests {+ #[test]+ fn lens_shader_compiles() {+ let shader = fframes::Shader::sksl(include_str!("../shaders/lens.sksl"));+ fframes_skia_renderer::render::compile_shader(&shader).expect("lens SkSL");+ }}diff --git a/tools/backend.rs b/tools/backend.rsnew file mode 100644index 0000000..8d1cb9f--- /dev/null+++ b/tools/backend.rs@@ -0,0 +1,113 @@+//! Shared by the two passes. CPU rendering never creates a GPU context.+use fframes::{RenderOptions, Video, cli, cpu::CpuRenderingBackend};+#[cfg(target_os = "macos")]+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;+#[cfg(not(target_os = "macos"))]+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;+use fframes_skia_renderer::{+ SkiaCpuCtx, SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,+};+use std::process::ExitCode;++#[derive(Debug, PartialEq)]+enum Backend {+ Auto,+ Gpu,+ Cpu,+}++impl Backend {+ fn parse(value: &str) -> Result<Self, &'static str> {+ match value {+ "auto" => Ok(Self::Auto),+ "gpu" => Ok(Self::Gpu),+ "cpu" => Ok(Self::Cpu),+ _ => Err("FF_BACKEND must be auto, gpu or cpu"),+ }+ }+}++pub fn run<'a, 'media: 'a, V: Video + Send + Sync>(+ video: &'a V,+ options: RenderOptions<'a, 'media>,+ output: &str,+ sksl: bool,+) -> ExitCode {+ let backend = match Backend::parse(&std::env::var("FF_BACKEND").unwrap_or("auto".into())) {+ Ok(backend) => backend,+ Err(err) => {+ eprintln!("{err}");+ return ExitCode::FAILURE;+ }+ };+ let pipeline = if cfg!(target_os = "macos") {+ SkiaPipelineConfig {+ concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,+ ..Default::default()+ }+ } else {+ // Bound GPU contexts and frame buffers on shared/discrete GPUs.+ SkiaPipelineConfig {+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),+ encoder_threads: 2,+ buffer_queue_size: 2,+ ..Default::default()+ }+ };+ let runner = cli::new(video, options)+ .preview(fframes_native_player::cli_preview)+ .default_output(output);+ if backend != Backend::Cpu {+ match SkiaGpuCtx::new(V::WIDTH, V::HEIGHT) {+ Ok(gpu) => {+ eprintln!(+ "FF_BACKEND=gpu ({})",+ if cfg!(target_os = "macos") {+ "Metal"+ } else {+ "Vulkan"+ }+ );+ return runner+ .backend(SkiaFFramesRenderer::new(pipeline, &gpu))+ .run();+ }+ Err(err) if backend == Backend::Auto => {+ eprintln!("GPU unavailable: {err}; falling back to CPU")+ }+ Err(err) => {+ eprintln!("GPU unavailable: {err}");+ return ExitCode::FAILURE;+ }+ }+ }+ if sksl {+ // tiny-skia cannot execute SkSL. Skia's raster backend can, including the lens.+ eprintln!("FF_BACKEND=cpu (Skia raster; full SkSL lens, no effects disabled)");+ let cpu = SkiaCpuCtx::new(V::WIDTH, V::HEIGHT);+ runner+ .backend(SkiaFFramesRenderer::new(pipeline, &cpu))+ .run()+ } else {+ eprintln!("FF_BACKEND=cpu (tiny-skia SVG)");+ runner+ .backend(CpuRenderingBackend {+ concurrency: 2,+ ..Default::default()+ })+ .run()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn backend_selection() {+ assert_eq!(Backend::parse("auto"), Ok(Backend::Auto));+ assert_eq!(Backend::parse("gpu"), Ok(Backend::Gpu));+ assert_eq!(Backend::parse("cpu"), Ok(Backend::Cpu));+ assert!(Backend::parse("typo").is_err());+ }+}diff --git a/tools/render.sh b/tools/render.shindex 58a3a15..d1c4f71 100755--- a/tools/render.sh+++ b/tools/render.sh@@ -3,10 +3,23 @@# Run from anywhere: tools/render.shset -euo pipefailcd "$(dirname "$0")/.."+export FF_BACKEND="${FF_BACKEND:-auto}" # auto, gpu (fail if unavailable), or cpu+case "$FF_BACKEND" in+ auto|gpu|cpu) ;;+ *) printf 'FF_BACKEND must be auto, gpu or cpu\n' >&2; exit 1 ;;+esac+command -v ffmpeg >/dev/null+command -v ffprobe >/dev/null+python=("${PYTHON:-python3}" -I)+if ! "${python[@]}" -c 'import numpy, scipy' 2>/dev/null; then+ command -v uv >/dev/null || { printf 'Install numpy/scipy for Python 3, or install uv\n' >&2; exit 1; }+ python=(uv run --no-project --with numpy --with scipy python -I)+fi+bin="${CARGO_TARGET_DIR:-target}/release"cargo build --release -p hud -p flat -p lensmkdir -p out/pass-target/release/tracks-target/release/flat render -o out/pass/flat.mp4-python3 -I tools/sfx.py-target/release/lens render -o out/tracking.mp4+"$bin"/tracks+"$bin"/flat render -o out/pass/flat.mp4+"${python[@]}" tools/sfx.py+"$bin"/lens render -o out/tracking.mp4ffprobe -v error -show_entries stream=codec_type,width,height,nb_frames,duration -of compact out/tracking.mp4