Skip to content

A virtual camera films an AI agent's screen while a tracker locks onto every glyph. Two fframes passes, one SkSL shader, all sound synthesized.

Render on Linux with Vulkan, and fall back to the CPU

Kevin Rajanad650545de25
diff --git a/README.md b/README.md
index c7d5ed6..85f0a51 100644
--- a/README.md
+++ b/README.md
@@ -99,11 +99,40 @@ red lock frames tear the bass. See [Slab score](#slab-score).
## Quick start
-macOS uses Metal; Windows uses Vulkan. For Windows requirements and the PowerShell
-render pipeline, see [Windows setup](docs/WINDOWS.md). The macOS instructions follow.
+macOS uses Metal; Linux and Windows use Vulkan. For Windows requirements and the
+PowerShell render pipeline, see [Windows setup](docs/WINDOWS.md).
-Requirements: macOS with Apple silicon (Skia on Metal), Rust, ffmpeg, Python 3 with `numpy`
-and `scipy`. The README images also need Pillow and `img2webp`.
+### Requirements
+
+- Rust (2024 edition), ffmpeg and ffprobe on `PATH`.
+- Python 3 with `numpy` and `scipy`, or [uv](https://docs.astral.sh/uv/)
+ (`render.sh` uses `uv run --with numpy --with scipy` if the packages are missing).
+- macOS: Apple silicon, Skia on Metal (the existing path).
+- Linux: a Vulkan loader and GPU driver, plus a C/C++ toolchain, pkg-config,
+ libclang, fontconfig and OpenGL development libraries for Skia. On Debian/Ubuntu:
+ `sudo apt install build-essential pkg-config libclang-dev libfontconfig1-dev libgl1-mesa-dev libvulkan1 ffmpeg`.
+ For NVIDIA, install your distribution's proprietary Vulkan-capable driver.
+ Rust Skia normally downloads a prebuilt; a source build also needs Python and Ninja.
+- README images additionally need Pillow and `img2webp`.
+
+Linux uses Vulkan, not Metal. The default `FF_BACKEND=auto` tries the platform GPU
+and reports a CPU fallback if context creation fails. To choose explicitly:
+
+```bash
+FF_BACKEND=gpu tools/render.sh # Vulkan on Linux/Windows, Metal on macOS; no silent fallback
+FF_BACKEND=cpu tools/render.sh # no GPU context is created (also works with GPUs hidden)
+```
+
+The CPU flat pass uses fframes' tiny-skia SVG renderer. The CPU lens pass uses
+fframes' `SkiaCpuCtx` raster compatibility backend, **not** tiny-skia, because
+tiny-skia cannot run SkSL. The original lens shader still runs: perspective,
+depth of field, palette, aberration, bloom, LED grid, tear and grain are retained.
+This is slower, and SVG antialiasing/filter kernels can differ from the GPU pass.
+Backend selection is printed to stderr; invalid values fail rather than choosing
+a different backend. `FF_BACKEND` also works when invoking either binary directly.
+`preview` still needs a GPU/window; the fallback is for offline rendering.
+
+### Render
```bash
git clone https://github.com/mrsarac/ff-tracking && cd ff-tracking
diff --git a/flat/Cargo.toml b/flat/Cargo.toml
index c239f00..2dba2ea 100644
--- a/flat/Cargo.toml
+++ b/flat/Cargo.toml
@@ -8,15 +8,18 @@ publish = false
hud = { path = "../hud" }
fframes = { workspace = true }
-[target.'cfg(windows)'.dependencies]
+[target.'cfg(not(target_os = "macos"))'.dependencies]
fframes_skia_renderer = { workspace = true, features = ["vulkan"] }
fframes_native_player = { workspace = true, features = ["vulkan"] }
-[target.'cfg(not(windows))'.dependencies]
+[target.'cfg(target_os = "macos")'.dependencies]
fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
fframes_skia_renderer = { workspace = true, features = ["metal"] }
fframes_native_player = { workspace = true, features = ["metal"] }
+[target.'cfg(target_os = "linux")'.dependencies]
+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
+
[lib]
name = "flat"
path = "src/lib.rs"
diff --git a/flat/src/main.rs b/flat/src/main.rs
index 24e1bc1..895c150 100644
--- a/flat/src/main.rs
+++ b/flat/src/main.rs
@@ -1,20 +1,14 @@
-use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider, cli};
-#[cfg(not(windows))]
-use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;
-#[cfg(windows)]
-use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;
-use fframes_skia_renderer::{
- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,
-};
-use flat::{FlatMedia, FlatVideo, HEIGHT, WIDTH};
+use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider};
+#[path = "../../tools/backend.rs"]
+mod backend;
+use flat::{FlatMedia, FlatVideo};
use std::process::ExitCode;
fn main() -> ExitCode {
let media = FlatMedia::prepare().expect("media");
let video = FlatVideo::new();
- let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");
- cli::new(
+ backend::run(
&video,
RenderOptions {
media: Some(&media),
@@ -27,25 +21,7 @@ fn main() -> ExitCode {
},
..Default::default()
},
+ "out/pass/flat.mp4",
+ false,
)
- .backend(SkiaFFramesRenderer::new(
- if cfg!(windows) {
- // Bound GPU contexts and frame buffers to limit memory use on Windows.
- SkiaPipelineConfig {
- concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),
- encoder_threads: 2,
- buffer_queue_size: 2,
- ..Default::default()
- }
- } else {
- SkiaPipelineConfig {
- concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,
- ..Default::default()
- }
- },
- &gpu,
- ))
- .preview(fframes_native_player::cli_preview)
- .default_output("out/pass/flat.mp4")
- .run()
}
diff --git a/lens/Cargo.toml b/lens/Cargo.toml
index fc3cf36..2d4d492 100644
--- a/lens/Cargo.toml
+++ b/lens/Cargo.toml
@@ -8,15 +8,18 @@ publish = false
hud = { path = "../hud" }
fframes = { workspace = true }
-[target.'cfg(windows)'.dependencies]
+[target.'cfg(not(target_os = "macos"))'.dependencies]
fframes_skia_renderer = { workspace = true, features = ["vulkan"] }
fframes_native_player = { workspace = true, features = ["vulkan"] }
-[target.'cfg(not(windows))'.dependencies]
+[target.'cfg(target_os = "macos")'.dependencies]
fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
fframes_skia_renderer = { workspace = true, features = ["metal"] }
fframes_native_player = { workspace = true, features = ["metal"] }
+[target.'cfg(target_os = "linux")'.dependencies]
+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
+
[lib]
name = "lens"
path = "src/lib.rs"
diff --git a/lens/src/main.rs b/lens/src/main.rs
index c5f6270..93c3271 100644
--- a/lens/src/main.rs
+++ b/lens/src/main.rs
@@ -1,14 +1,9 @@
use fframes::{
- CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions, cli,
+ CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions,
};
-#[cfg(not(windows))]
-use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;
-#[cfg(windows)]
-use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;
-use fframes_skia_renderer::{
- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,
-};
-use lens::{HEIGHT, LensVideo, WIDTH};
+#[path = "../../tools/backend.rs"]
+mod backend;
+use lens::LensVideo;
use std::process::ExitCode;
fn main() -> ExitCode {
@@ -21,9 +16,8 @@ fn main() -> ExitCode {
let fonts = fonts_dir.process_media_source().expect("fonts");
let media = CombinedMediaProvider::from([&pass as &dyn MediaProvider, &fonts]);
let video = LensVideo::new();
- let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");
- cli::new(
+ backend::run(
&video,
RenderOptions {
media: Some(&media),
@@ -35,25 +29,16 @@ fn main() -> ExitCode {
},
..Default::default()
},
+ "out/tracking.mp4",
+ true,
)
- .backend(SkiaFFramesRenderer::new(
- if cfg!(windows) {
- // Bound GPU contexts and frame buffers to limit memory use on Windows.
- SkiaPipelineConfig {
- concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),
- encoder_threads: 2,
- buffer_queue_size: 2,
- ..Default::default()
- }
- } else {
- SkiaPipelineConfig {
- concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,
- ..Default::default()
- }
- },
- &gpu,
- ))
- .preview(fframes_native_player::cli_preview)
- .default_output("out/tracking.mp4")
- .run()
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn lens_shader_compiles() {
+ let shader = fframes::Shader::sksl(include_str!("../shaders/lens.sksl"));
+ fframes_skia_renderer::render::compile_shader(&shader).expect("lens SkSL");
+ }
}
diff --git a/tools/backend.rs b/tools/backend.rs
new file mode 100644
index 0000000..8d1cb9f
--- /dev/null
+++ b/tools/backend.rs
@@ -0,0 +1,113 @@
+//! Shared by the two passes. CPU rendering never creates a GPU context.
+use fframes::{RenderOptions, Video, cli, cpu::CpuRenderingBackend};
+#[cfg(target_os = "macos")]
+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;
+#[cfg(not(target_os = "macos"))]
+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;
+use fframes_skia_renderer::{
+ SkiaCpuCtx, SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,
+};
+use std::process::ExitCode;
+
+#[derive(Debug, PartialEq)]
+enum Backend {
+ Auto,
+ Gpu,
+ Cpu,
+}
+
+impl Backend {
+ fn parse(value: &str) -> Result<Self, &'static str> {
+ match value {
+ "auto" => Ok(Self::Auto),
+ "gpu" => Ok(Self::Gpu),
+ "cpu" => Ok(Self::Cpu),
+ _ => Err("FF_BACKEND must be auto, gpu or cpu"),
+ }
+ }
+}
+
+pub fn run<'a, 'media: 'a, V: Video + Send + Sync>(
+ video: &'a V,
+ options: RenderOptions<'a, 'media>,
+ output: &str,
+ sksl: bool,
+) -> ExitCode {
+ let backend = match Backend::parse(&std::env::var("FF_BACKEND").unwrap_or("auto".into())) {
+ Ok(backend) => backend,
+ Err(err) => {
+ eprintln!("{err}");
+ return ExitCode::FAILURE;
+ }
+ };
+ let pipeline = if cfg!(target_os = "macos") {
+ SkiaPipelineConfig {
+ concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,
+ ..Default::default()
+ }
+ } else {
+ // Bound GPU contexts and frame buffers on shared/discrete GPUs.
+ SkiaPipelineConfig {
+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),
+ encoder_threads: 2,
+ buffer_queue_size: 2,
+ ..Default::default()
+ }
+ };
+ let runner = cli::new(video, options)
+ .preview(fframes_native_player::cli_preview)
+ .default_output(output);
+ if backend != Backend::Cpu {
+ match SkiaGpuCtx::new(V::WIDTH, V::HEIGHT) {
+ Ok(gpu) => {
+ eprintln!(
+ "FF_BACKEND=gpu ({})",
+ if cfg!(target_os = "macos") {
+ "Metal"
+ } else {
+ "Vulkan"
+ }
+ );
+ return runner
+ .backend(SkiaFFramesRenderer::new(pipeline, &gpu))
+ .run();
+ }
+ Err(err) if backend == Backend::Auto => {
+ eprintln!("GPU unavailable: {err}; falling back to CPU")
+ }
+ Err(err) => {
+ eprintln!("GPU unavailable: {err}");
+ return ExitCode::FAILURE;
+ }
+ }
+ }
+ if sksl {
+ // tiny-skia cannot execute SkSL. Skia's raster backend can, including the lens.
+ eprintln!("FF_BACKEND=cpu (Skia raster; full SkSL lens, no effects disabled)");
+ let cpu = SkiaCpuCtx::new(V::WIDTH, V::HEIGHT);
+ runner
+ .backend(SkiaFFramesRenderer::new(pipeline, &cpu))
+ .run()
+ } else {
+ eprintln!("FF_BACKEND=cpu (tiny-skia SVG)");
+ runner
+ .backend(CpuRenderingBackend {
+ concurrency: 2,
+ ..Default::default()
+ })
+ .run()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn backend_selection() {
+ assert_eq!(Backend::parse("auto"), Ok(Backend::Auto));
+ assert_eq!(Backend::parse("gpu"), Ok(Backend::Gpu));
+ assert_eq!(Backend::parse("cpu"), Ok(Backend::Cpu));
+ assert!(Backend::parse("typo").is_err());
+ }
+}
diff --git a/tools/render.sh b/tools/render.sh
index 58a3a15..d1c4f71 100755
--- a/tools/render.sh
+++ b/tools/render.sh
@@ -3,10 +3,23 @@
# Run from anywhere: tools/render.sh
set -euo pipefail
cd "$(dirname "$0")/.."
+export FF_BACKEND="${FF_BACKEND:-auto}" # auto, gpu (fail if unavailable), or cpu
+case "$FF_BACKEND" in
+ auto|gpu|cpu) ;;
+ *) printf 'FF_BACKEND must be auto, gpu or cpu\n' >&2; exit 1 ;;
+esac
+command -v ffmpeg >/dev/null
+command -v ffprobe >/dev/null
+python=("${PYTHON:-python3}" -I)
+if ! "${python[@]}" -c 'import numpy, scipy' 2>/dev/null; then
+ command -v uv >/dev/null || { printf 'Install numpy/scipy for Python 3, or install uv\n' >&2; exit 1; }
+ python=(uv run --no-project --with numpy --with scipy python -I)
+fi
+bin="${CARGO_TARGET_DIR:-target}/release"
cargo build --release -p hud -p flat -p lens
mkdir -p out/pass
-target/release/tracks
-target/release/flat render -o out/pass/flat.mp4
-python3 -I tools/sfx.py
-target/release/lens render -o out/tracking.mp4
+"$bin"/tracks
+"$bin"/flat render -o out/pass/flat.mp4
+"${python[@]}" tools/sfx.py
+"$bin"/lens render -o out/tracking.mp4
ffprobe -v error -show_entries stream=codec_type,width,height,nb_frames,duration -of compact out/tracking.mp4