Skip to content

A virtual camera films an AI agent's screen while a tracker locks onto every glyph. Two fframes passes, one SkSL shader, all sound synthesized.

Add Windows Vulkan rendering support

Tanzim6414d47458f8
diff --git a/.gitignore b/.gitignore
index 0383233..ff1267a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ strip.png
check.png
__pycache__/
.DS_Store
+.venv/
diff --git a/Cargo.lock b/Cargo.lock
index fde7998..829723c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -217,6 +217,15 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "ash"
+version = "0.38.0+1.3.281"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"
+dependencies = [
+ "libloading",
+]
+
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -1041,6 +1050,7 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28303fd0cfdbb9d937a07cd3091c859a5777cba4b6622ba37e8a4047cb01675e"
dependencies = [
+ "ash",
"core-foundation 0.10.1",
"fframes",
"foreign-types-shared",
diff --git a/Cargo.toml b/Cargo.toml
index 58b3e44..6541608 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,9 +3,9 @@ resolver = "3"
members = ["hud", "flat", "lens"]
[workspace.dependencies]
-fframes = { version = "=1.2.0", features = ["compile-time-svgtree", "cli", "h264", "libav-agree-gpl"] }
-fframes_skia_renderer = { version = "=1.2.0", features = ["metal"] }
-fframes_native_player = { version = "=1.2.0", default-features = false, features = ["metal", "audio"] }
+fframes = { version = "=1.2.0", features = ["compile-time-svgtree", "cli"] }
+fframes_skia_renderer = { version = "=1.2.0", default-features = true }
+fframes_native_player = { version = "=1.2.0", default-features = false, features = ["audio"] }
[profile.dev]
opt-level = 1
diff --git a/README.md b/README.md
index 8a49d72..5af6067 100644
--- a/README.md
+++ b/README.md
@@ -99,6 +99,9 @@ red lock frames tear the bass. See [Slab score](#slab-score).
## Quick start
+macOS uses Metal; Windows uses Vulkan. For Windows requirements and the PowerShell
+render pipeline, see [Windows setup](docs/WINDOWS.md). The macOS instructions follow.
+
Requirements: macOS with Apple silicon (Skia on Metal), Rust, ffmpeg, Python 3 with `numpy`
and `scipy`. The README images also need Pillow and `img2webp`.
diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md
new file mode 100644
index 0000000..1582655
--- /dev/null
+++ b/docs/WINDOWS.md
@@ -0,0 +1,58 @@
+# Windows setup
+
+The Windows build uses Vulkan in place of Metal. Scenes, camera math, shader and
+sound synthesis are unchanged.
+
+## Requirements
+
+- 64-bit Windows and a Vulkan-capable GPU with current drivers.
+- [Rust](https://rustup.rs/) with the MSVC toolchain.
+- Visual Studio Build Tools with **Desktop development with C++**, including a Windows SDK.
+- Python 3 with `numpy` and `scipy`.
+- LLVM/libclang, available from [LLVM releases](https://github.com/llvm/llvm-project/releases).
+- A **shared FFmpeg 9.0 development build**, with `bin`, `include` and `lib` directories.
+ For example, use `ffmpeg-n9.0-latest-win64-gpl-shared-9.0.zip` from
+ [BtbN/FFmpeg-Builds](https://github.com/BtbN/FFmpeg-Builds/releases/tag/latest).
+
+A standalone `ffmpeg.exe` is insufficient: Rust links against FFmpeg libraries,
+and build-time media processing loads the DLLs. See also
+[fframes' Windows requirements](https://github.com/dmtrKovalenko/fframes#requirements).
+
+## Render
+
+In PowerShell, adjust the installation paths to match your machine:
+
+```powershell
+$env:FFMPEG_DIR = "C:\tools\ffmpeg-n9.0-latest-win64-gpl-shared-9.0"
+$env:LIBCLANG_PATH = "C:\Program Files\LLVM\bin"
+$env:PATH = "$env:FFMPEG_DIR\bin;$env:PATH"
+
+git clone https://github.com/mrsarac/ff-tracking
+cd ff-tracking
+python -m venv .venv
+.\.venv\Scripts\python.exe -m pip install numpy scipy
+powershell -NoProfile -ExecutionPolicy Bypass -File tools/render.ps1 -Python .\.venv\Scripts\python.exe
+# -> out/tracking.mp4
+```
+
+The execution-policy option applies only to this PowerShell process. Keep FFmpeg's
+`bin` directory on `PATH` when running the built programs. The script accepts the
+Python executable via `-Python`, stops on failed steps, restores the caller's
+directory, and honors Cargo's configured target directory.
+
+After rendering, the usual per-pass tools work:
+
+```powershell
+.\target\release\lens.exe preview
+.\target\release\lens.exe strip all -n 12
+.\target\release\lens.exe audio analyze
+```
+
+If you configured a custom Cargo target directory, use its `release` folder instead.
+
+Windows uses the codecs from the shared FFmpeg installation, so the Cargo features
+that request building codecs from source are enabled only for the original
+non-Windows configuration. Windows limits the rendering pipeline to two GPU
+contexts, two encoder workers and a small queue: each worker decodes the 4K first
+pass, so unrestricted concurrency can exhaust memory, especially on longer edits.
+The original macOS backend and concurrency setting remain unchanged.
diff --git a/flat/Cargo.toml b/flat/Cargo.toml
index 85d653b..c239f00 100644
--- a/flat/Cargo.toml
+++ b/flat/Cargo.toml
@@ -7,8 +7,15 @@ publish = false
[dependencies]
hud = { path = "../hud" }
fframes = { workspace = true }
-fframes_skia_renderer = { workspace = true }
-fframes_native_player = { workspace = true }
+
+[target.'cfg(windows)'.dependencies]
+fframes_skia_renderer = { workspace = true, features = ["vulkan"] }
+fframes_native_player = { workspace = true, features = ["vulkan"] }
+
+[target.'cfg(not(windows))'.dependencies]
+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
+fframes_skia_renderer = { workspace = true, features = ["metal"] }
+fframes_native_player = { workspace = true, features = ["metal"] }
[lib]
name = "flat"
diff --git a/flat/src/main.rs b/flat/src/main.rs
index 9644f90..24e1bc1 100644
--- a/flat/src/main.rs
+++ b/flat/src/main.rs
@@ -1,6 +1,10 @@
use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider, cli};
+#[cfg(not(windows))]
+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;
+#[cfg(windows)]
+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;
use fframes_skia_renderer::{
- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig, metal::SkiaMetalCtx,
+ SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,
};
use flat::{FlatMedia, FlatVideo, HEIGHT, WIDTH};
use std::process::ExitCode;
@@ -8,7 +12,7 @@ use std::process::ExitCode;
fn main() -> ExitCode {
let media = FlatMedia::prepare().expect("media");
let video = FlatVideo::new();
- let gpu = SkiaMetalCtx::new(WIDTH, HEIGHT).expect("GPU context");
+ let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");
cli::new(
&video,
@@ -24,16 +28,23 @@ fn main() -> ExitCode {
..Default::default()
},
)
- .backend(
- SkiaFFramesRenderer::new_metal(
- &gpu,
+ .backend(SkiaFFramesRenderer::new(
+ if cfg!(windows) {
+ // Bound GPU contexts and frame buffers to limit memory use on Windows.
+ SkiaPipelineConfig {
+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),
+ encoder_threads: 2,
+ buffer_queue_size: 2,
+ ..Default::default()
+ }
+ } else {
SkiaPipelineConfig {
concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,
..Default::default()
- },
- )
- .expect("skia renderer"),
- )
+ }
+ },
+ &gpu,
+ ))
.preview(fframes_native_player::cli_preview)
.default_output("out/pass/flat.mp4")
.run()
diff --git a/lens/Cargo.toml b/lens/Cargo.toml
index 0fc116f..fc3cf36 100644
--- a/lens/Cargo.toml
+++ b/lens/Cargo.toml
@@ -7,8 +7,15 @@ publish = false
[dependencies]
hud = { path = "../hud" }
fframes = { workspace = true }
-fframes_skia_renderer = { workspace = true }
-fframes_native_player = { workspace = true }
+
+[target.'cfg(windows)'.dependencies]
+fframes_skia_renderer = { workspace = true, features = ["vulkan"] }
+fframes_native_player = { workspace = true, features = ["vulkan"] }
+
+[target.'cfg(not(windows))'.dependencies]
+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }
+fframes_skia_renderer = { workspace = true, features = ["metal"] }
+fframes_native_player = { workspace = true, features = ["metal"] }
[lib]
name = "lens"
diff --git a/lens/src/main.rs b/lens/src/main.rs
index 2ce9442..c5f6270 100644
--- a/lens/src/main.rs
+++ b/lens/src/main.rs
@@ -1,8 +1,12 @@
use fframes::{
CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions, cli,
};
+#[cfg(not(windows))]
+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;
+#[cfg(windows)]
+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;
use fframes_skia_renderer::{
- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig, metal::SkiaMetalCtx,
+ SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,
};
use lens::{HEIGHT, LensVideo, WIDTH};
use std::process::ExitCode;
@@ -17,7 +21,7 @@ fn main() -> ExitCode {
let fonts = fonts_dir.process_media_source().expect("fonts");
let media = CombinedMediaProvider::from([&pass as &dyn MediaProvider, &fonts]);
let video = LensVideo::new();
- let gpu = SkiaMetalCtx::new(WIDTH, HEIGHT).expect("GPU context");
+ let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");
cli::new(
&video,
@@ -32,16 +36,23 @@ fn main() -> ExitCode {
..Default::default()
},
)
- .backend(
- SkiaFFramesRenderer::new_metal(
- &gpu,
+ .backend(SkiaFFramesRenderer::new(
+ if cfg!(windows) {
+ // Bound GPU contexts and frame buffers to limit memory use on Windows.
+ SkiaPipelineConfig {
+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),
+ encoder_threads: 2,
+ buffer_queue_size: 2,
+ ..Default::default()
+ }
+ } else {
SkiaPipelineConfig {
concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,
..Default::default()
- },
- )
- .expect("skia renderer"),
- )
+ }
+ },
+ &gpu,
+ ))
.preview(fframes_native_player::cli_preview)
.default_output("out/tracking.mp4")
.run()
diff --git a/tools/render.ps1 b/tools/render.ps1
new file mode 100644
index 0000000..4d8228e
--- /dev/null
+++ b/tools/render.ps1
@@ -0,0 +1,38 @@
+# Windows pipeline. Run from any directory with: powershell -File tools/render.ps1
+param([string]$Python = 'python')
+
+$ErrorActionPreference = 'Stop'
+function Assert-Success([string]$Step) {
+ if ($LASTEXITCODE -ne 0) { throw "$Step failed (exit $LASTEXITCODE)." }
+}
+
+Push-Location (Split-Path $PSScriptRoot -Parent)
+try {
+ foreach ($command in @('cargo', 'ffmpeg', 'ffprobe', $Python)) {
+ if (-not (Get-Command $command -ErrorAction SilentlyContinue)) {
+ throw "Required command '$command' was not found. See the Windows setup in README.md."
+ }
+ }
+ & $Python -I -c 'import numpy, scipy'
+ Assert-Success 'Python dependencies'
+
+ cargo build --release -p hud -p flat -p lens
+ Assert-Success 'Build'
+ # Use Cargo metadata so custom target directories and .cargo/config.toml work too.
+ $metadata = cargo metadata --no-deps --format-version 1
+ Assert-Success 'Cargo metadata'
+ $bin = Join-Path (($metadata | ConvertFrom-Json).target_directory) 'release'
+ New-Item -ItemType Directory -Force out/pass | Out-Null
+ & (Join-Path $bin 'tracks.exe')
+ Assert-Success 'Track export'
+ & (Join-Path $bin 'flat.exe') render -o out/pass/flat.mp4
+ Assert-Success 'Flat render'
+ & $Python -I tools/sfx.py
+ Assert-Success 'Sound synthesis'
+ & (Join-Path $bin 'lens.exe') render -o out/tracking.mp4
+ Assert-Success 'Lens render'
+ ffprobe -v error -show_entries stream=codec_type,width,height,nb_frames,duration -of compact out/tracking.mp4
+ Assert-Success 'Output verification'
+} finally {
+ Pop-Location
+}