A virtual camera films an AI agent's screen while a tracker locks onto every glyph. Two fframes passes, one SkSL shader, all sound synthesized.
Add Windows Vulkan rendering support
Tanzim6414d47458f8
diff --git a/.gitignore b/.gitignoreindex 0383233..ff1267a 100644--- a/.gitignore+++ b/.gitignore@@ -7,3 +7,4 @@ strip.pngcheck.png__pycache__/.DS_Store+.venv/diff --git a/Cargo.lock b/Cargo.lockindex fde7998..829723c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -217,6 +217,15 @@ dependencies = ["stable_deref_trait",]+[[package]]+name = "ash"+version = "0.38.0+1.3.281"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f"+dependencies = [+ "libloading",+]+[[package]]name = "atomic-waker"version = "1.1.2"@@ -1041,6 +1050,7 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "28303fd0cfdbb9d937a07cd3091c859a5777cba4b6622ba37e8a4047cb01675e"dependencies = [+ "ash","core-foundation 0.10.1","fframes","foreign-types-shared",diff --git a/Cargo.toml b/Cargo.tomlindex 58b3e44..6541608 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -3,9 +3,9 @@ resolver = "3"members = ["hud", "flat", "lens"][workspace.dependencies]-fframes = { version = "=1.2.0", features = ["compile-time-svgtree", "cli", "h264", "libav-agree-gpl"] }-fframes_skia_renderer = { version = "=1.2.0", features = ["metal"] }-fframes_native_player = { version = "=1.2.0", default-features = false, features = ["metal", "audio"] }+fframes = { version = "=1.2.0", features = ["compile-time-svgtree", "cli"] }+fframes_skia_renderer = { version = "=1.2.0", default-features = true }+fframes_native_player = { version = "=1.2.0", default-features = false, features = ["audio"] }[profile.dev]opt-level = 1diff --git a/README.md b/README.mdindex 8a49d72..5af6067 100644--- a/README.md+++ b/README.md@@ -99,6 +99,9 @@ red lock frames tear the bass. See [Slab score](#slab-score).## Quick start+macOS uses Metal; Windows uses Vulkan. For Windows requirements and the PowerShell+render pipeline, see [Windows setup](docs/WINDOWS.md). The macOS instructions follow.+Requirements: macOS with Apple silicon (Skia on Metal), Rust, ffmpeg, Python 3 with `numpy`and `scipy`. The README images also need Pillow and `img2webp`.diff --git a/docs/WINDOWS.md b/docs/WINDOWS.mdnew file mode 100644index 0000000..1582655--- /dev/null+++ b/docs/WINDOWS.md@@ -0,0 +1,58 @@+# Windows setup++The Windows build uses Vulkan in place of Metal. Scenes, camera math, shader and+sound synthesis are unchanged.++## Requirements++- 64-bit Windows and a Vulkan-capable GPU with current drivers.+- [Rust](https://rustup.rs/) with the MSVC toolchain.+- Visual Studio Build Tools with **Desktop development with C++**, including a Windows SDK.+- Python 3 with `numpy` and `scipy`.+- LLVM/libclang, available from [LLVM releases](https://github.com/llvm/llvm-project/releases).+- A **shared FFmpeg 9.0 development build**, with `bin`, `include` and `lib` directories.+ For example, use `ffmpeg-n9.0-latest-win64-gpl-shared-9.0.zip` from+ [BtbN/FFmpeg-Builds](https://github.com/BtbN/FFmpeg-Builds/releases/tag/latest).++A standalone `ffmpeg.exe` is insufficient: Rust links against FFmpeg libraries,+and build-time media processing loads the DLLs. See also+[fframes' Windows requirements](https://github.com/dmtrKovalenko/fframes#requirements).++## Render++In PowerShell, adjust the installation paths to match your machine:++```powershell+$env:FFMPEG_DIR = "C:\tools\ffmpeg-n9.0-latest-win64-gpl-shared-9.0"+$env:LIBCLANG_PATH = "C:\Program Files\LLVM\bin"+$env:PATH = "$env:FFMPEG_DIR\bin;$env:PATH"++git clone https://github.com/mrsarac/ff-tracking+cd ff-tracking+python -m venv .venv+.\.venv\Scripts\python.exe -m pip install numpy scipy+powershell -NoProfile -ExecutionPolicy Bypass -File tools/render.ps1 -Python .\.venv\Scripts\python.exe+# -> out/tracking.mp4+```++The execution-policy option applies only to this PowerShell process. Keep FFmpeg's+`bin` directory on `PATH` when running the built programs. The script accepts the+Python executable via `-Python`, stops on failed steps, restores the caller's+directory, and honors Cargo's configured target directory.++After rendering, the usual per-pass tools work:++```powershell+.\target\release\lens.exe preview+.\target\release\lens.exe strip all -n 12+.\target\release\lens.exe audio analyze+```++If you configured a custom Cargo target directory, use its `release` folder instead.++Windows uses the codecs from the shared FFmpeg installation, so the Cargo features+that request building codecs from source are enabled only for the original+non-Windows configuration. Windows limits the rendering pipeline to two GPU+contexts, two encoder workers and a small queue: each worker decodes the 4K first+pass, so unrestricted concurrency can exhaust memory, especially on longer edits.+The original macOS backend and concurrency setting remain unchanged.diff --git a/flat/Cargo.toml b/flat/Cargo.tomlindex 85d653b..c239f00 100644--- a/flat/Cargo.toml+++ b/flat/Cargo.toml@@ -7,8 +7,15 @@ publish = false[dependencies]hud = { path = "../hud" }fframes = { workspace = true }-fframes_skia_renderer = { workspace = true }-fframes_native_player = { workspace = true }++[target.'cfg(windows)'.dependencies]+fframes_skia_renderer = { workspace = true, features = ["vulkan"] }+fframes_native_player = { workspace = true, features = ["vulkan"] }++[target.'cfg(not(windows))'.dependencies]+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }+fframes_skia_renderer = { workspace = true, features = ["metal"] }+fframes_native_player = { workspace = true, features = ["metal"] }[lib]name = "flat"diff --git a/flat/src/main.rs b/flat/src/main.rsindex 9644f90..24e1bc1 100644--- a/flat/src/main.rs+++ b/flat/src/main.rs@@ -1,6 +1,10 @@use fframes::{EncoderOptions, RenderOptions, StaticMediaProvider, cli};+#[cfg(not(windows))]+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;+#[cfg(windows)]+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;use fframes_skia_renderer::{- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig, metal::SkiaMetalCtx,+ SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,};use flat::{FlatMedia, FlatVideo, HEIGHT, WIDTH};use std::process::ExitCode;@@ -8,7 +12,7 @@ use std::process::ExitCode;fn main() -> ExitCode {let media = FlatMedia::prepare().expect("media");let video = FlatVideo::new();- let gpu = SkiaMetalCtx::new(WIDTH, HEIGHT).expect("GPU context");+ let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");cli::new(&video,@@ -24,16 +28,23 @@ fn main() -> ExitCode {..Default::default()},)- .backend(- SkiaFFramesRenderer::new_metal(- &gpu,+ .backend(SkiaFFramesRenderer::new(+ if cfg!(windows) {+ // Bound GPU contexts and frame buffers to limit memory use on Windows.+ SkiaPipelineConfig {+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),+ encoder_threads: 2,+ buffer_queue_size: 2,+ ..Default::default()+ }+ } else {SkiaPipelineConfig {concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,..Default::default()- },- )- .expect("skia renderer"),- )+ }+ },+ &gpu,+ )).preview(fframes_native_player::cli_preview).default_output("out/pass/flat.mp4").run()diff --git a/lens/Cargo.toml b/lens/Cargo.tomlindex 0fc116f..fc3cf36 100644--- a/lens/Cargo.toml+++ b/lens/Cargo.toml@@ -7,8 +7,15 @@ publish = false[dependencies]hud = { path = "../hud" }fframes = { workspace = true }-fframes_skia_renderer = { workspace = true }-fframes_native_player = { workspace = true }++[target.'cfg(windows)'.dependencies]+fframes_skia_renderer = { workspace = true, features = ["vulkan"] }+fframes_native_player = { workspace = true, features = ["vulkan"] }++[target.'cfg(not(windows))'.dependencies]+fframes = { workspace = true, features = ["h264", "libav-agree-gpl"] }+fframes_skia_renderer = { workspace = true, features = ["metal"] }+fframes_native_player = { workspace = true, features = ["metal"] }[lib]name = "lens"diff --git a/lens/src/main.rs b/lens/src/main.rsindex 2ce9442..c5f6270 100644--- a/lens/src/main.rs+++ b/lens/src/main.rs@@ -1,8 +1,12 @@use fframes::{CombinedMediaProvider, EncoderOptions, MediaDirectory, MediaProvider, RenderOptions, cli,};+#[cfg(not(windows))]+use fframes_skia_renderer::metal::SkiaMetalCtx as SkiaGpuCtx;+#[cfg(windows)]+use fframes_skia_renderer::vulkan::SkiaVulkanCtx as SkiaGpuCtx;use fframes_skia_renderer::{- SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig, metal::SkiaMetalCtx,+ SkiaFFramesRenderer, SkiaPipelineConcurrencyPolicy, SkiaPipelineConfig,};use lens::{HEIGHT, LensVideo, WIDTH};use std::process::ExitCode;@@ -17,7 +21,7 @@ fn main() -> ExitCode {let fonts = fonts_dir.process_media_source().expect("fonts");let media = CombinedMediaProvider::from([&pass as &dyn MediaProvider, &fonts]);let video = LensVideo::new();- let gpu = SkiaMetalCtx::new(WIDTH, HEIGHT).expect("GPU context");+ let gpu = SkiaGpuCtx::new(WIDTH, HEIGHT).expect("GPU context");cli::new(&video,@@ -32,16 +36,23 @@ fn main() -> ExitCode {..Default::default()},)- .backend(- SkiaFFramesRenderer::new_metal(- &gpu,+ .backend(SkiaFFramesRenderer::new(+ if cfg!(windows) {+ // Bound GPU contexts and frame buffers to limit memory use on Windows.+ SkiaPipelineConfig {+ concurrency_policy: SkiaPipelineConcurrencyPolicy::Concurrency(2),+ encoder_threads: 2,+ buffer_queue_size: 2,+ ..Default::default()+ }+ } else {SkiaPipelineConfig {concurrency_policy: SkiaPipelineConcurrencyPolicy::MaxPerformance,..Default::default()- },- )- .expect("skia renderer"),- )+ }+ },+ &gpu,+ )).preview(fframes_native_player::cli_preview).default_output("out/tracking.mp4").run()diff --git a/tools/render.ps1 b/tools/render.ps1new file mode 100644index 0000000..4d8228e--- /dev/null+++ b/tools/render.ps1@@ -0,0 +1,38 @@+# Windows pipeline. Run from any directory with: powershell -File tools/render.ps1+param([string]$Python = 'python')++$ErrorActionPreference = 'Stop'+function Assert-Success([string]$Step) {+ if ($LASTEXITCODE -ne 0) { throw "$Step failed (exit $LASTEXITCODE)." }+}++Push-Location (Split-Path $PSScriptRoot -Parent)+try {+ foreach ($command in @('cargo', 'ffmpeg', 'ffprobe', $Python)) {+ if (-not (Get-Command $command -ErrorAction SilentlyContinue)) {+ throw "Required command '$command' was not found. See the Windows setup in README.md."+ }+ }+ & $Python -I -c 'import numpy, scipy'+ Assert-Success 'Python dependencies'++ cargo build --release -p hud -p flat -p lens+ Assert-Success 'Build'+ # Use Cargo metadata so custom target directories and .cargo/config.toml work too.+ $metadata = cargo metadata --no-deps --format-version 1+ Assert-Success 'Cargo metadata'+ $bin = Join-Path (($metadata | ConvertFrom-Json).target_directory) 'release'+ New-Item -ItemType Directory -Force out/pass | Out-Null+ & (Join-Path $bin 'tracks.exe')+ Assert-Success 'Track export'+ & (Join-Path $bin 'flat.exe') render -o out/pass/flat.mp4+ Assert-Success 'Flat render'+ & $Python -I tools/sfx.py+ Assert-Success 'Sound synthesis'+ & (Join-Path $bin 'lens.exe') render -o out/tracking.mp4+ Assert-Success 'Lens render'+ ffprobe -v error -show_entries stream=codec_type,width,height,nb_frames,duration -of compact out/tracking.mp4+ Assert-Success 'Output verification'+} finally {+ Pop-Location+}