Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
//! Live background blur for the camera preview bubble -- the gpui counterpart
//! of the Tauri preview's blur path (`src-tauri/src/camera.rs:1328-1546`).
//!
//! The Tauri native preview renders every frame through its own wgpu device:
//! the camera IOSurface is imported as a texture, `cap_camera_effects::
//! BlurProcessor` segments + blurs it, and the output lands in the preview
//! surface. This app paints CVPixelBuffers directly (`paint_surface_fitted`),
//! so the same pipeline is bent into surface-in / surface-out form:
//!
//! 1. [`crate::camera_window`]'s `FrameConverter` hands over the converted
//! BGRA IOSurface pixel buffer (already downscaled to [`BLUR_MAX_DIMS`] and
//! mirrored when the toolbar says so).
//! 2. The buffer's IOSurface is imported as a `Bgra8Unorm` wgpu texture
//! through `cap-rendering`'s `iosurface_texture` seam -- the exact import
//! the Tauri `NativeFrameConverter` uses (`camera_native.rs:301-310`).
//! Imports are cached per ring slot (keyed by pixel-buffer pointer +
//! ring generation), so the steady state creates no textures.
//! 3. The same pixel buffer goes to Apple Vision person segmentation on its
//! own thread, and `BlurProcessor::process_into_encoder` composites with
//! the newest finished mask, refined against this frame by the guided
//! filter. Segmentation never blocks a preview frame.
//! 4. `cap_rendering::RgbaToBgraSurfaceConverter` blits the RGBA output into
//! a BGRA IOSurface-backed CVPixelBuffer from its own ring, and
//! `PendingSurface::wait` blocks until the GPU has finished writing it, so
//! the buffer handed back to the window is always complete when painted.
//!
//! Everything runs on one dedicated `camera-blur` thread (the Tauri preview
//! renders on a dedicated thread too, `camera.rs:466-490`), with the ~10ms
//! Neural Engine segmentation on a second one, so neither touches the UI
//! thread.
//! The channel back to the window is how "what you see is what records" holds:
//! this is the same `BlurProcessor` the editor/export camera layer runs
//! (`cap-rendering/src/lib.rs:5742`), fed by the same `BackgroundBlurMode`
//! the recording bridge writes into `project-config.json`.
//!
//! Failure degrades exactly like Tauri's `ensure_blur_processor`
//! (`camera.rs:1486-1519`): if the device, the ONNX runtime or the model
//! cannot be brought up, the worker exits, the window notices the closed
//! channel and keeps painting raw frames -- the toggle still cycles and
//! persists, nothing crashes.
#![cfg(target_os = "macos")]
use std::sync::OnceLock;
use std::time::Duration;
use anyhow::{Context as _, anyhow};
use cap_rendering::{
RgbaToBgraSurfaceConverter,
iosurface_texture::{IOSurfaceTextureCache, import_metal_texture_to_wgpu},
};
use cidre::{arc, cv};
/// With an effect on, the preview processes at most 720p: enough for a crisp
/// cutout edge in the largest bubble on a Retina display, while the guided
/// filter and composite stay around a millisecond of GPU time. The
/// `FrameConverter` applies this cap during its hardware conversion, so the
/// scale costs nothing extra.
pub const BLUR_MAX_DIMS: (usize, usize) = (1280, 720);
/// `CAMERA_PREVIEW_BLUR_INFERENCE_INTERVAL` (`camera.rs:50`).
const BLUR_INFERENCE_INTERVAL: Duration = Duration::from_millis(33);
/// `LOW_SPEC_PREVIEW_RAM_THRESHOLD_BYTES` (`camera.rs:66`): machines at or
/// under 8GB never spin up the ONNX/wgpu blur processor -- the heaviest cost
/// in the preview. The toggle still cycles and persists; it just takes no
/// visual effect, exactly like `ensure_blur_processor`'s low-spec early
/// return (`camera.rs:1491-1498`).
const LOW_SPEC_RAM_THRESHOLD_BYTES: u64 = 8 * 1024 * 1024 * 1024;
/// The Tauri check reads total RAM once at startup (`init_preview_profile`,
/// `camera.rs:75-87`, via sysinfo); `hw.memsize` is the same number without
/// the dependency.
pub fn is_low_spec_preview() -> bool {
static LOW_SPEC: OnceLock<bool> = OnceLock::new();
*LOW_SPEC.get_or_init(|| {
let mut bytes: u64 = 0;
let mut len = std::mem::size_of::<u64>();
let result = unsafe {
libc::sysctlbyname(
c"hw.memsize".as_ptr(),
&mut bytes as *mut u64 as *mut libc::c_void,
&mut len,
std::ptr::null_mut(),
0,
)
};
let low_spec = result == 0 && bytes > 0 && bytes <= LOW_SPEC_RAM_THRESHOLD_BYTES;
if low_spec {
tracing::info!(
total_ram_mb = bytes / 1_048_576,
"low-spec camera preview profile: background blur preview disabled"
);
}
low_spec
})
}
/// A CVPixelBuffer crossing the worker boundary. CF objects have atomic
/// refcounts; same contract as `FrameConverter` and cap-rendering's
/// `SurfaceFrame`.
pub struct SendPixelBuf(pub arc::R<cv::PixelBuf>);
unsafe impl Send for SendPixelBuf {}
pub struct BlurJob {
/// Converted BGRA IOSurface pixel buffer from the window's ring.
pub buffer: SendPixelBuf,
pub width: u32,
pub height: u32,
/// The `FrameConverter` ring generation the buffer belongs to; a rebuilt
/// ring invalidates the worker's imported-texture cache so a recycled
/// allocation address cannot alias a stale import.
pub ring_generation: u64,
pub mode: cap_camera_effects::BlurMode,
}
pub struct BlurOutput {
pub mode: cap_camera_effects::BlurMode,
pub cutout: Option<std::sync::Arc<gpui::RenderImage>>,
/// Blurred BGRA IOSurface pixel buffer, GPU work complete.
pub buffer: SendPixelBuf,
pub width: u32,
pub height: u32,
}
/// The worker loop. Runs until the job sender is dropped (blur switched off,
/// window closed) or an unrecoverable error; either way every GPU/ONNX
/// resource is dropped on exit, the same release-on-off behaviour as
/// `release_blur_resources` (`camera.rs:1477-1484`).
pub fn run(jobs: flume::Receiver<BlurJob>, results: flume::Sender<BlurOutput>) {
// Current-thread runtime for `request_adapter` and the timed blocking
// wait inside `PendingSurface::wait`.
let runtime = match tokio::runtime::Builder::new_current_thread()
.enable_time()
.build()
{
Ok(runtime) => runtime,
Err(error) => {
tracing::warn!("camera blur worker runtime failed: {error}");
return;
}
};
let Some((device, queue)) = shared_device(&runtime) else {
return;
};
let mut worker = match Worker::new(device, queue) {
Ok(worker) => worker,
Err(error) => {
tracing::warn!("camera blur preview unavailable: {error:#}");
return;
}
};
while let Ok(mut job) = jobs.recv() {
// Latest-wins, like the Tauri render loop draining `camera_rx`
// (`camera.rs:1018-1020`).
while let Ok(newer) = jobs.try_recv() {
job = newer;
}
match worker.process(&runtime, job) {
Ok(output) => {
if results.send(output).is_err() {
return;
}
}
Err(error) => {
tracing::warn!("camera blur preview stopped: {error:#}");
return;
}
}
}
}
/// One device + queue for the app's blur lifetime, latched like Tauri's
/// `blur_processor_init_attempted`: a failed bring-up is not retried on every
/// toggle. The Tauri preview keeps its device for the window's lifetime
/// (`camera.rs:573-583`, `LowPower`, downlevel limits); only the ONNX/blur
/// resources are per-enable.
fn shared_device(runtime: &tokio::runtime::Runtime) -> Option<(wgpu::Device, wgpu::Queue)> {
static DEVICE: OnceLock<Option<(wgpu::Device, wgpu::Queue)>> = OnceLock::new();
DEVICE
.get_or_init(|| {
runtime.block_on(async {
let instance = cap_rendering::create_wgpu_instance_sync();
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::LowPower,
force_fallback_adapter: cap_rendering::force_software_wgpu_adapter(),
compatible_surface: None,
})
.await
.map_err(|error| {
tracing::warn!("camera blur adapter unavailable: {error}");
})
.ok()?;
let (device, queue) = adapter
.request_device(&wgpu::DeviceDescriptor {
label: Some("Camera Preview Blur"),
required_features: wgpu::Features::empty(),
required_limits: wgpu::Limits::downlevel_webgl2_defaults()
.using_resolution(adapter.limits()),
memory_hints: Default::default(),
trace: wgpu::Trace::Off,
})
.await
.map_err(|error| {
tracing::warn!("camera blur device unavailable: {error}");
})
.ok()?;
cap_rendering::log_uncaptured_errors(&device, "camera-preview-blur");
Some((device, queue))
})
})
.clone()
}
struct Worker {
device: wgpu::Device,
queue: wgpu::Queue,
cache: IOSurfaceTextureCache,
processor: cap_camera_effects::BlurProcessor,
converter: RgbaToBgraSurfaceConverter,
/// Imported input textures keyed by pixel-buffer pointer. The converter
/// ring is four fixed buffers, so four entries cover the steady state and
/// nothing is created per frame (stricter than the Tauri path, which
/// re-imports each frame, `camera_native.rs:301-310`).
imported: Vec<(usize, wgpu::Texture)>,
imported_generation: u64,
frame_number: u32,
}
impl Worker {
fn new(device: wgpu::Device, queue: wgpu::Queue) -> anyhow::Result<Self> {
let mut processor =
cap_camera_effects::BlurProcessor::new(&device, wgpu::TextureFormat::Rgba8Unorm)
.context("blur processor init")?;
processor.set_inference_interval(BLUR_INFERENCE_INTERVAL);
let converter =
RgbaToBgraSurfaceConverter::new(&device).map_err(|error| anyhow!("{error}"))?;
let cache = IOSurfaceTextureCache::new().context("Metal device unavailable")?;
tracing::info!("camera blur preview initialized");
Ok(Self {
device,
queue,
cache,
processor,
converter,
imported: Vec::new(),
imported_generation: 0,
frame_number: 0,
})
}
fn process(
&mut self,
runtime: &tokio::runtime::Runtime,
job: BlurJob,
) -> anyhow::Result<BlurOutput> {
let mode = job.mode;
let (width, height) = (job.width, job.height);
let texture = self.input_texture(&job)?.clone();
// Vision takes the converted IOSurface directly (no GPU readback),
// and segmentation runs beside rendering instead of blocking it.
self.processor.set_segmentation_source(job.buffer.0.clone());
let mut encoder = self
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("Camera Preview Blur"),
});
self.processor.process_into_encoder(
&self.device,
&self.queue,
&texture,
&mut encoder,
mode,
);
let output = self
.processor
.process_returning_output()
.context("blur output missing")?;
self.frame_number = self.frame_number.wrapping_add(1);
let pending = runtime
.block_on(self.converter.encode(
&self.device,
&mut encoder,
output,
width,
height,
self.frame_number,
30,
))
.map_err(|error| anyhow!("{error}"))?;
let submission = self.queue.submit(std::iter::once(encoder.finish()));
let frame = runtime
.block_on(pending.wait(&self.device, submission))
.map_err(|error| anyhow!("{error}"))?;
let cutout = if mode == cap_camera_effects::BlurMode::Remove {
use core_foundation::base::TCFType as _;
use core_video::pixel_buffer::{CVPixelBuffer, CVPixelBufferRef};
let raw = &*frame.pixel_buffer as *const cv::PixelBuf as CVPixelBufferRef;
let buffer = unsafe { CVPixelBuffer::wrap_under_get_rule(raw) };
Some(
crate::camera_window::snapshot_cutout(&buffer)
.context("cutout preview pixels unavailable")?,
)
} else {
None
};
Ok(BlurOutput {
mode,
cutout,
buffer: SendPixelBuf(frame.pixel_buffer),
width: frame.width,
height: frame.height,
})
}
fn input_texture(&mut self, job: &BlurJob) -> anyhow::Result<&wgpu::Texture> {
if self.imported_generation != job.ring_generation {
self.imported.clear();
self.imported_generation = job.ring_generation;
}
let key = &*job.buffer.0 as *const cv::PixelBuf as usize;
if let Some(index) = self.imported.iter().position(|(k, _)| *k == key) {
return Ok(&self.imported[index].1);
}
let io_surface = job
.buffer
.0
.io_surf()
.context("pixel buffer has no IOSurface")?;
// A `Bgra8Unorm` binding samples as RGBA-ordered floats, so the blur
// pipelines (all `Float { filterable: true }` layouts) and the 256px
// segmentation downsample read correct channels with no extra
// conversion pass.
let metal = self
.cache
.create_bgra_texture(io_surface, job.width, job.height)
.map_err(|error| anyhow!("{error}"))?;
let texture = import_metal_texture_to_wgpu(
&self.device,
&metal,
wgpu::TextureFormat::Bgra8Unorm,
job.width,
job.height,
Some("Camera Blur Input"),
)
.map_err(|error| anyhow!("{error}"))?;
self.imported.push((key, texture));
Ok(&self.imported.last().expect("just pushed").1)
}
}
#[cfg(test)]
mod bench;
#[cfg(test)]
mod tests {
use super::*;
use cap_camera_effects::BlurMode;
use cidre::cf;
use core_foundation::base::TCFType as _;
use core_video::pixel_buffer::{CVPixelBuffer, CVPixelBufferRef};
use std::time::Instant;
#[test]
#[ignore = "requires CAP_CAMERA_EFFECT_TEST_FRAME containing a 1280x720 RGBA webcam frame"]
fn native_cutout_preserves_alpha_through_iosurface_and_preview_image() {
let pixels = std::fs::read(std::env::var("CAP_CAMERA_EFFECT_TEST_FRAME").unwrap()).unwrap();
let (width, height) = BLUR_MAX_DIMS;
assert_eq!(pixels.len(), width * height * 4);
let properties = cf::Dictionary::new();
let attributes = cf::Dictionary::with_keys_values(
&[
cv::pixel_buffer::keys::io_surf_props().as_ref(),
cv::pixel_buffer::keys::metal_compatibility().as_ref(),
],
&[properties.as_ref(), cf::Boolean::value_true().as_ref()],
)
.unwrap();
let buffer =
cv::PixelBuf::new(width, height, cv::PixelFormat::_32_BGRA, Some(&attributes)).unwrap();
let raw = &*buffer as *const cv::PixelBuf as CVPixelBufferRef;
let wrapped = unsafe { CVPixelBuffer::wrap_under_get_rule(raw) };
assert_eq!(wrapped.lock_base_address(0), 0);
let stride = wrapped.get_bytes_per_row();
let destination = unsafe {
std::slice::from_raw_parts_mut(wrapped.get_base_address().cast::<u8>(), stride * height)
};
for (source, target) in pixels
.chunks_exact(width * 4)
.zip(destination.chunks_exact_mut(stride))
{
for (source, target) in source.chunks_exact(4).zip(target.chunks_exact_mut(4)) {
target.copy_from_slice(&[source[2], source[1], source[0], source[3]]);
}
}
assert_eq!(wrapped.unlock_base_address(0), 0);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_time()
.build()
.unwrap();
let (device, queue) = shared_device(&runtime).unwrap();
let mut worker = Worker::new(device, queue).unwrap();
for mode in [
BlurMode::Remove,
BlurMode::Heavy,
BlurMode::Light,
BlurMode::Remove,
] {
let mut timings = Vec::new();
for _ in 0..30 {
let start = Instant::now();
let output = worker
.process(
&runtime,
BlurJob {
buffer: SendPixelBuf(buffer.clone()),
width: width as u32,
height: height as u32,
ring_generation: 1,
mode,
},
)
.unwrap();
timings.push(start.elapsed().as_secs_f64() * 1000.0);
assert_eq!(output.mode, mode);
if mode == BlurMode::Remove {
let image = output.cutout.unwrap();
let bytes = image.as_bytes(0).unwrap();
assert!(bytes.chunks_exact(4).any(|pixel| pixel[3] == 0));
assert!(bytes.chunks_exact(4).any(|pixel| pixel[3] == 255));
} else {
assert!(output.cutout.is_none());
}
std::thread::sleep(Duration::from_millis(33).saturating_sub(start.elapsed()));
}
timings.sort_by(f64::total_cmp);
eprintln!(
"native {mode:?} preview worker: p50={:.2}ms p95={:.2}ms",
timings[15], timings[28]
);
}
assert_eq!(worker.imported.len(), 1);
}
}