3D engine that AI agents drive
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
use std::time::Instant;
use napi::Result;
use wgpu::util::DeviceExt;
use crate::cycles::{unit_filter_table_words, CYCLES_LUT_WORDS};
use crate::host::accel::{
build_acceleration_structure, build_empty_cycles_acceleration, create_vertex_buffer, unpack_triangles,
INSTANCE_RECORD_BYTES, TRIANGLE_FLOATS, TRIANGLE_RECORD_BYTES,
};
use crate::host::dispatch::{DispatchPlanner, DISPATCH_BUDGET_SECONDS};
use crate::host::device::{failure, layout_entry, open_device, storage_buffer, storage_entry};
use crate::host::film::{image_accumulation_bytes, read_accumulation};
use crate::host::textures::upload_scene_buffers;
use crate::{NativeTraceInput, NativeTraceInputV2, ProbeCyclesLobeOptions};
// ---- In-process device and pipeline cache ----
//
// Opening a Vulkan device and compiling SPIR-V pipelines dominates per-call latency.
// We cache both across back-to-back calls in the same Node process.
//
// wgpu 23+ backs Device and Queue with Arc, making them Clone.
/// Most compiled pipelines kept at once. Each scene shape compiles its own specialized kernel, so a long
/// lived process that renders many shapes would otherwise grow without bound.
const MAX_CACHED_PIPELINES: usize = 32;
/// A pipeline cache key: the entry point plus a 64 bit hash of the specialized source, so the cache never
/// holds a second copy of a kernel text that is hundreds of kilobytes long.
pub(crate) type PipelineKey = (String, u64);
pub(crate) fn pipeline_key(entry_point: &str, specialized_source: &str) -> PipelineKey {
let mut hasher = DefaultHasher::new();
specialized_source.hash(&mut hasher);
(entry_point.to_string(), hasher.finish())
}
/// A map that holds at most `capacity` entries and starts over when a new key arrives at capacity.
struct BoundedCache<Value> {
capacity: usize,
entries: HashMap<PipelineKey, Value>,
}
impl<Value: Clone> BoundedCache<Value> {
fn new(capacity: usize) -> Self {
BoundedCache { capacity, entries: HashMap::new() }
}
fn get(&self, key: &PipelineKey) -> Option<Value> {
self.entries.get(key).cloned()
}
fn insert(&mut self, key: PipelineKey, value: Value) {
if self.entries.len() >= self.capacity && !self.entries.contains_key(&key) {
self.entries.clear();
}
self.entries.insert(key, value);
}
}
struct CachedDevice {
device: wgpu::Device,
queue: wgpu::Queue,
adapter_name: String,
backend_name: String,
features: String,
/// Set by the device lost callback; the next call then opens a fresh device and drops everything below.
device_lost: Arc<AtomicBool>,
/// What wgpu reported when it lost the device or rejected work, for the error a failed render raises.
device_error: Arc<Mutex<String>>,
/// Static 118,848-byte LUT storage buffer for Cycles binding 12.
lut_buffer: wgpu::Buffer,
/// Bind group layout for the Cycles kernel (bindings 0..=12).
cycles_bind_group_layout: wgpu::BindGroupLayout,
/// Compiled Cycles compute pipelines keyed by entry point and specialized source hash.
cycles_pipelines: BoundedCache<wgpu::ComputePipeline>,
}
static DEVICE_CACHE: OnceLock<Mutex<Option<CachedDevice>>> = OnceLock::new();
fn device_cache() -> &'static Mutex<Option<CachedDevice>> {
DEVICE_CACHE.get_or_init(|| Mutex::new(None))
}
/// Locks a cache. A panic while the lock was held leaves half updated state, so a poisoned lock is
/// recovered by resetting the cache to empty instead of failing every later call.
fn lock_resetting_when_poisoned<Value>(cache: &Mutex<Option<Value>>) -> MutexGuard<'_, Option<Value>> {
match cache.lock() {
Ok(guard) => guard,
Err(poisoned) => {
let mut guard = poisoned.into_inner();
*guard = None;
cache.clear_poison();
guard
}
}
}
fn lock_device_cache() -> MutexGuard<'static, Option<CachedDevice>> {
lock_resetting_when_poisoned(device_cache())
}
/// Returns cloned device/queue/metadata from the cache, opening once per process.
/// When the device was lost the cache is dropped and a fresh device is opened.
pub(crate) fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, String)> {
let mut guard = lock_device_cache();
let was_lost = guard.as_ref().is_some_and(|cached| cached.device_lost.load(Ordering::SeqCst));
if was_lost {
*guard = None;
}
if guard.is_none() {
let (device, queue, adapter_info) = open_device()?;
let device_lost = Arc::new(AtomicBool::new(false));
let lost_flag = Arc::clone(&device_lost);
let device_error = Arc::new(Mutex::new(String::new()));
let lost_message = Arc::clone(&device_error);
device.set_device_lost_callback(move |reason, message| {
if let Ok(mut stored) = lost_message.lock() {
*stored = format!("{reason:?}: {message}");
}
lost_flag.store(true, Ordering::SeqCst);
});
// The default handler panics on a lost device; record the report so the render can raise a readable error.
let error_flag = Arc::clone(&device_lost);
let error_message = Arc::clone(&device_error);
device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| {
if let Ok(mut stored) = error_message.lock() {
stored.push_str(&error.to_string());
}
error_flag.store(true, Ordering::SeqCst);
}));
let mut lut_words: Vec<u32> = CYCLES_LUT_WORDS.to_vec();
lut_words.extend(unit_filter_table_words());
let lut_bytes: &[u8] = bytemuck::cast_slice(&lut_words);
let lut_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("cycles LUT binding 12"),
contents: lut_bytes,
usage: wgpu::BufferUsages::STORAGE,
});
let cycles_bind_group_layout = create_cycles_bind_group_layout(&device);
let has_ray_query = device.features().contains(wgpu::Features::EXPERIMENTAL_RAY_QUERY);
*guard = Some(CachedDevice {
device,
queue,
adapter_name: adapter_info.name,
backend_name: format!("{:?}", adapter_info.backend),
features: if has_ray_query { "EXPERIMENTAL_RAY_QUERY".to_string() } else { String::new() },
device_lost,
device_error,
lut_buffer,
cycles_bind_group_layout,
cycles_pipelines: BoundedCache::new(MAX_CACHED_PIPELINES),
});
}
let cached = guard.as_ref().unwrap();
Ok((
cached.device.clone(),
cached.queue.clone(),
cached.adapter_name.clone(),
cached.backend_name.clone(),
cached.features.clone(),
))
}
/// The flag and message the device callbacks fill in when the cached device is lost.
fn cached_device_loss_state() -> Result<(Arc<AtomicBool>, Arc<Mutex<String>>)> {
let guard = lock_device_cache();
match guard.as_ref() {
Some(cached) => Ok((Arc::clone(&cached.device_lost), Arc::clone(&cached.device_error))),
None => Err(cache_was_reset_failure()),
}
}
fn cache_was_reset_failure() -> napi::Error {
failure("The GPU device was reset while the scene was being prepared. Run the render again.")
}
/// Looks up or compiles and caches a Cycles `ComputePipeline` for `(entry_point, specialized_source)`.
fn get_or_compile_cycles_pipeline(entry_point: &str, specialized_source: String) -> Result<wgpu::ComputePipeline> {
get_or_compile_cycles_pipeline_on(None, entry_point, specialized_source)
}
/// As `get_or_compile_cycles_pipeline`, for a caller that holds buffers on one device. `expected_device` is that device's
/// lost flag: when the cache has since moved to another device the call fails instead of returning a pipeline that
/// does not belong to the caller's device.
pub(crate) fn get_or_compile_cycles_pipeline_on(
expected_device: Option<&Arc<AtomicBool>>,
entry_point: &str,
specialized_source: String,
) -> Result<wgpu::ComputePipeline> {
let key = pipeline_key(entry_point, &specialized_source);
{
let guard = lock_device_cache();
let cached = guard.as_ref().ok_or_else(cache_was_reset_failure)?;
check_same_device(cached, expected_device)?;
if let Some(existing) = cached.cycles_pipelines.get(&key) {
return Ok(existing);
}
}
let compiled = with_cycles_cache_on(expected_device, |device, _queue, layout, _lut| {
compile_compute_pipeline(device, specialized_source, entry_point, layout)
})??;
let mut guard = lock_device_cache();
if let Some(ref mut cached) = *guard {
cached.cycles_pipelines.insert(key, compiled.clone());
}
Ok(compiled)
}
fn check_same_device(cached: &CachedDevice, expected_device: Option<&Arc<AtomicBool>>) -> Result<()> {
match expected_device {
Some(expected) if !Arc::ptr_eq(&cached.device_lost, expected) => Err(failure(
"The GPU device this session was created on has been replaced. Close the session and create a new one.",
)),
_ => Ok(()),
}
}
/// Returns the cached Cycles bind group layout and LUT buffer, with cloned device/queue.
/// Fails when the cache was reset since `get_cached_device` succeeded (device loss or a poisoned lock).
fn with_cycles_cache<F, T>(f: F) -> Result<T>
where
F: FnOnce(&wgpu::Device, &wgpu::Queue, &wgpu::BindGroupLayout, &wgpu::Buffer) -> T,
{
with_cycles_cache_on(None, f)
}
/// As `with_cycles_cache`, failing when the cache no longer holds the device whose lost flag is `expected_device`.
pub(crate) fn with_cycles_cache_on<F, T>(expected_device: Option<&Arc<AtomicBool>>, f: F) -> Result<T>
where
F: FnOnce(&wgpu::Device, &wgpu::Queue, &wgpu::BindGroupLayout, &wgpu::Buffer) -> T,
{
let guard = lock_device_cache();
let cached = guard.as_ref().ok_or_else(cache_was_reset_failure)?;
check_same_device(cached, expected_device)?;
Ok(f(&cached.device, &cached.queue, &cached.cycles_bind_group_layout, &cached.lut_buffer))
}
/// The cached device with everything a Cycles session needs to know about it. The loss flag identifies the device: two
/// contexts with the same flag (`Arc::ptr_eq`) share one device.
pub(crate) struct CyclesDeviceContext {
pub(crate) device: wgpu::Device,
pub(crate) queue: wgpu::Queue,
pub(crate) adapter_name: String,
pub(crate) backend_name: String,
pub(crate) features: String,
pub(crate) device_lost: Arc<AtomicBool>,
pub(crate) device_error: Arc<Mutex<String>>,
}
pub(crate) fn get_cycles_device_context() -> Result<CyclesDeviceContext> {
let (device, queue, adapter_name, backend_name, features) = get_cached_device()?;
let (device_lost, device_error) = cached_device_loss_state()?;
Ok(CyclesDeviceContext { device, queue, adapter_name, backend_name, features, device_lost, device_error })
}
const PARAMETER_BYTES: usize = 16 * 16;
const VOLUME_DENSITY_BYTE_OFFSET: usize = 51 * 4;
const SUBSURFACE_OFFSET_BYTE_OFFSET: usize = 60 * 4;
const SAMPLE_INDEX_BYTE_OFFSET: u64 = 43 * 4;
const SAMPLES_PER_WAIT: u32 = 4;
const KERNEL_PARAMS_V2_BYTES: usize = 320;
const FORMAT_V2_MAGIC: u32 = 0x36375054;
const FORMAT_V2_VERSION: u32 = 2;
const MATERIAL_RECORD_BYTES: usize = 256;
const EMISSIVE_TRIANGLE_RECORD_BYTES: usize = 96;
const LIGHT_RECORD_BYTES: usize = 128;
const LIGHT_TREE_NODE_BYTES: usize = 64;
const LIGHT_TREE_EMITTER_BYTES: usize = 32;
/// Buffers bound at 2..=10; the light tree emitter and light-to-tree buffers follow them for bindings 13 and 14.
const CYCLES_SCENE_BUFFER_COUNT: usize = 9;
const TEXTURE_DESCRIPTOR_BYTES: usize = 32;
const CYCLES_SAMPLE_OFFSET_BYTE_OFFSET: u64 = 16;
const LOBE_PROBE_WORKGROUP_SIZE: u32 = 64;
const MAX_COMPUTE_WORKGROUPS_PER_DIMENSION: u64 = 65_535;
/// A device with the scene's acceleration structure built and one compute pipeline bound to it.
pub(crate) struct PreparedScene {
pub(crate) device: wgpu::Device,
pub(crate) queue: wgpu::Queue,
pub(crate) pipeline: wgpu::ComputePipeline,
pub(crate) bind_group: wgpu::BindGroup,
pub(crate) accumulation: wgpu::Buffer,
pub(crate) parameters: wgpu::Buffer,
pub(crate) adapter_name: String,
pub(crate) backend_name: String,
pub(crate) features: String,
pub(crate) setup_seconds: f64,
pub(crate) compile_seconds: f64,
pub(crate) upload_seconds: f64,
pub(crate) acceleration_build_seconds: f64,
// The bind group keeps the buffers and the structure alive; holding them here makes that visible.
pub(crate) _buffers: Vec<wgpu::Buffer>,
pub(crate) _tlas: wgpu::Tlas,
}
/// Everything one run of Cycles dispatches touches, borrowed from the session that owns it.
pub(crate) struct CyclesDispatchTarget<'a> {
pub(crate) device: &'a wgpu::Device,
pub(crate) queue: &'a wgpu::Queue,
pub(crate) pipeline: &'a wgpu::ComputePipeline,
pub(crate) bind_group: &'a wgpu::BindGroup,
pub(crate) parameters: &'a wgpu::Buffer,
pub(crate) accumulation: &'a wgpu::Buffer,
pub(crate) device_lost: &'a AtomicBool,
pub(crate) device_error: &'a Mutex<String>,
}
/// What one kind of job adds to the shared scene setup: the kernel text, its entry point, the accumulation size and the
/// extra read only buffers bound after the accumulation (bindings 9 and up).
pub(crate) struct JobShape<'a> {
pub(crate) source: String,
pub(crate) entry_point: &'a str,
pub(crate) rewrite_specialization_flags: bool,
pub(crate) accumulation_bytes: u64,
pub(crate) extra_buffers: Vec<(&'a str, Vec<u8>)>,
}
fn specialize_legacy_kernel(source: &str, parameter_bytes: &[u8]) -> String {
// Fog and subsurface scattering compile in only for a scene that uses them: the legacy kernel declares both as false constants.
let fog_density_bytes: [u8; 4] = parameter_bytes[VOLUME_DENSITY_BYTE_OFFSET..VOLUME_DENSITY_BYTE_OFFSET + 4]
.try_into()
.unwrap();
let subsurface_offset_bytes: [u8; 4] = parameter_bytes[SUBSURFACE_OFFSET_BYTE_OFFSET..SUBSURFACE_OFFSET_BYTE_OFFSET + 4]
.try_into()
.unwrap();
let fog_density = f32::from_le_bytes(fog_density_bytes);
let subsurface_offset = u32::from_le_bytes(subsurface_offset_bytes);
source
.replace("const HAS_VOLUME: bool = false;", &format!("const HAS_VOLUME: bool = {};", fog_density > 0.0))
.replace("const HAS_SUBSURFACE: bool = false;", &format!("const HAS_SUBSURFACE: bool = {};", subsurface_offset > 0))
}
fn create_bind_group_layout(device: &wgpu::Device, extra_buffer_count: usize) -> wgpu::BindGroupLayout {
let mut layout_entries = vec![
layout_entry(
0,
wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
),
layout_entry(1, wgpu::BindingType::AccelerationStructure { vertex_return: false }),
];
layout_entries.extend((2..=7).map(|binding| storage_entry(binding, true)));
layout_entries.push(storage_entry(8, false));
layout_entries.extend((0..extra_buffer_count + 1).map(|offset| storage_entry(9 + offset as u32, true)));
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("path tracer"),
entries: &layout_entries,
})
}
fn create_cycles_bind_group_layout(device: &wgpu::Device) -> wgpu::BindGroupLayout {
let mut layout_entries = vec![
layout_entry(
0,
wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
),
layout_entry(1, wgpu::BindingType::AccelerationStructure { vertex_return: false }),
];
layout_entries.extend((2..=10).map(|binding| storage_entry(binding, true)));
layout_entries.push(storage_entry(11, false)); // film_accumulation (read_write)
layout_entries.push(storage_entry(12, true)); // cycles_lut (read-only: static LUTs, then the two pixel filter tables)
layout_entries.push(storage_entry(13, true)); // light_tree_emitters
layout_entries.push(storage_entry(14, true)); // light_to_tree
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("cycles path tracer"),
entries: &layout_entries,
})
}
pub(crate) fn compile_compute_pipeline(
device: &wgpu::Device,
shader_source: String,
entry_point: &str,
bind_group_layout: &wgpu::BindGroupLayout,
) -> Result<wgpu::ComputePipeline> {
let error_scope = device.push_error_scope(wgpu::ErrorFilter::Validation);
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("path tracer"),
source: wgpu::ShaderSource::Wgsl(shader_source.into()),
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("path tracer"),
bind_group_layouts: &[Some(bind_group_layout)],
immediate_size: 0,
});
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("path tracer"),
layout: Some(&pipeline_layout),
module: &module,
entry_point: Some(entry_point),
compilation_options: Default::default(),
cache: None,
});
if let Some(validation_error) = pollster::block_on(error_scope.pop()) {
return Err(failure(format!("Could not compile the native kernel: {validation_error}")));
}
Ok(pipeline)
}
fn create_scene_bind_group(
device: &wgpu::Device,
bind_group_layout: &wgpu::BindGroupLayout,
parameters: &wgpu::Buffer,
tlas: &wgpu::Tlas,
buffers: &[wgpu::Buffer],
accumulation: &wgpu::Buffer,
) -> wgpu::BindGroup {
let mut entries = vec![
wgpu::BindGroupEntry { binding: 0, resource: parameters.as_entire_binding() },
wgpu::BindGroupEntry { binding: 1, resource: tlas.as_binding() },
];
for (offset, buffer) in buffers[..6].iter().enumerate() {
entries.push(wgpu::BindGroupEntry {
binding: 2 + offset as u32,
resource: buffer.as_entire_binding(),
});
}
entries.push(wgpu::BindGroupEntry {
binding: 8,
resource: accumulation.as_entire_binding(),
});
for (offset, buffer) in buffers[6..].iter().enumerate() {
entries.push(wgpu::BindGroupEntry {
binding: 9 + offset as u32,
resource: buffer.as_entire_binding(),
});
}
device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("path tracer"),
layout: bind_group_layout,
entries: &entries,
})
}
pub(crate) fn create_cycles_bind_group(
device: &wgpu::Device,
bind_group_layout: &wgpu::BindGroupLayout,
parameters: &wgpu::Buffer,
tlas: &wgpu::Tlas,
buffers: &[wgpu::Buffer],
accumulation: &wgpu::Buffer,
lut_buffer: &wgpu::Buffer,
) -> wgpu::BindGroup {
let mut entries = vec![
wgpu::BindGroupEntry { binding: 0, resource: parameters.as_entire_binding() },
wgpu::BindGroupEntry { binding: 1, resource: tlas.as_binding() },
];
for (offset, buffer) in buffers[..CYCLES_SCENE_BUFFER_COUNT].iter().enumerate() {
entries.push(wgpu::BindGroupEntry {
binding: 2 + offset as u32,
resource: buffer.as_entire_binding(),
});
}
entries.push(wgpu::BindGroupEntry {
binding: 11,
resource: accumulation.as_entire_binding(),
});
entries.push(wgpu::BindGroupEntry {
binding: 12,
resource: lut_buffer.as_entire_binding(),
});
for (offset, buffer) in buffers[CYCLES_SCENE_BUFFER_COUNT..].iter().enumerate() {
entries.push(wgpu::BindGroupEntry {
binding: 13 + offset as u32,
resource: buffer.as_entire_binding(),
});
}
device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("cycles path tracer"),
layout: bind_group_layout,
entries: &entries,
})
}
pub(crate) fn prepare_scene(input: &NativeTraceInput, shape: JobShape) -> Result<PreparedScene> {
let setup_started = Instant::now();
let triangle_count = input.triangle_count as usize;
if triangle_count == 0 || input.triangles.len() < triangle_count * TRIANGLE_FLOATS {
return Err(failure("The triangle table must hold triangle_count triangles of 9 texels."));
}
let triangle_data: &[f32] = &input.triangles[..triangle_count * TRIANGLE_FLOATS];
if input.parameters.len() != PARAMETER_BYTES {
return Err(failure("The parameter block must be 256 bytes."));
}
let (device, queue, adapter_info) = open_device()?;
let setup_seconds = setup_started.elapsed().as_secs_f64();
let compile_started = Instant::now();
let shader_source = if shape.rewrite_specialization_flags {
specialize_legacy_kernel(&shape.source, &input.parameters)
} else {
shape.source
};
let bind_group_layout = create_bind_group_layout(&device, shape.extra_buffers.len());
let pipeline = compile_compute_pipeline(&device, shader_source, shape.entry_point, &bind_group_layout)?;
let compile_seconds = compile_started.elapsed().as_secs_f64();
let upload_started = Instant::now();
let unpacked = unpack_triangles(triangle_data, &input.materials, triangle_count);
let vertex_buffer = create_vertex_buffer(&device, &unpacked.vertices);
let uploaded = upload_scene_buffers(
&device,
input,
triangle_data,
&unpacked.slot_map,
unpacked.opaque_count,
shape.accumulation_bytes,
&shape.extra_buffers,
);
let upload_seconds = upload_started.elapsed().as_secs_f64();
let build_started = Instant::now();
let tlas = build_acceleration_structure(&device, &queue, &vertex_buffer, triangle_count, unpacked.opaque_count);
let acceleration_build_seconds = build_started.elapsed().as_secs_f64();
let bind_group = create_scene_bind_group(
&device,
&bind_group_layout,
&uploaded.parameters,
&tlas,
&uploaded.buffers,
&uploaded.accumulation,
);
let has_ray_query = device.features().contains(wgpu::Features::EXPERIMENTAL_RAY_QUERY);
let features = if has_ray_query { "EXPERIMENTAL_RAY_QUERY" } else { "" };
Ok(PreparedScene {
device,
queue,
pipeline,
bind_group,
accumulation: uploaded.accumulation,
parameters: uploaded.parameters,
adapter_name: adapter_info.name,
backend_name: format!("{:?}", adapter_info.backend),
features: features.to_string(),
setup_seconds,
compile_seconds,
upload_seconds,
acceleration_build_seconds,
_buffers: uploaded.buffers,
_tlas: tlas,
})
}
/// Runs every sample over `groups` workgroups and copies the accumulation back. `max_seconds` bounds the run: past it the
/// trace stops with an error instead of finishing, checked each time the CPU waits for the card.
pub(crate) fn trace_samples(
prepared: &PreparedScene,
samples: u32,
groups: (u32, u32),
accumulation_bytes: u64,
max_seconds: Option<f64>,
) -> Result<(Vec<f32>, f64)> {
let trace_started = Instant::now();
let device = &prepared.device;
let queue = &prepared.queue;
for sample_index in 0..samples {
queue.write_buffer(&prepared.parameters, SAMPLE_INDEX_BYTE_OFFSET, &sample_index.to_le_bytes());
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("sample"),
});
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: None,
timestamp_writes: None,
});
pass.set_pipeline(&prepared.pipeline);
pass.set_bind_group(0, &prepared.bind_group, &[]);
pass.dispatch_workgroups(groups.0, groups.1, 1);
}
queue.submit(Some(encoder.finish()));
if (sample_index + 1) % SAMPLES_PER_WAIT == 0 {
let _ = device.poll(wgpu::PollType::wait_indefinitely());
if let Some(limit) = max_seconds {
if trace_started.elapsed().as_secs_f64() > limit {
return Err(failure(format!(
"The native trace passed its time limit of {limit} s after {} of {samples} samples.",
sample_index + 1
)));
}
}
}
}
let sums = read_accumulation(device, queue, &prepared.accumulation, accumulation_bytes)?;
Ok((sums, trace_started.elapsed().as_secs_f64()))
}
pub(crate) fn validate_kernel_params_v2(parameter_bytes: &[u8]) -> Result<()> {
if parameter_bytes.len() != KERNEL_PARAMS_V2_BYTES {
return Err(failure(format!(
"KernelParamsV2 must be {KERNEL_PARAMS_V2_BYTES} bytes, got {}.",
parameter_bytes.len()
)));
}
let magic = u32::from_le_bytes(parameter_bytes[0..4].try_into().unwrap());
if magic != FORMAT_V2_MAGIC {
return Err(failure(format!(
"Invalid KernelParamsV2 magic 0x{magic:08x}: expected 0x{FORMAT_V2_MAGIC:08x}."
)));
}
let version = u32::from_le_bytes(parameter_bytes[4..8].try_into().unwrap());
if version != FORMAT_V2_VERSION {
return Err(failure(format!(
"Unsupported KernelParamsV2 format_version {version}: expected {FORMAT_V2_VERSION}."
)));
}
Ok(())
}
pub(crate) fn validate_record_buffer(label: &str, bytes: &[u8], count: u32, stride: usize) -> Result<()> {
let minimum_bytes = (count.max(1) as usize) * stride;
if bytes.len() < minimum_bytes || bytes.len() % stride != 0 {
return Err(failure(format!(
"FORMAT-v2 {label} buffer must be a multiple of {stride} bytes and at least {minimum_bytes} bytes for count {count}, got {} bytes.",
bytes.len()
)));
}
Ok(())
}
/// Byte offset of `kernel_flags` in `KernelParamsV2` and its light-tree bit (`KERNEL_FLAG_USE_LIGHT_TREE` in pack-input-v2.mjs).
const KERNEL_PARAMS_KERNEL_FLAGS_BYTE: usize = 32;
const KERNEL_FLAG_USE_LIGHT_TREE: u32 = 1;
/// Refuses kernel parameters that switch the light tree on for an input that carries tree nodes but no emitter tables.
pub(crate) fn validate_light_tree_flag_has_tables(
kernel_params: &[u8],
light_tree_node_count: u32,
has_emitter_tables: bool,
) -> Result<()> {
let tree_flag_set = kernel_params
.get(KERNEL_PARAMS_KERNEL_FLAGS_BYTE..KERNEL_PARAMS_KERNEL_FLAGS_BYTE + 4)
.map(|bytes| u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) & KERNEL_FLAG_USE_LIGHT_TREE != 0)
.unwrap_or(false);
if tree_flag_set && light_tree_node_count > 0 && !has_emitter_tables {
return Err(failure(
"The light-tree kernel flag is set and the input has light tree nodes, but lightTreeEmitters, lightTreeEmitterCount and lightToTree are missing. Repack the scene with buildNativeInputV2.",
));
}
Ok(())
}
/// The emitter table and the emitter-to-slot table come together or not at all, and the slot table must cover
/// every analytic light and emissive triangle of the input.
pub(crate) fn validate_light_tree_tables(input: &NativeTraceInputV2) -> Result<()> {
let (emitters, emitter_count, light_to_tree) = match (
&input.light_tree_emitters,
input.light_tree_emitter_count,
&input.light_to_tree,
) {
(None, None, None) => {
return validate_light_tree_flag_has_tables(&input.kernel_params, input.light_tree_node_count, false);
}
(Some(emitters), Some(emitter_count), Some(light_to_tree)) => (emitters, emitter_count, light_to_tree),
_ => {
return Err(failure(
"lightTreeEmitters, lightTreeEmitterCount and lightToTree must all be present or all absent. Pack them together with buildNativeInputV2.",
))
}
};
validate_record_buffer("lightTreeEmitters", emitters, emitter_count, LIGHT_TREE_EMITTER_BYTES)?;
let unified_count = input.light_count as u64 + input.emissive_triangle_count as u64;
if emitter_count as u64 != unified_count {
return Err(failure(format!(
"lightTreeEmitterCount is {emitter_count} but the input has {unified_count} lights and emissive triangles. The tree must hold every emitter once."
)));
}
if (light_to_tree.len() as u64) < unified_count.max(1) {
return Err(failure(format!(
"lightToTree holds {} entries, fewer than the {unified_count} emitters of the input.",
light_to_tree.len()
)));
}
Ok(())
}
/// Refuses a render size or sample count the kernel cannot run.
pub(crate) fn validate_render_extent(width: u32, height: u32, total_samples: u32) -> Result<()> {
if width == 0 || height == 0 {
return Err(failure("Trace width and height must both be >= 1."));
}
if total_samples == 0 {
return Err(failure("Trace samples must be >= 1."));
}
Ok(())
}
/// Checks a FORMAT-v2 input before a session uploads it: schema, parameter block and the size of every table.
pub(crate) fn validate_cycles_input(input: &NativeTraceInputV2) -> Result<()> {
if input.schema_version != FORMAT_V2_VERSION {
return Err(failure(format!(
"Invalid FORMAT-v2 input schemaVersion {}: expected {FORMAT_V2_VERSION}.",
input.schema_version
)));
}
validate_kernel_params_v2(&input.kernel_params)?;
validate_record_buffer("instances", &input.instances, input.instance_count, INSTANCE_RECORD_BYTES)?;
validate_record_buffer("triangles", &input.triangles, input.triangle_count, TRIANGLE_RECORD_BYTES)?;
validate_record_buffer("materials", &input.materials, input.material_count, MATERIAL_RECORD_BYTES)?;
validate_record_buffer(
"emissiveTriangles",
&input.emissive_triangles,
input.emissive_triangle_count,
EMISSIVE_TRIANGLE_RECORD_BYTES,
)?;
validate_record_buffer("lights", &input.lights, input.light_count, LIGHT_RECORD_BYTES)?;
validate_record_buffer(
"lightTreeNodes",
&input.light_tree_nodes,
input.light_tree_node_count,
LIGHT_TREE_NODE_BYTES,
)?;
validate_record_buffer(
"textureDescriptors",
&input.texture_descriptors,
input.texture_count,
TEXTURE_DESCRIPTOR_BYTES,
)?;
validate_light_tree_tables(input)
}
/// How `trace_cycles_samples` splits the samples into dispatches.
#[derive(Clone, Copy)]
pub(crate) enum DispatchPlan {
/// Every dispatch draws this many samples, as the caller asked.
Fixed(u32),
/// Each dispatch is sized from the time of the last so none runs past the time budget; no batch is larger than `largest_batch`.
TimeBudget { largest_batch: u32 },
}
pub(crate) struct CyclesTrace {
pub(crate) sums: Vec<f32>,
pub(crate) traced_seconds: f64,
/// The most samples one dispatch drew.
pub(crate) largest_batch: u32,
/// The longest single dispatch; only measured when the dispatches are sized by time.
pub(crate) longest_dispatch_seconds: Option<f64>,
}
/// Raises the error for a lost device, naming the dispatch that was running and how to lower the work per dispatch.
fn fail_if_device_lost(
target: &CyclesDispatchTarget<'_>,
sample_offset: u32,
batch_samples: u32,
samples: u32,
) -> Result<()> {
if !target.device_lost.load(Ordering::SeqCst) {
return Ok(());
}
let detail = target.device_error.lock().map(|stored| stored.clone()).unwrap_or_default();
let smaller_batch = (batch_samples / 2).max(1);
Err(failure(format!(
"The GPU device was lost during the render, while drawing samples {sample_offset} to {} of {samples} in one dispatch of \
{batch_samples} samples ({}). The driver resets a GPU when a single dispatch runs too long. Lower the work per dispatch: \
pass samplesPerDispatch {smaller_batch} or less, or leave it unset so the render splits itself into dispatches of \
about {DISPATCH_BUDGET_SECONDS} s, or lower the samples or the resolution. Then run the render again.",
sample_offset + batch_samples,
if detail.is_empty() { "no detail from the driver".to_string() } else { detail },
)))
}
/// Draws `samples` samples numbered `first_sample` onwards, adding them to what the film already holds. A one-shot render
/// and the first call of a progressive film start at sample 0.
pub(crate) fn trace_cycles_samples(
target: &CyclesDispatchTarget<'_>,
first_sample: u32,
samples: u32,
plan: DispatchPlan,
groups: (u32, u32),
accumulation_bytes: u64,
max_seconds: Option<f64>,
) -> Result<CyclesTrace> {
let trace_started = Instant::now();
let device = target.device;
let queue = target.queue;
let mut planner = match plan {
DispatchPlan::TimeBudget { largest_batch } => Some(DispatchPlanner::new(DISPATCH_BUDGET_SECONDS, largest_batch)),
DispatchPlan::Fixed(_) => None,
};
let end_sample = first_sample + samples;
let mut sample_offset = first_sample;
let mut dispatch_count = 0u32;
let mut largest_batch = 0u32;
let mut longest_dispatch_seconds: Option<f64> = None;
while sample_offset < end_sample {
let remaining_samples = end_sample - sample_offset;
let batch_samples = match (&planner, plan) {
(Some(planner), _) => planner.next_batch(remaining_samples),
(None, DispatchPlan::Fixed(step)) => remaining_samples.min(step.max(1)),
(None, DispatchPlan::TimeBudget { .. }) => unreachable!("a time budget always has a planner"),
};
largest_batch = largest_batch.max(batch_samples);
let dispatch_started = Instant::now();
let mut dispatch_words = [0u8; 8];
dispatch_words[0..4].copy_from_slice(&sample_offset.to_le_bytes());
dispatch_words[4..8].copy_from_slice(&batch_samples.to_le_bytes());
queue.write_buffer(target.parameters, CYCLES_SAMPLE_OFFSET_BYTE_OFFSET, &dispatch_words);
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("cycles sample batch"),
});
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: None,
timestamp_writes: None,
});
pass.set_pipeline(target.pipeline);
pass.set_bind_group(0, target.bind_group, &[]);
pass.dispatch_workgroups(groups.0, groups.1, 1);
}
queue.submit(Some(encoder.finish()));
let dispatch_offset = sample_offset;
sample_offset += batch_samples;
dispatch_count += 1;
// A planned render waits for every dispatch to time it. A fixed one, when no time limit is set, polls every 4
// dispatches (excluding the final batch, which `read_accumulation` waits on directly) to drain staging belts
// without stalling each batch.
let should_poll = if planner.is_some() {
true
} else if max_seconds.is_some() {
dispatch_count % SAMPLES_PER_WAIT == 0 || sample_offset >= end_sample
} else {
sample_offset < end_sample && dispatch_count % SAMPLES_PER_WAIT == 0
};
if should_poll {
let _ = device.poll(wgpu::PollType::wait_indefinitely());
fail_if_device_lost(target, dispatch_offset, batch_samples, end_sample)?;
if let Some(planner) = planner.as_mut() {
let dispatch_seconds = dispatch_started.elapsed().as_secs_f64();
planner.record(batch_samples, dispatch_seconds);
longest_dispatch_seconds = Some(longest_dispatch_seconds.map_or(dispatch_seconds, |longest| longest.max(dispatch_seconds)));
}
if let Some(limit) = max_seconds {
if trace_started.elapsed().as_secs_f64() > limit {
return Err(failure(format!(
"The native trace passed its time limit of {limit} s after {sample_offset} of {end_sample} samples."
)));
}
}
}
}
let sums = read_accumulation(device, queue, target.accumulation, accumulation_bytes);
fail_if_device_lost(target, end_sample.saturating_sub(largest_batch), largest_batch, end_sample)?;
Ok(CyclesTrace {
sums: sums?,
traced_seconds: trace_started.elapsed().as_secs_f64(),
largest_batch,
longest_dispatch_seconds,
})
}
pub(crate) fn run_cycles_lobe_probe(options: &ProbeCyclesLobeOptions, shader_source: String) -> Result<Vec<f32>> {
if options.width == 0 || options.height == 0 {
return Err(failure("probeCyclesLobe width and height must both be >= 1."));
}
let total_invocations = (options.width as u64) * (options.height as u64);
let groups_x = total_invocations.div_ceil(LOBE_PROBE_WORKGROUP_SIZE as u64);
if groups_x > MAX_COMPUTE_WORKGROUPS_PER_DIMENSION {
return Err(failure(format!(
"probeCyclesLobe dispatch requires {groups_x} workgroups in X, exceeding WebGPU/Vulkan limit {MAX_COMPUTE_WORKGROUPS_PER_DIMENSION}."
)));
}
validate_kernel_params_v2(&options.kernel_params)?;
if options.materials.len() < MATERIAL_RECORD_BYTES || options.materials.len() % MATERIAL_RECORD_BYTES != 0 {
return Err(failure(format!(
"probeCyclesLobe materials buffer must be a positive multiple of {MATERIAL_RECORD_BYTES} bytes, got {} bytes.",
options.materials.len()
)));
}
if let Some(ref descriptors) = options.texture_descriptors {
if descriptors.is_empty() || descriptors.len() % TEXTURE_DESCRIPTOR_BYTES != 0 {
return Err(failure(format!(
"probeCyclesLobe textureDescriptors must be a positive multiple of {TEXTURE_DESCRIPTOR_BYTES} bytes, got {} bytes.",
descriptors.len()
)));
}
}
let (device, queue, _, _, _) = get_cached_device()?;
let pipeline = get_or_compile_cycles_pipeline("test_lobe_samples", shader_source)?;
let acceleration = build_empty_cycles_acceleration(&device, &queue);
let accumulation_bytes = image_accumulation_bytes(options.width, options.height);
let mut parameter_bytes = options.kernel_params.to_vec();
parameter_bytes[8..12].copy_from_slice(&options.width.to_le_bytes());
parameter_bytes[12..16].copy_from_slice(&options.height.to_le_bytes());
let parameters = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("lobe probe params"),
contents: ¶meter_bytes,
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
});
let default_descriptors = [0u8; TEXTURE_DESCRIPTOR_BYTES];
let descriptor_slice: &[u8] = match options.texture_descriptors {
Some(ref bytes) => bytes,
None => &default_descriptors,
};
let default_texels = [0u32; 4];
let texel_slice: &[u32] = match options.texture_texels {
Some(ref words) if !words.is_empty() => words,
_ => &default_texels,
};
let buffers = vec![
storage_buffer(&device, "dummy instances", &[0u8; INSTANCE_RECORD_BYTES]),
storage_buffer(&device, "dummy triangles", &[0u8; TRIANGLE_RECORD_BYTES]),
storage_buffer(&device, "probe materials", &options.materials),
storage_buffer(&device, "dummy emissive triangles", &[0u8; EMISSIVE_TRIANGLE_RECORD_BYTES]),
storage_buffer(&device, "dummy lights", &[0u8; LIGHT_RECORD_BYTES]),
storage_buffer(&device, "dummy light tree", &[0u8; LIGHT_TREE_NODE_BYTES]),
storage_buffer(&device, "probe texture descriptors", descriptor_slice),
storage_buffer(&device, "probe texture texels", bytemuck::cast_slice(texel_slice)),
storage_buffer(&device, "dummy environment texels", &[0u8; 16]),
storage_buffer(&device, "dummy light tree emitters", &[0u8; LIGHT_TREE_EMITTER_BYTES]),
storage_buffer(&device, "dummy light to tree", &[0u8; 4]),
];
let accumulation = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("lobe probe accumulation"),
size: accumulation_bytes,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let bind_group = with_cycles_cache(|_device, _queue, layout, lut| {
create_cycles_bind_group(
&device,
layout,
¶meters,
&acceleration.tlas,
&buffers,
&accumulation,
lut,
)
})?;
let started = Instant::now();
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("lobe probe dispatch"),
});
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: None,
timestamp_writes: None,
});
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, &bind_group, &[]);
pass.dispatch_workgroups(groups_x as u32, 1, 1);
}
queue.submit(Some(encoder.finish()));
let _ = device.poll(wgpu::PollType::wait_indefinitely());
if let Some(limit) = options.max_seconds {
if started.elapsed().as_secs_f64() > limit {
return Err(failure(format!(
"probeCyclesLobe passed its time limit of {limit} s."
)));
}
}
read_accumulation(&device, &queue, &accumulation, accumulation_bytes)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pipeline_key_separates_entry_points_and_sources() {
assert_eq!(pipeline_key("main", "source a"), pipeline_key("main", "source a"));
assert_ne!(pipeline_key("main", "source a"), pipeline_key("main", "source b"));
assert_ne!(pipeline_key("main", "source a"), pipeline_key("other", "source a"));
}
#[test]
fn bounded_cache_never_exceeds_capacity() {
let mut cache: BoundedCache<u32> = BoundedCache::new(3);
for index in 0..10u32 {
cache.insert(pipeline_key("main", &index.to_string()), index);
assert!(cache.entries.len() <= 3);
}
assert_eq!(cache.get(&pipeline_key("main", "9")), Some(9));
}
#[test]
fn bounded_cache_keeps_entries_when_a_known_key_is_replaced() {
let mut cache: BoundedCache<u32> = BoundedCache::new(2);
cache.insert(pipeline_key("main", "a"), 1);
cache.insert(pipeline_key("main", "b"), 2);
cache.insert(pipeline_key("main", "a"), 3);
assert_eq!(cache.get(&pipeline_key("main", "a")), Some(3));
assert_eq!(cache.get(&pipeline_key("main", "b")), Some(2));
}
#[test]
fn poisoned_lock_is_reset_instead_of_failing_later_calls() {
let cache = Arc::new(Mutex::new(Some(7u32)));
let panicking = Arc::clone(&cache);
let outcome = std::thread::spawn(move || {
let _guard = panicking.lock().unwrap();
panic!("panic while the cache lock is held");
})
.join();
assert!(outcome.is_err());
assert!(cache.is_poisoned());
{
let guard = lock_resetting_when_poisoned(&cache);
assert!(guard.is_none());
}
assert!(!cache.is_poisoned());
*lock_resetting_when_poisoned(&cache) = Some(9);
assert_eq!(*lock_resetting_when_poisoned(&cache), Some(9));
}
fn kernel_params_with_flags(kernel_flags: u32) -> Vec<u8> {
let mut kernel_params = vec![0u8; KERNEL_PARAMS_V2_BYTES];
kernel_params[0..4].copy_from_slice(&FORMAT_V2_MAGIC.to_le_bytes());
kernel_params[4..8].copy_from_slice(&FORMAT_V2_VERSION.to_le_bytes());
kernel_params[KERNEL_PARAMS_KERNEL_FLAGS_BYTE..KERNEL_PARAMS_KERNEL_FLAGS_BYTE + 4]
.copy_from_slice(&kernel_flags.to_le_bytes());
kernel_params
}
#[test]
fn a_frame_may_switch_the_light_tree_on_only_when_the_session_holds_the_emitter_tables() {
let tree_on = kernel_params_with_flags(KERNEL_FLAG_USE_LIGHT_TREE);
let tree_off = kernel_params_with_flags(0);
assert!(validate_light_tree_flag_has_tables(&tree_on, 3, true).is_ok());
assert!(validate_light_tree_flag_has_tables(&tree_on, 0, false).is_ok());
assert!(validate_light_tree_flag_has_tables(&tree_off, 3, false).is_ok());
let message = validate_light_tree_flag_has_tables(&tree_on, 3, false)
.expect_err("the tree flag without emitter tables must be refused")
.to_string();
assert!(message.contains("buildNativeInputV2"), "{message}");
}
#[test]
fn a_frame_kernel_params_block_must_be_the_v2_block() {
assert!(validate_kernel_params_v2(&kernel_params_with_flags(0)).is_ok());
assert!(validate_kernel_params_v2(&[0u8; 12]).is_err());
let mut wrong_version = kernel_params_with_flags(0);
wrong_version[4..8].copy_from_slice(&1u32.to_le_bytes());
assert!(validate_kernel_params_v2(&wrong_version).is_err());
}
#[test]
fn an_image_of_zero_pixels_or_zero_samples_is_refused() {
assert!(validate_render_extent(16, 16, 1).is_ok());
assert!(validate_render_extent(0, 16, 1).is_err());
assert!(validate_render_extent(16, 0, 1).is_err());
assert!(validate_render_extent(16, 16, 0).is_err());
}
}