3D engine that AI agents drive
Split Cook native renders into dispatches under a time budget (#960)
Nguyen Xuan An3341248e7e2c
diff --git a/bench/cycles-parity/trace-native.mjs b/bench/cycles-parity/trace-native.mjsindex f62cdcac..2cb055bd 100644--- a/bench/cycles-parity/trace-native.mjs+++ b/bench/cycles-parity/trace-native.mjs@@ -168,6 +168,7 @@ async function runCli() {samples: options.samples,averageSamples: traced.averageSamples,samplesPerDispatch: traced.samplesPerDispatch ?? null,+ longestDispatchSeconds: traced.longestDispatchSeconds ?? null,bounces: options.bounces,seed: options.seed,sampler: options.sampler,diff --git a/changes/960-cook-dispatch-budget.md b/changes/960-cook-dispatch-budget.mdnew file mode 100644index 00000000..0158db93--- /dev/null+++ b/changes/960-cook-dispatch-budget.md@@ -0,0 +1 @@+- Cook native renders split themselves into dispatches that each stay under about half a second, so a heavy scene such as 500 lights at 960x540 and 256 samples no longer loses the Vulkan device, and a lost device now raises an error that names how to lower the work per dispatch (#960)diff --git a/packages/render-native/README.md b/packages/render-native/README.mdindex 750cc742..47c1914c 100644--- a/packages/render-native/README.md+++ b/packages/render-native/README.md@@ -33,3 +33,7 @@ const { radiance, escaped } = traceNativeTexels(input, { count: texelCount, poin`radiance` is the mean light that reaches each texel (three floats per texel) and `escaped` the share of first rays that left the scene. The agent runtime uses it for recipe bakes (`bake.backend: "native"`).Cook's sampler is an Owen-scrambled Sobol sequence (Burley 2020); `sampler: 'pcg'` selects the earlier white-noise hash. Lights are picked through a light tree (Conty and Kulla 2018) when a kind, analytic lights or emissive triangles, holds more than 8 emitters, no sun is among the lights and no instance restricts its light group; `useLightTree: true` forces it and `false` turns it off. Both change only the noise pattern of the image, not its mean. The pixel filter stays `box` unless `filterType: 'blackman-harris'` is passed ([FORMAT-v2.md](./FORMAT-v2.md)).++## Dispatches++The Cook kernel splits a render into dispatches that each aim for about 0.5 s, sized from the time of the previous one, so a heavy scene does not run past the driver timeout and lose the device. `samplesPerDispatch` fixes the batch instead. A lost device raises an error that names the dispatch and how to lower the work per dispatch. The split is tested in `tests/dispatch.rs` (`cargo test --test dispatch`).diff --git a/packages/render-native/src/host/dispatch.rs b/packages/render-native/src/host/dispatch.rsnew file mode 100644index 00000000..1f80a4f9--- /dev/null+++ b/packages/render-native/src/host/dispatch.rs@@ -0,0 +1,35 @@+//! How a render is split into GPU dispatches. A driver resets the device when one dispatch runs for more than a couple of+//! seconds (the Windows timeout is 2 s), so the render is cut into dispatches that each stay under a time budget.++/// The longest one dispatch should run. Well under the 2 s driver timeout, long enough that the wait between dispatches is noise.+pub(crate) const DISPATCH_BUDGET_SECONDS: f64 = 0.5;++/// A batch grows by at most this factor per dispatch, so one fast first measurement cannot jump to a dispatch that is too long.+const BATCH_GROWTH_LIMIT: u32 = 4;++/// Chooses how many samples the next dispatch draws from the measured time of the last one.+/// The first dispatch draws a single sample to measure the cost of one.+pub(crate) struct DispatchPlanner {+ budget_seconds: f64,+ largest_batch: u32,+ next_batch: u32,+}++impl DispatchPlanner {+ pub(crate) fn new(budget_seconds: f64, largest_batch: u32) -> Self {+ Self { budget_seconds, largest_batch: largest_batch.max(1), next_batch: 1 }+ }++ /// The samples to draw now, never more than `remaining_samples`.+ pub(crate) fn next_batch(&self, remaining_samples: u32) -> u32 {+ self.next_batch.min(remaining_samples).max(1)+ }++ /// Records that `batch_samples` took `seconds` and sizes the next batch to fit the budget.+ pub(crate) fn record(&mut self, batch_samples: u32, seconds: f64) {+ let seconds_per_sample = (seconds / batch_samples.max(1) as f64).max(1e-9);+ let fitting_samples = (self.budget_seconds / seconds_per_sample).min(u32::MAX as f64) as u32;+ let growth_ceiling = batch_samples.saturating_mul(BATCH_GROWTH_LIMIT).min(self.largest_batch);+ self.next_batch = fitting_samples.clamp(1, growth_ceiling.max(1));+ }+}diff --git a/packages/render-native/src/host/mod.rs b/packages/render-native/src/host/mod.rsindex 95fb9769..9d597bfd 100644--- a/packages/render-native/src/host/mod.rs+++ b/packages/render-native/src/host/mod.rs@@ -1,4 +1,5 @@pub(crate) mod accel;+pub(crate) mod dispatch;pub(crate) mod device;pub(crate) mod film;pub(crate) mod session;diff --git a/packages/render-native/src/host/session.rs b/packages/render-native/src/host/session.rsindex 033c2a03..9d877509 100644--- a/packages/render-native/src/host/session.rs+++ b/packages/render-native/src/host/session.rs@@ -14,6 +14,7 @@ use crate::host::accel::{create_vertex_buffer, unpack_triangles, CyclesAcceleration, INSTANCE_RECORD_BYTES, TRIANGLE_FLOATS,TRIANGLE_RECORD_BYTES,};+use crate::host::dispatch::{DispatchPlanner, DISPATCH_BUDGET_SECONDS};use crate::host::device::{failure, layout_entry, open_device, storage_buffer, storage_entry};use crate::host::film::{image_accumulation_bytes, read_accumulation};use crate::host::textures::{upload_cycles_scene_buffers, upload_scene_buffers};@@ -71,6 +72,8 @@ struct CachedDevice {features: String,/// Set by the device lost callback; the next call then opens a fresh device and drops everything below.device_lost: Arc<AtomicBool>,+ /// What wgpu reported when it lost the device or rejected work, for the error a failed render raises.+ device_error: Arc<Mutex<String>>,/// Static 118,848-byte LUT storage buffer for Cycles binding 12.lut_buffer: wgpu::Buffer,/// Bind group layout for the Cycles kernel (bindings 0..=12).@@ -115,9 +118,23 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Strlet (device, queue, adapter_info) = open_device()?;let device_lost = Arc::new(AtomicBool::new(false));let lost_flag = Arc::clone(&device_lost);- device.set_device_lost_callback(move |_reason, _message| {+ let device_error = Arc::new(Mutex::new(String::new()));+ let lost_message = Arc::clone(&device_error);+ device.set_device_lost_callback(move |reason, message| {+ if let Ok(mut stored) = lost_message.lock() {+ *stored = format!("{reason:?}: {message}");+ }lost_flag.store(true, Ordering::SeqCst);});+ // The default handler panics on a lost device; record the report so the render can raise a readable error.+ let error_flag = Arc::clone(&device_lost);+ let error_message = Arc::clone(&device_error);+ device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| {+ if let Ok(mut stored) = error_message.lock() {+ stored.push_str(&error.to_string());+ }+ error_flag.store(true, Ordering::SeqCst);+ }));let lut_bytes: &[u8] = bytemuck::cast_slice(&CYCLES_LUT_WORDS[..]);let lut_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {label: Some("cycles LUT binding 12"),@@ -133,6 +150,7 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Strbackend_name: format!("{:?}", adapter_info.backend),features: if has_ray_query { "EXPERIMENTAL_RAY_QUERY".to_string() } else { String::new() },device_lost,+ device_error,lut_buffer,cycles_bind_group_layout,cycles_pipelines: BoundedCache::new(MAX_CACHED_PIPELINES),@@ -148,6 +166,15 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Str))}+/// The flag and message the device callbacks fill in when the cached device is lost.+fn cached_device_loss_state() -> Result<(Arc<AtomicBool>, Arc<Mutex<String>>)> {+ let guard = lock_device_cache();+ match guard.as_ref() {+ Some(cached) => Ok((Arc::clone(&cached.device_lost), Arc::clone(&cached.device_error))),+ None => Err(cache_was_reset_failure()),+ }+}+fn cache_was_reset_failure() -> napi::Error {failure("The GPU device was reset while the scene was being prepared. Run the render again.")}@@ -231,6 +258,8 @@ pub(crate) struct PreparedCyclesScene {pub(crate) adapter_name: String,pub(crate) backend_name: String,pub(crate) features: String,+ pub(crate) device_lost: Arc<AtomicBool>,+ pub(crate) device_error: Arc<Mutex<String>>,pub(crate) setup_seconds: f64,pub(crate) compile_seconds: f64,pub(crate) upload_seconds: f64,@@ -598,6 +627,7 @@ pub(crate) fn prepare_cycles_scene(// Use cached device; open once per process.let (device, queue, adapter_name, backend_name, features) = get_cached_device()?;+ let (device_lost, device_error) = cached_device_loss_state()?;let setup_seconds = setup_started.elapsed().as_secs_f64();let compile_started = Instant::now();@@ -644,6 +674,8 @@ pub(crate) fn prepare_cycles_scene(adapter_name,backend_name,features,+ device_lost,+ device_error,setup_seconds,compile_seconds,upload_seconds,@@ -653,23 +685,75 @@ pub(crate) fn prepare_cycles_scene(})}+/// How `trace_cycles_samples` splits the samples into dispatches.+#[derive(Clone, Copy)]+pub(crate) enum DispatchPlan {+ /// Every dispatch draws this many samples, as the caller asked.+ Fixed(u32),+ /// Each dispatch is sized from the time of the last so none runs past the time budget; no batch is larger than `largest_batch`.+ TimeBudget { largest_batch: u32 },+}++pub(crate) struct CyclesTrace {+ pub(crate) sums: Vec<f32>,+ pub(crate) traced_seconds: f64,+ /// The most samples one dispatch drew.+ pub(crate) largest_batch: u32,+ /// The longest single dispatch; only measured when the dispatches are sized by time.+ pub(crate) longest_dispatch_seconds: Option<f64>,+}++/// Raises the error for a lost device, naming the dispatch that was running and how to lower the work per dispatch.+fn fail_if_device_lost(+ prepared: &PreparedCyclesScene,+ sample_offset: u32,+ batch_samples: u32,+ samples: u32,+) -> Result<()> {+ if !prepared.device_lost.load(Ordering::SeqCst) {+ return Ok(());+ }+ let detail = prepared.device_error.lock().map(|stored| stored.clone()).unwrap_or_default();+ let smaller_batch = (batch_samples / 2).max(1);+ Err(failure(format!(+ "The GPU device was lost during the render, while drawing samples {sample_offset} to {} of {samples} in one dispatch of \+ {batch_samples} samples ({}). The driver resets a GPU when a single dispatch runs too long. Lower the work per dispatch: \+ pass samplesPerDispatch {smaller_batch} or less, or leave it unset so the render splits itself into dispatches of \+ about {DISPATCH_BUDGET_SECONDS} s, or lower the samples or the resolution. Then run the render again.",+ sample_offset + batch_samples,+ if detail.is_empty() { "no detail from the driver".to_string() } else { detail },+ )))+}+pub(crate) fn trace_cycles_samples(prepared: &PreparedCyclesScene,samples: u32,- samples_per_dispatch: u32,+ plan: DispatchPlan,groups: (u32, u32),accumulation_bytes: u64,max_seconds: Option<f64>,-) -> Result<(Vec<f32>, f64)> {+) -> Result<CyclesTrace> {let trace_started = Instant::now();let device = &prepared.device;let queue = &prepared.queue;- let step = samples_per_dispatch.max(1);+ let mut planner = match plan {+ DispatchPlan::TimeBudget { largest_batch } => Some(DispatchPlanner::new(DISPATCH_BUDGET_SECONDS, largest_batch)),+ DispatchPlan::Fixed(_) => None,+ };let mut sample_offset = 0u32;let mut dispatch_count = 0u32;+ let mut largest_batch = 0u32;+ let mut longest_dispatch_seconds: Option<f64> = None;while sample_offset < samples {- let batch_samples = (samples - sample_offset).min(step);+ let remaining_samples = samples - sample_offset;+ let batch_samples = match (&planner, plan) {+ (Some(planner), _) => planner.next_batch(remaining_samples),+ (None, DispatchPlan::Fixed(step)) => remaining_samples.min(step.max(1)),+ (None, DispatchPlan::TimeBudget { .. }) => unreachable!("a time budget always has a planner"),+ };+ largest_batch = largest_batch.max(batch_samples);+ let dispatch_started = Instant::now();let mut dispatch_words = [0u8; 8];dispatch_words[0..4].copy_from_slice(&sample_offset.to_le_bytes());dispatch_words[4..8].copy_from_slice(&batch_samples.to_le_bytes());@@ -689,17 +773,27 @@ pub(crate) fn trace_cycles_samples(}queue.submit(Some(encoder.finish()));+ let dispatch_offset = sample_offset;sample_offset += batch_samples;dispatch_count += 1;- // When no time limit is set poll every 4 dispatches (excluding the final batch, which- // `read_accumulation` waits on directly) to drain staging belts without stalling each batch.- let should_poll = if max_seconds.is_some() {+ // A planned render waits for every dispatch to time it. A fixed one, when no time limit is set, polls every 4+ // dispatches (excluding the final batch, which `read_accumulation` waits on directly) to drain staging belts+ // without stalling each batch.+ let should_poll = if planner.is_some() {+ true+ } else if max_seconds.is_some() {dispatch_count % SAMPLES_PER_WAIT == 0 || sample_offset >= samples} else {sample_offset < samples && dispatch_count % SAMPLES_PER_WAIT == 0};if should_poll {let _ = device.poll(wgpu::PollType::wait_indefinitely());+ fail_if_device_lost(prepared, dispatch_offset, batch_samples, samples)?;+ if let Some(planner) = planner.as_mut() {+ let dispatch_seconds = dispatch_started.elapsed().as_secs_f64();+ planner.record(batch_samples, dispatch_seconds);+ longest_dispatch_seconds = Some(longest_dispatch_seconds.map_or(dispatch_seconds, |longest| longest.max(dispatch_seconds)));+ }if let Some(limit) = max_seconds {if trace_started.elapsed().as_secs_f64() > limit {return Err(failure(format!(@@ -710,8 +804,14 @@ pub(crate) fn trace_cycles_samples(}}- let sums = read_accumulation(device, queue, &prepared.accumulation, accumulation_bytes)?;- Ok((sums, trace_started.elapsed().as_secs_f64()))+ let sums = read_accumulation(device, queue, &prepared.accumulation, accumulation_bytes);+ fail_if_device_lost(prepared, samples.saturating_sub(largest_batch), largest_batch, samples)?;+ Ok(CyclesTrace {+ sums: sums?,+ traced_seconds: trace_started.elapsed().as_secs_f64(),+ largest_batch,+ longest_dispatch_seconds,+ })}pub(crate) fn run_cycles_lobe_probe(options: &ProbeCyclesLobeOptions, shader_source: String) -> Result<Vec<f32>> {diff --git a/packages/render-native/src/lib.rs b/packages/render-native/src/lib.rsindex 0ffe5796..079fedd4 100644--- a/packages/render-native/src/lib.rs+++ b/packages/render-native/src/lib.rs@@ -14,7 +14,7 @@ use crate::host::film::{texel_accumulation_bytes, texel_workgroups,};use crate::host::session::{- prepare_cycles_scene, prepare_scene, run_cycles_lobe_probe, trace_cycles_samples, trace_samples, JobShape,+ DispatchPlan, prepare_cycles_scene, prepare_scene, run_cycles_lobe_probe, trace_cycles_samples, trace_samples, JobShape,};const KERNEL_SOURCE: &str = include_str!("kernel.wgsl");@@ -138,6 +138,8 @@ pub struct NativeTraceResult {pub average_samples: f64,/// The samples each dispatch drew, as given or as chosen automatically (Cycles kernel only; absent for the legacy kernel).pub samples_per_dispatch: Option<u32>,+ /// The longest single dispatch in seconds, measured when dispatches are sized by time (absent for a fixed batch).+ pub longest_dispatch_seconds: Option<f64>,pub adapter: String,pub backend: String,/// "EXPERIMENTAL_RAY_QUERY" when the device the trace ran on was opened with ray queries (the tracer needs them).@@ -195,6 +197,7 @@ pub fn trace_scene(input: NativeTraceInput, options: NativeTraceOptions) -> Resuvariance: Float32Array::new(resolved.variance),average_samples: resolved.average_samples,samples_per_dispatch: None,+ longest_dispatch_seconds: None,adapter: prepared.adapter_name,backend: prepared.backend_name,features: prepared.features,@@ -221,34 +224,24 @@ pub fn trace_scene_cycles(input: NativeTraceInputV2, options: NativeCyclesTraceOaccumulation_bytes,)?;let groups = image_workgroups(options.width, options.height);- let samples_per_dispatch = match options.samples_per_dispatch {- Some(n) => n.max(1),- None => {- // Auto-select a batch that keeps GPU utilisation high without coarse progress steps.- // With a wall-clock limit use step=1 for tight responsiveness below 2 s; otherwise- // batch by pixel count so each dispatch is a worthwhile chunk.- let pixel_count = options.width as u64 * options.height as u64;- let batch = match options.max_seconds {- Some(limit) if limit < 2.0 => 1,- _ => {- if pixel_count <= 1_000_000 {- options.samples.min(32)- } else {- options.samples.min(16)- }- }- };- batch.max(1)- }+ // Unless the caller fixes the batch, each dispatch is sized from the time of the last so none runs long enough for the+ // driver to reset the device. A wall-clock limit under 2 s keeps one sample per dispatch for tight responsiveness.+ let plan = match options.samples_per_dispatch {+ Some(samples_per_dispatch) => DispatchPlan::Fixed(samples_per_dispatch.max(1)),+ None => match options.max_seconds {+ Some(limit) if limit < 2.0 => DispatchPlan::Fixed(1),+ _ => DispatchPlan::TimeBudget { largest_batch: options.samples.max(1) },+ },};- let (sums, traced_seconds) = trace_cycles_samples(+ let trace = trace_cycles_samples(&prepared,options.samples,- samples_per_dispatch,+ plan,groups,accumulation_bytes,options.max_seconds,)?;+ let (sums, traced_seconds, samples_per_dispatch) = (trace.sums, trace.traced_seconds, trace.largest_batch);let resolved = resolve_image(&sums, pixel_count);Ok(NativeTraceResult {rgba: Float32Array::new(resolved.rgba),@@ -258,6 +251,7 @@ pub fn trace_scene_cycles(input: NativeTraceInputV2, options: NativeCyclesTraceOvariance: Float32Array::new(resolved.variance),average_samples: resolved.average_samples,samples_per_dispatch: Some(samples_per_dispatch),+ longest_dispatch_seconds: trace.longest_dispatch_seconds,adapter: prepared.adapter_name,backend: prepared.backend_name,features: prepared.features,diff --git a/packages/render-native/tests/dispatch.rs b/packages/render-native/tests/dispatch.rsnew file mode 100644index 00000000..652b5cc0--- /dev/null+++ b/packages/render-native/tests/dispatch.rs@@ -0,0 +1,65 @@+//! Tests for how a render is split into dispatches. The planner is included by path because the napi library does not+//! link into a test binary.+#![allow(dead_code)]++#[path = "../src/host/dispatch.rs"]+mod dispatch;++use dispatch::DispatchPlanner;++fn batches_for(samples: u32, seconds_per_sample: f64, largest_batch: u32) -> Vec<u32> {+ let mut planner = DispatchPlanner::new(0.5, largest_batch);+ let mut remaining = samples;+ let mut batches = Vec::new();+ while remaining > 0 {+ let batch = planner.next_batch(remaining);+ planner.record(batch, batch as f64 * seconds_per_sample);+ remaining -= batch;+ batches.push(batch);+ }+ batches+}++#[test]+fn the_first_dispatch_draws_one_sample() {+ assert_eq!(DispatchPlanner::new(0.5, 64).next_batch(256), 1);+}++#[test]+fn every_sample_is_drawn_exactly_once() {+ let batches = batches_for(256, 0.02, 256);+ assert_eq!(batches.iter().sum::<u32>(), 256);+}++#[test]+fn a_slow_scene_gets_small_dispatches_that_fit_the_budget() {+ let batches = batches_for(256, 0.1, 256);+ assert!(batches.iter().all(|batch| *batch as f64 * 0.1 <= 0.5 + 1e-9), "{batches:?}");+}++#[test]+fn a_fast_scene_grows_by_a_bounded_factor() {+ let batches = batches_for(256, 0.0001, 256);+ assert_eq!(&batches[..4], &[1, 4, 16, 64]);+}++#[test]+fn a_scene_slower_than_the_budget_still_draws_one_sample_at_a_time() {+ let batches = batches_for(4, 3.0, 4);+ assert_eq!(batches, vec![1, 1, 1, 1]);+}++#[test]+fn the_batch_never_passes_the_largest_allowed() {+ let batches = batches_for(256, 0.0001, 8);+ assert!(batches.iter().all(|batch| *batch <= 8));+}++#[test]+fn a_dispatch_that_turns_slower_shrinks_the_next_one() {+ let mut planner = DispatchPlanner::new(0.5, 256);+ planner.record(1, 0.01);+ assert_eq!(planner.next_batch(256), 4);+ planner.record(4, 2.0);+ assert_eq!(planner.next_batch(256), 1);+}