Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Split Cook native renders into dispatches under a time budget (#960)

Nguyen Xuan An3341248e7e2c
diff --git a/bench/cycles-parity/trace-native.mjs b/bench/cycles-parity/trace-native.mjs
index f62cdcac..2cb055bd 100644
--- a/bench/cycles-parity/trace-native.mjs
+++ b/bench/cycles-parity/trace-native.mjs
@@ -168,6 +168,7 @@ async function runCli() {
samples: options.samples,
averageSamples: traced.averageSamples,
samplesPerDispatch: traced.samplesPerDispatch ?? null,
+ longestDispatchSeconds: traced.longestDispatchSeconds ?? null,
bounces: options.bounces,
seed: options.seed,
sampler: options.sampler,
diff --git a/changes/960-cook-dispatch-budget.md b/changes/960-cook-dispatch-budget.md
new file mode 100644
index 00000000..0158db93
--- /dev/null
+++ b/changes/960-cook-dispatch-budget.md
@@ -0,0 +1 @@
+- Cook native renders split themselves into dispatches that each stay under about half a second, so a heavy scene such as 500 lights at 960x540 and 256 samples no longer loses the Vulkan device, and a lost device now raises an error that names how to lower the work per dispatch (#960)
diff --git a/packages/render-native/README.md b/packages/render-native/README.md
index 750cc742..47c1914c 100644
--- a/packages/render-native/README.md
+++ b/packages/render-native/README.md
@@ -33,3 +33,7 @@ const { radiance, escaped } = traceNativeTexels(input, { count: texelCount, poin
`radiance` is the mean light that reaches each texel (three floats per texel) and `escaped` the share of first rays that left the scene. The agent runtime uses it for recipe bakes (`bake.backend: "native"`).
Cook's sampler is an Owen-scrambled Sobol sequence (Burley 2020); `sampler: 'pcg'` selects the earlier white-noise hash. Lights are picked through a light tree (Conty and Kulla 2018) when a kind, analytic lights or emissive triangles, holds more than 8 emitters, no sun is among the lights and no instance restricts its light group; `useLightTree: true` forces it and `false` turns it off. Both change only the noise pattern of the image, not its mean. The pixel filter stays `box` unless `filterType: 'blackman-harris'` is passed ([FORMAT-v2.md](./FORMAT-v2.md)).
+
+## Dispatches
+
+The Cook kernel splits a render into dispatches that each aim for about 0.5 s, sized from the time of the previous one, so a heavy scene does not run past the driver timeout and lose the device. `samplesPerDispatch` fixes the batch instead. A lost device raises an error that names the dispatch and how to lower the work per dispatch. The split is tested in `tests/dispatch.rs` (`cargo test --test dispatch`).
diff --git a/packages/render-native/src/host/dispatch.rs b/packages/render-native/src/host/dispatch.rs
new file mode 100644
index 00000000..1f80a4f9
--- /dev/null
+++ b/packages/render-native/src/host/dispatch.rs
@@ -0,0 +1,35 @@
+//! How a render is split into GPU dispatches. A driver resets the device when one dispatch runs for more than a couple of
+//! seconds (the Windows timeout is 2 s), so the render is cut into dispatches that each stay under a time budget.
+
+/// The longest one dispatch should run. Well under the 2 s driver timeout, long enough that the wait between dispatches is noise.
+pub(crate) const DISPATCH_BUDGET_SECONDS: f64 = 0.5;
+
+/// A batch grows by at most this factor per dispatch, so one fast first measurement cannot jump to a dispatch that is too long.
+const BATCH_GROWTH_LIMIT: u32 = 4;
+
+/// Chooses how many samples the next dispatch draws from the measured time of the last one.
+/// The first dispatch draws a single sample to measure the cost of one.
+pub(crate) struct DispatchPlanner {
+ budget_seconds: f64,
+ largest_batch: u32,
+ next_batch: u32,
+}
+
+impl DispatchPlanner {
+ pub(crate) fn new(budget_seconds: f64, largest_batch: u32) -> Self {
+ Self { budget_seconds, largest_batch: largest_batch.max(1), next_batch: 1 }
+ }
+
+ /// The samples to draw now, never more than `remaining_samples`.
+ pub(crate) fn next_batch(&self, remaining_samples: u32) -> u32 {
+ self.next_batch.min(remaining_samples).max(1)
+ }
+
+ /// Records that `batch_samples` took `seconds` and sizes the next batch to fit the budget.
+ pub(crate) fn record(&mut self, batch_samples: u32, seconds: f64) {
+ let seconds_per_sample = (seconds / batch_samples.max(1) as f64).max(1e-9);
+ let fitting_samples = (self.budget_seconds / seconds_per_sample).min(u32::MAX as f64) as u32;
+ let growth_ceiling = batch_samples.saturating_mul(BATCH_GROWTH_LIMIT).min(self.largest_batch);
+ self.next_batch = fitting_samples.clamp(1, growth_ceiling.max(1));
+ }
+}
diff --git a/packages/render-native/src/host/mod.rs b/packages/render-native/src/host/mod.rs
index 95fb9769..9d597bfd 100644
--- a/packages/render-native/src/host/mod.rs
+++ b/packages/render-native/src/host/mod.rs
@@ -1,4 +1,5 @@
pub(crate) mod accel;
+pub(crate) mod dispatch;
pub(crate) mod device;
pub(crate) mod film;
pub(crate) mod session;
diff --git a/packages/render-native/src/host/session.rs b/packages/render-native/src/host/session.rs
index 033c2a03..9d877509 100644
--- a/packages/render-native/src/host/session.rs
+++ b/packages/render-native/src/host/session.rs
@@ -14,6 +14,7 @@ use crate::host::accel::{
create_vertex_buffer, unpack_triangles, CyclesAcceleration, INSTANCE_RECORD_BYTES, TRIANGLE_FLOATS,
TRIANGLE_RECORD_BYTES,
};
+use crate::host::dispatch::{DispatchPlanner, DISPATCH_BUDGET_SECONDS};
use crate::host::device::{failure, layout_entry, open_device, storage_buffer, storage_entry};
use crate::host::film::{image_accumulation_bytes, read_accumulation};
use crate::host::textures::{upload_cycles_scene_buffers, upload_scene_buffers};
@@ -71,6 +72,8 @@ struct CachedDevice {
features: String,
/// Set by the device lost callback; the next call then opens a fresh device and drops everything below.
device_lost: Arc<AtomicBool>,
+ /// What wgpu reported when it lost the device or rejected work, for the error a failed render raises.
+ device_error: Arc<Mutex<String>>,
/// Static 118,848-byte LUT storage buffer for Cycles binding 12.
lut_buffer: wgpu::Buffer,
/// Bind group layout for the Cycles kernel (bindings 0..=12).
@@ -115,9 +118,23 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Str
let (device, queue, adapter_info) = open_device()?;
let device_lost = Arc::new(AtomicBool::new(false));
let lost_flag = Arc::clone(&device_lost);
- device.set_device_lost_callback(move |_reason, _message| {
+ let device_error = Arc::new(Mutex::new(String::new()));
+ let lost_message = Arc::clone(&device_error);
+ device.set_device_lost_callback(move |reason, message| {
+ if let Ok(mut stored) = lost_message.lock() {
+ *stored = format!("{reason:?}: {message}");
+ }
lost_flag.store(true, Ordering::SeqCst);
});
+ // The default handler panics on a lost device; record the report so the render can raise a readable error.
+ let error_flag = Arc::clone(&device_lost);
+ let error_message = Arc::clone(&device_error);
+ device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| {
+ if let Ok(mut stored) = error_message.lock() {
+ stored.push_str(&error.to_string());
+ }
+ error_flag.store(true, Ordering::SeqCst);
+ }));
let lut_bytes: &[u8] = bytemuck::cast_slice(&CYCLES_LUT_WORDS[..]);
let lut_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("cycles LUT binding 12"),
@@ -133,6 +150,7 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Str
backend_name: format!("{:?}", adapter_info.backend),
features: if has_ray_query { "EXPERIMENTAL_RAY_QUERY".to_string() } else { String::new() },
device_lost,
+ device_error,
lut_buffer,
cycles_bind_group_layout,
cycles_pipelines: BoundedCache::new(MAX_CACHED_PIPELINES),
@@ -148,6 +166,15 @@ fn get_cached_device() -> Result<(wgpu::Device, wgpu::Queue, String, String, Str
))
}
+/// The flag and message the device callbacks fill in when the cached device is lost.
+fn cached_device_loss_state() -> Result<(Arc<AtomicBool>, Arc<Mutex<String>>)> {
+ let guard = lock_device_cache();
+ match guard.as_ref() {
+ Some(cached) => Ok((Arc::clone(&cached.device_lost), Arc::clone(&cached.device_error))),
+ None => Err(cache_was_reset_failure()),
+ }
+}
+
fn cache_was_reset_failure() -> napi::Error {
failure("The GPU device was reset while the scene was being prepared. Run the render again.")
}
@@ -231,6 +258,8 @@ pub(crate) struct PreparedCyclesScene {
pub(crate) adapter_name: String,
pub(crate) backend_name: String,
pub(crate) features: String,
+ pub(crate) device_lost: Arc<AtomicBool>,
+ pub(crate) device_error: Arc<Mutex<String>>,
pub(crate) setup_seconds: f64,
pub(crate) compile_seconds: f64,
pub(crate) upload_seconds: f64,
@@ -598,6 +627,7 @@ pub(crate) fn prepare_cycles_scene(
// Use cached device; open once per process.
let (device, queue, adapter_name, backend_name, features) = get_cached_device()?;
+ let (device_lost, device_error) = cached_device_loss_state()?;
let setup_seconds = setup_started.elapsed().as_secs_f64();
let compile_started = Instant::now();
@@ -644,6 +674,8 @@ pub(crate) fn prepare_cycles_scene(
adapter_name,
backend_name,
features,
+ device_lost,
+ device_error,
setup_seconds,
compile_seconds,
upload_seconds,
@@ -653,23 +685,75 @@ pub(crate) fn prepare_cycles_scene(
})
}
+/// How `trace_cycles_samples` splits the samples into dispatches.
+#[derive(Clone, Copy)]
+pub(crate) enum DispatchPlan {
+ /// Every dispatch draws this many samples, as the caller asked.
+ Fixed(u32),
+ /// Each dispatch is sized from the time of the last so none runs past the time budget; no batch is larger than `largest_batch`.
+ TimeBudget { largest_batch: u32 },
+}
+
+pub(crate) struct CyclesTrace {
+ pub(crate) sums: Vec<f32>,
+ pub(crate) traced_seconds: f64,
+ /// The most samples one dispatch drew.
+ pub(crate) largest_batch: u32,
+ /// The longest single dispatch; only measured when the dispatches are sized by time.
+ pub(crate) longest_dispatch_seconds: Option<f64>,
+}
+
+/// Raises the error for a lost device, naming the dispatch that was running and how to lower the work per dispatch.
+fn fail_if_device_lost(
+ prepared: &PreparedCyclesScene,
+ sample_offset: u32,
+ batch_samples: u32,
+ samples: u32,
+) -> Result<()> {
+ if !prepared.device_lost.load(Ordering::SeqCst) {
+ return Ok(());
+ }
+ let detail = prepared.device_error.lock().map(|stored| stored.clone()).unwrap_or_default();
+ let smaller_batch = (batch_samples / 2).max(1);
+ Err(failure(format!(
+ "The GPU device was lost during the render, while drawing samples {sample_offset} to {} of {samples} in one dispatch of \
+ {batch_samples} samples ({}). The driver resets a GPU when a single dispatch runs too long. Lower the work per dispatch: \
+ pass samplesPerDispatch {smaller_batch} or less, or leave it unset so the render splits itself into dispatches of \
+ about {DISPATCH_BUDGET_SECONDS} s, or lower the samples or the resolution. Then run the render again.",
+ sample_offset + batch_samples,
+ if detail.is_empty() { "no detail from the driver".to_string() } else { detail },
+ )))
+}
+
pub(crate) fn trace_cycles_samples(
prepared: &PreparedCyclesScene,
samples: u32,
- samples_per_dispatch: u32,
+ plan: DispatchPlan,
groups: (u32, u32),
accumulation_bytes: u64,
max_seconds: Option<f64>,
-) -> Result<(Vec<f32>, f64)> {
+) -> Result<CyclesTrace> {
let trace_started = Instant::now();
let device = &prepared.device;
let queue = &prepared.queue;
- let step = samples_per_dispatch.max(1);
+ let mut planner = match plan {
+ DispatchPlan::TimeBudget { largest_batch } => Some(DispatchPlanner::new(DISPATCH_BUDGET_SECONDS, largest_batch)),
+ DispatchPlan::Fixed(_) => None,
+ };
let mut sample_offset = 0u32;
let mut dispatch_count = 0u32;
+ let mut largest_batch = 0u32;
+ let mut longest_dispatch_seconds: Option<f64> = None;
while sample_offset < samples {
- let batch_samples = (samples - sample_offset).min(step);
+ let remaining_samples = samples - sample_offset;
+ let batch_samples = match (&planner, plan) {
+ (Some(planner), _) => planner.next_batch(remaining_samples),
+ (None, DispatchPlan::Fixed(step)) => remaining_samples.min(step.max(1)),
+ (None, DispatchPlan::TimeBudget { .. }) => unreachable!("a time budget always has a planner"),
+ };
+ largest_batch = largest_batch.max(batch_samples);
+ let dispatch_started = Instant::now();
let mut dispatch_words = [0u8; 8];
dispatch_words[0..4].copy_from_slice(&sample_offset.to_le_bytes());
dispatch_words[4..8].copy_from_slice(&batch_samples.to_le_bytes());
@@ -689,17 +773,27 @@ pub(crate) fn trace_cycles_samples(
}
queue.submit(Some(encoder.finish()));
+ let dispatch_offset = sample_offset;
sample_offset += batch_samples;
dispatch_count += 1;
- // When no time limit is set poll every 4 dispatches (excluding the final batch, which
- // `read_accumulation` waits on directly) to drain staging belts without stalling each batch.
- let should_poll = if max_seconds.is_some() {
+ // A planned render waits for every dispatch to time it. A fixed one, when no time limit is set, polls every 4
+ // dispatches (excluding the final batch, which `read_accumulation` waits on directly) to drain staging belts
+ // without stalling each batch.
+ let should_poll = if planner.is_some() {
+ true
+ } else if max_seconds.is_some() {
dispatch_count % SAMPLES_PER_WAIT == 0 || sample_offset >= samples
} else {
sample_offset < samples && dispatch_count % SAMPLES_PER_WAIT == 0
};
if should_poll {
let _ = device.poll(wgpu::PollType::wait_indefinitely());
+ fail_if_device_lost(prepared, dispatch_offset, batch_samples, samples)?;
+ if let Some(planner) = planner.as_mut() {
+ let dispatch_seconds = dispatch_started.elapsed().as_secs_f64();
+ planner.record(batch_samples, dispatch_seconds);
+ longest_dispatch_seconds = Some(longest_dispatch_seconds.map_or(dispatch_seconds, |longest| longest.max(dispatch_seconds)));
+ }
if let Some(limit) = max_seconds {
if trace_started.elapsed().as_secs_f64() > limit {
return Err(failure(format!(
@@ -710,8 +804,14 @@ pub(crate) fn trace_cycles_samples(
}
}
- let sums = read_accumulation(device, queue, &prepared.accumulation, accumulation_bytes)?;
- Ok((sums, trace_started.elapsed().as_secs_f64()))
+ let sums = read_accumulation(device, queue, &prepared.accumulation, accumulation_bytes);
+ fail_if_device_lost(prepared, samples.saturating_sub(largest_batch), largest_batch, samples)?;
+ Ok(CyclesTrace {
+ sums: sums?,
+ traced_seconds: trace_started.elapsed().as_secs_f64(),
+ largest_batch,
+ longest_dispatch_seconds,
+ })
}
pub(crate) fn run_cycles_lobe_probe(options: &ProbeCyclesLobeOptions, shader_source: String) -> Result<Vec<f32>> {
diff --git a/packages/render-native/src/lib.rs b/packages/render-native/src/lib.rs
index 0ffe5796..079fedd4 100644
--- a/packages/render-native/src/lib.rs
+++ b/packages/render-native/src/lib.rs
@@ -14,7 +14,7 @@ use crate::host::film::{
texel_accumulation_bytes, texel_workgroups,
};
use crate::host::session::{
- prepare_cycles_scene, prepare_scene, run_cycles_lobe_probe, trace_cycles_samples, trace_samples, JobShape,
+ DispatchPlan, prepare_cycles_scene, prepare_scene, run_cycles_lobe_probe, trace_cycles_samples, trace_samples, JobShape,
};
const KERNEL_SOURCE: &str = include_str!("kernel.wgsl");
@@ -138,6 +138,8 @@ pub struct NativeTraceResult {
pub average_samples: f64,
/// The samples each dispatch drew, as given or as chosen automatically (Cycles kernel only; absent for the legacy kernel).
pub samples_per_dispatch: Option<u32>,
+ /// The longest single dispatch in seconds, measured when dispatches are sized by time (absent for a fixed batch).
+ pub longest_dispatch_seconds: Option<f64>,
pub adapter: String,
pub backend: String,
/// "EXPERIMENTAL_RAY_QUERY" when the device the trace ran on was opened with ray queries (the tracer needs them).
@@ -195,6 +197,7 @@ pub fn trace_scene(input: NativeTraceInput, options: NativeTraceOptions) -> Resu
variance: Float32Array::new(resolved.variance),
average_samples: resolved.average_samples,
samples_per_dispatch: None,
+ longest_dispatch_seconds: None,
adapter: prepared.adapter_name,
backend: prepared.backend_name,
features: prepared.features,
@@ -221,34 +224,24 @@ pub fn trace_scene_cycles(input: NativeTraceInputV2, options: NativeCyclesTraceO
accumulation_bytes,
)?;
let groups = image_workgroups(options.width, options.height);
- let samples_per_dispatch = match options.samples_per_dispatch {
- Some(n) => n.max(1),
- None => {
- // Auto-select a batch that keeps GPU utilisation high without coarse progress steps.
- // With a wall-clock limit use step=1 for tight responsiveness below 2 s; otherwise
- // batch by pixel count so each dispatch is a worthwhile chunk.
- let pixel_count = options.width as u64 * options.height as u64;
- let batch = match options.max_seconds {
- Some(limit) if limit < 2.0 => 1,
- _ => {
- if pixel_count <= 1_000_000 {
- options.samples.min(32)
- } else {
- options.samples.min(16)
- }
- }
- };
- batch.max(1)
- }
+ // Unless the caller fixes the batch, each dispatch is sized from the time of the last so none runs long enough for the
+ // driver to reset the device. A wall-clock limit under 2 s keeps one sample per dispatch for tight responsiveness.
+ let plan = match options.samples_per_dispatch {
+ Some(samples_per_dispatch) => DispatchPlan::Fixed(samples_per_dispatch.max(1)),
+ None => match options.max_seconds {
+ Some(limit) if limit < 2.0 => DispatchPlan::Fixed(1),
+ _ => DispatchPlan::TimeBudget { largest_batch: options.samples.max(1) },
+ },
};
- let (sums, traced_seconds) = trace_cycles_samples(
+ let trace = trace_cycles_samples(
&prepared,
options.samples,
- samples_per_dispatch,
+ plan,
groups,
accumulation_bytes,
options.max_seconds,
)?;
+ let (sums, traced_seconds, samples_per_dispatch) = (trace.sums, trace.traced_seconds, trace.largest_batch);
let resolved = resolve_image(&sums, pixel_count);
Ok(NativeTraceResult {
rgba: Float32Array::new(resolved.rgba),
@@ -258,6 +251,7 @@ pub fn trace_scene_cycles(input: NativeTraceInputV2, options: NativeCyclesTraceO
variance: Float32Array::new(resolved.variance),
average_samples: resolved.average_samples,
samples_per_dispatch: Some(samples_per_dispatch),
+ longest_dispatch_seconds: trace.longest_dispatch_seconds,
adapter: prepared.adapter_name,
backend: prepared.backend_name,
features: prepared.features,
diff --git a/packages/render-native/tests/dispatch.rs b/packages/render-native/tests/dispatch.rs
new file mode 100644
index 00000000..652b5cc0
--- /dev/null
+++ b/packages/render-native/tests/dispatch.rs
@@ -0,0 +1,65 @@
+//! Tests for how a render is split into dispatches. The planner is included by path because the napi library does not
+//! link into a test binary.
+#![allow(dead_code)]
+
+#[path = "../src/host/dispatch.rs"]
+mod dispatch;
+
+use dispatch::DispatchPlanner;
+
+fn batches_for(samples: u32, seconds_per_sample: f64, largest_batch: u32) -> Vec<u32> {
+ let mut planner = DispatchPlanner::new(0.5, largest_batch);
+ let mut remaining = samples;
+ let mut batches = Vec::new();
+ while remaining > 0 {
+ let batch = planner.next_batch(remaining);
+ planner.record(batch, batch as f64 * seconds_per_sample);
+ remaining -= batch;
+ batches.push(batch);
+ }
+ batches
+}
+
+#[test]
+fn the_first_dispatch_draws_one_sample() {
+ assert_eq!(DispatchPlanner::new(0.5, 64).next_batch(256), 1);
+}
+
+#[test]
+fn every_sample_is_drawn_exactly_once() {
+ let batches = batches_for(256, 0.02, 256);
+ assert_eq!(batches.iter().sum::<u32>(), 256);
+}
+
+#[test]
+fn a_slow_scene_gets_small_dispatches_that_fit_the_budget() {
+ let batches = batches_for(256, 0.1, 256);
+ assert!(batches.iter().all(|batch| *batch as f64 * 0.1 <= 0.5 + 1e-9), "{batches:?}");
+}
+
+#[test]
+fn a_fast_scene_grows_by_a_bounded_factor() {
+ let batches = batches_for(256, 0.0001, 256);
+ assert_eq!(&batches[..4], &[1, 4, 16, 64]);
+}
+
+#[test]
+fn a_scene_slower_than_the_budget_still_draws_one_sample_at_a_time() {
+ let batches = batches_for(4, 3.0, 4);
+ assert_eq!(batches, vec![1, 1, 1, 1]);
+}
+
+#[test]
+fn the_batch_never_passes_the_largest_allowed() {
+ let batches = batches_for(256, 0.0001, 8);
+ assert!(batches.iter().all(|batch| *batch <= 8));
+}
+
+#[test]
+fn a_dispatch_that_turns_slower_shrinks_the_next_one() {
+ let mut planner = DispatchPlanner::new(0.5, 256);
+ planner.record(1, 0.01);
+ assert_eq!(planner.next_batch(256), 4);
+ planner.record(4, 2.0);
+ assert_eq!(planner.next_batch(256), 1);
+}