LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
Issue #1764: running out of device memory falls back without stopping the GPU
Antonio Sokic99d4de32c7e9
diff --git a/crates/gpu/src/ctx.rs b/crates/gpu/src/ctx.rsindex 7bbbbc68d..b47dab4a0 100644--- a/crates/gpu/src/ctx.rs+++ b/crates/gpu/src/ctx.rs@@ -3,8 +3,6 @@use std::collections::HashMap;use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};-use wgpu::util::DeviceExt;-/// A kernel module: WGSL source, its storage bindings (name, writable, element type) after the/// parameter block `P` (binding 0), and its entry points.struct Module {@@ -520,13 +518,37 @@ impl Gpu {}// (dropping the view schedules the copy)drop(view);- return Buf { buf: Some(Tracked::new(buf)), len };}+ // no view: the buffer couldn't be allocated (out of device memory). The error scope has recorded why and+ // the render fails over to the CPU; allocating it again would only fail again (issue #1764)+ return Buf { buf: Some(Tracked::new(buf)), len };}- let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytes, usage });+ let buf = self.init_buffer(bytes, usage);Buf { buf: Some(Tracked::new(buf)), len }}+ /// A buffer holding `contents` (zero-padded to the copy alignment), written through the queue. Not+ /// `create_buffer_init`: it maps the new buffer, and mapping one that couldn't be allocated (out of device+ /// memory) panics inside wgpu instead of raising an error the render's scopes catch (issue #1764).+ fn init_buffer(&self, contents: &[u8], usage: wgpu::BufferUsages) -> wgpu::Buffer {+ let align = wgpu::COPY_BUFFER_ALIGNMENT as usize;+ let size = contents.len().max(align).next_multiple_of(align);+ let buf = self.device.create_buffer(&wgpu::BufferDescriptor {+ label: None,+ size: size as u64,+ usage: usage | wgpu::BufferUsages::COPY_DST,+ mapped_at_creation: false,+ });+ if size == contents.len() {+ self.queue.write_buffer(&buf, 0, contents);+ } else {+ let mut padded = contents.to_vec();+ padded.resize(size, 0);+ self.queue.write_buffer(&buf, 0, &padded);+ }+ buf+ }+fn over_limit(&self, size: u64) {fail(FailKind::Limit,@@ -570,11 +592,7 @@ impl Gpu {return;};assert_eq!(bufs.len(), k.nbuf, "{name}: binding count");- let params = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {- label: None,- contents: bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }),- usage: wgpu::BufferUsages::STORAGE,- });+ let params = self.init_buffer(bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }), wgpu::BufferUsages::STORAGE);let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: params.as_entire_binding() }];for (i, b) in bufs.iter().enumerate() {let buf = b.map(|b| b.raw()).unwrap_or(&self.dummy);@@ -902,6 +920,65 @@ mod tests {println!("CHILD: survived");}+ /// A render that runs out of device memory falls back to the CPU and leaves the GPU in use: the upload+ /// whose buffer couldn't be allocated used to try again through `create_buffer_init`, which panicked+ /// mapping the second failed buffer, and the panic stopped GPU use for the session (issue #1764).+ #[test]+ fn running_out_of_device_memory_falls_back_without_stopping_the_gpu() {+ if run_child("out_of_memory_child").is_none() {+ lightcraft_testkit::skip!(Gpu, "no GPU device, or its memory never ran out");+ }+ }++ #[test]+ #[ignore = "run in its own process by running_out_of_device_memory_falls_back_without_stopping_the_gpu"]+ fn out_of_memory_child() {+ let Some(g) = crate::device() else {+ println!("CHILD: no GPU");+ return;+ };+ // hold device memory until an allocation fails: 1 GiB at a time, then smaller, so little is left+ let mut held = Vec::new();+ let mut full = false;+ for size in [1u64 << 30, 1 << 28, 1 << 26] {+ for _ in 0..128 {+ let scope = g.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory);+ let usage = wgpu::BufferUsages::STORAGE;+ let buf = g.device.create_buffer(&wgpu::BufferDescriptor { label: None, size, usage, mapped_at_creation: false });+ if pollster::block_on(scope.pop()).is_some() {+ full = true;+ break;+ }+ held.push(buf);+ }+ }+ if !full {+ println!("CHILD: no GPU (its memory never ran out)");+ return;+ }+ // a render that needs more than is left: a larger one when the driver still finds room for the first+ let s = lightcraft_develop::DevelopSettings::default();+ let mut ran_out = false;+ for (w, h) in [(4000, 3000), (8000, 6000)] {+ let src = std::sync::Arc::new(lightcraft_raster::Rgb32f { width: w, height: h, data: vec![[0.18, 0.18, 0.18]; w * h] });+ let req = lightcraft_pipeline::RenderRequest::fit(w, h);+ match crate::try_render(&src, &lightcraft_pipeline::SourceInfo::default(), &s, &req, None) {+ Ok(_) => continue,+ Err(crate::Fallback::Failed(reason)) => assert!(reason.contains("out of device memory"), "{reason}"),+ Err(crate::Fallback::NotTried) => panic!("the GPU wasn't tried"),+ }+ ran_out = true;+ break;+ }+ assert_eq!(crate::unavailable_reason(), None, "the GPU stays in use");+ drop(held);+ if !ran_out {+ println!("CHILD: no GPU (the renders found room anyway)");+ return;+ }+ println!("CHILD: survived");+ }+/// Empties the buffer pool when dropped, as the UI does (`trim_pool(0)`) and as returning a/// buffer to a full pool does.struct TrimOnDrop(&'static super::Gpu);