SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Lost GPU: the old process exits instead of hanging in the driver, after the settings are saved (#9044) (#9054)
froglemonade2fc499441e27f
diff --git a/apps/solidcraft/src/device_lost.rs b/apps/solidcraft/src/device_lost.rsindex b892723c3c..2f598c69c1 100644--- a/apps/solidcraft/src/device_lost.rs+++ b/apps/solidcraft/src/device_lost.rs@@ -5,12 +5,25 @@//! for recovery as a forced quit does, and starts again ([`restart`]), and the new start says what//! happened ([`note`]). A device lost again soon after such a restart starts the next one on the//! software renderer, which doesn't depend on the graphics driver.+//!+//! The old process ends without tearing down its GPU objects ([`end`]): on a lost NVIDIA device+//! destroying the window's swap chain busy-waits in the driver and never returns (#9044).use crate::startup;/// Set on the start after a lost device: when the device was lost (seconds since 1970).pub const LOST_ENV: &str = "SOLIDCRAFT_DEVICE_LOST";+/// Set on the start after a lost device: why it was lost, as wgpu said it. The old process's+/// session journal, which recorded it, is gone by then.+pub const REASON_ENV: &str = "SOLIDCRAFT_DEVICE_LOST_REASON";++/// The longest reason passed on ([`REASON_ENV`]) and shown, in characters.+const REASON_CHARS: usize = 300;++/// How long the process may take to end ([`end`]) before it is aborted.+const END_GRACE: std::time::Duration = std::time::Duration::from_secs(3);+/// A loss this recent (seconds) counts for the start after it: its note, and a second loss/// within it starts on the software renderer. Inherited by later starts, an older stamp is/// ignored.@@ -32,27 +45,54 @@ pub fn restart_on_software(after_recent_loss: bool, cpu_renderer: bool) -> boolafter_recent_loss || cpu_renderer}+/// `reason` on one line, at most [`REASON_CHARS`] long; None when there is nothing to say.+pub fn reason(reason: Option<&str>) -> Option<String> {+ let line = reason?.split_whitespace().collect::<Vec<_>>().join(" ");+ (!line.is_empty()).then(|| line.chars().take(REASON_CHARS).collect())+}+/// What the start after a lost device tells the user, `software` when it runs on the software-/// renderer.-pub fn note(software: bool) -> String {+/// renderer, with what the driver reported ([`REASON_ENV`]).+pub fn note(software: bool, reason: Option<&str>) -> String {let mut note = String::from("The graphics device was reset (the graphics driver stopped responding), so SolidCraft closed and started again. Unsaved changes were kept for recovery.",);if software {note.push_str(" This start uses software rendering, which doesn't depend on the graphics driver.");}+ if let Some(reason) = self::reason(reason) {+ note.push_str(&format!(" The graphics driver reported: {reason}"));+ }note}-/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at`, on the software-/// renderer when `software`.-pub fn restart(software: bool, lost_at: u64) {- let Ok(exe) = std::env::current_exe() else { return };+/// The command that starts `exe` again with `args`, the loss stamped at `lost_at` and its+/// `reason`, on the software renderer when `software`.+fn command(+ exe: std::path::PathBuf,+ args: impl IntoIterator<Item = std::ffi::OsString>,+ software: bool,+ lost_at: u64,+ reason: &str,+) -> std::process::Command {let mut command = std::process::Command::new(exe);- command.args(std::env::args_os().skip(1)).env(LOST_ENV, lost_at.to_string());+ command.args(args).env(LOST_ENV, lost_at.to_string());+ match self::reason(Some(reason)) {+ Some(reason) => command.env(REASON_ENV, reason),+ // Not one inherited from an earlier loss.+ None => command.env_remove(REASON_ENV),+ };if software {command.env(startup::RENDERER_ENV, "cpu");}+ command+}++/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at` and its `reason`,+/// on the software renderer when `software`.+pub fn restart(software: bool, lost_at: u64, reason: &str) {+ let Ok(exe) = std::env::current_exe() else { return };+ let mut command = command(exe, std::env::args_os().skip(1), software, lost_at, reason);#[cfg(windows)]{use std::os::windows::process::CommandExt;@@ -64,6 +104,23 @@ pub fn restart(software: bool, lost_at: u64) {}}+/// End the process now, without tearing down what is left on the lost device: the window's swap+/// chain, the viewport's buffers and pipelines. On NVIDIA's Vulkan driver destroying the swap+/// chain of a lost device busy-waits forever, which left the old window frozen at 100% of a core+/// beside the new one (#9044). Should the exit itself get stuck in a library's exit handler, the+/// process is aborted after [`END_GRACE`].+pub fn end() -> ! {+ let watchdog = std::thread::Builder::new().name("solidcraft-exit".into()).spawn(|| {+ std::thread::sleep(END_GRACE);+ eprintln!("SolidCraft graphics: still closing after the device was lost; aborting");+ std::process::abort();+ });+ if let Err(e) = watchdog {+ log::warn!("graphics: no exit watchdog: {e}");+ }+ std::process::exit(0)+}+#[cfg(test)]mod tests {/// Only a recent, well-formed stamp counts: a later start that inherited it says nothing.@@ -84,8 +141,34 @@ mod tests {assert!(!super::restart_on_software(false, false));assert!(super::restart_on_software(true, false));assert!(super::restart_on_software(false, true));- assert!(super::note(true).contains("software rendering"));- assert!(!super::note(false).contains("software rendering"));- assert!(super::note(false).contains("kept for recovery"));+ assert!(super::note(true, None).contains("software rendering"));+ assert!(!super::note(false, None).contains("software rendering"));+ assert!(super::note(false, None).contains("kept for recovery"));+ }++ /// The start after a loss is told why (#9044): the old process's journal held the reason, and+ /// it is deleted as that process closes.+ #[test]+ fn the_next_start_is_told_why_the_device_was_lost() {+ let exe = std::path::PathBuf::from("solidcraft");+ let args = ["--control".into(), "7979".into()];+ let command = super::command(exe.clone(), args, true, 1_000, "Lost: Device lost\n (vkQueueSubmit)");+ let env = |command: &std::process::Command, name: &str| {+ command.get_envs().find(|(k, _)| *k == name).map(|(_, v)| v.map(|v| v.to_string_lossy().into_owned()))+ };+ assert_eq!(env(&command, super::LOST_ENV), Some(Some("1000".into())));+ assert_eq!(env(&command, super::REASON_ENV), Some(Some("Lost: Device lost (vkQueueSubmit)".into())), "on one line");+ assert_eq!(env(&command, crate::startup::RENDERER_ENV), Some(Some("cpu".into())));+ assert_eq!(command.get_args().collect::<Vec<_>>(), ["--control", "7979"]);+ // No reason: none inherited from an earlier loss either.+ let command = super::command(exe, Vec::new(), false, 1_000, " ");+ assert_eq!(env(&command, super::REASON_ENV), Some(None));+ assert_eq!(env(&command, crate::startup::RENDERER_ENV), None);++ let note = super::note(false, Some("Lost: Device lost"));+ assert!(note.ends_with("The graphics driver reported: Lost: Device lost"), "{note}");+ assert!(!super::note(false, Some(" \n ")).contains("reported"));+ let long = "x".repeat(1_000);+ assert_eq!(super::reason(Some(&long)).map(|r| r.chars().count()), Some(300));}}diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rsindex d9c6fbc9f6..c4cfcf40f8 100644--- a/apps/solidcraft/src/main.rs+++ b/apps/solidcraft/src/main.rs@@ -204,13 +204,29 @@ impl eframe::App for App {}solidcraft_ui_egui::recovery_ui::close(&mut self.solve);solidcraft_ui_egui::open_elsewhere::close(&mut self.solve);- // Before a restart below starts the next instance, which would otherwise get nothing from- // the 3D mouse driver (`solidcraft_3dx_client::stop`).+ // Before a restart (`Drop for App`) starts the next instance, which would otherwise get+ // nothing from the 3D mouse driver (`solidcraft_3dx_client::stop`).#[cfg(target_os = "macos")]solidcraft_3dx_client::stop();solidcraft_engine::crash::end_session();- if let Some(lost_at) = self.gpu.as_ref().and_then(|g| g.lost_at) {- device_lost::restart(self.gpu.as_ref().is_some_and(|g| g.restart_on_software), lost_at);+ // The next start (after a lost device, or into an update) begins once the app is dropped+ // (`Drop for App`), when the settings are on disk.+ }+}++impl Drop for App {+ /// eframe drops its settings storage first, which waits for `app.ron` to be written, then the+ /// app, then the window and its GPU surface. So the next start begins here: started from+ /// `on_exit`, it read `app.ron` while the old process was still writing it, found it+ /// unreadable and moved it aside, resetting every setting (#9044).+ fn drop(&mut self) {+ if let Some(gpu) = &self.gpu+ && let Some(lost_at) = gpu.lost_at+ {+ device_lost::restart(gpu.restart_on_software, lost_at, &gpu.reason);+ // The window's GPU surface is still to be dropped, and on a lost device that may+ // never return (#9044).+ device_lost::end();}// An update asked to restart into the new version or to run its installer.if let Some((program, args)) = solidcraft_ui_egui::update::take_launch(&mut self.solve) {@@ -238,6 +254,8 @@ struct LostWatch {restart_on_software: bool,/// When the loss was seen: the app is closing to start again.lost_at: Option<u64>,+ /// Why, as wgpu said it: passed to the next start ([`device_lost::REASON_ENV`]).+ reason: String,}impl App {@@ -251,6 +269,7 @@ impl App {eprintln!("SolidCraft graphics: the device was lost ({reason}); starting again");solidcraft_engine::crash::phase(&format!("graphics device lost: {reason}"));gpu.lost_at = Some(device_lost::now_secs());+ gpu.reason = reason;self.solve.quit_requested = true;}}@@ -839,7 +858,11 @@ fn main() -> eframe::Result {// The launch before this one closed because its graphics device was lost (`App::watch_device`).let after_loss = device_lost::recent(std::env::var(device_lost::LOST_ENV).ok().as_deref(), device_lost::now_secs());if after_loss {- notes.push(device_lost::note(cpu_renderer));+ let reason = std::env::var(device_lost::REASON_ENV).ok();+ if let Some(reason) = device_lost::reason(reason.as_deref()) {+ log::warn!("graphics: the last launch lost its device ({reason})");+ }+ notes.push(device_lost::note(cpu_renderer, reason.as_deref()));}if safe_mode {notes.push("Safe mode: default settings (yours are kept for the next normal start), no recovery offer, software rendering.".into());@@ -979,6 +1002,7 @@ fn main() -> eframe::Result {lost: rs.device_lost.clone(),restart_on_software: device_lost::restart_on_software(after_loss, cpu_renderer),lost_at: None,+ reason: String::new(),});if !safe_mode {app.preferences.graphics_adapter =diff --git a/docs/book/src/guide/troubleshooting.md b/docs/book/src/guide/troubleshooting.mdindex edfe0c9b49..685ee7ca21 100644--- a/docs/book/src/guide/troubleshooting.md+++ b/docs/book/src/guide/troubleshooting.md@@ -158,9 +158,12 @@ When the graphics driver stops responding, the system resets it (on Windows, "Distopped responding and has recovered"), and the graphics device SolidCraft draws with is gonefor good: the window can't draw anything more. SolidCraft then closes, keeps your unsavedchanges for recovery as it does after a crash, and starts again; the new window says what-happened and offers the changes back. If the device is reset again within ten minutes of that-restart, the next start uses software rendering, which doesn't depend on the graphics driver.-Updating the graphics driver from the GPU maker usually stops the resets.+happened, with what the graphics driver reported, and offers the changes back. The old window+closes on its own; SolidCraft doesn't wait for the driver to release it. If the device is reset+again within ten minutes of that restart, the next start uses software rendering, which doesn't+depend on the graphics driver. Updating the graphics driver from the GPU maker usually stops the+resets. Laptops that lose the device on waking from sleep (often with an NVIDIA GPU that doesn't+drive the screen) can keep SolidCraft on the integrated GPU instead: Preferences › Graphics.## A design saved by 0.10 won't open in an older version