SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Windows: start Direct3D 12 first, and Vulkan only without a D3D12 GPU (#1190)
bherbruck95ea05e7c80f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex e1730e2655..841b505e2e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -306,11 +306,38 @@ jobs:- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2with:key: windows-fxc- - run: cargo test -p solidcraft-gpu --profile ci --locked+ # One test at a time: WARP's shader JIT crashes now and then (a null read in+ # d3d10warp.dll's JIT optimizer, about 1 run in 60) when several devices compile compute+ # pipelines at once, which the parallel tests do. Minidumps showed it in WARP itself, not+ # in SolidCraft or FXC; serial runs never hit it.+ - run: cargo test -p solidcraft-gpu --profile ci --locked -- --test-threads=1++ # The Windows app starts D3D12 first and only the backend it uses (a Vulkan driver crashed the+ # app at startup on Intel). Start the real app on the runner (its D3D12 adapter is WARP) with+ # several saved settings and safe mode, and check each draws a frame and quits cleanly.+ windows-startup:+ name: Windows app startup+ needs: changes+ if: needs.changes.outputs.native == 'true'+ runs-on: windows-latest+ timeout-minutes: 60+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - uses: dtolnay/rust-toolchain@stable+ - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2+ with:+ key: windows-startup+ - name: Backend order and probing on Windows+ run: cargo test -p solidcraft --profile ci --locked -- startup::tests+ - name: Build the app+ run: cargo build -p solidcraft --profile ci --locked+ - name: Start the app under different settings+ shell: pwsh+ run: tools/windows-startup-smoke.ps1 -Exe target/ci/solidcraft.exe# Preserve the existing check name for branch protection, including docs-only PRs.ci:- needs: [changes, checks, clippy, build-tests, tests, doc-tests, oracle, windows-fxc]+ needs: [changes, checks, clippy, build-tests, tests, doc-tests, oracle, windows-fxc, windows-startup]if: always()runs-on: ubuntu-24.04steps:@@ -325,6 +352,7 @@ jobs:DOCS: ${{ needs.doc-tests.result }}ORACLE: ${{ needs.oracle.result }}WINDOWS_FXC: ${{ needs.windows-fxc.result }}+ WINDOWS_STARTUP: ${{ needs.windows-startup.result }}run: |test "$CHANGES" = successtest "$CHECKS" = success@@ -335,6 +363,7 @@ jobs:test "$DOCS" = successtest "$ORACLE" = successtest "$WINDOWS_FXC" = success+ test "$WINDOWS_STARTUP" = successfiwasm:diff --git a/apps/solidcraft/Cargo.toml b/apps/solidcraft/Cargo.tomlindex 45c60cba2a..55825d62cd 100644--- a/apps/solidcraft/Cargo.toml+++ b/apps/solidcraft/Cargo.toml@@ -29,6 +29,8 @@ log = { workspace = true }rfd = { workspace = true }# Checks eframe's settings file (app.ron) before eframe reads it (src/startup.rs).ron = "0.12"+# Waits for wgpu's adapter list when choosing the graphics backend before the window opens.+pollster = { workspace = true }getrandom = "0.3"# The update check (docs/updates.md): HTTPS with rustls, certificates from the operating system.ureq = { version = "3.4", default-features = false, features = ["rustls-no-provider", "platform-verifier"] }@@ -50,8 +52,5 @@ block2 = "0.6"[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies]winit = { version = "0.30", default-features = false, features = ["x11"] }-[dev-dependencies]-pollster = { workspace = true }-[build-dependencies]embed-resource = "3.0.11"diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rsindex 8609ddbe28..4896e2aa5c 100644--- a/apps/solidcraft/src/main.rs+++ b/apps/solidcraft/src/main.rs@@ -276,11 +276,19 @@ fn adopt_settings(Ok(())}+/// `graphics.json`, read before the window opens.+fn graphics() -> Option<serde_json::Value> {+ graphics_path().and_then(|p| std::fs::read_to_string(p).ok()).and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())+}++/// The graphics backend preference (`graphics.json`'s `backend`: `auto`, `dx12`, `vulkan`, `gl`).+fn backend_preference() -> Option<String> {+ graphics().and_then(|v| v.get("backend").and_then(serde_json::Value::as_str).map(str::to_string))+}+/// Anti-aliasing samples from Preferences (4 when unset).fn msaa() -> u16 {- graphics_path()- .and_then(|p| std::fs::read_to_string(p).ok())- .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())+ graphics().and_then(|v| v.get("msaa").and_then(serde_json::Value::as_u64)).and_then(|m| u16::try_from(m).ok()).filter(|m| [1, 2, 4, 8].contains(m))@@ -399,7 +407,7 @@ fn main() -> eframe::Result {settings.as_ref().map(|d| d.join("app.ron"))},persist_window: !safe_mode,- wgpu_options: wgpu_options(cpu_renderer),+ wgpu_options: wgpu_options(cpu_renderer, if safe_mode { None } else { backend_preference() }.as_deref()),..Default::default()};#[cfg(all(unix, not(target_os = "macos")))]@@ -504,11 +512,20 @@ fn main() -> eframe::Result {}}-/// The renderer settings: with `cpu`, the software adapter (WARP, lavapipe, llvmpipe).-fn wgpu_options(cpu: bool) -> eframe::egui_wgpu::WgpuConfiguration {+/// The renderer settings: with `cpu`, the software adapter (WARP, lavapipe, llvmpipe). The+/// backends come from `startup::choose_backends` (on Windows D3D12 first, and only the backend+/// in use is started); `preference` is `graphics.json`'s `backend`.+fn wgpu_options(cpu: bool, preference: Option<&str>) -> eframe::egui_wgpu::WgpuConfiguration {let mut options = eframe::egui_wgpu::WgpuConfiguration::default();- if cpu && let eframe::egui_wgpu::WgpuSetup::CreateNew(setup) = &mut options.wgpu_setup {- setup.native_adapter_selector = Some(std::sync::Arc::new(startup::pick_cpu_adapter));+ if let eframe::egui_wgpu::WgpuSetup::CreateNew(setup) = &mut options.wgpu_setup {+ let env_set = std::env::var(startup::WGPU_BACKEND_ENV).is_ok_and(|v| !v.trim().is_empty());+ if let Some(backends) = startup::choose_backends(cfg!(target_os = "windows"), env_set, cpu, preference, startup::has_hardware_adapter) {+ eprintln!("SolidCraft graphics: starting {backends:?}");+ setup.instance_descriptor.backends = backends;+ }+ if cpu {+ setup.native_adapter_selector = Some(std::sync::Arc::new(startup::pick_cpu_adapter));+ }}options}diff --git a/apps/solidcraft/src/startup.rs b/apps/solidcraft/src/startup.rsindex d24fd52ae4..0fda0365b1 100644--- a/apps/solidcraft/src/startup.rs+++ b/apps/solidcraft/src/startup.rs@@ -110,6 +110,68 @@ pub fn pick_cpu_adapter(adapters: &[eframe::wgpu::Adapter], surface: Option<&efr})}+/// `WGPU_BACKEND`: when set, wgpu picks the backends from it and the order below is not used.+pub const WGPU_BACKEND_ENV: &str = "WGPU_BACKEND";++/// The graphics backends to start with, or `None` to leave it to wgpu (`WGPU_BACKEND`, or all+/// the backends it was built with).+///+/// On Windows, D3D12 comes first, then Vulkan, then OpenGL, and only the first one with a+/// hardware adapter is started at all. wgpu would otherwise start Vulkan, D3D12 and OpenGL+/// together and prefer Vulkan, and starting Vulkan loads every installed Vulkan driver: Intel's+/// (`igvk64.dll`) crashed SolidCraft at startup that way, inside the driver, before anything+/// could be reported. D3D12 is Windows' own graphics API, the one its drivers are tested with.+/// `has_hardware(backends)` says whether those backends have an adapter that isn't software+/// (it is only asked about the backends being tried, one at a time).+///+/// - `WGPU_BACKEND` set (`env_set`): `None`, whatever it says wins.+/// - The software renderer (`cpu`, safe mode): D3D12 alone on Windows (WARP); `None` elsewhere.+/// - `preference` (`graphics.json`'s `backend`: `dx12`, `vulkan`, `gl` or a comma list; `auto`+/// or empty for the order above): those backends, on any system.+/// - Elsewhere than Windows: `None`.+/// - No hardware adapter at all: D3D12 (WARP, Windows' software renderer).+pub fn choose_backends(+ windows: bool,+ env_set: bool,+ cpu: bool,+ preference: Option<&str>,+ mut has_hardware: impl FnMut(eframe::wgpu::Backends) -> bool,+) -> Option<eframe::wgpu::Backends> {+ use eframe::wgpu::Backends;+ if env_set {+ return None;+ }+ if cpu {+ return windows.then_some(Backends::DX12);+ }+ if let Some(p) = preference.map(str::trim).filter(|p| !p.is_empty() && !p.eq_ignore_ascii_case("auto")) {+ let chosen = Backends::from_comma_list(p);+ if !chosen.is_empty() {+ return Some(chosen);+ }+ log::warn!("graphics: unknown backend preference `{p}`; using the default order");+ }+ if !windows {+ return None;+ }+ Some(WINDOWS_ORDER.into_iter().find(|b| has_hardware(*b)).unwrap_or(Backends::DX12))+}++/// The order backends are tried in on Windows.+pub const WINDOWS_ORDER: [eframe::wgpu::Backends; 3] = [eframe::wgpu::Backends::DX12, eframe::wgpu::Backends::VULKAN, eframe::wgpu::Backends::GL];++/// Whether `backends` (started on their own, nothing else) have a hardware adapter.+pub fn has_hardware_adapter(backends: eframe::wgpu::Backends) -> bool {+ let mut desc = eframe::wgpu::InstanceDescriptor::new_without_display_handle();+ desc.backends = backends;+ desc.flags = eframe::wgpu::InstanceFlags::from_build_config().with_env();+ let instance = eframe::wgpu::Instance::new(desc);+ let adapters = pollster::block_on(instance.enumerate_adapters(backends));+ let found: Vec<String> = adapters.iter().map(|a| format!("{} ({:?})", a.get_info().name, a.get_info().device_type)).collect();+ log::info!("graphics: {backends:?} adapters: {found:?}");+ adapters.iter().any(|a| a.get_info().device_type != eframe::wgpu::DeviceType::Cpu)+}+/// The message shown when no window could be opened at all.pub fn no_window_message(error: &str, settings: Option<&Path>) -> String {let folder = settings.map(|d| d.display().to_string()).unwrap_or_else(|| "the SolidCraft settings folder".into());@@ -182,6 +244,65 @@ mod tests {let _ = std::fs::remove_dir_all(&d);}+ #[test]+ fn windows_starts_d3d12_first_and_vulkan_only_when_d3d12_has_no_gpu() {+ use eframe::wgpu::Backends;+ let probe = |with: &'static [Backends]| {+ move |b: Backends| {+ assert!(b == Backends::DX12 || b == Backends::VULKAN || b == Backends::GL, "one backend at a time: {b:?}");+ with.contains(&b)+ }+ };+ // D3D12 works: Vulkan and OpenGL are never even asked about (starting Vulkan loads+ // its drivers, which is what crashed).+ let mut asked = Vec::new();+ let chosen = choose_backends(true, false, false, None, |b| {+ asked.push(b);+ b == Backends::DX12+ });+ assert_eq!(chosen, Some(Backends::DX12));+ assert_eq!(asked, [Backends::DX12]);+ // Everything works: still D3D12 alone.+ assert_eq!(choose_backends(true, false, false, None, probe(&[Backends::DX12, Backends::VULKAN, Backends::GL])), Some(Backends::DX12));+ // No D3D12 hardware adapter: Vulkan, else OpenGL, else D3D12's software adapter (WARP).+ assert_eq!(choose_backends(true, false, false, None, probe(&[Backends::VULKAN, Backends::GL])), Some(Backends::VULKAN));+ assert_eq!(choose_backends(true, false, false, None, probe(&[Backends::GL])), Some(Backends::GL));+ assert_eq!(choose_backends(true, false, false, None, probe(&[])), Some(Backends::DX12));+ // `auto` and blanks are the default order.+ assert_eq!(choose_backends(true, false, false, Some(" auto "), probe(&[Backends::DX12])), Some(Backends::DX12));+ assert_eq!(choose_backends(true, false, false, Some(""), probe(&[Backends::GL])), Some(Backends::GL));+ }++ #[test]+ fn wgpu_backend_the_preference_and_the_software_renderer_override_the_order() {+ use eframe::wgpu::Backends;+ let never = |b: Backends| -> bool { panic!("no probing expected, asked about {b:?}") };+ // WGPU_BACKEND: wgpu reads it itself, nothing is probed.+ assert_eq!(choose_backends(true, true, false, Some("gl"), never), None);+ assert_eq!(choose_backends(true, true, true, None, never), None);+ // The software renderer (safe mode, the retry): WARP through D3D12 alone, never Vulkan.+ assert_eq!(choose_backends(true, false, true, Some("vulkan"), never), Some(Backends::DX12));+ assert_eq!(choose_backends(false, false, true, None, never), None);+ // The preference, on any system, without probing.+ assert_eq!(choose_backends(true, false, false, Some("vulkan"), never), Some(Backends::VULKAN));+ assert_eq!(choose_backends(true, false, false, Some("GL"), never), Some(Backends::GL));+ assert_eq!(choose_backends(true, false, false, Some("dx12,vulkan"), never), Some(Backends::DX12 | Backends::VULKAN));+ assert_eq!(choose_backends(false, false, false, Some("vulkan"), never), Some(Backends::VULKAN));+ // An unknown preference is the default order.+ assert_eq!(choose_backends(true, false, false, Some("glide"), |b| b == Backends::DX12), Some(Backends::DX12));+ // Elsewhere: wgpu's own choice.+ assert_eq!(choose_backends(false, false, false, None, never), None);+ assert_eq!(choose_backends(false, false, false, Some("auto"), never), None);+ }++ #[test]+ fn probing_a_backend_alone_never_panics() {+ // Whatever this machine has (often nothing): an answer, not a crash.+ for b in WINDOWS_ORDER {+ let _ = has_hardware_adapter(b);+ }+ }+#[test]fn the_message_names_the_settings_folder_and_safe_mode() {let m = no_window_message("no adapter", Some(Path::new("/x/SolidCraft")));diff --git a/crates/ui-egui/src/prefs.rs b/crates/ui-egui/src/prefs.rsindex 840108995a..b8ab371362 100644--- a/crates/ui-egui/src/prefs.rs+++ b/crates/ui-egui/src/prefs.rs@@ -629,7 +629,14 @@ pub fn show_mm_bare(app: &SolveApp, v: f64) -> String {/// change while it is open).fn save_graphics(app: &SolveApp) {if let Some(path) = &app.services.graphics_path {- let _ = std::fs::write(path, json!({ "msaa": app.preferences.msaa }).to_string());+ // Other settings in the file (the graphics `backend`) stay as they are.+ let mut graphics = std::fs::read_to_string(path)+ .ok()+ .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())+ .filter(serde_json::Value::is_object)+ .unwrap_or_else(|| json!({}));+ graphics["msaa"] = json!(app.preferences.msaa);+ let _ = std::fs::write(path, graphics.to_string());}}@@ -1215,10 +1222,12 @@ mod tests {// WebGPU's guarantee for the surface format: 8x saved, the device can only do 1x and 4x.app.msaa_supported = vec![1, 4];app.preferences.msaa = 8;+ std::fs::write(&path, r#"{"msaa":8,"backend":"vulkan"}"#).unwrap();check_msaa(&mut app, 8);assert_eq!(app.preferences.msaa, 4);let saved: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap_or_default()).unwrap_or_default();assert_eq!(saved["msaa"], 4);+ assert_eq!(saved["backend"], "vulkan", "the backend preference is kept");assert_eq!(app.settings_reset, [SettingReset::Msaa { requested: 8, reset: 4, supported: vec![1, 4] }]);assert_eq!(app.settings_reset[0].message(),diff --git a/docs/book/src/guide/troubleshooting.md b/docs/book/src/guide/troubleshooting.mdindex 4ace05d76e..582f454add 100644--- a/docs/book/src/guide/troubleshooting.md+++ b/docs/book/src/guide/troubleshooting.md@@ -62,6 +62,28 @@ saying what failed (earlier versions closed without a word). Things to try:- Run `solidcraft --crash-log` to list recent crash reports, and attach the newest one to anissue.+## SolidCraft closes right away on Windows++A graphics driver can crash the app while it starts, before SolidCraft can show anything (an+old Intel Vulkan driver, `igvk64.dll`, did). Windows' Event Viewer names the driver under+*Windows Logs › Application* ("Faulting module name").++From 0.11.1, SolidCraft on Windows starts Direct3D 12 first and loads no other graphics API+when Direct3D 12 has a graphics card; Vulkan and then OpenGL are tried only on a computer+without one. Earlier versions started Vulkan, Direct3D 12 and OpenGL together and preferred+Vulkan. To choose the graphics API yourself, in any version, open a Command Prompt and start+SolidCraft from it:++```+set WGPU_BACKEND=dx12+"C:\Program Files\SolidCraft\solidcraft.exe"+```++`vulkan` and `gl` are the other choices. `setx WGPU_BACKEND dx12` keeps it for apps started+from the Start menu after you sign in again. From 0.11.1 the setting can also live in+`graphics.json` in the settings folder, as `{"msaa": 4, "backend": "dx12"}` (`auto` is the+order above). Safe mode ignores it and always uses the software renderer.+## A design saved by 0.10 won't open in an older versionDesigns saved by a newer SolidCraft use a newer file format. Older versions refuse them withdiff --git a/tools/windows-startup-smoke.ps1 b/tools/windows-startup-smoke.ps1new file mode 100644index 0000000000..bb7e945c42--- /dev/null+++ b/tools/windows-startup-smoke.ps1@@ -0,0 +1,91 @@+# Start the Windows app under different graphics settings and check each one gets its window up,+# draws a frame and quits cleanly, with no crash report. The CI runner has no GPU: its D3D12 adapter+# is WARP, so this exercises the backend order's fallback and the D3D12 path end to end.+#+# Usage: tools/windows-startup-smoke.ps1 -Exe target\ci\solidcraft.exe+param([Parameter(Mandatory = $true)][string]$Exe)+$ErrorActionPreference = 'Stop'++$cases = @(+ @{ Name = 'default order'; Graphics = $null; Env = @{}; Args = @(); Expect = 'graphics: Dx12' },+ @{ Name = 'saved 8x anti-aliasing'; Graphics = '{"msaa":8}'; Env = @{}; Args = @(); Expect = '(requested 8x)' },+ @{ Name = 'saved 2x, backend preference dx12'; Graphics = '{"msaa":2,"backend":"dx12"}'; Env = @{}; Args = @(); Expect = '(requested 2x)' },+ @{ Name = 'damaged graphics.json'; Graphics = '{"msaa":'; Env = @{}; Args = @(); Expect = 'graphics: Dx12' },+ @{ Name = 'WGPU_BACKEND=dx12 with FXC'; Graphics = $null; Env = @{ WGPU_BACKEND = 'dx12'; WGPU_DX12_COMPILER = 'fxc' }; Args = @(); Expect = 'graphics: Dx12' },+ @{ Name = 'safe mode'; Graphics = '{"msaa":8,"backend":"vulkan"}'; Env = @{}; Args = @('--safe-mode'); Expect = 'graphics: Dx12' }+)++$failed = @()+$port = 47811+foreach ($case in $cases) {+ $port += 1+ $root = Join-Path $env:RUNNER_TEMP ("smoke-" + $port)+ Remove-Item -Recurse -Force $root -ErrorAction SilentlyContinue+ # eframe asks Windows for the roaming folder (not %APPDATA%), so the settings go in the+ # runner's real one, emptied for each case.+ $settings = Join-Path ([Environment]::GetFolderPath('ApplicationData')) 'SolidCraft\data'+ Remove-Item -Recurse -Force $settings -ErrorAction SilentlyContinue+ $local = Join-Path $root 'local'+ $crashes = Join-Path $root 'crashes'+ New-Item -ItemType Directory -Force $settings, $local, $crashes | Out-Null+ if ($case.Graphics) { Set-Content -NoNewline -Path (Join-Path $settings 'graphics.json') -Value $case.Graphics }+ $token = -join ((1..64) | ForEach-Object { '{0:x}' -f (Get-Random -Maximum 16) })++ $psi = New-Object System.Diagnostics.ProcessStartInfo+ $psi.FileName = (Resolve-Path $Exe).Path+ foreach ($a in (@('--sample', '--control', "$port") + $case.Args)) { [void]$psi.ArgumentList.Add($a) }+ $psi.UseShellExecute = $false+ $psi.RedirectStandardError = $true+ $psi.RedirectStandardOutput = $true+ $psi.Environment['LOCALAPPDATA'] = $local+ $psi.Environment['SOLIDCRAFT_CRASH_DIR'] = $crashes+ $psi.Environment['SOLIDCRAFT_CONTROL_TOKEN'] = $token+ $psi.Environment.Remove('WGPU_BACKEND') | Out-Null+ $psi.Environment.Remove('WGPU_DX12_COMPILER') | Out-Null+ foreach ($k in $case.Env.Keys) { $psi.Environment[$k] = $case.Env[$k] }+ Write-Host "=== $($case.Name)"+ $p = [System.Diagnostics.Process]::Start($psi)+ $stderr = $p.StandardError.ReadToEndAsync()+ $stdout = $p.StandardOutput.ReadToEndAsync()++ $ok = $false+ $why = ''+ try {+ $client = $null+ $deadline = (Get-Date).AddSeconds(180)+ while (-not $client -and (Get-Date) -lt $deadline -and -not $p.HasExited) {+ try { $client = New-Object System.Net.Sockets.TcpClient('127.0.0.1', $port) } catch { Start-Sleep -Milliseconds 500 }+ }+ if (-not $client) { throw "no control channel (exited: $($p.HasExited))" }+ $client.ReceiveTimeout = 120000+ $stream = $client.GetStream()+ $writer = New-Object System.IO.StreamWriter($stream)+ $writer.AutoFlush = $true+ $reader = New-Object System.IO.StreamReader($stream)+ $shot = Join-Path $root 'shot.png'+ $writer.WriteLine((@{ id = 1; token = $token; method = 'ui.screenshot'; params = @{ path = $shot } } | ConvertTo-Json -Compress))+ $reply = $reader.ReadLine()+ Write-Host "screenshot: $reply"+ if (-not ($reply -like '*"ok":true*') -or -not (Test-Path $shot)) { throw "no frame drawn: $reply" }+ $writer.WriteLine((@{ id = 2; method = 'app.quit'; params = @{ force = $true } } | ConvertTo-Json -Compress))+ Write-Host "quit: $($reader.ReadLine())"+ $client.Close()+ if (-not $p.WaitForExit(60000)) { throw 'did not exit after app.quit' }+ if ($p.ExitCode -ne 0) { throw ('exit code 0x{0:x}' -f $p.ExitCode) }+ $ok = $true+ } catch {+ $why = $_.Exception.Message+ } finally {+ if (-not $p.HasExited) { $p.Kill(); $p.WaitForExit() }+ }+ $err = $stderr.Result+ Write-Host $err+ if ($ok -and -not $err.Contains($case.Expect)) { $ok = $false; $why = "stderr does not say '$($case.Expect)'" }+ $reports = @(Get-ChildItem -Recurse -Filter report.json $crashes -ErrorAction SilentlyContinue)+ if ($reports.Count -gt 0) {+ foreach ($r in $reports) { Get-Content $r.FullName | Write-Host }+ if ($ok) { $ok = $false; $why = 'a crash report was written' }+ }+ if ($ok) { Write-Host "PASS: $($case.Name)" } else { Write-Host "FAIL: $($case.Name): $why"; $failed += $case.Name }+}+if ($failed.Count -gt 0) { Write-Error ("failed: " + ($failed -join ', ')) }