LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
feat(xtask): #1037 ci reports the machine, GPU adapters and skipped tests
Chad Elliott82045f4d2cd0
diff --git a/crates/gpu/tests/gpu_probe.rs b/crates/gpu/tests/gpu_probe.rsnew file mode 100644index 000000000..e006fbc80--- /dev/null+++ b/crates/gpu/tests/gpu_probe.rs@@ -0,0 +1,87 @@+//! What GPU this machine offers the develop pipeline, as one JSON line: the capability probe of+//! `cargo xtask ci`, which is std + serde_json and has no wgpu of its own. Ignored in normal runs;+//! xtask runs it after the workspace tests, so it costs no extra build:+//! `cargo test --workspace --test gpu_probe -- --ignored --nocapture`.+//!+//! - `LIGHTCRAFT_GPU_PROBE` unset: the compute device the pipeline picks (`available()`,+//! `adapter_name()`, `unavailable_reason()`), the backends it may use and the backends this build+//! contains.+//! - `LIGHTCRAFT_GPU_PROBE=<backend>` (`vulkan`, `metal`, `dx12`, `gl`): every adapter that+//! backend lists, software ones (lavapipe, llvmpipe, WARP) included. One backend per process, so+//! a driver that crashes while it is enumerated (issue #136) costs only its own line.+//!+//! Only reports: which adapters the pipeline accepts is `ctx.rs`'s policy.+#![cfg(not(target_arch = "wasm32"))]++use lightcraft_gpu::backend::{self, Backends};++const NAMES: [(&str, Backends); 4] = [("vulkan", Backends::VULKAN), ("metal", Backends::METAL), ("dx12", Backends::DX12), ("gl", Backends::GL)];++#[test]+#[ignore = "the capability probe of `cargo xtask ci`"]+fn gpu_probe() {+ let line = match std::env::var("LIGHTCRAFT_GPU_PROBE").ok().filter(|b| !b.is_empty()) {+ None => pipeline(),+ Some(name) => adapters(&name),+ };+ // its own line, prefixed: libtest may have printed `test gpu_probe ... ` on this one+ println!("\ngpu_probe: {line}");+}++fn names(set: Backends) -> String {+ let list: Vec<String> = NAMES.iter().filter(|(_, b)| set.contains(*b)).map(|(n, _)| json_str(n)).collect();+ format!("[{}]", list.join(","))+}++fn pipeline() -> String {+ let available = lightcraft_gpu::available();+ let opt = |s: Option<String>| s.map_or_else(|| "null".to_owned(), |s| json_str(&s));+ let compiled = wgpu::Instance::enabled_backend_features();+ format!(+ "{{\"pipeline\":{{\"available\":{available},\"adapter\":{},\"reason\":{},\"backends\":{},\"compiled\":{}}}}}",+ opt(lightcraft_gpu::adapter_name()),+ opt(lightcraft_gpu::unavailable_reason()),+ names(backend::compute_backends().unwrap_or(Backends::empty()) & compiled),+ names(compiled),+ )+}++fn adapters(name: &str) -> String {+ let (_, set) = NAMES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("unknown backend `{name}` (vulkan, metal, dx12, gl)"));+ let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();+ desc.backends = *set;+ desc.backend_options = backend::backend_options();+ desc.flags = backend::instance_flags_from_env(desc.flags);+ let instance = wgpu::Instance::new(desc);+ let adapters: Vec<String> = pollster::block_on(instance.enumerate_adapters(*set))+ .iter()+ .map(|a| {+ let i = a.get_info();+ format!(+ "{{\"name\":{},\"type\":{},\"vendor\":\"0x{:04x}\",\"device\":\"0x{:04x}\",\"driver\":{},\"driver_info\":{}}}",+ json_str(&i.name),+ json_str(&format!("{:?}", i.device_type)),+ i.vendor,+ i.device,+ json_str(&i.driver),+ json_str(&i.driver_info),+ )+ })+ .collect();+ format!("{{\"backend\":{},\"adapters\":[{}]}}", json_str(name), adapters.join(","))+}++/// A JSON string literal.+fn json_str(s: &str) -> String {+ let mut out = String::from("\"");+ for c in s.chars() {+ match c {+ '"' => out.push_str("\\\""),+ '\\' => out.push_str("\\\\"),+ c if u32::from(c) < 0x20 => out.push_str(&format!("\\u{:04x}", u32::from(c))),+ c => out.push(c),+ }+ }+ out.push('"');+ out+}diff --git a/xtask/src/ci_report.rs b/xtask/src/ci_report.rsnew file mode 100644index 000000000..dfbd9f5f9--- /dev/null+++ b/xtask/src/ci_report.rs@@ -0,0 +1,599 @@+//! What a `cargo xtask ci` run covered on this machine: the capability probe (OS, CPUs, RAM, GPU+//! adapters per backend, corpus, craft-fonts), each step's result and time, and the tests that+//! skipped for want of hardware or data. Written to `target/ci/report.json` and+//! `target/ci/summary.md` (a block to paste into a PR) and printed after the CI summary.+//!+//! Tests record skips with `lightcraft_testkit::skip!`; each test step points+//! `LIGHTCRAFT_SKIP_LOG` at `target/ci/skips-<step>.jsonl`. The GPU adapters come from the ignored+//! `gpu_probe` test of `lightcraft-gpu` (xtask has no wgpu). Nothing here is sent anywhere.+//!+//! The report holds no host name, user name or absolute path: the workspace root and the home+//! directory are cut from skip reasons.++use std::collections::BTreeMap;+use std::fmt::Write as _;+use std::path::{Path, PathBuf};+use std::process::Command;++use serde_json::{Value, json};++/// Bumped when a field changes meaning.+pub const SCHEMA: u64 = 1;++/// The environment variable `lightcraft-testkit` appends skips to.+pub const SKIP_LOG_ENV: &str = "LIGHTCRAFT_SKIP_LOG";++/// `target/ci` under the workspace root (not `CARGO_TARGET_DIR`: one place to look, like+/// `target/bench`).+pub fn dir(root: &Path) -> PathBuf {+ root.join("target").join("ci")+}++/// Create `target/ci` and remove the previous run's skip logs, report and summary.+pub fn prepare(root: &Path) -> Result<PathBuf, String> {+ let dir = dir(root);+ std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;+ for entry in std::fs::read_dir(&dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {+ let name = entry.file_name().to_string_lossy().into_owned();+ if (name.starts_with("skips-") && name.ends_with(".jsonl")) || name == "report.json" || name == "summary.md" {+ let _ = std::fs::remove_file(entry.path());+ }+ }+ Ok(dir)+}++/// The skip log of one step.+pub fn skip_log(root: &Path, step: &str) -> PathBuf {+ let safe: String = step.chars().map(|c| if c.is_ascii_alphanumeric() || c == '-' { c } else { '-' }).collect();+ dir(root).join(format!("skips-{safe}.jsonl"))+}++fn output(program: &str, args: &[&str], cwd: Option<&Path>) -> Option<String> {+ let mut c = Command::new(program);+ c.args(args);+ if let Some(cwd) = cwd {+ c.current_dir(cwd);+ }+ let out = c.output().ok()?;+ out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_owned())+}++/// Commit, branch and whether the work tree has uncommitted changes.+pub fn git(root: &Path) -> Value {+ let commit = output("git", &["rev-parse", "HEAD"], Some(root));+ let branch = output("git", &["rev-parse", "--abbrev-ref", "HEAD"], Some(root));+ let dirty = output("git", &["status", "--porcelain"], Some(root)).map(|s| !s.is_empty());+ json!({"commit": commit, "branch": branch, "dirty": dirty})+}++/// OS, version, architecture, CPUs and RAM.+pub fn machine(ram_total_gb: Option<u64>, ram_available_mb: Option<u64>) -> Value {+ json!({+ "os": std::env::consts::OS,+ "os_version": os_version(),+ "arch": std::env::consts::ARCH,+ "cpus": std::thread::available_parallelism().map(|n| n.get()).ok(),+ "ram_total_gb": ram_total_gb,+ "ram_available_mb": ram_available_mb,+ })+}++fn os_version() -> Option<String> {+ if cfg!(target_os = "linux") {+ let release = std::fs::read_to_string("/etc/os-release")+ .ok()+ .and_then(|s| s.lines().find_map(|l| l.strip_prefix("PRETTY_NAME=")).map(|v| v.trim_matches('"').to_owned()));+ let kernel = output("uname", &["-r"], None);+ match (release, kernel) {+ (Some(r), Some(k)) => Some(format!("{r}, kernel {k}")),+ (r, k) => r.or(k),+ }+ } else if cfg!(target_os = "macos") {+ output("sw_vers", &["-productVersion"], None).map(|v| format!("macOS {v}"))+ } else if cfg!(windows) {+ output("cmd", &["/C", "ver"], None)+ } else {+ output("uname", &["-sr"], None)+ }+}++/// `rustc -V` of the toolchain cargo uses.+pub fn toolchain() -> Option<String> {+ output(&std::env::var("RUSTC").unwrap_or_else(|_| "rustc".into()), &["-V"], None)+}++/// Test data present: how many of the pinned raw samples, other corpus files, craft-fonts.+pub fn data(root: &Path, pinned_raw: &[&str]) -> Value {+ let corpus = std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| root.join("corpus"));+ let count = |sub: &str| -> usize { std::fs::read_dir(corpus.join(sub)).map(|d| d.flatten().filter(|e| e.path().is_file()).count()).unwrap_or(0) };+ let pinned = pinned_raw.iter().filter(|name| corpus.join("raw").join(name).is_file()).count();+ let fonts = std::env::var_os("CRAFT_FONTS_DIR").filter(|d| !d.is_empty()).map(PathBuf::from);+ json!({+ "corpus": {+ "custom_dir": std::env::var_os("LIGHTCRAFT_CORPUS").is_some(),+ "raw_pinned": pinned,+ "raw_pinned_total": pinned_raw.len(),+ "raw_files": count("raw"),+ "images": count("images"),+ "pngsuite": count("pngsuite"),+ },+ "craft_fonts": {+ "set": fonts.is_some(),+ // crates/engine/build.rs embeds the fonts of `fonts/manifest.txt` (a relative dir may mean the root)+ "found": fonts.as_deref().is_some_and(|d| [d.to_path_buf(), root.join(d)].iter().any(|d| d.join("fonts/manifest.txt").is_file())),+ },+ })+}++/// Backends wgpu can probe, in the order reports list them.+const BACKENDS: [&str; 4] = ["vulkan", "metal", "dx12", "gl"];++/// Run `gpu_probe` (the pipeline's device), then once per backend: the backends the pipeline may+/// use, or with `all` every backend in the build (`--hardware`; on Windows that loads the Vulkan+/// driver, which the app avoids by default, issue #136).+pub fn gpu(mut probe: impl FnMut(Option<&str>) -> Result<String, String>, all: bool) -> Value {+ let pipeline = match probe(None).and_then(|s| first_json(&s)) {+ Ok(v) => v.get("pipeline").cloned().unwrap_or(Value::Null),+ Err(e) => return json!({"error": e}),+ };+ let names = |key: &str| -> Vec<String> {+ pipeline[key].as_array().map(|a| a.iter().filter_map(|v| v.as_str().map(str::to_owned)).collect()).unwrap_or_default()+ };+ let (compiled, used) = (names("compiled"), names("backends"));+ let mut backends = Vec::new();+ for b in BACKENDS.iter().filter(|b| compiled.iter().any(|c| c == *b) && (all || used.iter().any(|u| u == *b))) {+ backends.push(match probe(Some(b)).and_then(|s| first_json(&s)) {+ Ok(v) => v,+ Err(e) => json!({"backend": b, "adapters": [], "error": e}),+ });+ }+ json!({"pipeline": pipeline, "backends": backends})+}++/// The JSON of the `gpu_probe: {…}` line in the probe's output (libtest prints around it).+fn first_json(s: &str) -> Result<Value, String> {+ let line = s.lines().find_map(|l| l.trim().strip_prefix("gpu_probe: ")).ok_or_else(|| format!("gpu_probe printed no result: {s:?}"))?;+ serde_json::from_str(line).map_err(|e| format!("gpu_probe: bad JSON ({e}): {line}"))+}++/// Backends with at least one hardware (not software) adapter: `--hardware` runs the GPU tests+/// on each.+pub fn hardware_backends(gpu: &Value) -> Vec<(String, String)> {+ let mut out = Vec::new();+ for b in gpu["backends"].as_array().into_iter().flatten() {+ let Some(name) = b["backend"].as_str() else { continue };+ let hw = b["adapters"].as_array().into_iter().flatten().find(|a| a["type"].as_str().is_some_and(|t| t != "Cpu"));+ if let Some(a) = hw {+ out.push((name.to_owned(), a["name"].as_str().unwrap_or("?").to_owned()));+ }+ }+ out+}++/// One test that skipped, possibly in several steps.+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]+pub struct Skip {+ pub need: String,+ pub reason: String,+ pub test: String,+ pub binary: String,+ pub location: String,+ pub steps: Vec<String>,+}++/// The skips of every `skips-<step>.jsonl` in `dir`, deduplicated (the HEIF step reruns some test+/// binaries), sorted by need, then reason. Unreadable lines are dropped.+pub fn read_skips(dir: &Path, root: &Path) -> Vec<Skip> {+ let mut by_key: BTreeMap<(String, String, String, String, String), Vec<String>> = BTreeMap::new();+ let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() };+ let mut files: Vec<PathBuf> = entries.flatten().map(|e| e.path()).collect();+ files.sort();+ for path in files {+ let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();+ let Some(step) = name.strip_prefix("skips-").and_then(|n| n.strip_suffix(".jsonl")) else { continue };+ let Ok(text) = std::fs::read_to_string(&path) else { continue };+ for line in text.lines() {+ let Ok(v) = serde_json::from_str::<Value>(line) else { continue };+ let field = |k: &str| v[k].as_str().unwrap_or("").to_owned();+ let key = (field("need"), sanitize(&field("reason"), root), field("test"), field("binary"), field("location"));+ let steps = by_key.entry(key).or_default();+ if !steps.iter().any(|s| s == step) {+ steps.push(step.to_owned());+ }+ }+ }+ by_key.into_iter().map(|((need, reason, test, binary, location), steps)| Skip { need, reason, test, binary, location, steps }).collect()+}++/// Cut the workspace root and the home directory from `s`, and fold `dir/../` in paths.+pub fn sanitize(s: &str, root: &Path) -> String {+ let mut out = s.to_owned();+ let root = root.display().to_string();+ if !root.is_empty() {+ for sep in ['/', '\\'] {+ out = out.replace(&format!("{root}{sep}"), "");+ }+ out = out.replace(&root, ".");+ }+ let home = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")).map(|h| h.to_string_lossy().into_owned());+ if let Some(home) = home.filter(|h| h.len() > 1) {+ out = out.replace(&home, "~");+ }+ out.split(' ').map(fold_parent_dirs).collect::<Vec<_>>().join(" ")+}++/// `crates/raw/../../corpus/raw` → `corpus/raw` (lexically; leading `..` stay).+fn fold_parent_dirs(token: &str) -> String {+ if !token.contains("/../") && !token.contains("\\..\\") {+ return token.to_owned();+ }+ let sep = if token.contains("/../") { '/' } else { '\\' };+ let mut parts: Vec<&str> = Vec::new();+ for part in token.split(sep) {+ if part == ".." && parts.last().is_some_and(|p| *p != ".." && !p.is_empty()) {+ parts.pop();+ } else {+ parts.push(part);+ }+ }+ parts.join(&sep.to_string())+}++pub fn skips_json(skips: &[Skip]) -> Value {+ Value::Array(+ skips+ .iter()+ .map(|s| json!({"need": s.need, "reason": s.reason, "test": s.test, "binary": s.binary, "location": s.location, "steps": s.steps}))+ .collect(),+ )+}++/// What an empty skip list means: no test skipped, or the tests never ran.+fn no_skips(report: &Value) -> &'static str {+ let test_ran = report["steps"].as_array().into_iter().flatten().any(|s| s["name"] == "test" && s["status"] != "not run");+ if test_ran { "none recorded" } else { "unknown, the test step did not run" }+}++fn skips_from(report: &Value) -> Vec<Skip> {+ report["skips"]+ .as_array()+ .into_iter()+ .flatten()+ .map(|v| {+ let f = |k: &str| v[k].as_str().unwrap_or("").to_owned();+ let steps = v["steps"].as_array().map(|a| a.iter().filter_map(|s| s.as_str().map(str::to_owned)).collect()).unwrap_or_default();+ Skip { need: f("need"), reason: f("reason"), test: f("test"), binary: f("binary"), location: f("location"), steps }+ })+ .collect()+}++/// Skip counts per need, most first: `[("corpus", 40), ("gpu", 5)]`.+pub fn counts_by_need(skips: &[Skip]) -> Vec<(String, usize)> {+ let mut counts: BTreeMap<&str, usize> = BTreeMap::new();+ for s in skips {+ *counts.entry(&s.need).or_default() += 1;+ }+ let mut v: Vec<(String, usize)> = counts.into_iter().map(|(k, n)| (k.to_owned(), n)).collect();+ v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));+ v+}++/// What gets the skipped tests of `need` running.+fn hint(need: &str) -> &'static str {+ match need {+ "gpu" => "a hardware GPU adapter the pipeline accepts",+ "corpus" => "the corpus (`cargo xtask corpus --download`; some files are private)",+ "fonts" => "craft-fonts (`CRAFT_FONTS_DIR`, see the README)",+ "model" => "the model files the reasons name",+ "input" => "the input files the reasons name",+ "system" => "what the operating system provides",+ "feature" => "support this build doesn't have yet",+ _ => "what the reasons name",+ }+}++/// The skips of `need`, grouped by reason, largest group first.+fn by_reason<'a>(skips: &'a [Skip], need: &str) -> Vec<(&'a str, Vec<&'a Skip>)> {+ let mut groups: BTreeMap<&str, Vec<&Skip>> = BTreeMap::new();+ for k in skips.iter().filter(|k| k.need == need) {+ groups.entry(&k.reason).or_default().push(k);+ }+ let mut groups: Vec<(&str, Vec<&Skip>)> = groups.into_iter().collect();+ groups.sort_by(|a, b| b.1.len().cmp(&a.1.len()).then_with(|| a.0.cmp(b.0)));+ groups+}++pub fn seconds(s: f64) -> String {+ if s < 60.0 { format!("{s:.1} s") } else { format!("{}m {:02}s", (s / 60.0).floor() as u64, (s % 60.0).floor() as u64) }+}++fn test_name(s: &Skip) -> String {+ if s.test.is_empty() || s.test == "main" { format!("{} ({})", s.binary, s.location) } else { format!("{}::{}", s.binary, s.test) }+}++fn adapter_line(a: &Value) -> String {+ let name = a["name"].as_str().unwrap_or("?");+ let kind = a["type"].as_str().unwrap_or("?");+ let driver = [a["driver"].as_str().unwrap_or(""), a["driver_info"].as_str().unwrap_or("")]+ .into_iter()+ .filter(|s| !s.is_empty())+ .collect::<Vec<_>>()+ .join(" ");+ if driver.is_empty() { format!("{name} ({kind})") } else { format!("{name} ({kind}; {driver})") }+}++/// "the pipeline's device" line.+fn gpu_line(report: &Value) -> String {+ let gpu = &report["capabilities"]["gpu"];+ if let Some(e) = gpu["error"].as_str() {+ return format!("probe failed: {e}");+ }+ let p = &gpu["pipeline"];+ match (p["available"].as_bool(), p["adapter"].as_str(), p["reason"].as_str()) {+ (Some(true), Some(a), _) => a.to_owned(),+ (_, _, Some(r)) => format!("none ({r})"),+ _ => "not probed".to_owned(),+ }+}++fn adapters_lines(report: &Value) -> Vec<String> {+ let mut out = Vec::new();+ for b in report["capabilities"]["gpu"]["backends"].as_array().into_iter().flatten() {+ let name = b["backend"].as_str().unwrap_or("?");+ let list: Vec<String> = b["adapters"].as_array().into_iter().flatten().map(adapter_line).collect();+ let text = match (b["error"].as_str(), list.is_empty()) {+ (Some(e), _) => format!("probe failed: {e}"),+ (None, true) => "no adapter".to_owned(),+ (None, false) => list.join(", "),+ };+ out.push(format!("{name}: {text}"));+ }+ out+}++fn machine_line(report: &Value) -> String {+ let m = &report["machine"];+ let mut s = format!("{} {}", m["os"].as_str().unwrap_or("?"), m["arch"].as_str().unwrap_or("?"));+ if let Some(v) = m["os_version"].as_str() {+ let _ = write!(s, " ({v})");+ }+ if let Some(n) = m["cpus"].as_u64() {+ let _ = write!(s, ", {n} CPUs");+ }+ if let Some(gb) = m["ram_total_gb"].as_u64() {+ let _ = write!(s, ", {gb} GB RAM");+ }+ if let Some(mb) = m["ram_available_mb"].as_u64() {+ let _ = write!(s, " ({:.1} GB available)", mb as f64 / 1024.0);+ }+ s+}++fn data_line(report: &Value) -> String {+ let d = &report["capabilities"]["data"];+ let c = &d["corpus"];+ let n = |k: &str| c[k].as_u64().unwrap_or(0);+ let fonts = match (d["craft_fonts"]["set"].as_bool(), d["craft_fonts"]["found"].as_bool()) {+ (Some(true), Some(true)) => "CRAFT_FONTS_DIR set",+ (Some(true), _) => "CRAFT_FONTS_DIR set but not found",+ _ => "no CRAFT_FONTS_DIR",+ };+ format!("corpus: {} of {} pinned raws, {} images, {} PngSuite files; {fonts}", n("raw_pinned"), n("raw_pinned_total"), n("images"), n("pngsuite"))+}++fn commit_line(report: &Value) -> String {+ let g = &report["git"];+ let commit = g["commit"].as_str().map_or("unknown commit".to_owned(), |c| c.chars().take(10).collect());+ let state = match g["dirty"].as_bool() {+ Some(true) => "with uncommitted changes",+ Some(false) => "clean",+ None => "state unknown",+ };+ format!("{commit} ({state})")+}++/// The lines printed after the CI summary.+pub fn summary_text(report: &Value) -> String {+ let mut s = String::new();+ let _ = writeln!(s, "\nMachine: {}", machine_line(report));+ let _ = writeln!(s, "GPU for the pipeline: {}", gpu_line(report));+ for line in adapters_lines(report) {+ let _ = writeln!(s, " adapters on {line}");+ }+ let _ = writeln!(s, "Data: {}", data_line(report));+ for h in report["hardware"].as_array().into_iter().flatten() {+ let _ = writeln!(+ s,+ "GPU tests on {} ({}): {}",+ h["backend"].as_str().unwrap_or("?"),+ h["adapter"].as_str().unwrap_or("?"),+ h["status"].as_str().unwrap_or("?")+ );+ }+ let skips = skips_from(report);+ if skips.is_empty() {+ let _ = writeln!(s, "Skipped tests: {}", no_skips(report));+ } else {+ let counts: Vec<String> = counts_by_need(&skips).iter().map(|(k, n)| format!("{k} {n}")).collect();+ let _ = writeln!(s, "Skipped tests: {} passed without running their checks ({}):", skips.len(), counts.join(", "));+ for (need, n) in counts_by_need(&skips) {+ let _ = writeln!(s, " {need} ({n}): needs {}", hint(&need));+ let groups = by_reason(&skips, &need);+ const SHOWN: usize = 4;+ for (reason, tests) in groups.iter().take(SHOWN) {+ let first = tests.first().map(|t| test_name(t)).unwrap_or_default();+ let more = if tests.len() > 1 { ", …" } else { "" };+ let _ = writeln!(s, " {:>3}× {reason} ({first}{more})", tests.len());+ }+ if groups.len() > SHOWN {+ let rest: usize = groups.iter().skip(SHOWN).map(|(_, t)| t.len()).sum();+ let _ = writeln!(s, " {rest:>3}× {} other reasons (target/ci/report.json)", groups.len() - SHOWN);+ }+ }+ }+ s+}++fn cell(s: &str) -> String {+ s.replace('|', "\\|").replace('\n', " ")+}++/// A Markdown block for a PR description or comment.+pub fn summary_md(report: &Value) -> String {+ let mut s = String::new();+ let result = report["result"].as_str().unwrap_or("?");+ let command = report["command"].as_str().unwrap_or("cargo xtask ci");+ let _ = writeln!(s, "<!-- lightcraft ci report, schema {SCHEMA} -->");+ let _ = writeln!(s, "**`{command}`: {result}** at {}\n", commit_line(report));+ let _ = writeln!(s, "| | |\n|---|---|");+ let _ = writeln!(s, "| Machine | {} |", cell(&machine_line(report)));+ if let Some(t) = report["toolchain"].as_str() {+ let _ = writeln!(s, "| Rust | {} |", cell(t));+ }+ let _ = writeln!(s, "| GPU for the pipeline | {} |", cell(&gpu_line(report)));+ let adapters = adapters_lines(report);+ if !adapters.is_empty() {+ let _ = writeln!(s, "| Adapters | {} |", cell(&adapters.join("; ")));+ }+ let _ = writeln!(s, "| Data | {} |", cell(&data_line(report)));+ let _ = writeln!(s, "\n| Step | Result | Time |\n|---|---|---|");+ for step in report["steps"].as_array().into_iter().flatten() {+ let time = step["seconds"].as_f64().map(seconds).unwrap_or_default();+ let _ = writeln!(s, "| {} | {} | {time} |", cell(step["name"].as_str().unwrap_or("?")), step["status"].as_str().unwrap_or("?"));+ }+ for h in report["hardware"].as_array().into_iter().flatten() {+ let time = h["seconds"].as_f64().map(seconds).unwrap_or_default();+ let name = format!("GPU tests on {} ({})", h["backend"].as_str().unwrap_or("?"), h["adapter"].as_str().unwrap_or("?"));+ let _ = writeln!(s, "| {} | {} | {time} |", cell(&name), h["status"].as_str().unwrap_or("?"));+ }+ let skips = skips_from(report);+ if skips.is_empty() {+ let _ = writeln!(s, "\nSkipped tests: {}.", no_skips(report));+ } else {+ let counts: Vec<String> = counts_by_need(&skips).iter().map(|(k, n)| format!("{k} {n}")).collect();+ let _ = writeln!(s, "\n<details><summary>{} tests skipped ({})</summary>\n", skips.len(), counts.join(", "));+ let _ = writeln!(s, "| Need | Tests | Reason | For example |\n|---|---|---|---|");+ for (need, _) in counts_by_need(&skips) {+ let groups = by_reason(&skips, &need);+ const SHOWN: usize = 12;+ for (reason, tests) in groups.iter().take(SHOWN) {+ let first = tests.first().map(|t| test_name(t)).unwrap_or_default();+ let _ = writeln!(s, "| {} | {} | {} | `{}` |", cell(&need), tests.len(), cell(reason), cell(&first));+ }+ if groups.len() > SHOWN {+ let rest: usize = groups.iter().skip(SHOWN).map(|(_, t)| t.len()).sum();+ let _ = writeln!(s, "| {} | {rest} | {} other reasons | |", cell(&need), groups.len() - SHOWN);+ }+ }+ let _ = writeln!(s, "\n</details>");+ }+ s+}++#[cfg(test)]+mod tests {+ use super::*;++ fn tmp(name: &str) -> PathBuf {+ let d = std::env::temp_dir().join(format!("xtask-ci-report-{name}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&d);+ std::fs::create_dir_all(&d).unwrap();+ d+ }++ #[test]+ fn skips_are_read_per_step_deduplicated_and_cut_free_of_the_root() {+ let root = Path::new("/work/lightcraft");+ let dir = tmp("skips");+ let gpu =+ r#"{"need":"gpu","reason":"no GPU adapter","test":"camera_tone","binary":"equivalence","location":"crates/gpu/tests/equivalence.rs:22"}"#;+ let corpus = r#"{"need":"corpus","reason":"skip /work/lightcraft/crates/raw/../../corpus/raw absent","test":"decodes","binary":"corpus","location":"crates/raw/tests/corpus.rs:28"}"#;+ std::fs::write(dir.join("skips-heif.jsonl"), format!("{corpus}\n")).unwrap();+ std::fs::write(dir.join("skips-test.jsonl"), format!("{gpu}\n{corpus}\nnot json\n")).unwrap();+ std::fs::write(dir.join("other.jsonl"), format!("{gpu}\n")).unwrap();+ let skips = read_skips(&dir, root);+ assert_eq!(skips.len(), 2, "{skips:?}");+ assert_eq!(skips[0].need, "corpus");+ assert_eq!(skips[0].reason, "skip corpus/raw absent");+ assert_eq!(skips[0].steps, ["heif", "test"]);+ assert_eq!(skips[1].steps, ["test"]);+ assert_eq!(counts_by_need(&skips), [("corpus".to_owned(), 1), ("gpu".to_owned(), 1)]);+ std::fs::remove_dir_all(&dir).unwrap();+ }++ #[test]+ fn prepare_clears_the_last_runs_files_only() {+ let root = tmp("prepare");+ let dir = dir(&root);+ std::fs::create_dir_all(&dir).unwrap();+ for f in ["skips-test.jsonl", "report.json", "summary.md", "keep.txt"] {+ std::fs::write(dir.join(f), "x").unwrap();+ }+ prepare(&root).unwrap();+ let left: Vec<String> = std::fs::read_dir(&dir).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect();+ assert_eq!(left, ["keep.txt"]);+ assert_eq!(skip_log(&root, "gpu:dx12").file_name().unwrap(), "skips-gpu-dx12.jsonl");+ std::fs::remove_dir_all(&root).unwrap();+ }++ #[test]+ fn parent_dirs_fold_in_path_tokens_only() {+ assert_eq!(fold_parent_dirs("crates/raw/../../corpus/raw"), "corpus/raw");+ assert_eq!(fold_parent_dirs("../corpus/raw"), "../corpus/raw");+ assert_eq!(fold_parent_dirs("C:\\w\\crates\\raw\\..\\..\\corpus"), "C:\\w\\corpus");+ assert_eq!(fold_parent_dirs("absent"), "absent");+ }++ #[test]+ fn the_probe_asks_the_pipelines_backends_or_all_with_hardware() {+ let pipeline = r#"{"pipeline":{"available":false,"adapter":null,"reason":"software adapter (llvmpipe) skipped","backends":["vulkan","dx12"],"compiled":["vulkan","dx12","gl"]}}"#;+ let asked = std::cell::RefCell::new(Vec::new());+ let probe = |b: Option<&str>| -> Result<String, String> {+ asked.borrow_mut().push(b.map(str::to_owned));+ Ok(match b {+ None => format!("running 1 test\ntest gpu_probe ... \ngpu_probe: {pipeline}\nok\n"),+ Some("vulkan") => r#"gpu_probe: {"backend":"vulkan","adapters":[{"name":"llvmpipe","type":"Cpu","driver":"llvmpipe","driver_info":"Mesa"}]}"#.to_owned(),+ Some("dx12") => r#"gpu_probe: {"backend":"dx12","adapters":[{"name":"Intel(R) UHD Graphics 620","type":"IntegratedGpu","driver":"","driver_info":"31.0.101"}]}"#.to_owned(),+ Some(_) => return Err("crashed".into()),+ })+ };+ let gpu_default = gpu(probe, false);+ assert_eq!(*asked.borrow(), [None, Some("vulkan".to_owned()), Some("dx12".to_owned())]);+ assert_eq!(hardware_backends(&gpu_default), [("dx12".to_owned(), "Intel(R) UHD Graphics 620".to_owned())]);+ asked.borrow_mut().clear();+ let gpu_all = gpu(probe, true);+ assert_eq!(asked.borrow().len(), 4, "gl too");+ assert_eq!(gpu_all["backends"][2]["error"], "crashed");++ let report = json!({+ "command": "cargo xtask ci",+ "result": "pass",+ "git": {"commit": "0123456789abcdef", "dirty": false},+ "machine": {"os": "windows", "arch": "x86_64", "cpus": 8, "ram_total_gb": 16},+ "capabilities": {"gpu": gpu_all, "data": {"corpus": {"raw_pinned": 3, "raw_pinned_total": 30}, "craft_fonts": {"set": false}}},+ "steps": [{"name": "fmt", "status": "ok", "seconds": 1.25}, {"name": "test", "status": "ok", "seconds": 125.0}],+ "skips": [{"need": "gpu", "reason": "no GPU | adapter", "test": "memory", "binary": "memory", "location": "x.rs:1", "steps": ["test"]}],+ });+ let text = summary_text(&report);+ assert!(text.contains("GPU for the pipeline: none (software adapter (llvmpipe) skipped)"), "{text}");+ assert!(text.contains("adapters on dx12: Intel(R) UHD Graphics 620 (IntegratedGpu; 31.0.101)"), "{text}");+ assert!(text.contains("adapters on gl: probe failed: crashed"), "{text}");+ assert!(text.contains("Skipped tests: 1 passed without running their checks (gpu 1)"), "{text}");+ assert!(text.contains(" gpu (1): needs a hardware GPU adapter"), "{text}");+ assert!(text.contains(" 1× no GPU | adapter (memory::memory)"), "{text}");+ assert!(text.contains("corpus: 3 of 30 pinned raws"), "{text}");+ let md = summary_md(&report);+ assert!(md.contains("**`cargo xtask ci`: pass** at 0123456789 (clean)"), "{md}");+ assert!(md.contains("| test | ok | 2m 05s |"), "{md}");+ assert!(md.contains("| gpu | 1 | no GPU \\| adapter | `memory::memory` |"), "{md}");+ assert!(md.contains("1 tests skipped (gpu 1)"), "{md}");+ }++ #[test]+ fn no_skips_is_not_claimed_when_the_tests_never_ran() {+ let stopped = json!({"steps": [{"name": "clippy", "status": "fail"}, {"name": "test", "status": "not run"}], "skips": []});+ assert!(summary_text(&stopped).contains("Skipped tests: unknown, the test step did not run"));+ let ran = json!({"steps": [{"name": "test", "status": "ok"}], "skips": []});+ assert!(summary_md(&ran).contains("Skipped tests: none recorded."));+ }+}diff --git a/xtask/src/main.rs b/xtask/src/main.rsindex 08ef435c5..0bc3631ba 100644--- a/xtask/src/main.rs+++ b/xtask/src/main.rs@@ -5,6 +5,7 @@mod assets;mod bench;+mod ci_report;mod ico;mod layers;mod parity;@@ -33,7 +34,11 @@ commands:web [--serve [port]] [--dev]build the browser app (apps/lightcraft-web) into <target>/web-site/;--serve serves it on http://127.0.0.1:<port> (default 8080)- ci fmt --check, clippy -D warnings, heif, test, parity refs, layers, assets, wasm (stops at first failure)+ ci [--hardware] [--summary-md]+ fmt --check, clippy -D warnings, heif, test, GPU probe, parity refs, layers, assets, wasm+ (stops at first failure); writes target/ci/report.json (machine, GPU adapters, data,+ steps, skipped tests) and target/ci/summary.md; --hardware also runs the GPU tests on+ every backend with a hardware adapter; --summary-md prints the Markdown summarycorpus [--download]show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/ and checks their sha256stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`)@@ -51,7 +56,7 @@ fn main() -> ExitCode {Some("parity") => parity::run(&root(), rest.contains(&"--write")),Some("wasm") => cmd_wasm(),Some("web") => web::run(&rest),- Some("ci") => cmd_ci(),+ Some("ci") => cmd_ci(&rest),Some("corpus") => cmd_corpus(rest.contains(&"--download")),Some("stats") => stats::run(&root(), rest.contains(&"--exact")),Some("-h" | "--help" | "help") | None => {@@ -240,9 +245,38 @@ fn cmd_wasm() -> Result<(), String> {if failed == 0 { Ok(()) } else { Err(format!("{failed} crate(s) failed the wasm check")) }}-fn cmd_ci() -> Result<(), String> {- type Step = (&'static str, Box<dyn Fn() -> Result<(), String>>);- let steps: Vec<Step> = vec![+fn cmd_ci(args: &[&str]) -> Result<(), String> {+ let hardware = args.contains(&"--hardware");+ let print_md = args.contains(&"--summary-md");+ if let Some(other) = args.iter().find(|a| !matches!(**a, "--hardware" | "--summary-md")) {+ return Err(format!("ci: unknown option `{other}`\n\n{USAGE}"));+ }+ let root = root();+ let report_dir = ci_report::prepare(&root)?;+ let started = std::time::Instant::now();+ let started_unix = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);+ // test runs append the skips their tests record (`lightcraft_testkit::skip!`) to target/ci/skips-<step>.jsonl+ let test = |cargo_args: &[&str], step: &str| {+ let mut c = cargo();+ c.args(cargo_args).env(ci_report::SKIP_LOG_ENV, ci_report::skip_log(&root, step));+ run(c, &format!("cargo {}", cargo_args.join(" ")))+ };+ let gpu = std::cell::RefCell::new(serde_json::Value::Null);+ let hardware_runs = std::cell::RefCell::new(Vec::new());+ // What this machine offers the GPU tests, once, after the workspace tests (whose build the+ // probe reuses). Never fails the run: the report says what went wrong.+ let probe = || {+ if gpu.borrow().is_null() {+ eprintln!("\n=== ci: GPU probe ===");+ let t = std::time::Instant::now();+ let mut found = ci_report::gpu(probe_gpu, hardware);+ found["seconds"] = serde_json::json!(t.elapsed().as_secs_f64());+ *gpu.borrow_mut() = found;+ }+ };++ type Step<'a> = (&'static str, Box<dyn Fn() -> Result<(), String> + 'a>);+ let mut steps: Vec<Step> = vec![("fmt",Box::new(|| {@@ -266,46 +300,144 @@ fn cmd_ci() -> Result<(), String> {let mut c = cargo();c.args(["clippy", "-p", "lightcraft-codecs", "--features", "heif", "--all-targets", "--", "-D", "warnings"]);run(c, "cargo clippy -p lightcraft-codecs --features heif --all-targets -- -D warnings")?;- let mut c = cargo();- c.args(["test", "-p", "lightcraft-codecs", "-p", "lightcraft-heif", "--features", "lightcraft-codecs/heif"]);- run(c, "cargo test -p lightcraft-codecs -p lightcraft-heif --features lightcraft-codecs/heif")?;+ test(&["test", "-p", "lightcraft-codecs", "-p", "lightcraft-heif", "--features", "lightcraft-codecs/heif"], "heif")?;// the engine's HEIC import test, with the decoder (the workspace run checks the error)- let mut c = cargo();- c.args(["test", "-p", "lightcraft-engine", "--lib", "--features", "lightcraft-codecs/heif", "heic"]);- run(c, "cargo test -p lightcraft-engine --lib --features lightcraft-codecs/heif heic")+ test(&["test", "-p", "lightcraft-engine", "--lib", "--features", "lightcraft-codecs/heif", "heic"], "heif")}),),- (- "test",+ ("test", Box::new(|| test(&["test", "--workspace"], "test"))),+ ];+ if hardware {+ steps.push((+ "gpu-backends",Box::new(|| {- let mut c = cargo();- c.args(["test", "--workspace"]);- run(c, "cargo test --workspace")+ // the GPU tests once per backend with a hardware adapter (`LIGHTCRAFT_GPU_BACKEND`)+ probe();+ let backends = ci_report::hardware_backends(&gpu.borrow());+ if backends.is_empty() {+ eprintln!("no hardware GPU adapter on any backend: nothing to run");+ }+ let mut failed = Vec::new();+ for (backend, adapter) in backends {+ let step = format!("gpu-{backend}");+ let mut c = cargo();+ c.args(["test", "-p", "lightcraft-gpu", "--features", "denoise"])+ .env("LIGHTCRAFT_GPU_BACKEND", &backend)+ .env(ci_report::SKIP_LOG_ENV, ci_report::skip_log(&root, &step));+ let t = std::time::Instant::now();+ let ok = run(c, &format!("LIGHTCRAFT_GPU_BACKEND={backend} cargo test -p lightcraft-gpu --features denoise")).is_ok();+ hardware_runs.borrow_mut().push(serde_json::json!({+ "backend": backend,+ "adapter": adapter,+ "status": if ok { "ok" } else { "fail" },+ "seconds": t.elapsed().as_secs_f64(),+ }));+ if !ok {+ failed.push(backend);+ }+ }+ if failed.is_empty() { Ok(()) } else { Err(format!("the GPU tests failed on {}", failed.join(", "))) }}),- ),- ("parity", Box::new(|| parity::run(&root(), false))),- ("layers", Box::new(cmd_layers)),- ("assets", Box::new(|| assets::run(&root()))),- ("wasm", Box::new(cmd_wasm)),- ];- let mut done = Vec::new();+ ));+ }+ steps.push(("parity", Box::new(|| parity::run(&root, false))));+ steps.push(("layers", Box::new(cmd_layers)));+ steps.push(("assets", Box::new(|| assets::run(&root))));+ steps.push(("wasm", Box::new(cmd_wasm)));++ let mut results = Vec::new();+ let mut failure = None;for (name, f) in &steps {eprintln!("\n=== ci: {name} ===");- if let Err(e) = f() {+ let t = std::time::Instant::now();+ let r = f();+ let status = if r.is_ok() { "ok" } else { "fail" };+ results.push(serde_json::json!({"name": name, "status": status, "seconds": t.elapsed().as_secs_f64()}));+ if *name == "test" {+ // right after the workspace tests, before parity and wasm build other feature sets+ probe();+ }+ if let Err(e) = r {+ failure = Some((*name, e));+ break;+ }+ }+ if gpu.borrow().is_null() {+ *gpu.borrow_mut() = serde_json::json!({"error": "not probed: the test step did not run"});+ }+ let done: Vec<&str> = steps.iter().take(results.len()).map(|(n, _)| *n).collect();+ match &failure {+ Some((name, e)) => {println!("\nCI summary:");- for d in &done {+ for d in done.iter().filter(|d| *d != name) {println!(" ok {d}");}println!(" FAIL {name}: {e}");- for (n, _) in steps.iter().skip(done.len() + 1) {+ for (n, _) in steps.iter().skip(results.len()) {println!(" skip {n}");}- return Err(format!("ci failed at `{name}`"));}- done.push(*name);+ None => println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", ")),+ }+ for (n, _) in steps.iter().skip(results.len()) {+ results.push(serde_json::json!({"name": n, "status": "not run"}));+ }++ let skips = ci_report::read_skips(&report_dir, &root);+ let ram_mb = available_ram_mb();+ let report = serde_json::json!({+ "schema": ci_report::SCHEMA,+ "command": std::iter::once("cargo xtask ci").chain(args.iter().copied()).collect::<Vec<_>>().join(" "),+ "result": if failure.is_some() { "fail" } else { "pass" },+ "failed_step": failure.as_ref().map(|(n, _)| *n),+ "started_unix": started_unix,+ "seconds": started.elapsed().as_secs_f64(),+ "git": ci_report::git(&root),+ "machine": ci_report::machine(total_ram_gb(), ram_mb),+ "toolchain": ci_report::toolchain(),+ "capabilities": {"gpu": gpu.take(), "data": ci_report::data(&root, &RAW_SAMPLES.iter().map(|s| s.0).collect::<Vec<_>>())},+ "steps": results,+ "hardware": hardware_runs.take(),+ "skips": ci_report::skips_json(&skips),+ });+ print!("{}", ci_report::summary_text(&report));+ let md = ci_report::summary_md(&report);+ let written = serde_json::to_string_pretty(&report)+ .map_err(|e| e.to_string())+ .and_then(|json| std::fs::write(report_dir.join("report.json"), json + "\n").map_err(|e| e.to_string()))+ .and_then(|()| std::fs::write(report_dir.join("summary.md"), &md).map_err(|e| e.to_string()));+ match written {+ Ok(()) => println!("Report: target/ci/report.json; to paste into a PR: target/ci/summary.md (or run with --summary-md)"),+ Err(e) => eprintln!("ci: could not write the report to {}: {e}", report_dir.display()),+ }+ if print_md {+ println!("\n{md}");+ }+ match failure {+ Some((name, _)) => Err(format!("ci failed at `{name}`")),+ None => Ok(()),+ }+}++/// Run the ignored `gpu_probe` test of lightcraft-gpu: the pipeline's device (`None`) or one+/// backend's adapters. `--workspace`, like the test step, so its build is reused. Its stdout, or why+/// it failed (a driver that crashes while it is enumerated, issue #136, ends up here, not in xtask).+fn probe_gpu(backend: Option<&str>) -> Result<String, String> {+ let mut c = cargo();+ c.args(["test", "-q", "--workspace", "--test", "gpu_probe", "--", "--ignored", "--exact", "gpu_probe", "--nocapture"]);+ c.env("LIGHTCRAFT_GPU_PROBE", backend.unwrap_or(""));+ let out = c.output().map_err(|e| format!("gpu_probe: {e}"))?;+ if !out.status.success() {+ let err = String::from_utf8_lossy(&out.stderr);+ let tail: Vec<&str> = err.lines().rev().filter(|l| !l.trim().is_empty()).take(3).collect();+ return Err(format!(+ "gpu_probe {}exited with {}: {}",+ backend.map(|b| format!("{b} ")).unwrap_or_default(),+ out.status,+ tail.into_iter().rev().collect::<Vec<_>>().join(" / ")+ ));}- println!("\nCI summary: all {} steps passed ({})", done.len(), done.join(", "));- Ok(())+ Ok(String::from_utf8_lossy(&out.stdout).into_owned())}/// CC0 raw samples (raw.pixls.us). Small, representative set; extend freely (CC0 only).