Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: scope expectation proposals to recorded source hosts [skip ci]
Richie McIlroy4fbe8ce4beba
diff --git a/crates/verify-expect/src/lib.rs b/crates/verify-expect/src/lib.rsindex 160c94548..33030b5cf 100644--- a/crates/verify-expect/src/lib.rs+++ b/crates/verify-expect/src/lib.rs@@ -200,8 +200,13 @@ pub struct Xfail {impl Xfail {pub fn applies_on(&self, platform: &str) -> bool {+ self.applies_on_host(platform, host().as_deref())+ }++ fn applies_on_host(&self, platform: &str, source_host: Option<&str>) -> bool {(self.platforms.is_empty() || self.platforms.iter().any(|p| p == platform))- && (self.hosts.is_empty() || host().is_some_and(|h| self.hosts.contains(&h)))+ && (self.hosts.is_empty()+ || source_host.is_some_and(|h| self.hosts.iter().any(|candidate| candidate == h)))}pub fn matches(&self, case: &str, platform: &str) -> bool {@@ -623,12 +628,16 @@ fn covered_by<'a>(case: &str,class: &str,platform: &str,+ source_host: Option<&str>,seeds: &BTreeSet<u64>,evidence: &Evidence,) -> BTreeSet<u64> {let mut covered = BTreeSet::new();for x in xfails.filter(|x| {- x.class == class && x.matches(case, platform) && excuses(x, case, class, evidence).is_ok()+ x.class == class+ && x.applies_on_host(platform, source_host)+ && glob_match(&x.case, case)+ && excuses(x, case, class, evidence).is_ok()}) {match &x.seeds {None => return seeds.clone(),@@ -694,6 +703,7 @@ pub fn judge_with(case,class,platform,+ host().as_deref(),seeds,evidence,);@@ -759,9 +769,9 @@ pub fn judge_with(verdict}-/// Rewrites `xfails` so that this platform's run passes exactly, without-/// touching what other platforms or cases that did not run expect. Links-/// of replaced entries are kept; new proposals get the reason+/// Proposes changes for this platform and host without narrowing entries+/// for unobserved hosts, platforms or cases. Links of replaced entries+/// are kept; new proposals get the reason/// `TODO: <problem>` from `problems`.pub fn bless(xfails: &[Xfail],@@ -794,15 +804,26 @@ pub fn bless_observing(unobserved: unobserved.clone(),..Default::default()};- bless_with(xfails, platform, ran, observed, &evidence, problems)+ bless_with(+ xfails,+ platform,+ host().as_deref(),+ ran,+ observed,+ &evidence,+ problems,+ )}/// [`bless`] with measurements: a failure outside every entry's bound gets a/// new entry (TODO reason) whose bound is the measured range, to be-/// reviewed against the mechanism.+/// reviewed against the mechanism. Unknown-host observations cannot remove+/// entries. All-host entries are retained because finite host observations+/// cannot establish their behavior on every possible host.pub fn bless_with(xfails: &[Xfail],platform: &str,+ source_host: Option<&str>,ran: &Ran,observed: &Observed,evidence: &Evidence,@@ -819,9 +840,15 @@ pub fn bless_with(let unmeasured = matched.iter().any(|(case, _)| unobserved.contains(&((*case).clone(), x.class.clone())));- if !x.applies_on(platform) || matched.is_empty() || x.flaky || unmeasured {+ if source_host.is_none()+ || x.hosts.is_empty()+ || !x.applies_on_host(platform, source_host)+ || matched.is_empty()+ || x.flaky+ || unmeasured+ {out.push(x.clone());- if x.applies_on(platform) {+ if x.applies_on_host(platform, source_host) {kept_here.push(x.clone());}continue;@@ -847,6 +874,19 @@ pub fn bless_with(if !others.is_empty() {out.push(x.with_platforms(others));}+ let other_hosts: Vec<String> = x+ .hosts+ .iter()+ .filter(|h| Some(h.as_str()) != source_host)+ .cloned()+ .collect();+ if !other_hosts.is_empty() {+ out.push(Xfail {+ platforms: vec![platform.to_string()],+ hosts: other_hosts,+ ..x.clone()+ });+ }if let Some(listed) = &x.seeds {for (case, ran_seeds) in &matched {let not_ran: BTreeSet<u64> = listed@@ -859,6 +899,7 @@ pub fn bless_with(case: (*case).clone(),seeds: Some(not_ran),platforms: vec![platform.to_string()],+ hosts: source_host.into_iter().map(str::to_string).collect(),..x.clone()};out.push(entry.clone());@@ -868,10 +909,19 @@ pub fn bless_with(}}for ((case, class), seeds) in observed {- let covered = covered_by(kept_here.iter(), case, class, platform, seeds, evidence);+ let covered = covered_by(+ kept_here.iter(),+ case,+ class,+ platform,+ source_host,+ seeds,+ evidence,+ );let fully = kept_here.iter().any(|x| {x.class == *class- && x.matches(case, platform)+ && x.applies_on_host(platform, source_host)+ && glob_match(&x.case, case)&& x.seeds.is_none()&& excuses(x, case, class, evidence).is_ok()});@@ -879,9 +929,11 @@ pub fn bless_with(continue;}let missing: BTreeSet<u64> = seeds.difference(&covered).copied().collect();- let previous = xfails- .iter()- .find(|x| x.class == *class && glob_match(&x.case, case));+ let previous = xfails.iter().find(|x| {+ x.class == *class+ && x.applies_on_host(platform, source_host)+ && glob_match(&x.case, case)+ });let entry = Xfail {case: case.clone(),class: class.clone(),@@ -889,7 +941,7 @@ pub fn bless_with(seeds: (!seeds.is_empty()).then_some(missing),flaky: false,starved: false,- hosts: Vec::new(),+ hosts: source_host.into_iter().map(str::to_string).collect(),bound: if seeds.is_empty() {evidence.measures@@ -1055,6 +1107,7 @@ fn write_records_to(dir: &Path, records: &[(Record, serde_json::Value)]) {.expect("open report file");for (record, extra) in records {let mut value = serde_json::to_value(record).expect("serialize report record");+ value["host"] = serde_json::to_value(host()).expect("serialize report host");if let (Some(map), Some(extra)) = (value.as_object_mut(), extra.as_object()) {for (k, v) in extra {map.entry(k.clone()).or_insert_with(|| v.clone());@@ -1082,7 +1135,7 @@ mod tests {seeds: seeds.map(set),flaky: false,starved: false,- hosts: Vec::new(),+ hosts: host().into_iter().collect(),bound: Bound::new(),reason: "r".into(),link: String::new(),@@ -1122,7 +1175,13 @@ mod tests {repro: String::new(),seconds: 0.0,};- write_records_to(dir, &[(record, serde_json::json!({"writer": writer}))]);+ write_records_to(+ dir,+ &[(+ record,+ serde_json::json!({"writer": writer, "host": "incorrect"}),+ )],+ );}});}@@ -1132,6 +1191,8 @@ mod tests {let cases: BTreeSet<String> = text.lines().map(|line| {+ let value: serde_json::Value = serde_json::from_str(line).unwrap();+ assert_eq!(value["host"], serde_json::json!(host()));let record: Record = serde_json::from_str(line).unwrap();assert_eq!(record.outcome, Outcome::Unsupported);record.case@@ -1333,6 +1394,90 @@ mod tests {assert_eq!(blessed, vec![xfails[0].clone(), proposed]);}+ #[test]+ fn bless_preserves_unobserved_host_and_platform_slices() {+ let xfail = Xfail {+ hosts: vec!["github-hosted".into(), "local".into()],+ ..entry("a", "stale", &["macos", "windows"], None)+ };+ let ran = Ran::from([("a".into(), None)]);+ let blessed = bless_with(+ &[xfail.clone()],+ "macos",+ Some("local"),+ &ran,+ &Observed::new(),+ &Evidence::default(),+ &BTreeMap::new(),+ );+ assert_eq!(blessed.len(), 2);+ assert!(+ blessed+ .iter()+ .any(|x| x.platforms == ["windows"] && x.hosts == xfail.hosts)+ );+ assert!(+ blessed+ .iter()+ .any(|x| x.platforms == ["macos"] && x.hosts == ["github-hosted"])+ );+ assert!(+ !blessed+ .iter()+ .any(|x| x.applies_on_host("macos", Some("local")))+ );++ let untested = bless_with(+ &[xfail.clone()],+ "macos",+ Some("blacksmith"),+ &ran,+ &Observed::new(),+ &Evidence::default(),+ &BTreeMap::new(),+ );+ assert_eq!(untested, vec![xfail]);+ }++ #[test]+ fn bless_unknown_hosts_cannot_remove_or_narrow_entries() {+ let xfails = vec![+ entry("a", "stale", &["macos"], Some(&[1, 2])),+ Xfail {+ hosts: Vec::new(),+ ..entry("b", "stale", &["macos"], None)+ },+ ];+ let ran = Ran::from([("a".into(), Some(set(&[1, 2]))), ("b".into(), None)]);+ for observed in [+ Observed::new(),+ Observed::from([(("a".into(), "stale".into()), set(&[1]))]),+ ] {+ let blessed = bless_with(+ &xfails,+ "macos",+ None,+ &ran,+ &observed,+ &Evidence::default(),+ &BTreeMap::new(),+ );+ for entry in &xfails {+ assert!(blessed.contains(entry));+ }+ }+ let blessed = bless_with(+ &xfails[1..],+ "macos",+ Some("blacksmith"),+ &ran,+ &Observed::new(),+ &Evidence::default(),+ &BTreeMap::new(),+ );+ assert_eq!(blessed, xfails[1..]);+ }+fn measured(case: &str, class: &str, metrics: &[(&str, f64, f64)]) -> Evidence {let mut evidence = Evidence::default();evidence.measures.insert(@@ -1450,7 +1595,15 @@ mod tests {let ran = Ran::from([("a".to_string(), None)]);let observed = Observed::from([(("a".into(), "sync".into()), set(&[]))]);let evidence = measured("a", "sync", &[("ms", -201.0, -199.0)]);- let blessed = bless_with(&[], "macos", &ran, &observed, &evidence, &BTreeMap::new());+ let blessed = bless_with(+ &[],+ "macos",+ host().as_deref(),+ &ran,+ &observed,+ &evidence,+ &BTreeMap::new(),+ );assert_eq!(blessed.len(), 1);assert_eq!(blessed[0].bound["ms"],diff --git a/crates/verify-expect/src/main.rs b/crates/verify-expect/src/main.rsindex c0dad0602..f1cdf6342 100644--- a/crates/verify-expect/src/main.rs+++ b/crates/verify-expect/src/main.rs@@ -44,6 +44,90 @@ struct Run {problems: BTreeMap<(String, String), String>,}+#[derive(serde::Deserialize)]+struct SourceRecord {+ #[serde(flatten)]+ record: Record,+ #[serde(default)]+ host: Option<String>,+}++type HostRuns = BTreeMap<(String, Option<String>), Run>;+type Runs = BTreeMap<String, HostRuns>;++fn add_record(runs: &mut Runs, source: SourceRecord) {+ let SourceRecord { record, host } = source;+ if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {+ return;+ }+ let host = host+ .map(|h| h.trim().to_string())+ .filter(|h| !h.is_empty() && h != "unknown");+ let run = runs+ .entry(record.expectations.clone())+ .or_default()+ .entry((record.platform.clone(), host))+ .or_default();+ let ran = run.ran.entry(record.case.clone()).or_insert(None);+ if let Some(seeds) = &record.seeds_ran {+ ran.get_or_insert_with(BTreeSet::new).extend(seeds);+ }+ for class in record.unobserved.keys() {+ run.evidence+ .unobserved+ .insert((record.case.clone(), class.clone()));+ }+ if record.starved {+ run.evidence.starved.insert(record.case.clone());+ }+ for (class, metrics) in &record.measures {+ let entry = run+ .evidence+ .measures+ .entry((record.case.clone(), class.clone()))+ .or_default();+ for (metric, range) in metrics {+ entry+ .entry(metric.clone())+ .and_modify(|r| {+ r.include(range.min);+ r.include(range.max);+ })+ .or_insert(*range);+ }+ }+ for (class, seeds) in &record.failures {+ run.observed+ .entry((record.case.clone(), class.clone()))+ .or_default()+ .extend(seeds);+ if let Some(problem) = record.problems.get(class) {+ run.problems+ .entry((record.case.clone(), class.clone()))+ .or_insert_with(|| problem.clone());+ }+ }+}++fn proposals(+ entries: &[cap_verify_expect::Xfail],+ runs: &HostRuns,+) -> Vec<cap_verify_expect::Xfail> {+ let mut entries = entries.to_vec();+ for ((platform, host), run) in runs {+ entries = bless_with(+ &entries,+ platform,+ host.as_deref(),+ &run.ran,+ &run.observed,+ &run.evidence,+ &run.problems,+ );+ }+ entries+}+fn bless_command(inputs: &[String]) -> ExitCode {let mut files = Vec::new();for input in inputs {@@ -53,64 +137,18 @@ fn bless_command(inputs: &[String]) -> ExitCode {eprintln!("no .jsonl records under {inputs:?}");return ExitCode::FAILURE;}- let mut runs: BTreeMap<String, BTreeMap<String, Run>> = BTreeMap::new();+ let mut runs = Runs::new();for file in &files {let text = std::fs::read_to_string(file).unwrap_or_default();for line in text.lines().filter(|l| !l.trim().is_empty()) {- let record: Record = match serde_json::from_str(line) {+ let record: SourceRecord = match serde_json::from_str(line) {Ok(record) => record,Err(e) => {eprintln!("{}: skipping a record: {e}", file.display());continue;}};- if matches!(record.outcome, Outcome::Unsupported | Outcome::Inconclusive) {- continue;- }- let run = runs- .entry(record.expectations.clone())- .or_default()- .entry(record.platform.clone())- .or_default();- let ran = run.ran.entry(record.case.clone()).or_insert(None);- if let Some(seeds) = &record.seeds_ran {- ran.get_or_insert_with(BTreeSet::new).extend(seeds);- }- for class in record.unobserved.keys() {- run.evidence- .unobserved- .insert((record.case.clone(), class.clone()));- }- if record.starved {- run.evidence.starved.insert(record.case.clone());- }- for (class, metrics) in &record.measures {- let entry = run- .evidence- .measures- .entry((record.case.clone(), class.clone()))- .or_default();- for (metric, range) in metrics {- entry- .entry(metric.clone())- .and_modify(|r| {- r.include(range.min);- r.include(range.max);- })- .or_insert(*range);- }- }- for (class, seeds) in &record.failures {- run.observed- .entry((record.case.clone(), class.clone()))- .or_default()- .extend(seeds);- if let Some(problem) = record.problems.get(class) {- run.problems- .entry((record.case.clone(), class.clone()))- .or_insert_with(|| problem.clone());- }- }+ add_record(&mut runs, record);}}let root = repo_root();@@ -118,20 +156,10 @@ fn bless_command(inputs: &[String]) -> ExitCode {let path = root.join(expectations);let text = std::fs::read_to_string(&path).unwrap_or_default();let before = load(&path);- let mut entries = before.clone();- for (platform, run) in platforms {- entries = bless_with(- &entries,- platform,- &run.ran,- &run.observed,- &run.evidence,- &run.problems,- );- }+ let entries = proposals(&before, platforms);let added = entries.iter().filter(|e| !before.contains(e)).count();let removed = before.iter().filter(|e| !entries.contains(e)).count();- let platform_list: Vec<&String> = platforms.keys().collect();+ let platform_list: Vec<_> = platforms.keys().collect();if added == 0 && removed == 0 {println!("{expectations}: unchanged ({platform_list:?})");continue;@@ -198,6 +226,62 @@ mod tests {use super::*;use cap_verify_expect::{Metrics, Range};+ fn source_record(source_host: Option<&str>, failure: Option<f64>) -> SourceRecord {+ let mut value = serde_json::json!({+ "suite": "recording",+ "expectations": "expectations.toml",+ "case": "a",+ "platform": "macos",+ "outcome": if failure.is_some() { "xfail" } else { "pass" },+ });+ if let Some(host) = source_host {+ value["host"] = host.into();+ }+ if let Some(value_seen) = failure {+ value["failures"] = serde_json::json!({ "sync": [] });+ value["measures"] = serde_json::json!({ "sync": { "ms": [value_seen, value_seen] } });+ }+ serde_json::from_value(value).unwrap()+ }++ #[test]+ fn mixed_host_records_keep_separate_measurements_and_expectations() {+ let entries = parse(+ "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nplatforms = [\"macos\"]\nhosts = [\"local\", \"github-hosted\"]\nbound = { ms = [-5, 5] }\nreason = \"known offset\"\n",+ )+ .unwrap();+ let mut runs = Runs::new();+ add_record(&mut runs, source_record(Some("local"), Some(3.0)));+ add_record(&mut runs, source_record(Some("blacksmith"), Some(10.0)));+ add_record(&mut runs, source_record(Some("github-hosted"), None));+ let host_runs = &runs["expectations.toml"];+ assert_eq!(host_runs.len(), 3);+ let proposed = proposals(&entries, host_runs);+ assert_eq!(proposed.len(), 2);+ let local = proposed.iter().find(|x| x.hosts == ["local"]).unwrap();+ assert_eq!(local.bound, entries[0].bound);+ assert_eq!(local.reason, entries[0].reason);+ let blacksmith = proposed.iter().find(|x| x.hosts == ["blacksmith"]).unwrap();+ assert_eq!(blacksmith.bound["ms"], Range::at(10.0));+ assert!(blacksmith.reason.starts_with("TODO:"));+ }++ #[test]+ fn legacy_and_unknown_host_records_never_use_the_invoking_host() {+ let entries = parse(+ "[[xfail]]\ncase = \"a\"\nclass = \"sync\"\nhosts = [\"local\", \"github-hosted\"]\nreason = \"known offset\"\n",+ )+ .unwrap();+ for host in [None, Some(""), Some("unknown")] {+ let mut runs = Runs::new();+ add_record(&mut runs, source_record(host, None));+ add_record(&mut runs, source_record(Some("blacksmith"), None));+ let host_runs = &runs["expectations.toml"];+ assert!(host_runs.contains_key(&("macos".into(), None)));+ assert_eq!(proposals(&entries, host_runs), entries);+ }+ }+#[test]fn expanded_signature_requires_a_new_explanation() {let entries = parse(@@ -216,6 +300,7 @@ mod tests {let proposals = bless_with(&entries,"macos",+ Some("local"),&ran,&observed,&evidence,@@ -239,6 +324,7 @@ mod tests {let retained = bless_with(&entries,"macos",+ Some("local"),&ran,&observed,&Evidence {