Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): B29 - measure the ranking against an uninformed ordering, and report the modest answer
CarterPerez-dev06ad208b42ef
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rsnew file mode 100644index 00000000..dc1a19d5--- /dev/null+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rs@@ -0,0 +1,389 @@+// ©AngelaMos | 2026+// eval.rs++use std::collections::BTreeSet;++use crate::harness::Harness;+use crate::rank::Ranking;+use crate::surface::Surface;++pub const DEFAULT_CUTOFFS: [usize; 3] = [5, 10, 20];+pub const BASELINE_ROUNDS: u64 = 200;+pub const BASELINE_SEED: u64 = 0x6C69_7364_6578_0001;++#[derive(Debug, Clone, Copy, PartialEq)]+pub struct AtCutoff {+ pub k: usize,+ pub precision: f64,+ pub recall: f64,+}++#[derive(Debug, Clone, PartialEq)]+pub struct Evaluation {+ pub labels: BTreeSet<String>,+ pub ranked: BTreeSet<String>,+ pub dropped: BTreeSet<String>,+ pub first_rank: Option<usize>,+ pub reciprocal_rank: f64,+ pub cutoffs: Vec<AtCutoff>,+ pub baseline_reciprocal_rank: f64,+ pub baseline_hits: Vec<f64>,+ pub ordered: usize,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Unmeasurable {+ NoHarness,+ NoLabels { calls: usize },+ NoCandidates,+}++impl std::fmt::Display for Unmeasurable {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Unmeasurable::NoHarness => write!(+ formatter,+ "this tree ships no in-tree fuzz harness, so there is no author-chosen entry point to score the ranking against"+ ),+ Unmeasurable::NoLabels { calls } => write!(+ formatter,+ "none of the {calls} call(s) mined from the in-tree harnesses is a function this library declares, so there are no usable labels"+ ),+ Unmeasurable::NoCandidates => {+ write!(+ formatter,+ "the ranking is empty, so there is nothing to score"+ )+ }+ }+ }+}++impl Evaluation {+ #[must_use]+ #[allow(clippy::cast_precision_loss)]+ pub fn selection_recall(&self) -> f64 {+ if self.labels.is_empty() {+ return 0.0;+ }+ self.ranked.len() as f64 / self.labels.len() as f64+ }+}++// The universe is every function the ranker CONSIDERED, candidates plus the ones it+// deliberately excluded. Restricting labels to it drops the noise B36 documents in the+// call miner - a licence header mines `Copyright` as a call - without also dropping a+// genuine entry point the ranker chose to exclude, which has to count against it.+#[must_use]+pub fn considered(ranking: &Ranking) -> BTreeSet<String> {+ ranking+ .candidates+ .iter()+ .map(|candidate| candidate.name.clone())+ .chain(ranking.excluded.iter().map(|item| item.name.clone()))+ .collect()+}++#[must_use]+pub fn labels_from(harnesses: &[Harness], ranking: &Ranking) -> BTreeSet<String> {+ let universe = considered(ranking);+ harnesses+ .iter()+ .flat_map(|harness| harness.calls.iter())+ .filter(|call| universe.contains(*call))+ .cloned()+ .collect()+}++// A harness calls far more than its entry point: libarchive's own fuzzer walks through+// archive_entry_set_uid and archive_write_set_format_zip, and libpng's through+// png_set_gray_to_rgb. Labelling every call as "a parser entry point" is over-inclusive,+// which is why precision@k here is not a quality number and MUST NOT be read as one. The+// question the label set CAN answer is whether lisdex's ordering puts an author-chosen+// function nearer the top than an uninformed ordering does, and that is answered by+// comparing against the shuffled baseline below over the SAME labels.+#[must_use]+#[allow(clippy::cast_precision_loss)]+pub fn score_order(+ order: &[&str],+ labels: &BTreeSet<String>,+ cutoffs: &[usize],+) -> (f64, Vec<usize>) {+ let reciprocal = order+ .iter()+ .position(|name| labels.contains(*name))+ .map_or(0.0, |index| 1.0 / (index + 1) as f64);+ let hits = cutoffs+ .iter()+ .map(|k| {+ let taken = order.len().min(*k);+ order[..taken]+ .iter()+ .filter(|name| labels.contains(**name))+ .count()+ })+ .collect();+ (reciprocal, hits)+}++// A deterministic permutation so a reported baseline can be reproduced exactly. Seeded+// per round by the caller; the constants are the SplitMix64 finalizer.+#[must_use]+#[allow(clippy::cast_possible_truncation)]+pub fn shuffled<'a>(order: &[&'a str], seed: u64) -> Vec<&'a str> {+ let mut state = seed;+ let mut next = || {+ state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);+ let mut z = state;+ z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);+ z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);+ z ^ (z >> 31)+ };+ let mut out = order.to_vec();+ for index in (1..out.len()).rev() {+ let pick = (next() % (index as u64 + 1)) as usize;+ out.swap(index, pick);+ }+ out+}++#[allow(clippy::cast_precision_loss)]+pub fn evaluate(surface: &Surface, cutoffs: &[usize]) -> Result<Evaluation, Unmeasurable> {+ if surface.harnesses.is_empty() {+ return Err(Unmeasurable::NoHarness);+ }+ if surface.ranking.candidates.is_empty() {+ return Err(Unmeasurable::NoCandidates);+ }+ let labels = labels_from(&surface.harnesses, &surface.ranking);+ if labels.is_empty() {+ let calls = surface+ .harnesses+ .iter()+ .map(|harness| harness.calls.len())+ .sum();+ return Err(Unmeasurable::NoLabels { calls });+ }++ let order: Vec<&str> = surface+ .ranking+ .candidates+ .iter()+ .map(|candidate| candidate.name.as_str())+ .collect();+ let ranked: BTreeSet<String> = order+ .iter()+ .filter(|name| labels.contains(**name))+ .map(|name| (*name).to_owned())+ .collect();+ let dropped = labels.difference(&ranked).cloned().collect();++ let first_rank = order+ .iter()+ .position(|name| labels.contains(*name))+ .map(|index| index + 1);+ let reciprocal_rank = first_rank.map_or(0.0, |rank| 1.0 / rank as f64);++ let cutoff_ks: Vec<usize> = cutoffs.to_vec();+ let cutoffs: Vec<AtCutoff> = cutoffs+ .iter()+ .map(|k| {+ let taken = order.len().min(*k);+ let hits = order[..taken]+ .iter()+ .filter(|name| labels.contains(**name))+ .count();+ AtCutoff {+ k: *k,+ precision: if taken == 0 {+ 0.0+ } else {+ hits as f64 / taken as f64+ },+ recall: hits as f64 / labels.len() as f64,+ }+ })+ .collect();++ let mut baseline_reciprocal = 0.0_f64;+ let mut baseline_hits = vec![0.0_f64; cutoffs.len()];+ for round in 0..BASELINE_ROUNDS {+ let permuted = shuffled(&order, BASELINE_SEED ^ round);+ let (reciprocal, hits) = score_order(&permuted, &labels, &cutoff_ks);+ baseline_reciprocal += reciprocal;+ for (slot, hit) in hits.iter().enumerate() {+ baseline_hits[slot] += *hit as f64;+ }+ }+ let rounds = BASELINE_ROUNDS as f64;++ Ok(Evaluation {+ labels,+ ranked,+ dropped,+ first_rank,+ reciprocal_rank,+ cutoffs,+ baseline_reciprocal_rank: baseline_reciprocal / rounds,+ baseline_hits: baseline_hits.iter().map(|hit| hit / rounds).collect(),+ ordered: order.len(),+ })+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::rank::{Candidate, Excluded, ExclusionReason};+ use crate::signature::Analysis;+ use std::path::PathBuf;++ fn candidate(name: &str) -> Candidate {+ Candidate {+ name: name.to_owned(),+ params: Vec::new(),+ result: crate::signature::Returns {+ spelling: "void".to_owned(),+ shape: crate::signature::Shape::Integral,+ },+ header: PathBuf::from("lib.h"),+ line: 1,+ fuzzability: crate::signature::Fuzzability::CastBuffer,+ analysis: Analysis {+ fuzzability: crate::signature::Fuzzability::CastBuffer,+ roles: Vec::new(),+ context_args: 0,+ extents: std::collections::BTreeMap::new(),+ },+ complexity: crate::rank::Complexity {+ own: 0,+ accumulated: 0,+ undiscovered: 0,+ },+ exported: true,+ reachable_from_harness: false,+ score: 0.0,+ reasons: Vec::new(),+ }+ }++ fn ranking(names: &[&str], excluded: &[&str]) -> Ranking {+ Ranking {+ candidates: names.iter().map(|name| candidate(name)).collect(),+ excluded: excluded+ .iter()+ .map(|name| Excluded {+ name: (*name).to_owned(),+ reason: ExclusionReason::Primitive,+ })+ .collect(),+ }+ }++ fn harness(calls: &[&str]) -> Harness {+ Harness {+ path: PathBuf::from("fuzz.c"),+ calls: calls.iter().map(|call| (*call).to_owned()).collect(),+ }+ }++ #[test]+ fn a_call_the_library_does_not_declare_is_not_a_label() {+ let ranking = ranking(&["png_read_png", "png_read_info"], &["png_create"]);+ let harnesses = [harness(&[+ "png_read_png",+ "Copyright",+ "memcpy",+ "png_create",+ ])];+ let labels = labels_from(&harnesses, &ranking);+ assert!(labels.contains("png_read_png"));+ assert!(+ labels.contains("png_create"),+ "a deliberately EXCLUDED function is still a real label; missing it must count against the ranker"+ );+ assert!(!labels.contains("Copyright"), "licence-header noise");+ assert!(!labels.contains("memcpy"), "libc is not this library");+ }++ #[test]+ fn a_tree_with_no_harness_cannot_be_scored_at_all() {+ let surface = surface_with(ranking(&["a"], &[]), Vec::new());+ assert_eq!(+ evaluate(&surface, &DEFAULT_CUTOFFS),+ Err(Unmeasurable::NoHarness)+ );+ }++ #[test]+ fn harness_calls_that_are_all_foreign_leave_nothing_to_measure() {+ let surface = surface_with(ranking(&["a"], &[]), vec![harness(&["printf", "malloc"])]);+ assert_eq!(+ evaluate(&surface, &DEFAULT_CUTOFFS),+ Err(Unmeasurable::NoLabels { calls: 2 })+ );+ }++ #[test]+ fn a_perfect_ranking_scores_one_at_every_cutoff_it_covers() {+ let surface = surface_with(+ ranking(&["hit_a", "hit_b", "miss_c", "miss_d"], &[]),+ vec![harness(&["hit_a", "hit_b"])],+ );+ let evaluation = evaluate(&surface, &[2, 4]).unwrap();+ assert_eq!(evaluation.first_rank, Some(1));+ assert!((evaluation.reciprocal_rank - 1.0).abs() < 1e-12);+ assert!((evaluation.cutoffs[0].precision - 1.0).abs() < 1e-12);+ assert!((evaluation.cutoffs[0].recall - 1.0).abs() < 1e-12);+ assert!((evaluation.cutoffs[1].precision - 0.5).abs() < 1e-12);+ assert!((evaluation.cutoffs[1].recall - 1.0).abs() < 1e-12);+ }++ #[test]+ fn a_ranking_that_buries_the_entry_point_scores_badly() {+ let surface = surface_with(+ ranking(&["noise1", "noise2", "noise3", "real"], &[]),+ vec![harness(&["real"])],+ );+ let evaluation = evaluate(&surface, &[1, 4]).unwrap();+ assert_eq!(evaluation.first_rank, Some(4));+ assert!((evaluation.reciprocal_rank - 0.25).abs() < 1e-12);+ assert!((evaluation.cutoffs[0].recall - 0.0).abs() < 1e-12);+ assert!((evaluation.cutoffs[1].recall - 1.0).abs() < 1e-12);+ }++ // A label the ranker EXCLUDED never appears in the ordered list, so it can never be+ // recalled at any cutoff. That is a selection failure and has to be visible separately.+ #[test]+ fn a_label_the_ranker_excluded_is_reported_as_dropped_not_silently_forgiven() {+ let surface = surface_with(+ ranking(&["kept"], &["thrown_away"]),+ vec![harness(&["kept", "thrown_away"])],+ );+ let evaluation = evaluate(&surface, &[10]).unwrap();+ assert_eq!(evaluation.labels.len(), 2);+ assert_eq!(evaluation.ranked.len(), 1);+ assert!(evaluation.dropped.contains("thrown_away"));+ assert!((evaluation.selection_recall() - 0.5).abs() < 1e-12);+ assert!(+ (evaluation.cutoffs[0].recall - 0.5).abs() < 1e-12,+ "recall must be against ALL labels, not only the rankable ones"+ );+ }++ fn surface_with(ranking: Ranking, harnesses: Vec<Harness>) -> Surface {+ Surface {+ source: PathBuf::from("/tree"),+ header_root: PathBuf::from("/tree"),+ origin: crate::headers::HeaderOrigin::Tree,+ headers_read: 1,+ headers: Vec::new(),+ skipped: Vec::new(),+ harnesses,+ ladder: crate::ladder::Ladder::default(),+ declared: 1,+ defined: 1,+ skipped_sources: Vec::new(),+ ranking,+ }+ }+}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rsindex 54bb0e10..b5847244 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs@@ -2,6 +2,7 @@// lib.rspub mod complexity;+pub mod eval;pub mod extract;pub mod harness;pub mod headers;diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rsindex 48b4da0f..c93cde1b 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs@@ -39,6 +39,7 @@ use lisdex_monitor::snapshot::Source as MonitorSource;use lisdex_report::advisory::Advisory;use lisdex_report::bundle::{self, Reproducer};use lisdex_report::{json, markdown};+use lisdex_surface::eval;use lisdex_surface::extract::{self, Language};use lisdex_surface::surface::{self, SurfaceOptions};use lisdex_triage::classify::classify;@@ -84,6 +85,23 @@ const LLM_PROBE_BALLOT: [&str; 2] = ["byte_buffer", "nul_terminated_string"];const SAMPLE_CSV_HEADER: &str ="relative_time,corpus_count,edges_found,execs_per_sec,total_execs,saved_crashes,map_size_pct";+#[derive(clap::Args)]+struct RankEvalArgs {+ #[arg(help = "source tree of the target library, repeatable")]+ sources: Vec<PathBuf>,+ #[arg(long = "include", help = "extra include directory, repeatable")]+ includes: Vec<PathBuf>,+ #[arg(long = "define", help = "extra preprocessor define, repeatable")]+ defines: Vec<String>,+ #[arg(long, help = "parse headers as C++ instead of C")]+ cpp: bool,+ #[arg(+ long = "at",+ help = "cutoff to report precision and recall at, repeatable"+ )]+ cutoffs: Vec<usize>,+}+#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]enum MetricChoice {Edges,@@ -194,6 +212,10 @@ enum Command {about = "compare two builds over repeated trials, with a Mann-Whitney U test and an effect size")]Bench(BenchArgs),+ #[command(+ about = "score the cold-start ranking against the entry points this project's own fuzz harnesses call"+ )]+ RankEval(RankEvalArgs),#[command(about = "watch a campaign live in a terminal dashboard")]Tui(TuiArgs),#[command(about = "serve a local web console over the same campaign state")]@@ -714,6 +736,7 @@ fn dispatch() -> Result<(), Box<dyn Error>> {Command::Minimize(args) => minimize_campaign(args),Command::Llm(args) => llm(args),Command::Bench(args) => run_bench(args),+ Command::RankEval(args) => rank_eval(args),Command::Tui(args) => tui(args),Command::Serve(args) => serve(args),}@@ -802,6 +825,146 @@ fn library_vocabulary(merged}+#[allow(+ clippy::cast_precision_loss,+ clippy::cast_possible_truncation,+ clippy::cast_sign_loss+)]+fn report_tree(+ label: &str,+ evaluation: &eval::Evaluation,+ hits: &mut [usize],+ baseline_hits: &mut [f64],+) {+ println!(+ " {label:<12} candidates={:<5} labels={:<4} reached ranking={:<4} first={:<5}",+ evaluation.ordered,+ evaluation.labels.len(),+ evaluation.ranked.len(),+ evaluation+ .first_rank+ .map_or_else(|| "none".to_owned(), |rank| rank.to_string()),+ );+ print!(+ " {:<12} lisdex rr={:.3}",+ "", evaluation.reciprocal_rank+ );+ for (index, cutoff) in evaluation.cutoffs.iter().enumerate() {+ let hit = (cutoff.recall * evaluation.labels.len() as f64).round() as usize;+ hits[index] += hit;+ print!(" hits@{}={hit}", cutoff.k);+ }+ println!();+ print!(+ " {:<12} shuffled rr={:.3}",+ "", evaluation.baseline_reciprocal_rank+ );+ for (index, cutoff) in evaluation.cutoffs.iter().enumerate() {+ print!(" hits@{}={:.2}", cutoff.k, evaluation.baseline_hits[index]);+ baseline_hits[index] += evaluation.baseline_hits[index];+ }+ println!();+ if !evaluation.dropped.is_empty() {+ let mut dropped: Vec<&str> = evaluation.dropped.iter().map(String::as_str).collect();+ dropped.sort_unstable();+ println!(+ " {:<12} dropped before ranking: {}",+ "",+ dropped.join(", ")+ );+ }+}++fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {+ if args.sources.is_empty() {+ return Err("name at least one source tree to score".into());+ }+ let cutoffs: Vec<usize> = if args.cutoffs.is_empty() {+ eval::DEFAULT_CUTOFFS.to_vec()+ } else {+ args.cutoffs.clone()+ };++ let mut scored = 0usize;+ let mut reciprocal_total = 0.0_f64;+ let mut label_total = 0usize;+ let mut ranked_total = 0usize;+ let mut hits: Vec<usize> = vec![0; cutoffs.len()];+ let mut baseline_hits: Vec<f64> = vec![0.0; cutoffs.len()];+ let mut baseline_total = 0.0_f64;++ for source in &args.sources {+ let mut options = SurfaceOptions::new(source);+ options.includes.clone_from(&args.includes);+ options.defines.clone_from(&args.defines);+ options.language = if args.cpp { Language::Cpp } else { Language::C };+ // Cold start on purpose. With ignore_harnesses the reachability seed set is empty,+ // so the ranking never sees the harnesses the labels are drawn from. Scoring a+ // warm ranking against harness-derived labels would be measuring the ranker+ // against its own input.+ options.ignore_harnesses = true;++ let label = source.file_name().map_or_else(+ || source.display().to_string(),+ |name| name.to_string_lossy().into_owned(),+ );+ let analysis = match surface::analyze(&options) {+ Ok(analysis) => analysis,+ Err(err) => {+ println!(" {label:<12} NOT ANALYSED {err}");+ continue;+ }+ };+ match eval::evaluate(&analysis, &cutoffs) {+ Ok(evaluation) => {+ scored += 1;+ reciprocal_total += evaluation.reciprocal_rank;+ label_total += evaluation.labels.len();+ ranked_total += evaluation.ranked.len();+ baseline_total += evaluation.baseline_reciprocal_rank;+ report_tree(&label, &evaluation, &mut hits, &mut baseline_hits);+ }+ Err(reason) => println!(" {label:<12} NOT SCORED {reason}"),+ }+ }++ if scored == 0 {+ return Err("no tree could be scored, so there is no ranking measurement".into());+ }++ #[allow(clippy::cast_precision_loss)]+ {+ let mrr = reciprocal_total / scored as f64;+ let baseline_mrr = baseline_total / scored as f64;+ println!(+ "\ntrees scored {scored} labels {label_total} reached ranking {ranked_total} \+ selection recall {:.3}",+ ranked_total as f64 / label_total as f64+ );+ let lift = if baseline_mrr > 0.0 && mrr > 0.0 {+ format!("{:.2}x", mrr / baseline_mrr)+ } else {+ "n/a (nothing was found by either ordering)".to_owned()+ };+ println!(" MRR lisdex {mrr:.3} shuffled {baseline_mrr:.3} lift {lift}");+ for (index, k) in cutoffs.iter().enumerate() {+ println!(+ " hits@{k} lisdex {} ({:.3} of {label_total}) shuffled {:.2} ({:.3})",+ hits[index],+ hits[index] as f64 / label_total as f64,+ baseline_hits[index],+ baseline_hits[index] / label_total as f64+ );+ }+ println!(+ "\nprecision@k is deliberately NOT reported as a quality number: a project's own\n\+ harness calls its setup and teardown too, so most labels are not entry points.\n\+ The shuffled baseline faces the SAME labels, so the lift is the comparable figure."+ );+ }+ Ok(())+}+fn parse_arm(spec: &str) -> Result<(String, PathBuf), Box<dyn Error>> {let (name, path) = spec.split_once('=')diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.shindex fac5960e..1ac4fa36 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh@@ -896,6 +896,41 @@ elsefirm -rf "$BARE"+banner "B29 RANKING MEASURED AGAINST AN UNINFORMED ORDERING"+# The band assertions above check six hand-picked functions. They cannot say whether the+# ORDERING carries information, because a ranking that put those six first and everything+# else at random would pass all of them. This scores the cold-start ranking against the+# entry points each project's OWN fuzz harnesses call, and against a shuffled ordering of+# the same candidates facing the same labels.+#+# Cold start matters: with harness reachability seeded, the ranker would be scored against+# its own input. `rank-eval` forces ignore_harnesses, so the ordering never sees the+# harnesses the labels come from.+RANKEVAL="$OUT/rank-eval.txt"+if "$LISDEX" rank-eval "$SRC/libpng" "$SRC/libarchive" "$SRC/lz4" "$SRC/brotli" \+ >"$RANKEVAL" 2>&1; then+ LIS_MRR="$(awk '/^ MRR/ {print $3}' "$RANKEVAL")"+ SHUF_MRR="$(awk '/^ MRR/ {print $5}' "$RANKEVAL")"+ if [ -n "$LIS_MRR" ] && [ -n "$SHUF_MRR" ]; then+ # The baseline must be NON-ZERO, or "lisdex beat it" is satisfied by a baseline+ # that never finds anything and the comparison proves nothing.+ if awk "BEGIN{exit !($SHUF_MRR > 0)}"; then+ ok "the shuffled baseline finds labels too (MRR $SHUF_MRR), so the comparison is live"+ else+ bad "the shuffled baseline scored 0, so it cannot discriminate anything"+ fi+ if awk "BEGIN{exit !($LIS_MRR > $SHUF_MRR)}"; then+ ok "the ranking beats an uninformed ordering on MRR ($LIS_MRR vs $SHUF_MRR)"+ else+ bad "the ranking is no better than a shuffle of its own candidates ($LIS_MRR vs $SHUF_MRR)"+ fi+ else+ bad "rank-eval produced no MRR line (see $RANKEVAL)"+ fi+else+ bad "rank-eval failed (see $RANKEVAL)"+fi+banner "RESULT"echo " passed: $PASS"echo " failed: $FAIL"