Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇

feat(lisdex): B29 - measure the ranking against an uninformed ordering, and report the modest answer

CarterPerez-dev06ad208b42ef
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rs
new file mode 100644
index 00000000..dc1a19d5
--- /dev/null
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/eval.rs
@@ -0,0 +1,389 @@
+// ©AngelaMos | 2026
+// eval.rs
+
+use std::collections::BTreeSet;
+
+use crate::harness::Harness;
+use crate::rank::Ranking;
+use crate::surface::Surface;
+
+pub const DEFAULT_CUTOFFS: [usize; 3] = [5, 10, 20];
+pub const BASELINE_ROUNDS: u64 = 200;
+pub const BASELINE_SEED: u64 = 0x6C69_7364_6578_0001;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct AtCutoff {
+ pub k: usize,
+ pub precision: f64,
+ pub recall: f64,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Evaluation {
+ pub labels: BTreeSet<String>,
+ pub ranked: BTreeSet<String>,
+ pub dropped: BTreeSet<String>,
+ pub first_rank: Option<usize>,
+ pub reciprocal_rank: f64,
+ pub cutoffs: Vec<AtCutoff>,
+ pub baseline_reciprocal_rank: f64,
+ pub baseline_hits: Vec<f64>,
+ pub ordered: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Unmeasurable {
+ NoHarness,
+ NoLabels { calls: usize },
+ NoCandidates,
+}
+
+impl std::fmt::Display for Unmeasurable {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Unmeasurable::NoHarness => write!(
+ formatter,
+ "this tree ships no in-tree fuzz harness, so there is no author-chosen entry point to score the ranking against"
+ ),
+ Unmeasurable::NoLabels { calls } => write!(
+ formatter,
+ "none of the {calls} call(s) mined from the in-tree harnesses is a function this library declares, so there are no usable labels"
+ ),
+ Unmeasurable::NoCandidates => {
+ write!(
+ formatter,
+ "the ranking is empty, so there is nothing to score"
+ )
+ }
+ }
+ }
+}
+
+impl Evaluation {
+ #[must_use]
+ #[allow(clippy::cast_precision_loss)]
+ pub fn selection_recall(&self) -> f64 {
+ if self.labels.is_empty() {
+ return 0.0;
+ }
+ self.ranked.len() as f64 / self.labels.len() as f64
+ }
+}
+
+// The universe is every function the ranker CONSIDERED, candidates plus the ones it
+// deliberately excluded. Restricting labels to it drops the noise B36 documents in the
+// call miner - a licence header mines `Copyright` as a call - without also dropping a
+// genuine entry point the ranker chose to exclude, which has to count against it.
+#[must_use]
+pub fn considered(ranking: &Ranking) -> BTreeSet<String> {
+ ranking
+ .candidates
+ .iter()
+ .map(|candidate| candidate.name.clone())
+ .chain(ranking.excluded.iter().map(|item| item.name.clone()))
+ .collect()
+}
+
+#[must_use]
+pub fn labels_from(harnesses: &[Harness], ranking: &Ranking) -> BTreeSet<String> {
+ let universe = considered(ranking);
+ harnesses
+ .iter()
+ .flat_map(|harness| harness.calls.iter())
+ .filter(|call| universe.contains(*call))
+ .cloned()
+ .collect()
+}
+
+// A harness calls far more than its entry point: libarchive's own fuzzer walks through
+// archive_entry_set_uid and archive_write_set_format_zip, and libpng's through
+// png_set_gray_to_rgb. Labelling every call as "a parser entry point" is over-inclusive,
+// which is why precision@k here is not a quality number and MUST NOT be read as one. The
+// question the label set CAN answer is whether lisdex's ordering puts an author-chosen
+// function nearer the top than an uninformed ordering does, and that is answered by
+// comparing against the shuffled baseline below over the SAME labels.
+#[must_use]
+#[allow(clippy::cast_precision_loss)]
+pub fn score_order(
+ order: &[&str],
+ labels: &BTreeSet<String>,
+ cutoffs: &[usize],
+) -> (f64, Vec<usize>) {
+ let reciprocal = order
+ .iter()
+ .position(|name| labels.contains(*name))
+ .map_or(0.0, |index| 1.0 / (index + 1) as f64);
+ let hits = cutoffs
+ .iter()
+ .map(|k| {
+ let taken = order.len().min(*k);
+ order[..taken]
+ .iter()
+ .filter(|name| labels.contains(**name))
+ .count()
+ })
+ .collect();
+ (reciprocal, hits)
+}
+
+// A deterministic permutation so a reported baseline can be reproduced exactly. Seeded
+// per round by the caller; the constants are the SplitMix64 finalizer.
+#[must_use]
+#[allow(clippy::cast_possible_truncation)]
+pub fn shuffled<'a>(order: &[&'a str], seed: u64) -> Vec<&'a str> {
+ let mut state = seed;
+ let mut next = || {
+ state = state.wrapping_add(0x9E37_79B9_7F4A_7C15);
+ let mut z = state;
+ z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+ z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+ z ^ (z >> 31)
+ };
+ let mut out = order.to_vec();
+ for index in (1..out.len()).rev() {
+ let pick = (next() % (index as u64 + 1)) as usize;
+ out.swap(index, pick);
+ }
+ out
+}
+
+#[allow(clippy::cast_precision_loss)]
+pub fn evaluate(surface: &Surface, cutoffs: &[usize]) -> Result<Evaluation, Unmeasurable> {
+ if surface.harnesses.is_empty() {
+ return Err(Unmeasurable::NoHarness);
+ }
+ if surface.ranking.candidates.is_empty() {
+ return Err(Unmeasurable::NoCandidates);
+ }
+ let labels = labels_from(&surface.harnesses, &surface.ranking);
+ if labels.is_empty() {
+ let calls = surface
+ .harnesses
+ .iter()
+ .map(|harness| harness.calls.len())
+ .sum();
+ return Err(Unmeasurable::NoLabels { calls });
+ }
+
+ let order: Vec<&str> = surface
+ .ranking
+ .candidates
+ .iter()
+ .map(|candidate| candidate.name.as_str())
+ .collect();
+ let ranked: BTreeSet<String> = order
+ .iter()
+ .filter(|name| labels.contains(**name))
+ .map(|name| (*name).to_owned())
+ .collect();
+ let dropped = labels.difference(&ranked).cloned().collect();
+
+ let first_rank = order
+ .iter()
+ .position(|name| labels.contains(*name))
+ .map(|index| index + 1);
+ let reciprocal_rank = first_rank.map_or(0.0, |rank| 1.0 / rank as f64);
+
+ let cutoff_ks: Vec<usize> = cutoffs.to_vec();
+ let cutoffs: Vec<AtCutoff> = cutoffs
+ .iter()
+ .map(|k| {
+ let taken = order.len().min(*k);
+ let hits = order[..taken]
+ .iter()
+ .filter(|name| labels.contains(**name))
+ .count();
+ AtCutoff {
+ k: *k,
+ precision: if taken == 0 {
+ 0.0
+ } else {
+ hits as f64 / taken as f64
+ },
+ recall: hits as f64 / labels.len() as f64,
+ }
+ })
+ .collect();
+
+ let mut baseline_reciprocal = 0.0_f64;
+ let mut baseline_hits = vec![0.0_f64; cutoffs.len()];
+ for round in 0..BASELINE_ROUNDS {
+ let permuted = shuffled(&order, BASELINE_SEED ^ round);
+ let (reciprocal, hits) = score_order(&permuted, &labels, &cutoff_ks);
+ baseline_reciprocal += reciprocal;
+ for (slot, hit) in hits.iter().enumerate() {
+ baseline_hits[slot] += *hit as f64;
+ }
+ }
+ let rounds = BASELINE_ROUNDS as f64;
+
+ Ok(Evaluation {
+ labels,
+ ranked,
+ dropped,
+ first_rank,
+ reciprocal_rank,
+ cutoffs,
+ baseline_reciprocal_rank: baseline_reciprocal / rounds,
+ baseline_hits: baseline_hits.iter().map(|hit| hit / rounds).collect(),
+ ordered: order.len(),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::rank::{Candidate, Excluded, ExclusionReason};
+ use crate::signature::Analysis;
+ use std::path::PathBuf;
+
+ fn candidate(name: &str) -> Candidate {
+ Candidate {
+ name: name.to_owned(),
+ params: Vec::new(),
+ result: crate::signature::Returns {
+ spelling: "void".to_owned(),
+ shape: crate::signature::Shape::Integral,
+ },
+ header: PathBuf::from("lib.h"),
+ line: 1,
+ fuzzability: crate::signature::Fuzzability::CastBuffer,
+ analysis: Analysis {
+ fuzzability: crate::signature::Fuzzability::CastBuffer,
+ roles: Vec::new(),
+ context_args: 0,
+ extents: std::collections::BTreeMap::new(),
+ },
+ complexity: crate::rank::Complexity {
+ own: 0,
+ accumulated: 0,
+ undiscovered: 0,
+ },
+ exported: true,
+ reachable_from_harness: false,
+ score: 0.0,
+ reasons: Vec::new(),
+ }
+ }
+
+ fn ranking(names: &[&str], excluded: &[&str]) -> Ranking {
+ Ranking {
+ candidates: names.iter().map(|name| candidate(name)).collect(),
+ excluded: excluded
+ .iter()
+ .map(|name| Excluded {
+ name: (*name).to_owned(),
+ reason: ExclusionReason::Primitive,
+ })
+ .collect(),
+ }
+ }
+
+ fn harness(calls: &[&str]) -> Harness {
+ Harness {
+ path: PathBuf::from("fuzz.c"),
+ calls: calls.iter().map(|call| (*call).to_owned()).collect(),
+ }
+ }
+
+ #[test]
+ fn a_call_the_library_does_not_declare_is_not_a_label() {
+ let ranking = ranking(&["png_read_png", "png_read_info"], &["png_create"]);
+ let harnesses = [harness(&[
+ "png_read_png",
+ "Copyright",
+ "memcpy",
+ "png_create",
+ ])];
+ let labels = labels_from(&harnesses, &ranking);
+ assert!(labels.contains("png_read_png"));
+ assert!(
+ labels.contains("png_create"),
+ "a deliberately EXCLUDED function is still a real label; missing it must count against the ranker"
+ );
+ assert!(!labels.contains("Copyright"), "licence-header noise");
+ assert!(!labels.contains("memcpy"), "libc is not this library");
+ }
+
+ #[test]
+ fn a_tree_with_no_harness_cannot_be_scored_at_all() {
+ let surface = surface_with(ranking(&["a"], &[]), Vec::new());
+ assert_eq!(
+ evaluate(&surface, &DEFAULT_CUTOFFS),
+ Err(Unmeasurable::NoHarness)
+ );
+ }
+
+ #[test]
+ fn harness_calls_that_are_all_foreign_leave_nothing_to_measure() {
+ let surface = surface_with(ranking(&["a"], &[]), vec![harness(&["printf", "malloc"])]);
+ assert_eq!(
+ evaluate(&surface, &DEFAULT_CUTOFFS),
+ Err(Unmeasurable::NoLabels { calls: 2 })
+ );
+ }
+
+ #[test]
+ fn a_perfect_ranking_scores_one_at_every_cutoff_it_covers() {
+ let surface = surface_with(
+ ranking(&["hit_a", "hit_b", "miss_c", "miss_d"], &[]),
+ vec![harness(&["hit_a", "hit_b"])],
+ );
+ let evaluation = evaluate(&surface, &[2, 4]).unwrap();
+ assert_eq!(evaluation.first_rank, Some(1));
+ assert!((evaluation.reciprocal_rank - 1.0).abs() < 1e-12);
+ assert!((evaluation.cutoffs[0].precision - 1.0).abs() < 1e-12);
+ assert!((evaluation.cutoffs[0].recall - 1.0).abs() < 1e-12);
+ assert!((evaluation.cutoffs[1].precision - 0.5).abs() < 1e-12);
+ assert!((evaluation.cutoffs[1].recall - 1.0).abs() < 1e-12);
+ }
+
+ #[test]
+ fn a_ranking_that_buries_the_entry_point_scores_badly() {
+ let surface = surface_with(
+ ranking(&["noise1", "noise2", "noise3", "real"], &[]),
+ vec![harness(&["real"])],
+ );
+ let evaluation = evaluate(&surface, &[1, 4]).unwrap();
+ assert_eq!(evaluation.first_rank, Some(4));
+ assert!((evaluation.reciprocal_rank - 0.25).abs() < 1e-12);
+ assert!((evaluation.cutoffs[0].recall - 0.0).abs() < 1e-12);
+ assert!((evaluation.cutoffs[1].recall - 1.0).abs() < 1e-12);
+ }
+
+ // A label the ranker EXCLUDED never appears in the ordered list, so it can never be
+ // recalled at any cutoff. That is a selection failure and has to be visible separately.
+ #[test]
+ fn a_label_the_ranker_excluded_is_reported_as_dropped_not_silently_forgiven() {
+ let surface = surface_with(
+ ranking(&["kept"], &["thrown_away"]),
+ vec![harness(&["kept", "thrown_away"])],
+ );
+ let evaluation = evaluate(&surface, &[10]).unwrap();
+ assert_eq!(evaluation.labels.len(), 2);
+ assert_eq!(evaluation.ranked.len(), 1);
+ assert!(evaluation.dropped.contains("thrown_away"));
+ assert!((evaluation.selection_recall() - 0.5).abs() < 1e-12);
+ assert!(
+ (evaluation.cutoffs[0].recall - 0.5).abs() < 1e-12,
+ "recall must be against ALL labels, not only the rankable ones"
+ );
+ }
+
+ fn surface_with(ranking: Ranking, harnesses: Vec<Harness>) -> Surface {
+ Surface {
+ source: PathBuf::from("/tree"),
+ header_root: PathBuf::from("/tree"),
+ origin: crate::headers::HeaderOrigin::Tree,
+ headers_read: 1,
+ headers: Vec::new(),
+ skipped: Vec::new(),
+ harnesses,
+ ladder: crate::ladder::Ladder::default(),
+ declared: 1,
+ defined: 1,
+ skipped_sources: Vec::new(),
+ ranking,
+ }
+ }
+}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs
index 54bb0e10..b5847244 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/lib.rs
@@ -2,6 +2,7 @@
// lib.rs
pub mod complexity;
+pub mod eval;
pub mod extract;
pub mod harness;
pub mod headers;
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
index 48b4da0f..c93cde1b 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
@@ -39,6 +39,7 @@ use lisdex_monitor::snapshot::Source as MonitorSource;
use lisdex_report::advisory::Advisory;
use lisdex_report::bundle::{self, Reproducer};
use lisdex_report::{json, markdown};
+use lisdex_surface::eval;
use lisdex_surface::extract::{self, Language};
use lisdex_surface::surface::{self, SurfaceOptions};
use lisdex_triage::classify::classify;
@@ -84,6 +85,23 @@ const LLM_PROBE_BALLOT: [&str; 2] = ["byte_buffer", "nul_terminated_string"];
const SAMPLE_CSV_HEADER: &str =
"relative_time,corpus_count,edges_found,execs_per_sec,total_execs,saved_crashes,map_size_pct";
+#[derive(clap::Args)]
+struct RankEvalArgs {
+ #[arg(help = "source tree of the target library, repeatable")]
+ sources: Vec<PathBuf>,
+ #[arg(long = "include", help = "extra include directory, repeatable")]
+ includes: Vec<PathBuf>,
+ #[arg(long = "define", help = "extra preprocessor define, repeatable")]
+ defines: Vec<String>,
+ #[arg(long, help = "parse headers as C++ instead of C")]
+ cpp: bool,
+ #[arg(
+ long = "at",
+ help = "cutoff to report precision and recall at, repeatable"
+ )]
+ cutoffs: Vec<usize>,
+}
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
enum MetricChoice {
Edges,
@@ -194,6 +212,10 @@ enum Command {
about = "compare two builds over repeated trials, with a Mann-Whitney U test and an effect size"
)]
Bench(BenchArgs),
+ #[command(
+ about = "score the cold-start ranking against the entry points this project's own fuzz harnesses call"
+ )]
+ RankEval(RankEvalArgs),
#[command(about = "watch a campaign live in a terminal dashboard")]
Tui(TuiArgs),
#[command(about = "serve a local web console over the same campaign state")]
@@ -714,6 +736,7 @@ fn dispatch() -> Result<(), Box<dyn Error>> {
Command::Minimize(args) => minimize_campaign(args),
Command::Llm(args) => llm(args),
Command::Bench(args) => run_bench(args),
+ Command::RankEval(args) => rank_eval(args),
Command::Tui(args) => tui(args),
Command::Serve(args) => serve(args),
}
@@ -802,6 +825,146 @@ fn library_vocabulary(
merged
}
+#[allow(
+ clippy::cast_precision_loss,
+ clippy::cast_possible_truncation,
+ clippy::cast_sign_loss
+)]
+fn report_tree(
+ label: &str,
+ evaluation: &eval::Evaluation,
+ hits: &mut [usize],
+ baseline_hits: &mut [f64],
+) {
+ println!(
+ " {label:<12} candidates={:<5} labels={:<4} reached ranking={:<4} first={:<5}",
+ evaluation.ordered,
+ evaluation.labels.len(),
+ evaluation.ranked.len(),
+ evaluation
+ .first_rank
+ .map_or_else(|| "none".to_owned(), |rank| rank.to_string()),
+ );
+ print!(
+ " {:<12} lisdex rr={:.3}",
+ "", evaluation.reciprocal_rank
+ );
+ for (index, cutoff) in evaluation.cutoffs.iter().enumerate() {
+ let hit = (cutoff.recall * evaluation.labels.len() as f64).round() as usize;
+ hits[index] += hit;
+ print!(" hits@{}={hit}", cutoff.k);
+ }
+ println!();
+ print!(
+ " {:<12} shuffled rr={:.3}",
+ "", evaluation.baseline_reciprocal_rank
+ );
+ for (index, cutoff) in evaluation.cutoffs.iter().enumerate() {
+ print!(" hits@{}={:.2}", cutoff.k, evaluation.baseline_hits[index]);
+ baseline_hits[index] += evaluation.baseline_hits[index];
+ }
+ println!();
+ if !evaluation.dropped.is_empty() {
+ let mut dropped: Vec<&str> = evaluation.dropped.iter().map(String::as_str).collect();
+ dropped.sort_unstable();
+ println!(
+ " {:<12} dropped before ranking: {}",
+ "",
+ dropped.join(", ")
+ );
+ }
+}
+
+fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {
+ if args.sources.is_empty() {
+ return Err("name at least one source tree to score".into());
+ }
+ let cutoffs: Vec<usize> = if args.cutoffs.is_empty() {
+ eval::DEFAULT_CUTOFFS.to_vec()
+ } else {
+ args.cutoffs.clone()
+ };
+
+ let mut scored = 0usize;
+ let mut reciprocal_total = 0.0_f64;
+ let mut label_total = 0usize;
+ let mut ranked_total = 0usize;
+ let mut hits: Vec<usize> = vec![0; cutoffs.len()];
+ let mut baseline_hits: Vec<f64> = vec![0.0; cutoffs.len()];
+ let mut baseline_total = 0.0_f64;
+
+ for source in &args.sources {
+ let mut options = SurfaceOptions::new(source);
+ options.includes.clone_from(&args.includes);
+ options.defines.clone_from(&args.defines);
+ options.language = if args.cpp { Language::Cpp } else { Language::C };
+ // Cold start on purpose. With ignore_harnesses the reachability seed set is empty,
+ // so the ranking never sees the harnesses the labels are drawn from. Scoring a
+ // warm ranking against harness-derived labels would be measuring the ranker
+ // against its own input.
+ options.ignore_harnesses = true;
+
+ let label = source.file_name().map_or_else(
+ || source.display().to_string(),
+ |name| name.to_string_lossy().into_owned(),
+ );
+ let analysis = match surface::analyze(&options) {
+ Ok(analysis) => analysis,
+ Err(err) => {
+ println!(" {label:<12} NOT ANALYSED {err}");
+ continue;
+ }
+ };
+ match eval::evaluate(&analysis, &cutoffs) {
+ Ok(evaluation) => {
+ scored += 1;
+ reciprocal_total += evaluation.reciprocal_rank;
+ label_total += evaluation.labels.len();
+ ranked_total += evaluation.ranked.len();
+ baseline_total += evaluation.baseline_reciprocal_rank;
+ report_tree(&label, &evaluation, &mut hits, &mut baseline_hits);
+ }
+ Err(reason) => println!(" {label:<12} NOT SCORED {reason}"),
+ }
+ }
+
+ if scored == 0 {
+ return Err("no tree could be scored, so there is no ranking measurement".into());
+ }
+
+ #[allow(clippy::cast_precision_loss)]
+ {
+ let mrr = reciprocal_total / scored as f64;
+ let baseline_mrr = baseline_total / scored as f64;
+ println!(
+ "\ntrees scored {scored} labels {label_total} reached ranking {ranked_total} \
+ selection recall {:.3}",
+ ranked_total as f64 / label_total as f64
+ );
+ let lift = if baseline_mrr > 0.0 && mrr > 0.0 {
+ format!("{:.2}x", mrr / baseline_mrr)
+ } else {
+ "n/a (nothing was found by either ordering)".to_owned()
+ };
+ println!(" MRR lisdex {mrr:.3} shuffled {baseline_mrr:.3} lift {lift}");
+ for (index, k) in cutoffs.iter().enumerate() {
+ println!(
+ " hits@{k} lisdex {} ({:.3} of {label_total}) shuffled {:.2} ({:.3})",
+ hits[index],
+ hits[index] as f64 / label_total as f64,
+ baseline_hits[index],
+ baseline_hits[index] / label_total as f64
+ );
+ }
+ println!(
+ "\nprecision@k is deliberately NOT reported as a quality number: a project's own\n\
+ harness calls its setup and teardown too, so most labels are not entry points.\n\
+ The shuffled baseline faces the SAME labels, so the lift is the comparable figure."
+ );
+ }
+ Ok(())
+}
+
fn parse_arm(spec: &str) -> Result<(String, PathBuf), Box<dyn Error>> {
let (name, path) = spec
.split_once('=')
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
index fac5960e..1ac4fa36 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
@@ -896,6 +896,41 @@ else
fi
rm -rf "$BARE"
+banner "B29 RANKING MEASURED AGAINST AN UNINFORMED ORDERING"
+# The band assertions above check six hand-picked functions. They cannot say whether the
+# ORDERING carries information, because a ranking that put those six first and everything
+# else at random would pass all of them. This scores the cold-start ranking against the
+# entry points each project's OWN fuzz harnesses call, and against a shuffled ordering of
+# the same candidates facing the same labels.
+#
+# Cold start matters: with harness reachability seeded, the ranker would be scored against
+# its own input. `rank-eval` forces ignore_harnesses, so the ordering never sees the
+# harnesses the labels come from.
+RANKEVAL="$OUT/rank-eval.txt"
+if "$LISDEX" rank-eval "$SRC/libpng" "$SRC/libarchive" "$SRC/lz4" "$SRC/brotli" \
+ >"$RANKEVAL" 2>&1; then
+ LIS_MRR="$(awk '/^ MRR/ {print $3}' "$RANKEVAL")"
+ SHUF_MRR="$(awk '/^ MRR/ {print $5}' "$RANKEVAL")"
+ if [ -n "$LIS_MRR" ] && [ -n "$SHUF_MRR" ]; then
+ # The baseline must be NON-ZERO, or "lisdex beat it" is satisfied by a baseline
+ # that never finds anything and the comparison proves nothing.
+ if awk "BEGIN{exit !($SHUF_MRR > 0)}"; then
+ ok "the shuffled baseline finds labels too (MRR $SHUF_MRR), so the comparison is live"
+ else
+ bad "the shuffled baseline scored 0, so it cannot discriminate anything"
+ fi
+ if awk "BEGIN{exit !($LIS_MRR > $SHUF_MRR)}"; then
+ ok "the ranking beats an uninformed ordering on MRR ($LIS_MRR vs $SHUF_MRR)"
+ else
+ bad "the ranking is no better than a shuffle of its own candidates ($LIS_MRR vs $SHUF_MRR)"
+ fi
+ else
+ bad "rank-eval produced no MRR line (see $RANKEVAL)"
+ fi
+else
+ bad "rank-eval failed (see $RANKEVAL)"
+fi
+
banner "RESULT"
echo " passed: $PASS"
echo " failed: $FAIL"