Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

feat(lisdex): B34 - a score is a property of the function, and the row's own constant clamps

CarterPerez-dev3b64e08165f5
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/rank.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/rank.rs
index c2cc56c2..351511d7 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/rank.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/rank.rs
@@ -108,12 +108,31 @@ fn exclusion_for(role: Role) -> Option<ExclusionReason> {
}
}
+// The fixed scale a danger score is measured against, so that a score is a property of the
+// FUNCTION rather than of whatever else happened to be analysed beside it. Normalising
+// against the candidate set's own maximum made it a property of the SET: adding one generated
+// candidate to libpng moved every top-10 score - png_image_begin_read_from_memory went 0.9150
+// to 0.9045 while its accumulated danger stayed at 5086 in both runs.
+//
+// The value is MEASURED, not guessed, and the obvious guess is wrong. Over the six validation
+// trees the largest accumulated danger is libarchive's 14646, then libpng 5088, libtiff 4543,
+// openjpeg 4402, brotli 832, lz4 439. A scale of 10000 therefore CLAMPS: libarchive's whole
+// top would saturate at 1.0 and stop discriminating, on the one tree where the numbers were
+// hardest won. This leaves 2.2x headroom over anything measured.
+//
+// A saturating ratio was tried first because it can never clamp at all, and was rejected on
+// measurement: it cannot discriminate across a range this wide. At the magnitudes a small
+// synthetic tree produces it scored a parser-driving reader 0.012 against a trivial
+// comparator's 0.001, a gap of one hundredth, which handed the ordering to the signature
+// grade. A logarithm keeps the same pair 0.247 against 0.067.
+pub const DANGER_SCALE: u32 = 32_768;
+
#[must_use]
-pub fn complexity_score(accumulated: u32, ceiling: u32) -> f64 {
- if accumulated == 0 || ceiling == 0 {
+pub fn complexity_score(accumulated: u32) -> f64 {
+ if accumulated == 0 {
return 0.0;
}
- let scaled = f64::from(accumulated + 1).ln() / f64::from(ceiling + 1).ln();
+ let scaled = f64::from(accumulated + 1).ln() / f64::from(DANGER_SCALE + 1).ln();
scaled.clamp(0.0, 1.0)
}
@@ -128,11 +147,9 @@ fn score_candidate(
signal: &NameSignal,
analysis: Analysis,
weights: &Weights,
- accumulated_ceiling: u32,
- undiscovered_ceiling: u32,
) -> Candidate {
- let accumulated = complexity_score(function.complexity.accumulated, accumulated_ceiling);
- let undiscovered = complexity_score(function.complexity.undiscovered, undiscovered_ceiling);
+ let accumulated = complexity_score(function.complexity.accumulated);
+ let undiscovered = complexity_score(function.complexity.undiscovered);
let fuzzability = analysis.fuzzability.score();
let export = if function.exported {
EXPORT_PUBLIC
@@ -156,12 +173,12 @@ fn score_candidate(
let reasons = vec![
format!(
- "undiscovered danger {} of {undiscovered_ceiling} -> {undiscovered:.3}",
+ "undiscovered danger {} -> {undiscovered:.3}",
function.complexity.undiscovered
),
format!("signature {:?} -> {fuzzability:.3}", analysis.fuzzability),
format!(
- "accumulated danger {} of {accumulated_ceiling} -> {accumulated:.3}",
+ "accumulated danger {} -> {accumulated:.3}",
function.complexity.accumulated
),
format!(
@@ -222,29 +239,9 @@ pub fn rank(functions: Vec<Function>, weights: &Weights) -> Ranking {
kept.push((function, signal, analysis));
}
- let accumulated_ceiling = kept
- .iter()
- .map(|(function, _, _)| function.complexity.accumulated)
- .max()
- .unwrap_or(0);
- let undiscovered_ceiling = kept
- .iter()
- .map(|(function, _, _)| function.complexity.undiscovered)
- .max()
- .unwrap_or(0);
-
let mut candidates: Vec<Candidate> = kept
.into_iter()
- .map(|(function, signal, analysis)| {
- score_candidate(
- function,
- &signal,
- analysis,
- weights,
- accumulated_ceiling,
- undiscovered_ceiling,
- )
- })
+ .map(|(function, signal, analysis)| score_candidate(function, &signal, analysis, weights))
.collect();
candidates.sort_by(|left, right| {
@@ -643,4 +640,65 @@ mod tests {
assert!(ranking.candidates.is_empty());
assert!(ranking.excluded.is_empty());
}
+
+ #[test]
+ fn the_same_danger_scores_the_same_whatever_else_was_analysed() {
+ let alone = complexity_score(5086);
+ let beside_a_giant = complexity_score(5086);
+ assert!((alone - beside_a_giant).abs() < f64::EPSILON);
+ assert!(
+ complexity_score(5086) > complexity_score(2745),
+ "a score must still order two functions by their danger"
+ );
+ }
+
+ #[test]
+ fn the_scale_still_separates_the_largest_library_measured() {
+ // libarchive's largest measured accumulated danger is 14646, and a scale of 10000 -
+ // the obvious guess - saturates it and everything above it to a single 1.0.
+ let biggest = complexity_score(14646);
+ let mid = complexity_score(5088);
+ let small = complexity_score(439);
+ assert!(small < mid && mid < biggest && biggest < 1.0);
+ assert!(
+ biggest - mid > 0.05,
+ "the top of the largest tree must stay distinguishable: {mid} then {biggest}"
+ );
+ }
+
+ #[test]
+ fn the_scale_still_separates_the_magnitudes_a_small_tree_produces() {
+ let reader = complexity_score(12);
+ let comparator = complexity_score(1);
+ assert!(
+ reader - comparator > 0.1,
+ "a saturating ratio put these one hundredth apart and handed the ordering to the \
+ signature grade: {comparator} then {reader}"
+ );
+ }
+
+ #[test]
+ fn a_function_that_reaches_nothing_scores_nothing() {
+ assert!((complexity_score(0) - 0.0).abs() < f64::EPSILON);
+ }
+
+ #[test]
+ fn adding_a_more_dangerous_candidate_leaves_every_other_score_untouched() {
+ let modest = function("modest", buffer_pair(), 100);
+ let giant = function("giant", buffer_pair(), 40_000);
+ let small_set = rank(vec![modest.clone()], &Weights::default());
+ let large_set = rank(vec![modest, giant], &Weights::default());
+ let before = small_set.candidates[0].score;
+ let after = large_set
+ .candidates
+ .iter()
+ .find(|candidate| candidate.name == "modest")
+ .unwrap()
+ .score;
+ assert!(
+ (before - after).abs() < f64::EPSILON,
+ "normalising against the set's own maximum moved libpng's whole top ten while \
+ every accumulated danger stayed identical: {before} then {after}"
+ );
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
index 23c2d026..95d19a6a 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
@@ -426,7 +426,21 @@ static int chunk_parse(struct ctx *c, int tag) {
if (tag == 2) { return 2; }
for (int i = 0; i < tag; i++) { c->state += i; }
while (c->state > 100) { c->state--; }
- switch (tag) { case 9: return 9; case 8: return 8; }
+ switch (tag) {
+ case 100: case 101: case 102: case 103: case 104: case 105: case 106: case 107:
+ case 108: case 109: case 110: case 111: case 112: case 113: case 114: case 115:
+ case 116: case 117: case 118: case 119: case 120: case 121: case 122: case 123:
+ case 124: case 125: case 126: case 127: case 128: case 129: case 130: case 131:
+ case 132: case 133: case 134: case 135: case 136: case 137: case 138: case 139:
+ case 140: case 141: case 142: case 143: case 144: case 145: case 146: case 147:
+ case 148: case 149: case 150: case 151: case 152: case 153: case 154: case 155:
+ case 156: case 157: case 158: case 159: case 160: case 161: case 162: case 163:
+ case 164: case 165: case 166: case 167: case 168: case 169: case 170: case 171:
+ case 172: case 173: case 174: case 175: case 176: case 177: case 178: case 179:
+ return tag;
+ case 9: return 9;
+ case 8: return 8;
+ }
return 0;
}
int ctx_read_header(struct ctx *c) {