Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(lisdex): M4 audit - a harness was truncating files, and two oracles alibied each other
CarterPerez-dev49adb53ca2a1
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rsindex 1435b1fc..477ea311 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs@@ -220,7 +220,31 @@ mod tests {Path::new("/seeds/a.png"),LeakCheck::Enforce,);- assert_eq!(command.env_value(ASAN_OPTIONS), Some(ASAN_OPTIONS_LEAKS));+ assert_eq!(+ command.env_value(ASAN_OPTIONS),+ Some("symbolize=0:detect_leaks=1")+ );+ }++ #[test]+ fn every_oracle_path_disables_in_process_symbolization() {+ // Symbolizing one report cost twelve seconds of CPU, so a single crashing+ // candidate burned four minutes across the corpus before the timeout fired.+ for options in [ASAN_OPTIONS_VALUE, ASAN_OPTIONS_LEAKS] {+ assert!(+ options.contains("symbolize=0"),+ "{options} must disable symbolization"+ );+ }+ assert_eq!(+ showmap_command(+ Path::new("/w/h.bin"),+ Path::new("/seeds/a.png"),+ Path::new("/tmp/map")+ )+ .env_value(ASAN_OPTIONS),+ Some("symbolize=0:detect_leaks=0")+ );}#[test]diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rsindex ffbf39d4..7e7a06fc 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs@@ -3,7 +3,7 @@use std::collections::BTreeMap;-use lisdex_surface::extract::{Vocabulary, identifier_words};+use lisdex_surface::extract::{Vocabulary, carries_word, identifier_words};use lisdex_surface::rank::Candidate;use lisdex_surface::signature::{Fuzzability, Param, ParamRole, Pointee, Shape};@@ -17,6 +17,10 @@ pub const EXTENT_WORDS: [&str; 15] = ["size", "sz", "len", "length", "count", "capacity", "cap", "num", "nbytes", "bytes", "max","limit", "n", "buflen", "bufsize",];+pub const FILESYSTEM_VERBS: [&str; 12] = [+ "open", "fopen", "unlink", "remove", "rename", "mkdir", "rmdir", "chdir", "chmod", "symlink",+ "truncate", "creat",+];#[derive(Debug, Clone, PartialEq, Eq)]pub enum Refusal {@@ -27,6 +31,8 @@ pub enum Refusal {UnboundExtent { param: String },NeedsCallback { param: String },PathEntry,+ UnsizedTextParam { param: String },+ FilesystemVerb { verb: String },UnsupportedShape { param: String },}@@ -54,6 +60,14 @@ impl std::fmt::Display for Refusal {write!(formatter, "parameter {param} needs a synthesized callback")}Refusal::PathEntry => write!(formatter, "takes a filesystem path"),+ Refusal::UnsizedTextParam { param } => write!(+ formatter,+ "char parameter {param} has no length beside it, so it is a string rather than a byte buffer; handing it fuzz bytes has already made a target open and truncate files named by the corpus"+ ),+ Refusal::FilesystemVerb { verb } => write!(+ formatter,+ "the entry point is named for a filesystem operation ({verb}), so an argument it receives may be taken as a path"+ ),Refusal::UnsupportedShape { param } => {write!(formatter, "parameter {param} has no binding rule")}@@ -78,11 +92,7 @@ fn shared_leading_words(left: &str, right: &str) -> usize {#[must_use]pub fn is_extent_name(name: &str) -> bool {- identifier_words(name).iter().any(|word| {- EXTENT_WORDS- .iter()- .any(|extent| word.eq_ignore_ascii_case(extent))- })+ carries_word(name, &EXTENT_WORDS)}#[must_use]@@ -225,6 +235,7 @@ fn bind(param: &Param,role: ParamRole,locals: &BTreeMap<String, String>,+ sized: bool,) -> Result<Value, Refusal> {let label = param.name.clone().unwrap_or_else(|| param.spelling.clone());match role {@@ -232,7 +243,17 @@ fn bind(Shape::Pointer {pointee: Pointee::Char | Pointee::SignedChar,..- } => Ok(Value::NulTerminated),+ } => {+ // A char pointer with no length beside it is a STRING, not a byte buffer.+ // zlib's real gzopen prototype is unnamed, so the path detector could not+ // fire and both arguments took the fuzz copy: the harness then created and+ // TRUNCATED files in the working directory named by corpus bytes.+ if sized {+ Ok(Value::NulTerminated)+ } else {+ Err(Refusal::UnsizedTextParam { param: label })+ }+ }_ => Ok(Value::Data {cast: cast_for(param)?,}),@@ -298,14 +319,30 @@ pub fn variant_masks(fields: usize) -> Vec<u32> {masks}+fn admissible(candidate: &Candidate) -> Result<(), Refusal> {+ if candidate.fuzzability == Fuzzability::NoInput {+ return Err(Refusal::NoDataInput);+ }+ // zlib's gzopen takes two unnamed char pointers, so no parameter-name rule could see+ // the path. The function's own name is the signal that an argument becomes a path.+ if let Some(verb) = FILESYSTEM_VERBS+ .iter()+ .find(|verb| carries_word(&candidate.name, &[verb]))+ {+ return Err(Refusal::FilesystemVerb {+ verb: (*verb).to_owned(),+ });+ }+ Ok(())+}+pub fn plan_candidate(candidate: &Candidate,include: &str,vocabulary: &Vocabulary,) -> Result<Vec<HarnessPlan>, Refusal> {- if candidate.fuzzability == Fuzzability::NoInput {- return Err(Refusal::NoDataInput);- }+ admissible(candidate)?;+ let sized = candidate.analysis.roles.contains(&ParamRole::Length);let mut locals: BTreeMap<String, String> = BTreeMap::new();let mut ordered: Vec<(String, String)> = Vec::new();@@ -342,7 +379,7 @@ pub fn plan_candidate(let mut args = Vec::with_capacity(candidate.params.len());for (param, role) in candidate.params.iter().zip(&candidate.analysis.roles) {- args.push(bind(param, *role, &locals)?);+ args.push(bind(param, *role, &locals, sized)?);}let mut base = HarnessPlan::new(candidate.name.clone()).with_include(include);@@ -929,6 +966,15 @@ mod tests {);}+ #[test]+ fn a_glued_lowercase_extent_name_is_still_an_extent() {+ assert!(is_extent_name("maxlen"));+ assert!(is_extent_name("bufsize"));+ assert!(is_extent_name("dstCapacity"));+ assert!(!is_extent_name("isFinal"));+ assert!(!is_extent_name("color_type"));+ }+#[test]fn a_config_that_is_not_an_extent_still_binds() {let expat = candidate(@@ -1084,7 +1130,15 @@ mod tests {assert_eq!(variant_masks(0), vec![0]);assert_eq!(variant_masks(1), vec![1, 0]);assert_eq!(variant_masks(2), vec![3, 2, 1, 0]);- assert_eq!(variant_masks(4).len(), MAX_PLAN_VARIANTS);+ assert_eq!(variant_masks(3), vec![7, 6, 5, 3, 4, 2, 1, 0]);+ }++ #[test]+ fn more_bindable_fields_than_the_cap_still_produce_a_bounded_plan_set() {+ let masks = variant_masks(8);+ assert!(masks.len() <= MAX_PLAN_VARIANTS);+ assert_eq!(masks[0].count_ones() as usize, MAX_POPULATED_FIELDS);+ assert_eq!(*masks.last().expect("the empty subset is always last"), 0);}#[test]diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rsindex 57ec730e..a63cb846 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs@@ -18,6 +18,7 @@ pub const DIAGNOSTIC_LINES: usize = 12;pub const MAX_SEEDS_PER_ORACLE: usize = 12;pub const MAPS_DIR: &str = "maps";pub const BINARIES_DIR: &str = "bin";+pub const PEN_DIR: &str = "pen";#[derive(Debug, thiserror::Error)]pub enum VerifyError {@@ -54,9 +55,14 @@ pub fn sample(mut files: Vec<PathBuf>, limit: usize) -> Vec<PathBuf> {return files;}// Taking the first N of a sorted corpus takes N near-identical neighbours; an even- // stride keeps the spread the inter-seed divergence oracle depends on.+ // stride keeps the spread the inter-seed divergence oracle depends on. The stride is+ // inclusive of BOTH ends: `index * len / limit` never reaches the last file, so a+ // 13-file corpus silently dropped the one seed that carried the divergence.+ if limit == 1 {+ return vec![files[0].clone()];+ }(0..limit)- .map(|index| files[index * files.len() / limit].clone())+ .map(|index| files[index * (files.len() - 1) / (limit - 1)].clone()).collect()}@@ -79,7 +85,17 @@ pub struct Corpus {impl Corpus {pub fn read(valid: &Path, invalid: Option<&Path>) -> Result<Self, VerifyError> {- let found = sample(read_dir(valid)?, MAX_SEEDS_PER_ORACLE);+ let all = read_dir(valid)?;+ let total = all.len();+ let found = sample(all, MAX_SEEDS_PER_ORACLE);+ if total > found.len() {+ eprintln!(+ "warning: {total} files in {} sampled down to {} for the oracles; a seed the \+ oracles never run cannot reject a harness",+ valid.display(),+ found.len()+ );+ }if found.len() < MIN_VALID_SEEDS {return Err(VerifyError::TooFewSeeds {path: valid.to_path_buf(),@@ -183,12 +199,17 @@ impl Bench {seed: &Path,map: &Path,leaks: LeakCheck,+ pen: &Path,) -> Result<SeedRun, VerifyError> {// afl-showmap returns 0 on a genuine SEGV, so the exit status has to come from a// direct run. That run is also the only path where LeakSanitizer fires, because// afl-fuzz injects detect_leaks=0 into every child.+ //+ // Both runs happen inside the pen: a candidate that takes a corpus string as a+ // path has already created and truncated files named by corpus bytes, and it did+ // it in whatever directory the user launched lisdex from.let direct = process::run(- &compile::exercise_command(binary, seed, leaks),+ &compile::exercise_command(binary, seed, leaks).cwd(pen),Duration::from_secs(EXEC_TIMEOUT_SECS),)?;let status = if direct.timed_out {@@ -201,7 +222,7 @@ impl Bench {let _ = std::fs::remove_file(map);process::run(- &compile::showmap_command(binary, seed, map),+ &compile::showmap_command(binary, seed, map).cwd(pen),Duration::from_secs(EXEC_TIMEOUT_SECS * 2),)?;let coverage = match std::fs::read_to_string(map) {@@ -225,10 +246,12 @@ impl Bench {fn observe(&self, label: &str, binary: &Path) -> Result<Observation, VerifyError> {let maps = self.work.join(MAPS_DIR).join(label);std::fs::create_dir_all(&maps).map_err(io(&maps))?;+ let pen = self.work.join(PEN_DIR).join(label);+ std::fs::create_dir_all(&pen).map_err(io(&pen))?;let mut observation = Observation::default();for (index, seed) in self.corpus.valid.iter().enumerate() {let map = maps.join(format!("valid-{index}.map"));- let run = Self::exercise(binary, seed, &map, LeakCheck::Enforce)?;+ let run = Self::exercise(binary, seed, &map, LeakCheck::Enforce, &pen)?;// The execution oracle runs before any coverage oracle, so once one seed is// dirty the verdict is settled and the remaining runs are wasted work.let dirty = run.status != RunStatus::Clean;@@ -243,7 +266,7 @@ impl Bench {let map = maps.join(format!("invalid-{index}.map"));observation.invalid- .push(Self::exercise(binary, seed, &map, LeakCheck::Ignore)?);+ .push(Self::exercise(binary, seed, &map, LeakCheck::Ignore, &pen)?);}Ok(observation)}@@ -321,7 +344,30 @@ mod tests {let taken = sample(files, 10);assert_eq!(taken.len(), 10);assert_eq!(taken[0], PathBuf::from("000"));- assert_eq!(taken[9], PathBuf::from("090"));+ assert_eq!(taken[9], PathBuf::from("099"));+ }++ #[test]+ fn the_last_file_is_always_reachable() {+ // `index * len / limit` never selects the last entry, so a 13-file corpus dropped+ // the seed that carried the divergence and the harness was falsely rejected.+ for total in MAX_SEEDS_PER_ORACLE + 1..MAX_SEEDS_PER_ORACLE * 3 {+ let files: Vec<PathBuf> = (0..total)+ .map(|i| PathBuf::from(format!("{i:04}")))+ .collect();+ let last = files[total - 1].clone();+ let taken = sample(files, MAX_SEEDS_PER_ORACLE);+ assert_eq!(taken.len(), MAX_SEEDS_PER_ORACLE);+ assert!(taken.contains(&last), "{total} files dropped the last one");+ }+ }++ #[test]+ fn a_sample_never_repeats_a_file() {+ let files: Vec<PathBuf> = (0..13).map(|i| PathBuf::from(format!("{i:03}"))).collect();+ let taken = sample(files, 12);+ let unique: std::collections::BTreeSet<&PathBuf> = taken.iter().collect();+ assert_eq!(unique.len(), taken.len());}#[test]@@ -352,6 +398,15 @@ mod tests {#[test]fn a_source_that_does_not_compile_is_a_rejection_not_an_error() {+ // The host has no afl-clang-lto, so this asserted nothing there. It now states+ // that plainly instead of passing silently.+ if lisdex_core::process::resolve_on_path(compile::CC).is_none() {+ eprintln!(+ "skipped: {} is not on PATH, run this in the container",+ compile::CC+ );+ return;+ }let work = scratch("nocompile");let seeds = seeded("nocompile-seeds", 3);let source = work.join("broken.c");@@ -364,12 +419,23 @@ mod tests {..Toolchain::default()},);- let Ok(verdict) = bench.verify(&source, "broken", None, &EdgeSet::new()) else {- return;- };+ let verdict = bench+ .verify(&source, "broken", None, &EdgeSet::new())+ .expect("a source that does not compile is a rejection, never an error");assert!(matches!(verdict.outcome,Err(Rejection::CompileFailed { .. })));}++ #[test]+ fn a_run_is_penned_so_a_target_cannot_write_into_the_callers_directory() {+ let command = compile::exercise_command(+ Path::new("/w/h.bin"),+ Path::new("/seeds/a.png"),+ LeakCheck::Ignore,+ )+ .cwd("/w/pen");+ assert_eq!(command.cwd.as_deref(), Some(Path::new("/w/pen")));+ }}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rsindex 9fdd64f3..8e07597b 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs@@ -25,9 +25,9 @@ impl Drop for ReentryGuard {pub const C_ARGS: [&str; 3] = ["-x", "c", "-std=c11"];pub const CPP_ARGS: [&str; 3] = ["-x", "c++", "-std=c++17"];-pub const TEARDOWN_VERBS: [&str; 12] = [+pub const TEARDOWN_VERBS: [&str; 15] = ["free", "destroy", "release", "delete", "close", "cleanup", "end", "fini", "finish", "dispose",- "uninit", "deinit",+ "uninit", "deinit", "done", "unref", "discard",];#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]@@ -350,12 +350,42 @@ pub fn identifier_words(name: &str) -> Vec<String> {words}-fn is_teardown_name(name: &str) -> bool {- identifier_words(name).iter().any(|word| {- TEARDOWN_VERBS+#[must_use]+pub fn carries_word(name: &str, vocabulary: &[&str]) -> bool {+ let words = identifier_words(name);+ if words.iter().any(|word| {+ vocabulary.iter()- .any(|verb| word.eq_ignore_ascii_case(verb))- })+ .any(|entry| word.eq_ignore_ascii_case(entry))+ }) {+ return true;+ }+ // An all-lowercase identifier has no boundary to split on, so `gzclose` and `maxlen`+ // read as one word. Fall back to a suffix match, which is where the verb or the+ // extent lands in every case measured.+ match words.as_slice() {+ [only] => vocabulary+ .iter()+ .any(|entry| only.len() > entry.len() && only.to_lowercase().ends_with(entry)),+ _ => false,+ }+}++fn is_teardown_name(name: &str) -> bool {+ carries_word(name, &TEARDOWN_VERBS)+}++fn record_behind_two_pointers(ty: &Type) -> Option<String> {+ let outer = ty.get_canonical_type();+ if outer.get_kind() != TypeKind::Pointer {+ return None;+ }+ let inner = outer.get_pointee_type()?.get_canonical_type();+ if inner.get_kind() != TypeKind::Pointer {+ return None;+ }+ let target = inner.get_pointee_type()?.get_canonical_type();+ (target.get_kind() == TypeKind::Record).then(|| record_name(&target))}fn sole_record_parameter(entity: &Entity) -> Option<String> {@@ -379,13 +409,27 @@ fn collect_relations(entity: &Entity, target: &Path, into: &mut Vocabulary) {if declared_in(entity, target).is_none() {return;}+ // A const return is a borrow, not an allocation: libwebp's WebPIDecodedArea hands back+ // a `const WebPDecBuffer *` while WebPInitDecBuffer exists precisely so the caller can+ // own one, and archive_entry_stat hands back a `const struct stat *`.if let Shape::Pointer {pointee: Pointee::Record(record),- ..+ is_const: false,} = result_of(entity).shape{into.produced.insert(record);}+ // An allocator does not have to RETURN the record: `int X_create(X **out)` hands it+ // back through a double pointer, and a record whose allocation we never see reads as a+ // stack context. Freeing that stack address then reports "attempting free on address+ // which was not malloc()-ed" with the frames inside the target library.+ for argument in entity.get_arguments().unwrap_or_default() {+ if let Some(ty) = argument.get_type()+ && let Some(record) = record_behind_two_pointers(&ty)+ {+ into.produced.insert(record);+ }+ }if is_teardown_name(&name)&& let Some(record) = sole_record_parameter(entity){@@ -634,6 +678,52 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);assert!(is_teardown_name("archive_read_free"));}+ #[test]+ fn an_all_lowercase_name_falls_back_to_a_suffix_match() {+ assert!(+ is_teardown_name("gzclose"),+ "a glued lowercase identifier has no boundary for the word split"+ );+ assert!(is_teardown_name("bzclose"));+ assert!(!is_teardown_name("gzread"));+ }++ #[test]+ fn the_teardown_verbs_cover_the_shapes_measured_on_real_libraries() {+ for name in [+ "FT_Done_Face",+ "g_object_unref",+ "zip_discard",+ "sqlite3_close_v2",+ "XML_ParserFree",+ "WebPFreeDecBuffer",+ "LZ4F_freeDecompressionContext",+ ] {+ assert!(is_teardown_name(name), "{name} should read as a teardown");+ }+ }++ #[test]+ fn a_const_return_is_a_borrow_not_an_allocation() {+ let vocab = vocabulary(+ "borrowed",+ "struct buf { int n; };\nvoid init_buf(struct buf *b);\nconst struct buf *peek_buf(void);\n",+ );+ assert!(+ !vocab.produced.contains("buf"),+ "libwebp's WebPIDecodedArea returns a const WebPDecBuffer * over a buffer the caller owns"+ );+ }++ #[test]+ fn a_non_const_return_is_still_an_allocation() {+ let vocab = vocabulary(+ "allocated",+ "struct buf { int n; };\nstruct buf *new_buf(void);\n",+ );+ assert!(vocab.produced.contains("buf"));+ }+#[test]fn a_multi_parameter_teardown_is_not_paired() {let vocab = vocabulary(diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rsindex ffc454a2..b4e6fb1a 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs@@ -97,10 +97,20 @@ pub fn is_excluded(name: &str) -> bool {}#[must_use]-pub fn is_vendored_tree(dir: &Path) -> bool {- // A name list never ends: libjpeg-turbo vendors a whole PNG decoder at src/spng and- // ranked eight spng_* functions in its own top 20. The structural signal is that the- // subdirectory carries its own licence while the project's licence sits at the root.+pub fn project_token(name: &str) -> String {+ let lowered = name.to_lowercase();+ let trimmed = lowered.strip_prefix("lib").unwrap_or(&lowered);+ trimmed+ .chars()+ .filter(char::is_ascii_alphanumeric)+ .collect()+}++fn kin(one: &str, other: &str) -> bool {+ !one.is_empty() && !other.is_empty() && (one.starts_with(other) || other.starts_with(one))+}++fn carries_own_licence(dir: &Path) -> bool {let Ok(entries) = std::fs::read_dir(dir) else {return false;};@@ -114,7 +124,43 @@ pub fn is_vendored_tree(dir: &Path) -> bool {})}-fn collect(root: &Path, depth: usize, found: &mut Vec<PathBuf>) -> Result<(), HeaderError> {+fn names_itself_after(dir: &Path, project: &str) -> bool {+ if dir+ .file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| kin(&project_token(name), project))+ {+ return true;+ }+ let Ok(entries) = std::fs::read_dir(dir) else {+ return false;+ };+ entries.flatten().any(|entry| {+ let path = entry.path();+ is_header(&path)+ && path+ .file_stem()+ .and_then(|stem| stem.to_str())+ .is_some_and(|stem| kin(&project_token(stem), project))+ })+}++#[must_use]+pub fn is_vendored_tree(dir: &Path, project: &str) -> bool {+ // A name list never ends: libjpeg-turbo vendors a whole PNG decoder at src/spng and+ // ranked eight spng_* functions in its own top 20. But a licence file alone is not the+ // signal - lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their OWN api,+ // and excluding those made both libraries un-analysable. A tree is vendored only when+ // it carries its own licence AND nothing in it is named after the project.+ carries_own_licence(dir) && !names_itself_after(dir, project)+}++fn collect(+ root: &Path,+ project: &str,+ depth: usize,+ found: &mut Vec<PathBuf>,+) -> Result<(), HeaderError> {if depth > MAX_DEPTH {return Ok(());}@@ -130,8 +176,8 @@ fn collect(root: &Path, depth: usize, found: &mut Vec<PathBuf>) -> Result<(), Helet path = entry.path();let name = entry.file_name().to_string_lossy().into_owned();if path.is_dir() {- if !is_excluded(&name) && !is_vendored_tree(&path) {- collect(&path, depth + 1, found)?;+ if !is_excluded(&name) && !is_vendored_tree(&path, project) {+ collect(&path, project, depth + 1, found)?;}} else if is_header(&path) && !is_private_header(&path) {found.push(path);@@ -159,22 +205,33 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {Ok(found)}-fn gathered(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {+fn gathered(root: &Path, project: &str) -> Result<Vec<PathBuf>, HeaderError> {let mut found = Vec::new();- collect(root, 0, &mut found)?;+ collect(root, project, 0, &mut found)?;found.sort();Ok(found)}+#[must_use]+pub fn project_of(source: &Path) -> String {+ source+ .file_name()+ .and_then(|name| name.to_str())+ .map(project_token)+ .unwrap_or_default()+}+pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, HeaderError> {if !source.is_dir() {return Err(HeaderError::NotADirectory(source.to_path_buf()));}+ let project = project_of(source);+if let Some(install) = install&& install.is_dir(){- let headers = gathered(install)?;+ let headers = gathered(install, &project)?;if !headers.is_empty() {return Ok(HeaderSet {root: install.to_path_buf(),@@ -186,7 +243,7 @@ pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, Headlet include = source.join(INCLUDE_DIR);if include.is_dir() {- let headers = gathered(&include)?;+ let headers = gathered(&include, &project)?;if !headers.is_empty() {return Ok(HeaderSet {root: include,@@ -206,7 +263,7 @@ pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, Head});}- let headers = gathered(source)?;+ let headers = gathered(source, &project)?;if headers.is_empty() {return Err(HeaderError::NoHeaders(source.to_path_buf()));}@@ -266,7 +323,10 @@ mod tests {"include/spng/spng.h",],);- let set = discover(&root, None).unwrap();+ let renamed = root.with_file_name("libjpeg-turbo-vendored");+ let _ = std::fs::remove_dir_all(&renamed);+ std::fs::rename(&root, &renamed).unwrap();+ let set = discover(&renamed, None).unwrap();assert_eq!(names(&set),vec!["turbojpeg.h"],@@ -275,10 +335,53 @@ mod tests {}#[test]- fn the_projects_own_licence_at_the_root_excludes_nothing() {- let root = fixture("root_licence", &["COPYING", "include/archive.h"]);+ fn a_licence_over_the_projects_own_api_excludes_nothing() {+ // lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their own headers.+ // Treating a licence file as proof of vendoring made both un-analysable.+ let root = fixture("own_licence", &["include/COPYING", "include/own_licence.h"]);let set = discover(&root, None).unwrap();- assert_eq!(names(&set), vec!["archive.h"]);+ assert_eq!(names(&set), vec!["own_licence.h"]);+ }++ #[test]+ fn a_licenced_subdirectory_named_after_the_project_is_kept() {+ let root = fixture(+ "lz4shape",+ &["lib/LICENSE", "lib/lz4frame.h", "ossfuzz/x.h"],+ );+ let renamed = root.with_file_name("lz4");+ let _ = std::fs::remove_dir_all(&renamed);+ std::fs::rename(&root, &renamed).unwrap();+ let set = discover(&renamed, None).unwrap();+ assert!(+ names(&set).contains(&"lz4frame.h".to_owned()),+ "lib/ carries lz4's own api and lz4frame.h is named after the project"+ );+ }++ #[test]+ fn a_licenced_subdirectory_named_after_nothing_local_is_vendored() {+ let root = fixture(+ "turboshape",+ &[+ "LICENSE.md",+ "src/turbojpeg.h",+ "src/spng/LICENSE",+ "src/spng/spng.h",+ ],+ );+ let renamed = root.with_file_name("libjpeg-turbo");+ let _ = std::fs::remove_dir_all(&renamed);+ std::fs::rename(&root, &renamed).unwrap();+ let set = discover(&renamed, None).unwrap();+ assert_eq!(names(&set), vec!["turbojpeg.h"]);+ }++ #[test]+ fn a_project_token_drops_the_lib_prefix_and_punctuation() {+ assert_eq!(project_token("libexpat"), "expat");+ assert_eq!(project_token("libjpeg-turbo"), "jpegturbo");+ assert_eq!(project_token("lz4"), "lz4");}#[test]diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rsindex 397586d0..dbab0148 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs@@ -112,7 +112,7 @@ fn is_source(path: &Path) -> bool {.is_some_and(|extension| SOURCE_EXTENSIONS.contains(&extension.to_lowercase().as_str()))}-fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {+fn walk_sources(root: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {if depth > MAX_DEPTH {return;}@@ -123,8 +123,8 @@ fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {let path = entry.path();let name = entry.file_name().to_string_lossy().to_lowercase();if path.is_dir() {- if !headers::is_excluded(&name) && !headers::is_vendored_tree(&path) {- walk_sources(&path, depth + 1, found);+ if !headers::is_excluded(&name) && !headers::is_vendored_tree(&path, project) {+ walk_sources(&path, project, depth + 1, found);}} else if is_source(&path) {found.push(path);@@ -135,7 +135,7 @@ fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {#[must_use]pub fn library_sources(root: &Path) -> Vec<PathBuf> {let mut found = Vec::new();- walk_sources(root, 0, &mut found);+ walk_sources(root, &headers::project_of(root), 0, &mut found);found.sort();found}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rsindex 6eabee99..2df9eb6e 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs@@ -304,6 +304,30 @@ fn dispatch() -> Result<(), Box<dyn Error>> {}}+fn library_vocabulary(+ analysis: &surface::Surface,+ parse: &lisdex_surface::extract::ParseOptions,+) -> lisdex_surface::extract::Vocabulary {+ // Mined across every header that declared a candidate, not just the one the candidate+ // lives in: a constructor or destructor in a sibling header is invisible otherwise, and+ // then a heap-produced record reads as a stack context and an allocation goes unpaired.+ let mut merged = lisdex_surface::extract::Vocabulary::default();+ let mut seen = std::collections::BTreeSet::new();+ for candidate in &analysis.ranking.candidates {+ if !seen.insert(candidate.header.clone()) {+ continue;+ }+ match extract::parse_vocabulary(&candidate.header, parse) {+ Ok(vocabulary) => merged.absorb(vocabulary),+ Err(err) => eprintln!(+ "warning: no vocabulary from {}: {err}",+ candidate.header.display()+ ),+ }+ }+ merged+}+fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {let mut options = SurfaceOptions::new(&args.source);options.install.clone_from(&args.install);@@ -328,12 +352,14 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {};let candidates = args.out.join(CANDIDATES_DIR);+ // Stale variants from an earlier run are indistinguishable from this run's evidence.+ let _ = std::fs::remove_dir_all(&candidates);std::fs::create_dir_all(&candidates)?;let mut rejected = Vec::new();let mut accepted = EdgeSet::new();+ let vocabulary = library_vocabulary(&analysis, &parse);for candidate in analysis.ranking.candidates.iter().take(args.top) {- let vocabulary = extract::parse_vocabulary(&candidate.header, &parse)?;let include = candidate.header.strip_prefix(&root)@@ -368,10 +394,18 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {// call sequence it EXTENDS, and variants run most-populated first, so no// later variant extends an earlier one. Feeding it a sibling would reject the// fallback for a constant gain of zero, which is true and meaningless.- let verdict = bench.verify(source, label, None, &accepted)?;+ // One candidate's toolchain failure must not kill the other nineteen.+ let verdict = match bench.verify(source, label, None, &accepted) {+ Ok(verdict) => verdict,+ Err(err) => {+ rejected.push(format!(" {label} - could not be judged: {err}"));+ continue;+ }+ };match &verdict.outcome {Ok(reached) => {accepted.absorb(reached);+ report_rejections(&rejected);return promote(plan,candidate.name.as_str(),@@ -386,12 +420,27 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {}println!("no candidate in the top {} survived the oracles:", args.top);- for line in rejected {+ for line in &rejected {println!("{line}");}Err("no harness synthesized".into())}+fn report_rejections(rejected: &[String]) {+ // Printed on SUCCESS too: a libarchive user handed a date parser needs to see that the+ // archive readers above it were refused, not just the winner.+ if rejected.is_empty() {+ return;+ }+ println!(+ "{} higher-ranked candidate(s) did not survive:",+ rejected.len()+ );+ for line in rejected {+ println!("{line}");+ }+}+fn promote(plan: &lisdex_harness::plan::HarnessPlan,name: &str,@@ -411,19 +460,38 @@ fn promote(Some(verdict) => {let reached = verdict.outcome.as_ref().map_or(0, EdgeSet::len);let observation = verdict.observation.as_ref();+ let invalid = observation.map_or(0, |o| o.invalid.len());println!(- "accepted by all four oracles: {} edges over {} valid and {} invalid seed(s), binary {}",+ "accepted: {} edges over {} valid and {invalid} invalid seed(s), binary {}",reached,observation.map_or(0, |o| o.valid.len()),- observation.map_or(0, |o| o.invalid.len()),verdict.binary.display());+ // Naming the oracles that actually ran, because two of D4's four cannot+ // discriminate in a single-harness run and saying "all four" was a lie.+ println!("oracles run: {}", oracles_run(invalid > 0));}None => println!("{UNVERIFIED_BANNER}"),}Ok(())}+fn oracles_run(differential: bool) -> String {+ let mut ran = vec!["1 compilation", "2 execution", "3a inter-seed divergence"];+ if differential {+ ran.push("3c valid-vs-invalid differential");+ }+ let mut line = ran.join(", ");+ if !differential {+ line.push_str("; 3c SKIPPED, no --invalid-seeds given");+ }+ line.push_str(+ "; 3b non-linear delta needs a call-sequence parent and 4 global novelty needs an \+ already-accepted harness, neither of which exists in a single-harness run",+ );+ line+}+fn verify(args: &VerifyArgs) -> Result<(), Box<dyn Error>> {let parent = args.sourcediff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.shindex 90d9ee33..7dc9b41a 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh@@ -15,6 +15,7 @@ INVALID="$GATE/invalid"CONTROLS="$GATE/controls"FINDINGS="$GATE/findings"DB="$GATE/lisdex.sqlite"+GATE_LOG="$GATE/stage.log"DRIVER="${LISDEX_AFL_DRIVER:-/usr/local/lib/afl/libAFLDriver.a}"LISDEX_BIN="${CARGO_TARGET_DIR:-/cargo-target}/debug/lisdex"GATE_DURATION="${LISDEX_M4_DURATION:-600}"@@ -38,11 +39,25 @@ fi[ -f "$OUT_ROOT/afl/libpng16.a" ] || die "magma libpng not built; run: just magma-libpng"[ -f "$DRIVER" ] || die "libAFLDriver.a missing at $DRIVER"-log "stage 1/10: build lisdex"+mkdir -p "$GATE"++log "stage 1/11: build lisdex"cargo build -p lisdex[ -x "$LISDEX_BIN" ] || die "lisdex binary not found at $LISDEX_BIN"-log "stage 2/10: assemble the two corpora the oracles need"+log "stage 2/11: every oracle answers for itself"+# The end-to-end controls below cannot isolate one coverage oracle from another: the stub+# and the unpopulated variant are each rejected by BOTH 3a and 3c, so either one alone+# keeps the gate green while the other is gutted. Each oracle owns unit tests that no+# other oracle can satisfy, so running them here is what makes a single gutted oracle red.+if cargo test -p lisdex-harness --lib > "$GATE_LOG" 2>&1; then+ pass "$(grep -m1 'test result' "$GATE_LOG") in lisdex-harness"+else+ tail -20 "$GATE_LOG" >&2+ fail "the oracle crate's own tests do not pass - one of the four oracles is not doing its job"+fi++log "stage 3/11: assemble the two corpora the oracles need"mkdir -p "$GATE"rm -rf "$CORPUS" "$INVALID" "$CONTROLS"mkdir -p "$CORPUS" "$INVALID" "$CONTROLS"@@ -83,7 +98,7 @@ lisdex_verify() {"$LISDEX_BIN" verify "$SRC" --include "$REPO" --work "$SCRATCH" "${FLAGS[@]}"}-log "stage 3/10: lisdex synthesizes AND self-verifies a harness with NO LLM"+log "stage 4/11: lisdex synthesizes AND self-verifies a harness with NO LLM"rm -rf "$SYNTH"mapfile -d '' HARNESS_FLAGS < <(oracle_flags)"$LISDEX_BIN" harness "$REPO" --out "$SYNTH" --top "$TOP" "${HARNESS_FLAGS[@]}" \@@ -91,10 +106,16 @@ mapfile -d '' HARNESS_FLAGS < <(oracle_flags)ENTRY="$(find "$SYNTH" -maxdepth 1 -name '*.c' | head -1)"[ -n "$ENTRY" ] || die "no harness was synthesized"ENTRY_NAME="$(basename "$ENTRY" .c)"-if grep -q 'accepted by all four oracles' "$GATE/synth.log"; then+if grep -q '^oracles run:' "$GATE/synth.log"; thenpass "lisdex accepted $ENTRY_NAME through its own oracle layer"+ printf ' %s\n' "$(grep -m1 '^oracles run:' "$GATE/synth.log" | cut -c1-120)"else- fail "lisdex emitted a harness without reporting an oracle verdict"+ fail "lisdex emitted a harness without reporting which oracles judged it"+fi+if grep -q '3c valid-vs-invalid differential' "$GATE/synth.log"; then+ pass "the valid-versus-invalid differential ran (an invalid corpus was supplied)"+else+ fail "oracle 3c did not run - the gate must exercise the differential"fiif grep -q 'LLVMFuzzerTestOneInput' "$ENTRY"; then@@ -119,7 +140,7 @@ build_harness() {)}-log "stage 4/10: independent cross-check - it compiles and every valid seed runs clean"+log "stage 5/11: independent cross-check - it compiles and every valid seed runs clean"if build_harness "$ENTRY" "$GATE/harness" afl 2>"$GATE/compile.log"; thenpass "synthesized harness compiles and links"else@@ -143,15 +164,21 @@ elsefail "$CRASHED/$SEEDS valid seeds crash or leak - the oracle would be unsound"fi-log "stage 5/10: the shipped judge ACCEPTS the harness it chose"+log "stage 6/11: the shipped judge ACCEPTS the harness it chose"if lisdex_verify "$ENTRY" "$GATE/verify-accepted" > "$GATE/verify-accepted.log" 2>&1; then- pass "$(grep -m1 ACCEPTED "$GATE/verify-accepted.log" || echo 'accepted')"+ EDGES="$(grep -m1 -oE 'ACCEPTED.* - [0-9]+ edges' "$GATE/verify-accepted.log" \+ | grep -oE '[0-9]+ edges' | grep -oE '^[0-9]+' || echo 0)"+ if [ "${EDGES:-0}" -gt 0 ]; then+ pass "$(grep -m1 ACCEPTED "$GATE/verify-accepted.log")"+ else+ fail "accepted with $EDGES edges - a harness that reaches nothing is not a harness"+ fielsetail -5 "$GATE/verify-accepted.log" >&2fail "the shipped judge rejects the harness lisdex itself accepted"fi-log "stage 6/10: NEGATIVE CONTROL - a stub that ignores its input must be REJECTED"+log "stage 7/11: NEGATIVE CONTROL - a stub that ignores its input must be REJECTED"cat > "$CONTROLS/stub.c" <<'STUB'// ©AngelaMos | 2026// stub.c@@ -179,7 +206,7 @@ elsefifi-log "stage 7/10: NEGATIVE CONTROL - the unpopulated struct variant must be REJECTED"+log "stage 8/11: NEGATIVE CONTROL - the unpopulated struct variant must be REJECTED"VARIANTS="$(find "$SYNTH/candidates" -maxdepth 1 -name "$ENTRY_NAME.v*.c" | sort -V)"VARIANT_COUNT="$(printf '%s\n' "$VARIANTS" | grep -c . || true)"if [ "${VARIANT_COUNT:-0}" -lt 2 ]; then@@ -194,7 +221,7 @@ elsefifi-log "stage 8/10: NEGATIVE CONTROL - a harness that crashes on a valid seed must be REJECTED"+log "stage 9/11: NEGATIVE CONTROL - a harness that crashes on a valid seed must be REJECTED"cat > "$CONTROLS/crasher.c" <<'CRASHER'// ©AngelaMos | 2026// crasher.c@@ -232,7 +259,7 @@ elsefifi-log "stage 9/10: NEGATIVE CONTROL - a harness that leaks on a valid seed must be REJECTED"+log "stage 10/11: NEGATIVE CONTROL - a harness that leaks on a valid seed must be REJECTED"cat > "$CONTROLS/leaker.c" <<'LEAKER'// ©AngelaMos | 2026// leaker.c@@ -269,7 +296,7 @@ elsefifi-log "stage 10/10: lisdex drives a campaign against the SYNTHESIZED harness"+log "stage 11/11: lisdex drives a campaign against the SYNTHESIZED harness"rm -rf "$FINDINGS" "$DB""$LISDEX_BIN" fuzz \--target "$GATE/harness" \