Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

fix(lisdex): M4 audit - a harness was truncating files, and two oracles alibied each other

CarterPerez-dev49adb53ca2a1
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs
index 1435b1fc..477ea311 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/compile.rs
@@ -220,7 +220,31 @@ mod tests {
Path::new("/seeds/a.png"),
LeakCheck::Enforce,
);
- assert_eq!(command.env_value(ASAN_OPTIONS), Some(ASAN_OPTIONS_LEAKS));
+ assert_eq!(
+ command.env_value(ASAN_OPTIONS),
+ Some("symbolize=0:detect_leaks=1")
+ );
+ }
+
+ #[test]
+ fn every_oracle_path_disables_in_process_symbolization() {
+ // Symbolizing one report cost twelve seconds of CPU, so a single crashing
+ // candidate burned four minutes across the corpus before the timeout fired.
+ for options in [ASAN_OPTIONS_VALUE, ASAN_OPTIONS_LEAKS] {
+ assert!(
+ options.contains("symbolize=0"),
+ "{options} must disable symbolization"
+ );
+ }
+ assert_eq!(
+ showmap_command(
+ Path::new("/w/h.bin"),
+ Path::new("/seeds/a.png"),
+ Path::new("/tmp/map")
+ )
+ .env_value(ASAN_OPTIONS),
+ Some("symbolize=0:detect_leaks=0")
+ );
}
#[test]
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs
index ffbf39d4..7e7a06fc 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/synth.rs
@@ -3,7 +3,7 @@
use std::collections::BTreeMap;
-use lisdex_surface::extract::{Vocabulary, identifier_words};
+use lisdex_surface::extract::{Vocabulary, carries_word, identifier_words};
use lisdex_surface::rank::Candidate;
use lisdex_surface::signature::{Fuzzability, Param, ParamRole, Pointee, Shape};
@@ -17,6 +17,10 @@ pub const EXTENT_WORDS: [&str; 15] = [
"size", "sz", "len", "length", "count", "capacity", "cap", "num", "nbytes", "bytes", "max",
"limit", "n", "buflen", "bufsize",
];
+pub const FILESYSTEM_VERBS: [&str; 12] = [
+ "open", "fopen", "unlink", "remove", "rename", "mkdir", "rmdir", "chdir", "chmod", "symlink",
+ "truncate", "creat",
+];
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Refusal {
@@ -27,6 +31,8 @@ pub enum Refusal {
UnboundExtent { param: String },
NeedsCallback { param: String },
PathEntry,
+ UnsizedTextParam { param: String },
+ FilesystemVerb { verb: String },
UnsupportedShape { param: String },
}
@@ -54,6 +60,14 @@ impl std::fmt::Display for Refusal {
write!(formatter, "parameter {param} needs a synthesized callback")
}
Refusal::PathEntry => write!(formatter, "takes a filesystem path"),
+ Refusal::UnsizedTextParam { param } => write!(
+ formatter,
+ "char parameter {param} has no length beside it, so it is a string rather than a byte buffer; handing it fuzz bytes has already made a target open and truncate files named by the corpus"
+ ),
+ Refusal::FilesystemVerb { verb } => write!(
+ formatter,
+ "the entry point is named for a filesystem operation ({verb}), so an argument it receives may be taken as a path"
+ ),
Refusal::UnsupportedShape { param } => {
write!(formatter, "parameter {param} has no binding rule")
}
@@ -78,11 +92,7 @@ fn shared_leading_words(left: &str, right: &str) -> usize {
#[must_use]
pub fn is_extent_name(name: &str) -> bool {
- identifier_words(name).iter().any(|word| {
- EXTENT_WORDS
- .iter()
- .any(|extent| word.eq_ignore_ascii_case(extent))
- })
+ carries_word(name, &EXTENT_WORDS)
}
#[must_use]
@@ -225,6 +235,7 @@ fn bind(
param: &Param,
role: ParamRole,
locals: &BTreeMap<String, String>,
+ sized: bool,
) -> Result<Value, Refusal> {
let label = param.name.clone().unwrap_or_else(|| param.spelling.clone());
match role {
@@ -232,7 +243,17 @@ fn bind(
Shape::Pointer {
pointee: Pointee::Char | Pointee::SignedChar,
..
- } => Ok(Value::NulTerminated),
+ } => {
+ // A char pointer with no length beside it is a STRING, not a byte buffer.
+ // zlib's real gzopen prototype is unnamed, so the path detector could not
+ // fire and both arguments took the fuzz copy: the harness then created and
+ // TRUNCATED files in the working directory named by corpus bytes.
+ if sized {
+ Ok(Value::NulTerminated)
+ } else {
+ Err(Refusal::UnsizedTextParam { param: label })
+ }
+ }
_ => Ok(Value::Data {
cast: cast_for(param)?,
}),
@@ -298,14 +319,30 @@ pub fn variant_masks(fields: usize) -> Vec<u32> {
masks
}
+fn admissible(candidate: &Candidate) -> Result<(), Refusal> {
+ if candidate.fuzzability == Fuzzability::NoInput {
+ return Err(Refusal::NoDataInput);
+ }
+ // zlib's gzopen takes two unnamed char pointers, so no parameter-name rule could see
+ // the path. The function's own name is the signal that an argument becomes a path.
+ if let Some(verb) = FILESYSTEM_VERBS
+ .iter()
+ .find(|verb| carries_word(&candidate.name, &[verb]))
+ {
+ return Err(Refusal::FilesystemVerb {
+ verb: (*verb).to_owned(),
+ });
+ }
+ Ok(())
+}
+
pub fn plan_candidate(
candidate: &Candidate,
include: &str,
vocabulary: &Vocabulary,
) -> Result<Vec<HarnessPlan>, Refusal> {
- if candidate.fuzzability == Fuzzability::NoInput {
- return Err(Refusal::NoDataInput);
- }
+ admissible(candidate)?;
+ let sized = candidate.analysis.roles.contains(&ParamRole::Length);
let mut locals: BTreeMap<String, String> = BTreeMap::new();
let mut ordered: Vec<(String, String)> = Vec::new();
@@ -342,7 +379,7 @@ pub fn plan_candidate(
let mut args = Vec::with_capacity(candidate.params.len());
for (param, role) in candidate.params.iter().zip(&candidate.analysis.roles) {
- args.push(bind(param, *role, &locals)?);
+ args.push(bind(param, *role, &locals, sized)?);
}
let mut base = HarnessPlan::new(candidate.name.clone()).with_include(include);
@@ -929,6 +966,15 @@ mod tests {
);
}
+ #[test]
+ fn a_glued_lowercase_extent_name_is_still_an_extent() {
+ assert!(is_extent_name("maxlen"));
+ assert!(is_extent_name("bufsize"));
+ assert!(is_extent_name("dstCapacity"));
+ assert!(!is_extent_name("isFinal"));
+ assert!(!is_extent_name("color_type"));
+ }
+
#[test]
fn a_config_that_is_not_an_extent_still_binds() {
let expat = candidate(
@@ -1084,7 +1130,15 @@ mod tests {
assert_eq!(variant_masks(0), vec![0]);
assert_eq!(variant_masks(1), vec![1, 0]);
assert_eq!(variant_masks(2), vec![3, 2, 1, 0]);
- assert_eq!(variant_masks(4).len(), MAX_PLAN_VARIANTS);
+ assert_eq!(variant_masks(3), vec![7, 6, 5, 3, 4, 2, 1, 0]);
+ }
+
+ #[test]
+ fn more_bindable_fields_than_the_cap_still_produce_a_bounded_plan_set() {
+ let masks = variant_masks(8);
+ assert!(masks.len() <= MAX_PLAN_VARIANTS);
+ assert_eq!(masks[0].count_ones() as usize, MAX_POPULATED_FIELDS);
+ assert_eq!(*masks.last().expect("the empty subset is always last"), 0);
}
#[test]
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs
index 57ec730e..a63cb846 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-harness/src/verify.rs
@@ -18,6 +18,7 @@ pub const DIAGNOSTIC_LINES: usize = 12;
pub const MAX_SEEDS_PER_ORACLE: usize = 12;
pub const MAPS_DIR: &str = "maps";
pub const BINARIES_DIR: &str = "bin";
+pub const PEN_DIR: &str = "pen";
#[derive(Debug, thiserror::Error)]
pub enum VerifyError {
@@ -54,9 +55,14 @@ pub fn sample(mut files: Vec<PathBuf>, limit: usize) -> Vec<PathBuf> {
return files;
}
// Taking the first N of a sorted corpus takes N near-identical neighbours; an even
- // stride keeps the spread the inter-seed divergence oracle depends on.
+ // stride keeps the spread the inter-seed divergence oracle depends on. The stride is
+ // inclusive of BOTH ends: `index * len / limit` never reaches the last file, so a
+ // 13-file corpus silently dropped the one seed that carried the divergence.
+ if limit == 1 {
+ return vec![files[0].clone()];
+ }
(0..limit)
- .map(|index| files[index * files.len() / limit].clone())
+ .map(|index| files[index * (files.len() - 1) / (limit - 1)].clone())
.collect()
}
@@ -79,7 +85,17 @@ pub struct Corpus {
impl Corpus {
pub fn read(valid: &Path, invalid: Option<&Path>) -> Result<Self, VerifyError> {
- let found = sample(read_dir(valid)?, MAX_SEEDS_PER_ORACLE);
+ let all = read_dir(valid)?;
+ let total = all.len();
+ let found = sample(all, MAX_SEEDS_PER_ORACLE);
+ if total > found.len() {
+ eprintln!(
+ "warning: {total} files in {} sampled down to {} for the oracles; a seed the \
+ oracles never run cannot reject a harness",
+ valid.display(),
+ found.len()
+ );
+ }
if found.len() < MIN_VALID_SEEDS {
return Err(VerifyError::TooFewSeeds {
path: valid.to_path_buf(),
@@ -183,12 +199,17 @@ impl Bench {
seed: &Path,
map: &Path,
leaks: LeakCheck,
+ pen: &Path,
) -> Result<SeedRun, VerifyError> {
// afl-showmap returns 0 on a genuine SEGV, so the exit status has to come from a
// direct run. That run is also the only path where LeakSanitizer fires, because
// afl-fuzz injects detect_leaks=0 into every child.
+ //
+ // Both runs happen inside the pen: a candidate that takes a corpus string as a
+ // path has already created and truncated files named by corpus bytes, and it did
+ // it in whatever directory the user launched lisdex from.
let direct = process::run(
- &compile::exercise_command(binary, seed, leaks),
+ &compile::exercise_command(binary, seed, leaks).cwd(pen),
Duration::from_secs(EXEC_TIMEOUT_SECS),
)?;
let status = if direct.timed_out {
@@ -201,7 +222,7 @@ impl Bench {
let _ = std::fs::remove_file(map);
process::run(
- &compile::showmap_command(binary, seed, map),
+ &compile::showmap_command(binary, seed, map).cwd(pen),
Duration::from_secs(EXEC_TIMEOUT_SECS * 2),
)?;
let coverage = match std::fs::read_to_string(map) {
@@ -225,10 +246,12 @@ impl Bench {
fn observe(&self, label: &str, binary: &Path) -> Result<Observation, VerifyError> {
let maps = self.work.join(MAPS_DIR).join(label);
std::fs::create_dir_all(&maps).map_err(io(&maps))?;
+ let pen = self.work.join(PEN_DIR).join(label);
+ std::fs::create_dir_all(&pen).map_err(io(&pen))?;
let mut observation = Observation::default();
for (index, seed) in self.corpus.valid.iter().enumerate() {
let map = maps.join(format!("valid-{index}.map"));
- let run = Self::exercise(binary, seed, &map, LeakCheck::Enforce)?;
+ let run = Self::exercise(binary, seed, &map, LeakCheck::Enforce, &pen)?;
// The execution oracle runs before any coverage oracle, so once one seed is
// dirty the verdict is settled and the remaining runs are wasted work.
let dirty = run.status != RunStatus::Clean;
@@ -243,7 +266,7 @@ impl Bench {
let map = maps.join(format!("invalid-{index}.map"));
observation
.invalid
- .push(Self::exercise(binary, seed, &map, LeakCheck::Ignore)?);
+ .push(Self::exercise(binary, seed, &map, LeakCheck::Ignore, &pen)?);
}
Ok(observation)
}
@@ -321,7 +344,30 @@ mod tests {
let taken = sample(files, 10);
assert_eq!(taken.len(), 10);
assert_eq!(taken[0], PathBuf::from("000"));
- assert_eq!(taken[9], PathBuf::from("090"));
+ assert_eq!(taken[9], PathBuf::from("099"));
+ }
+
+ #[test]
+ fn the_last_file_is_always_reachable() {
+ // `index * len / limit` never selects the last entry, so a 13-file corpus dropped
+ // the seed that carried the divergence and the harness was falsely rejected.
+ for total in MAX_SEEDS_PER_ORACLE + 1..MAX_SEEDS_PER_ORACLE * 3 {
+ let files: Vec<PathBuf> = (0..total)
+ .map(|i| PathBuf::from(format!("{i:04}")))
+ .collect();
+ let last = files[total - 1].clone();
+ let taken = sample(files, MAX_SEEDS_PER_ORACLE);
+ assert_eq!(taken.len(), MAX_SEEDS_PER_ORACLE);
+ assert!(taken.contains(&last), "{total} files dropped the last one");
+ }
+ }
+
+ #[test]
+ fn a_sample_never_repeats_a_file() {
+ let files: Vec<PathBuf> = (0..13).map(|i| PathBuf::from(format!("{i:03}"))).collect();
+ let taken = sample(files, 12);
+ let unique: std::collections::BTreeSet<&PathBuf> = taken.iter().collect();
+ assert_eq!(unique.len(), taken.len());
}
#[test]
@@ -352,6 +398,15 @@ mod tests {
#[test]
fn a_source_that_does_not_compile_is_a_rejection_not_an_error() {
+ // The host has no afl-clang-lto, so this asserted nothing there. It now states
+ // that plainly instead of passing silently.
+ if lisdex_core::process::resolve_on_path(compile::CC).is_none() {
+ eprintln!(
+ "skipped: {} is not on PATH, run this in the container",
+ compile::CC
+ );
+ return;
+ }
let work = scratch("nocompile");
let seeds = seeded("nocompile-seeds", 3);
let source = work.join("broken.c");
@@ -364,12 +419,23 @@ mod tests {
..Toolchain::default()
},
);
- let Ok(verdict) = bench.verify(&source, "broken", None, &EdgeSet::new()) else {
- return;
- };
+ let verdict = bench
+ .verify(&source, "broken", None, &EdgeSet::new())
+ .expect("a source that does not compile is a rejection, never an error");
assert!(matches!(
verdict.outcome,
Err(Rejection::CompileFailed { .. })
));
}
+
+ #[test]
+ fn a_run_is_penned_so_a_target_cannot_write_into_the_callers_directory() {
+ let command = compile::exercise_command(
+ Path::new("/w/h.bin"),
+ Path::new("/seeds/a.png"),
+ LeakCheck::Ignore,
+ )
+ .cwd("/w/pen");
+ assert_eq!(command.cwd.as_deref(), Some(Path::new("/w/pen")));
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
index 9fdd64f3..8e07597b 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
@@ -25,9 +25,9 @@ impl Drop for ReentryGuard {
pub const C_ARGS: [&str; 3] = ["-x", "c", "-std=c11"];
pub const CPP_ARGS: [&str; 3] = ["-x", "c++", "-std=c++17"];
-pub const TEARDOWN_VERBS: [&str; 12] = [
+pub const TEARDOWN_VERBS: [&str; 15] = [
"free", "destroy", "release", "delete", "close", "cleanup", "end", "fini", "finish", "dispose",
- "uninit", "deinit",
+ "uninit", "deinit", "done", "unref", "discard",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -350,12 +350,42 @@ pub fn identifier_words(name: &str) -> Vec<String> {
words
}
-fn is_teardown_name(name: &str) -> bool {
- identifier_words(name).iter().any(|word| {
- TEARDOWN_VERBS
+#[must_use]
+pub fn carries_word(name: &str, vocabulary: &[&str]) -> bool {
+ let words = identifier_words(name);
+ if words.iter().any(|word| {
+ vocabulary
.iter()
- .any(|verb| word.eq_ignore_ascii_case(verb))
- })
+ .any(|entry| word.eq_ignore_ascii_case(entry))
+ }) {
+ return true;
+ }
+ // An all-lowercase identifier has no boundary to split on, so `gzclose` and `maxlen`
+ // read as one word. Fall back to a suffix match, which is where the verb or the
+ // extent lands in every case measured.
+ match words.as_slice() {
+ [only] => vocabulary
+ .iter()
+ .any(|entry| only.len() > entry.len() && only.to_lowercase().ends_with(entry)),
+ _ => false,
+ }
+}
+
+fn is_teardown_name(name: &str) -> bool {
+ carries_word(name, &TEARDOWN_VERBS)
+}
+
+fn record_behind_two_pointers(ty: &Type) -> Option<String> {
+ let outer = ty.get_canonical_type();
+ if outer.get_kind() != TypeKind::Pointer {
+ return None;
+ }
+ let inner = outer.get_pointee_type()?.get_canonical_type();
+ if inner.get_kind() != TypeKind::Pointer {
+ return None;
+ }
+ let target = inner.get_pointee_type()?.get_canonical_type();
+ (target.get_kind() == TypeKind::Record).then(|| record_name(&target))
}
fn sole_record_parameter(entity: &Entity) -> Option<String> {
@@ -379,13 +409,27 @@ fn collect_relations(entity: &Entity, target: &Path, into: &mut Vocabulary) {
if declared_in(entity, target).is_none() {
return;
}
+ // A const return is a borrow, not an allocation: libwebp's WebPIDecodedArea hands back
+ // a `const WebPDecBuffer *` while WebPInitDecBuffer exists precisely so the caller can
+ // own one, and archive_entry_stat hands back a `const struct stat *`.
if let Shape::Pointer {
pointee: Pointee::Record(record),
- ..
+ is_const: false,
} = result_of(entity).shape
{
into.produced.insert(record);
}
+ // An allocator does not have to RETURN the record: `int X_create(X **out)` hands it
+ // back through a double pointer, and a record whose allocation we never see reads as a
+ // stack context. Freeing that stack address then reports "attempting free on address
+ // which was not malloc()-ed" with the frames inside the target library.
+ for argument in entity.get_arguments().unwrap_or_default() {
+ if let Some(ty) = argument.get_type()
+ && let Some(record) = record_behind_two_pointers(&ty)
+ {
+ into.produced.insert(record);
+ }
+ }
if is_teardown_name(&name)
&& let Some(record) = sole_record_parameter(entity)
{
@@ -634,6 +678,52 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);
assert!(is_teardown_name("archive_read_free"));
}
+ #[test]
+ fn an_all_lowercase_name_falls_back_to_a_suffix_match() {
+ assert!(
+ is_teardown_name("gzclose"),
+ "a glued lowercase identifier has no boundary for the word split"
+ );
+ assert!(is_teardown_name("bzclose"));
+ assert!(!is_teardown_name("gzread"));
+ }
+
+ #[test]
+ fn the_teardown_verbs_cover_the_shapes_measured_on_real_libraries() {
+ for name in [
+ "FT_Done_Face",
+ "g_object_unref",
+ "zip_discard",
+ "sqlite3_close_v2",
+ "XML_ParserFree",
+ "WebPFreeDecBuffer",
+ "LZ4F_freeDecompressionContext",
+ ] {
+ assert!(is_teardown_name(name), "{name} should read as a teardown");
+ }
+ }
+
+ #[test]
+ fn a_const_return_is_a_borrow_not_an_allocation() {
+ let vocab = vocabulary(
+ "borrowed",
+ "struct buf { int n; };\nvoid init_buf(struct buf *b);\nconst struct buf *peek_buf(void);\n",
+ );
+ assert!(
+ !vocab.produced.contains("buf"),
+ "libwebp's WebPIDecodedArea returns a const WebPDecBuffer * over a buffer the caller owns"
+ );
+ }
+
+ #[test]
+ fn a_non_const_return_is_still_an_allocation() {
+ let vocab = vocabulary(
+ "allocated",
+ "struct buf { int n; };\nstruct buf *new_buf(void);\n",
+ );
+ assert!(vocab.produced.contains("buf"));
+ }
+
#[test]
fn a_multi_parameter_teardown_is_not_paired() {
let vocab = vocabulary(
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
index ffc454a2..b4e6fb1a 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
@@ -97,10 +97,20 @@ pub fn is_excluded(name: &str) -> bool {
}
#[must_use]
-pub fn is_vendored_tree(dir: &Path) -> bool {
- // A name list never ends: libjpeg-turbo vendors a whole PNG decoder at src/spng and
- // ranked eight spng_* functions in its own top 20. The structural signal is that the
- // subdirectory carries its own licence while the project's licence sits at the root.
+pub fn project_token(name: &str) -> String {
+ let lowered = name.to_lowercase();
+ let trimmed = lowered.strip_prefix("lib").unwrap_or(&lowered);
+ trimmed
+ .chars()
+ .filter(char::is_ascii_alphanumeric)
+ .collect()
+}
+
+fn kin(one: &str, other: &str) -> bool {
+ !one.is_empty() && !other.is_empty() && (one.starts_with(other) || other.starts_with(one))
+}
+
+fn carries_own_licence(dir: &Path) -> bool {
let Ok(entries) = std::fs::read_dir(dir) else {
return false;
};
@@ -114,7 +124,43 @@ pub fn is_vendored_tree(dir: &Path) -> bool {
})
}
-fn collect(root: &Path, depth: usize, found: &mut Vec<PathBuf>) -> Result<(), HeaderError> {
+fn names_itself_after(dir: &Path, project: &str) -> bool {
+ if dir
+ .file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| kin(&project_token(name), project))
+ {
+ return true;
+ }
+ let Ok(entries) = std::fs::read_dir(dir) else {
+ return false;
+ };
+ entries.flatten().any(|entry| {
+ let path = entry.path();
+ is_header(&path)
+ && path
+ .file_stem()
+ .and_then(|stem| stem.to_str())
+ .is_some_and(|stem| kin(&project_token(stem), project))
+ })
+}
+
+#[must_use]
+pub fn is_vendored_tree(dir: &Path, project: &str) -> bool {
+ // A name list never ends: libjpeg-turbo vendors a whole PNG decoder at src/spng and
+ // ranked eight spng_* functions in its own top 20. But a licence file alone is not the
+ // signal - lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their OWN api,
+ // and excluding those made both libraries un-analysable. A tree is vendored only when
+ // it carries its own licence AND nothing in it is named after the project.
+ carries_own_licence(dir) && !names_itself_after(dir, project)
+}
+
+fn collect(
+ root: &Path,
+ project: &str,
+ depth: usize,
+ found: &mut Vec<PathBuf>,
+) -> Result<(), HeaderError> {
if depth > MAX_DEPTH {
return Ok(());
}
@@ -130,8 +176,8 @@ fn collect(root: &Path, depth: usize, found: &mut Vec<PathBuf>) -> Result<(), He
let path = entry.path();
let name = entry.file_name().to_string_lossy().into_owned();
if path.is_dir() {
- if !is_excluded(&name) && !is_vendored_tree(&path) {
- collect(&path, depth + 1, found)?;
+ if !is_excluded(&name) && !is_vendored_tree(&path, project) {
+ collect(&path, project, depth + 1, found)?;
}
} else if is_header(&path) && !is_private_header(&path) {
found.push(path);
@@ -159,22 +205,33 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {
Ok(found)
}
-fn gathered(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {
+fn gathered(root: &Path, project: &str) -> Result<Vec<PathBuf>, HeaderError> {
let mut found = Vec::new();
- collect(root, 0, &mut found)?;
+ collect(root, project, 0, &mut found)?;
found.sort();
Ok(found)
}
+#[must_use]
+pub fn project_of(source: &Path) -> String {
+ source
+ .file_name()
+ .and_then(|name| name.to_str())
+ .map(project_token)
+ .unwrap_or_default()
+}
+
pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, HeaderError> {
if !source.is_dir() {
return Err(HeaderError::NotADirectory(source.to_path_buf()));
}
+ let project = project_of(source);
+
if let Some(install) = install
&& install.is_dir()
{
- let headers = gathered(install)?;
+ let headers = gathered(install, &project)?;
if !headers.is_empty() {
return Ok(HeaderSet {
root: install.to_path_buf(),
@@ -186,7 +243,7 @@ pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, Head
let include = source.join(INCLUDE_DIR);
if include.is_dir() {
- let headers = gathered(&include)?;
+ let headers = gathered(&include, &project)?;
if !headers.is_empty() {
return Ok(HeaderSet {
root: include,
@@ -206,7 +263,7 @@ pub fn discover(source: &Path, install: Option<&Path>) -> Result<HeaderSet, Head
});
}
- let headers = gathered(source)?;
+ let headers = gathered(source, &project)?;
if headers.is_empty() {
return Err(HeaderError::NoHeaders(source.to_path_buf()));
}
@@ -266,7 +323,10 @@ mod tests {
"include/spng/spng.h",
],
);
- let set = discover(&root, None).unwrap();
+ let renamed = root.with_file_name("libjpeg-turbo-vendored");
+ let _ = std::fs::remove_dir_all(&renamed);
+ std::fs::rename(&root, &renamed).unwrap();
+ let set = discover(&renamed, None).unwrap();
assert_eq!(
names(&set),
vec!["turbojpeg.h"],
@@ -275,10 +335,53 @@ mod tests {
}
#[test]
- fn the_projects_own_licence_at_the_root_excludes_nothing() {
- let root = fixture("root_licence", &["COPYING", "include/archive.h"]);
+ fn a_licence_over_the_projects_own_api_excludes_nothing() {
+ // lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their own headers.
+ // Treating a licence file as proof of vendoring made both un-analysable.
+ let root = fixture("own_licence", &["include/COPYING", "include/own_licence.h"]);
let set = discover(&root, None).unwrap();
- assert_eq!(names(&set), vec!["archive.h"]);
+ assert_eq!(names(&set), vec!["own_licence.h"]);
+ }
+
+ #[test]
+ fn a_licenced_subdirectory_named_after_the_project_is_kept() {
+ let root = fixture(
+ "lz4shape",
+ &["lib/LICENSE", "lib/lz4frame.h", "ossfuzz/x.h"],
+ );
+ let renamed = root.with_file_name("lz4");
+ let _ = std::fs::remove_dir_all(&renamed);
+ std::fs::rename(&root, &renamed).unwrap();
+ let set = discover(&renamed, None).unwrap();
+ assert!(
+ names(&set).contains(&"lz4frame.h".to_owned()),
+ "lib/ carries lz4's own api and lz4frame.h is named after the project"
+ );
+ }
+
+ #[test]
+ fn a_licenced_subdirectory_named_after_nothing_local_is_vendored() {
+ let root = fixture(
+ "turboshape",
+ &[
+ "LICENSE.md",
+ "src/turbojpeg.h",
+ "src/spng/LICENSE",
+ "src/spng/spng.h",
+ ],
+ );
+ let renamed = root.with_file_name("libjpeg-turbo");
+ let _ = std::fs::remove_dir_all(&renamed);
+ std::fs::rename(&root, &renamed).unwrap();
+ let set = discover(&renamed, None).unwrap();
+ assert_eq!(names(&set), vec!["turbojpeg.h"]);
+ }
+
+ #[test]
+ fn a_project_token_drops_the_lib_prefix_and_punctuation() {
+ assert_eq!(project_token("libexpat"), "expat");
+ assert_eq!(project_token("libjpeg-turbo"), "jpegturbo");
+ assert_eq!(project_token("lz4"), "lz4");
}
#[test]
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
index 397586d0..dbab0148 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
@@ -112,7 +112,7 @@ fn is_source(path: &Path) -> bool {
.is_some_and(|extension| SOURCE_EXTENSIONS.contains(&extension.to_lowercase().as_str()))
}
-fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {
+fn walk_sources(root: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {
if depth > MAX_DEPTH {
return;
}
@@ -123,8 +123,8 @@ fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {
let path = entry.path();
let name = entry.file_name().to_string_lossy().to_lowercase();
if path.is_dir() {
- if !headers::is_excluded(&name) && !headers::is_vendored_tree(&path) {
- walk_sources(&path, depth + 1, found);
+ if !headers::is_excluded(&name) && !headers::is_vendored_tree(&path, project) {
+ walk_sources(&path, project, depth + 1, found);
}
} else if is_source(&path) {
found.push(path);
@@ -135,7 +135,7 @@ fn walk_sources(root: &Path, depth: usize, found: &mut Vec<PathBuf>) {
#[must_use]
pub fn library_sources(root: &Path) -> Vec<PathBuf> {
let mut found = Vec::new();
- walk_sources(root, 0, &mut found);
+ walk_sources(root, &headers::project_of(root), 0, &mut found);
found.sort();
found
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
index 6eabee99..2df9eb6e 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
@@ -304,6 +304,30 @@ fn dispatch() -> Result<(), Box<dyn Error>> {
}
}
+fn library_vocabulary(
+ analysis: &surface::Surface,
+ parse: &lisdex_surface::extract::ParseOptions,
+) -> lisdex_surface::extract::Vocabulary {
+ // Mined across every header that declared a candidate, not just the one the candidate
+ // lives in: a constructor or destructor in a sibling header is invisible otherwise, and
+ // then a heap-produced record reads as a stack context and an allocation goes unpaired.
+ let mut merged = lisdex_surface::extract::Vocabulary::default();
+ let mut seen = std::collections::BTreeSet::new();
+ for candidate in &analysis.ranking.candidates {
+ if !seen.insert(candidate.header.clone()) {
+ continue;
+ }
+ match extract::parse_vocabulary(&candidate.header, parse) {
+ Ok(vocabulary) => merged.absorb(vocabulary),
+ Err(err) => eprintln!(
+ "warning: no vocabulary from {}: {err}",
+ candidate.header.display()
+ ),
+ }
+ }
+ merged
+}
+
fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
let mut options = SurfaceOptions::new(&args.source);
options.install.clone_from(&args.install);
@@ -328,12 +352,14 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
};
let candidates = args.out.join(CANDIDATES_DIR);
+ // Stale variants from an earlier run are indistinguishable from this run's evidence.
+ let _ = std::fs::remove_dir_all(&candidates);
std::fs::create_dir_all(&candidates)?;
let mut rejected = Vec::new();
let mut accepted = EdgeSet::new();
+ let vocabulary = library_vocabulary(&analysis, &parse);
for candidate in analysis.ranking.candidates.iter().take(args.top) {
- let vocabulary = extract::parse_vocabulary(&candidate.header, &parse)?;
let include = candidate
.header
.strip_prefix(&root)
@@ -368,10 +394,18 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
// call sequence it EXTENDS, and variants run most-populated first, so no
// later variant extends an earlier one. Feeding it a sibling would reject the
// fallback for a constant gain of zero, which is true and meaningless.
- let verdict = bench.verify(source, label, None, &accepted)?;
+ // One candidate's toolchain failure must not kill the other nineteen.
+ let verdict = match bench.verify(source, label, None, &accepted) {
+ Ok(verdict) => verdict,
+ Err(err) => {
+ rejected.push(format!(" {label} - could not be judged: {err}"));
+ continue;
+ }
+ };
match &verdict.outcome {
Ok(reached) => {
accepted.absorb(reached);
+ report_rejections(&rejected);
return promote(
plan,
candidate.name.as_str(),
@@ -386,12 +420,27 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
}
println!("no candidate in the top {} survived the oracles:", args.top);
- for line in rejected {
+ for line in &rejected {
println!("{line}");
}
Err("no harness synthesized".into())
}
+fn report_rejections(rejected: &[String]) {
+ // Printed on SUCCESS too: a libarchive user handed a date parser needs to see that the
+ // archive readers above it were refused, not just the winner.
+ if rejected.is_empty() {
+ return;
+ }
+ println!(
+ "{} higher-ranked candidate(s) did not survive:",
+ rejected.len()
+ );
+ for line in rejected {
+ println!("{line}");
+ }
+}
+
fn promote(
plan: &lisdex_harness::plan::HarnessPlan,
name: &str,
@@ -411,19 +460,38 @@ fn promote(
Some(verdict) => {
let reached = verdict.outcome.as_ref().map_or(0, EdgeSet::len);
let observation = verdict.observation.as_ref();
+ let invalid = observation.map_or(0, |o| o.invalid.len());
println!(
- "accepted by all four oracles: {} edges over {} valid and {} invalid seed(s), binary {}",
+ "accepted: {} edges over {} valid and {invalid} invalid seed(s), binary {}",
reached,
observation.map_or(0, |o| o.valid.len()),
- observation.map_or(0, |o| o.invalid.len()),
verdict.binary.display()
);
+ // Naming the oracles that actually ran, because two of D4's four cannot
+ // discriminate in a single-harness run and saying "all four" was a lie.
+ println!("oracles run: {}", oracles_run(invalid > 0));
}
None => println!("{UNVERIFIED_BANNER}"),
}
Ok(())
}
+fn oracles_run(differential: bool) -> String {
+ let mut ran = vec!["1 compilation", "2 execution", "3a inter-seed divergence"];
+ if differential {
+ ran.push("3c valid-vs-invalid differential");
+ }
+ let mut line = ran.join(", ");
+ if !differential {
+ line.push_str("; 3c SKIPPED, no --invalid-seeds given");
+ }
+ line.push_str(
+ "; 3b non-linear delta needs a call-sequence parent and 4 global novelty needs an \
+ already-accepted harness, neither of which exists in a single-harness run",
+ );
+ line
+}
+
fn verify(args: &VerifyArgs) -> Result<(), Box<dyn Error>> {
let parent = args
.source
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh
index 90d9ee33..7dc9b41a 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m4-gate.sh
@@ -15,6 +15,7 @@ INVALID="$GATE/invalid"
CONTROLS="$GATE/controls"
FINDINGS="$GATE/findings"
DB="$GATE/lisdex.sqlite"
+GATE_LOG="$GATE/stage.log"
DRIVER="${LISDEX_AFL_DRIVER:-/usr/local/lib/afl/libAFLDriver.a}"
LISDEX_BIN="${CARGO_TARGET_DIR:-/cargo-target}/debug/lisdex"
GATE_DURATION="${LISDEX_M4_DURATION:-600}"
@@ -38,11 +39,25 @@ fi
[ -f "$OUT_ROOT/afl/libpng16.a" ] || die "magma libpng not built; run: just magma-libpng"
[ -f "$DRIVER" ] || die "libAFLDriver.a missing at $DRIVER"
-log "stage 1/10: build lisdex"
+mkdir -p "$GATE"
+
+log "stage 1/11: build lisdex"
cargo build -p lisdex
[ -x "$LISDEX_BIN" ] || die "lisdex binary not found at $LISDEX_BIN"
-log "stage 2/10: assemble the two corpora the oracles need"
+log "stage 2/11: every oracle answers for itself"
+# The end-to-end controls below cannot isolate one coverage oracle from another: the stub
+# and the unpopulated variant are each rejected by BOTH 3a and 3c, so either one alone
+# keeps the gate green while the other is gutted. Each oracle owns unit tests that no
+# other oracle can satisfy, so running them here is what makes a single gutted oracle red.
+if cargo test -p lisdex-harness --lib > "$GATE_LOG" 2>&1; then
+ pass "$(grep -m1 'test result' "$GATE_LOG") in lisdex-harness"
+else
+ tail -20 "$GATE_LOG" >&2
+ fail "the oracle crate's own tests do not pass - one of the four oracles is not doing its job"
+fi
+
+log "stage 3/11: assemble the two corpora the oracles need"
mkdir -p "$GATE"
rm -rf "$CORPUS" "$INVALID" "$CONTROLS"
mkdir -p "$CORPUS" "$INVALID" "$CONTROLS"
@@ -83,7 +98,7 @@ lisdex_verify() {
"$LISDEX_BIN" verify "$SRC" --include "$REPO" --work "$SCRATCH" "${FLAGS[@]}"
}
-log "stage 3/10: lisdex synthesizes AND self-verifies a harness with NO LLM"
+log "stage 4/11: lisdex synthesizes AND self-verifies a harness with NO LLM"
rm -rf "$SYNTH"
mapfile -d '' HARNESS_FLAGS < <(oracle_flags)
"$LISDEX_BIN" harness "$REPO" --out "$SYNTH" --top "$TOP" "${HARNESS_FLAGS[@]}" \
@@ -91,10 +106,16 @@ mapfile -d '' HARNESS_FLAGS < <(oracle_flags)
ENTRY="$(find "$SYNTH" -maxdepth 1 -name '*.c' | head -1)"
[ -n "$ENTRY" ] || die "no harness was synthesized"
ENTRY_NAME="$(basename "$ENTRY" .c)"
-if grep -q 'accepted by all four oracles' "$GATE/synth.log"; then
+if grep -q '^oracles run:' "$GATE/synth.log"; then
pass "lisdex accepted $ENTRY_NAME through its own oracle layer"
+ printf ' %s\n' "$(grep -m1 '^oracles run:' "$GATE/synth.log" | cut -c1-120)"
else
- fail "lisdex emitted a harness without reporting an oracle verdict"
+ fail "lisdex emitted a harness without reporting which oracles judged it"
+fi
+if grep -q '3c valid-vs-invalid differential' "$GATE/synth.log"; then
+ pass "the valid-versus-invalid differential ran (an invalid corpus was supplied)"
+else
+ fail "oracle 3c did not run - the gate must exercise the differential"
fi
if grep -q 'LLVMFuzzerTestOneInput' "$ENTRY"; then
@@ -119,7 +140,7 @@ build_harness() {
)
}
-log "stage 4/10: independent cross-check - it compiles and every valid seed runs clean"
+log "stage 5/11: independent cross-check - it compiles and every valid seed runs clean"
if build_harness "$ENTRY" "$GATE/harness" afl 2>"$GATE/compile.log"; then
pass "synthesized harness compiles and links"
else
@@ -143,15 +164,21 @@ else
fail "$CRASHED/$SEEDS valid seeds crash or leak - the oracle would be unsound"
fi
-log "stage 5/10: the shipped judge ACCEPTS the harness it chose"
+log "stage 6/11: the shipped judge ACCEPTS the harness it chose"
if lisdex_verify "$ENTRY" "$GATE/verify-accepted" > "$GATE/verify-accepted.log" 2>&1; then
- pass "$(grep -m1 ACCEPTED "$GATE/verify-accepted.log" || echo 'accepted')"
+ EDGES="$(grep -m1 -oE 'ACCEPTED.* - [0-9]+ edges' "$GATE/verify-accepted.log" \
+ | grep -oE '[0-9]+ edges' | grep -oE '^[0-9]+' || echo 0)"
+ if [ "${EDGES:-0}" -gt 0 ]; then
+ pass "$(grep -m1 ACCEPTED "$GATE/verify-accepted.log")"
+ else
+ fail "accepted with $EDGES edges - a harness that reaches nothing is not a harness"
+ fi
else
tail -5 "$GATE/verify-accepted.log" >&2
fail "the shipped judge rejects the harness lisdex itself accepted"
fi
-log "stage 6/10: NEGATIVE CONTROL - a stub that ignores its input must be REJECTED"
+log "stage 7/11: NEGATIVE CONTROL - a stub that ignores its input must be REJECTED"
cat > "$CONTROLS/stub.c" <<'STUB'
// ©AngelaMos | 2026
// stub.c
@@ -179,7 +206,7 @@ else
fi
fi
-log "stage 7/10: NEGATIVE CONTROL - the unpopulated struct variant must be REJECTED"
+log "stage 8/11: NEGATIVE CONTROL - the unpopulated struct variant must be REJECTED"
VARIANTS="$(find "$SYNTH/candidates" -maxdepth 1 -name "$ENTRY_NAME.v*.c" | sort -V)"
VARIANT_COUNT="$(printf '%s\n' "$VARIANTS" | grep -c . || true)"
if [ "${VARIANT_COUNT:-0}" -lt 2 ]; then
@@ -194,7 +221,7 @@ else
fi
fi
-log "stage 8/10: NEGATIVE CONTROL - a harness that crashes on a valid seed must be REJECTED"
+log "stage 9/11: NEGATIVE CONTROL - a harness that crashes on a valid seed must be REJECTED"
cat > "$CONTROLS/crasher.c" <<'CRASHER'
// ©AngelaMos | 2026
// crasher.c
@@ -232,7 +259,7 @@ else
fi
fi
-log "stage 9/10: NEGATIVE CONTROL - a harness that leaks on a valid seed must be REJECTED"
+log "stage 10/11: NEGATIVE CONTROL - a harness that leaks on a valid seed must be REJECTED"
cat > "$CONTROLS/leaker.c" <<'LEAKER'
// ©AngelaMos | 2026
// leaker.c
@@ -269,7 +296,7 @@ else
fi
fi
-log "stage 10/10: lisdex drives a campaign against the SYNTHESIZED harness"
+log "stage 11/11: lisdex drives a campaign against the SYNTHESIZED harness"
rm -rf "$FINDINGS" "$DB"
"$LISDEX_BIN" fuzz \
--target "$GATE/harness" \