Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
fix(lisdex): B30 again - the bracket on an include is evidence of nothing
CarterPerez-dev5f3eeea3cc48
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rsindex 14ddea2c..ee2cae72 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs@@ -119,12 +119,18 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,.parser(path).arguments(&options.argv()).skip_function_bodies(false)+ .detailed_preprocessing_record(true).parse().map_err(|source| ExtractError::Parse {path: path.to_path_buf(),reason: source.to_string(),})?;+ // This path tolerates every diagnostic on purpose, because a partly-parsed source+ // still yields usable call edges. A foreign header is not that: an audit measured+ // it moving libjpeg-turbo's #1 ranked entry point with the run still exiting 0.+ extract::refuse_foreign_includes(&unit.get_entity(), path, options)?;+let mut found = Vec::new();let mut seen = BTreeSet::new();for child in unit.get_entity().get_children() {diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rsindex 12fb4583..d15712f6 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs@@ -23,6 +23,7 @@ impl Drop for ReentryGuard {}}+pub const MIN_PROJECT_STEM: usize = 3;pub const C_ARGS: [&str; 3] = ["-x", "c", "-std=c11"];pub const CPP_ARGS: [&str; 3] = ["-x", "c++", "-std=c++17"];pub const TEARDOWN_VERBS: [&str; 15] = [@@ -53,6 +54,8 @@ pub struct ParseOptions {pub defines: Vec<String>,pub extra_args: Vec<String>,pub language: Language,+ pub tree_headers: Vec<PathBuf>,+ pub project: String,}impl ParseOptions {@@ -143,10 +146,11 @@ pub enum ExtractError {first: String,},#[error(- "{path} resolved its own quoted include {spelling:?} to {resolved}, which lies outside the source tree and every include path lisdex was given - that is a different copy of this library, so the declarations parsed here need not match the sources being built"+ "{includer} resolved {spelling:?} to {resolved}, which lies outside the source tree and every include path lisdex was given - that is a different copy of a header this library owns, so the declarations parsed here need not match the sources being built")]ForeignInclude {path: PathBuf,+ includer: PathBuf,spelling: String,resolved: PathBuf,},@@ -501,60 +505,137 @@ fn trusted_roots(target: &Path, options: &ParseOptions) -> Vec<PathBuf> {.collect()}-// A quoted include means "my own file"; an angled one means "somebody else's". Only the-// quoted form is evidence of the wrong library, and libclang exposes the difference nowhere-// except the raw tokens - an angled path is punctuation, a quoted one is a single literal.-fn is_quoted_include(entity: &Entity) -> bool {- entity.get_range().is_some_and(|range| {- range- .tokenize()- .iter()- .any(|token| token.get_spelling().starts_with('"'))- })+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum IncludeStyle {+ Quoted,+ Angled,+ Computed,}-fn located_in(path: &Path, roots: &[PathBuf]) -> bool {- roots.iter().any(|root| path.starts_with(root))+// The bracket is NOT the signal. An audit proved it on two real libraries: brotli+// angle-includes its own public headers (`#include <brotli/port.h>`, 30+ sites) and+// libarchive ships `#include PLATFORM_CONFIG_H`, so a quoted-only rule let a foreign copy+// drive the whole analysis. libclang exposes the spelling nowhere but the raw tokens - an+// angled path is punctuation, a quoted one is one literal, and a macro is neither.+fn include_style(entity: &Entity) -> IncludeStyle {+ let Some(range) = entity.get_range() else {+ return IncludeStyle::Computed;+ };+ let tokens = range.tokenize();+ if tokens+ .iter()+ .any(|token| token.get_spelling().starts_with('"'))+ {+ return IncludeStyle::Quoted;+ }+ if tokens.iter().any(|token| token.get_spelling() == "<") {+ return IncludeStyle::Angled;+ }+ IncludeStyle::Computed}-fn foreign_include(entity: &Entity, roots: &[PathBuf]) -> Option<(String, PathBuf)> {+// The tree shipping a header at this very sub-path means the include names something this+// library owns, so resolving it anywhere else is the wrong copy. This is what catches an+// angled self-include, which no bracket test can see.+fn tree_provides(spelling: &str, tree_headers: &[PathBuf]) -> bool {+ if spelling.is_empty() {+ return false;+ }+ let wanted = Path::new(spelling);+ tree_headers.iter().any(|header| header.ends_with(wanted))+}++// The other half of ownership, for a header the tree does NOT ship because the build+// generates it: libpng's pnglibconf.h and libexpat's expat_config.h both carry the project's+// own name. This is also what keeps libpng analysable - libpng writes `#include "zlib.h"`+// with quotes for a genuine external dependency, so the bracket says "mine" when it is not.+// Matched against the project token AND against its leading run of letters, because trees+// are cloned into directories carrying a version or a fork suffix - `libpng-1.6.40` tokenises+// to `png1640`, which `pnglibconf.h` does not contain while plain `png` does.+fn carries_project_token(spelling: &str, project: &str) -> bool {+ // A token this short matches half the headers in existence: zlib cloned as `libz`+ // tokenises to `z`, which would mark every header carrying a z as ours. Below the floor+ // there is no usable name signal, so ownership falls back to what the tree ships.+ if project.len() < MIN_PROJECT_STEM {+ return false;+ }+ let lowered = spelling.to_lowercase();+ let squashed: String = lowered+ .chars()+ .filter(char::is_ascii_alphanumeric)+ .collect();+ if squashed.contains(project) {+ return true;+ }+ let stem: String = project+ .chars()+ .take_while(char::is_ascii_alphabetic)+ .collect();+ stem.len() >= MIN_PROJECT_STEM && squashed.contains(&stem)+}++// A symlink from inside the tree to a generated header elsewhere is a normal out-of-tree+// build layout, so the raw path counts as trusted even when its target does not.+fn located_in(raw: &Path, canonical: &Path, roots: &[PathBuf]) -> bool {+ roots+ .iter()+ .any(|root| raw.starts_with(root) || canonical.starts_with(root))+}++fn foreign_include(+ entity: &Entity,+ roots: &[PathBuf],+ tree_headers: &[PathBuf],+ project: &str,+) -> Option<(PathBuf, String, PathBuf)> {for child in entity.get_children() {if child.get_kind() != EntityKind::InclusionDirective {continue;}- let Some(includer) = child+ // The includer must be ours. System headers quote their own siblings, and judging+ // those would refuse every target that includes stdio.h.+ let Some(raw_includer) = child.get_location().and_then(|location| location.get_file_location().file)- .and_then(|file| std::fs::canonicalize(file.get_path()).ok())+ .map(|file| file.get_path())else {continue;};- if !located_in(&includer, roots) {+ let includer = std::fs::canonicalize(&raw_includer).unwrap_or(raw_includer.clone());+ if !located_in(&raw_includer, &includer, roots) {continue;}- let Some(resolved) = child- .get_file()- .and_then(|file| std::fs::canonicalize(file.get_path()).ok())- else {+ let Some(raw_resolved) = child.get_file().map(|file| file.get_path()) else {continue;};- if located_in(&resolved, roots) || !is_quoted_include(&child) {+ let resolved = std::fs::canonicalize(&raw_resolved).unwrap_or(raw_resolved.clone());+ if located_in(&raw_resolved, &resolved, roots) {continue;}- return Some((child.get_name().unwrap_or_default(), resolved));+ let spelling = child.get_name().unwrap_or_default();+ // The bracket is evidence of nothing, so it is not consulted. Ownership is: the tree+ // ships this exact sub-path, or the name carries the project's own token, or the+ // spelling came from a macro and cannot be judged at all.+ let owned = include_style(&child) == IncludeStyle::Computed+ || tree_provides(&spelling, tree_headers)+ || carries_project_token(&spelling, project);+ if owned {+ return Some((includer, spelling, resolved));+ }}None}-fn refuse_foreign_includes(+pub(crate) fn refuse_foreign_includes(entity: &Entity,target: &Path,options: &ParseOptions,) -> Result<(), ExtractError> {let roots = trusted_roots(target, options);- match foreign_include(entity, &roots) {- Some((spelling, resolved)) => Err(ExtractError::ForeignInclude {+ match foreign_include(entity, &roots, &options.tree_headers, &options.project) {+ Some((includer, spelling, resolved)) => Err(ExtractError::ForeignInclude {path: target.to_path_buf(),+ includer,spelling,resolved,}),@@ -591,6 +672,10 @@ pub fn parse_vocabulary(path: &Path, options: &ParseOptions) -> Result<Vocabular}).map(Diagnostic::get_text).collect();+ // Ahead of the diagnostics verdict on purpose: a foreign copy is usually WHY the+ // header does not type check, and reporting the type error instead hides it.+ refuse_foreign_includes(&unit.get_entity(), &target, options)?;+if let Some(first) = failures.first() {return Err(ExtractError::Diagnostics {path: target.clone(),@@ -599,8 +684,6 @@ pub fn parse_vocabulary(path: &Path, options: &ParseOptions) -> Result<Vocabular});}- refuse_foreign_includes(&unit.get_entity(), &target, options)?;-let mut vocabulary = Vocabulary::default();collect_vocabulary(&unit.get_entity(), &target, &mut vocabulary);Ok(vocabulary)@@ -636,6 +719,8 @@ pub fn parse_header(path: &Path, options: &ParseOptions) -> Result<Vec<Declared>}).map(Diagnostic::get_text).collect();+ refuse_foreign_includes(&unit.get_entity(), &target, options)?;+if let Some(first) = failures.first() {return Err(ExtractError::Diagnostics {path: target.clone(),@@ -644,8 +729,6 @@ pub fn parse_header(path: &Path, options: &ParseOptions) -> Result<Vec<Declared>});}- refuse_foreign_includes(&unit.get_entity(), &target, options)?;-let mut found = Vec::new();let mut seen = BTreeSet::new();collect(&unit.get_entity(), &target, &mut found, &mut seen);@@ -1157,6 +1240,200 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);const GENERATED_CONFIG: &str = "typedef unsigned char CONFIGURED;\n";+ fn tree(label: &str, files: &[(&str, &str)]) -> PathBuf {+ let root = std::env::temp_dir().join(format!("lisdex-tree-{label}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&root);+ for (relative, body) in files {+ let path = root.join(relative);+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(&path, body).unwrap();+ }+ root+ }++ // F1. brotli angle-includes its OWN public headers, so an angled include is not proof of+ // somebody else's file. The tree shipping a header at that sub-path is what settles it.+ #[test]+ fn an_angled_include_of_a_header_the_tree_owns_is_refused() {+ let root = tree(+ "angledowned",+ &[+ ("inc/proj/port.h", GENERATED_CONFIG),+ (+ "api.h",+ "#include <proj/port.h>\nint parse(CONFIGURED *d, unsigned long n);\n",+ ),+ ],+ );+ let foreign = tree("angledforeign", &[("proj/port.h", GENERATED_CONFIG)]);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", foreign.display())],+ tree_headers: vec![root.join("inc/proj/port.h")],+ ..ParseOptions::default()+ };+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();+ let ExtractError::ForeignInclude { spelling, .. } = &error else {+ panic!("an angled include of a header the tree owns is the brotli case: {error:?}");+ };+ assert_eq!(spelling, "proj/port.h");+ }++ // The isolating control for the rule above: byte-identical fixture, the tree simply does+ // not own a matching header, so this is an ordinary system include and must be accepted.+ #[test]+ fn an_angled_include_the_tree_does_not_own_is_still_accepted() {+ let root = tree(+ "angledunowned",+ &[(+ "api.h",+ "#include <proj/port.h>\nint parse(CONFIGURED *d, unsigned long n);\n",+ )],+ );+ let foreign = tree("angledunownedforeign", &[("proj/port.h", GENERATED_CONFIG)]);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", foreign.display())],+ ..ParseOptions::default()+ };+ let found = parse_header(&root.join("api.h"), &options).unwrap();+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");+ }++ // F2. libarchive ships `#include PLATFORM_CONFIG_H`. A computed include carries neither a+ // quote token nor an angle token, so it must not fall through to the permissive branch.+ // The dominant B30 shape: a config the build generates, whose name carries no project+ // token at all (jconfig.h, tiffconf.h, opj_config.h). The tree shipping jconfig.h.in is+ // what proves the header is this build's own.+ #[test]+ fn a_header_the_tree_only_ships_a_generator_for_is_still_owned() {+ let root = tree(+ "generated",+ &[(+ "api.h",+ "#include \"genconf.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",+ )],+ );+ let foreign = tree("generatedforeign", &[("genconf.h", GENERATED_CONFIG)]);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", foreign.display())],+ tree_headers: vec![root.join("src/genconf.h")],+ ..ParseOptions::default()+ };+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();+ assert!(+ matches!(error, ExtractError::ForeignInclude { .. }),+ "the tree generates this header, so a copy from elsewhere is the wrong one: {error:?}"+ );+ }++ #[test]+ fn a_project_token_too_short_to_discriminate_marks_nothing_as_owned() {+ let path = header(+ "shorttoken",+ "#include \"zconf.h\"\nint parse_frame(CONFIGURED *d, unsigned long n);\n",+ );+ let elsewhere = outside_tree("shorttoken", "zconf.h", GENERATED_CONFIG);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", elsewhere.display())],+ project: "z".to_owned(),+ ..ParseOptions::default()+ };+ let found = parse_header(&path, &options).unwrap();+ assert!(+ found.iter().any(|item| item.name == "parse_frame"),+ "a one-letter token would mark half of every include set as ours: {:?}",+ names(&found)+ );+ }++ #[test]+ fn a_computed_include_resolved_outside_the_tree_is_refused() {+ let root = tree(+ "computed",+ &[(+ "api.h",+ "#define PLATFORM_CONFIG \"conf.h\"\n#include PLATFORM_CONFIG\n\+ int parse(CONFIGURED *d, unsigned long n);\n",+ )],+ );+ let foreign = tree("computedforeign", &[("conf.h", GENERATED_CONFIG)]);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", foreign.display())],+ ..ParseOptions::default()+ };+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();+ assert!(+ matches!(error, ExtractError::ForeignInclude { .. }),+ "a macro-spelled include still names a file this library owns: {error:?}"+ );+ }++ #[test]+ fn a_computed_include_resolved_inside_the_tree_is_accepted() {+ let root = tree(+ "computedlocal",+ &[+ ("conf.h", GENERATED_CONFIG),+ (+ "api.h",+ "#define PLATFORM_CONFIG \"conf.h\"\n#include PLATFORM_CONFIG\n\+ int parse(CONFIGURED *d, unsigned long n);\n",+ ),+ ],+ );+ let found = parse_header(&root.join("api.h"), &ParseOptions::default()).unwrap();+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");+ }++ // F6. An out-of-tree build that symlinks its generated config into the source tree is a+ // normal layout, and the link lives inside the tree even though its target does not.+ #[test]+ fn a_symlink_from_inside_the_tree_to_a_generated_header_is_not_foreign() {+ let root = tree(+ "symlinked",+ &[(+ "api.h",+ "#include \"conf.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",+ )],+ );+ let built = tree("symlinkedbuild", &[("conf.h", GENERATED_CONFIG)]);+ std::os::unix::fs::symlink(built.join("conf.h"), root.join("conf.h")).unwrap();+ let found = parse_header(&root.join("api.h"), &ParseOptions::default()).unwrap();+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");+ }++ // F7. The refusal has to name the file that WROTE the include, not the file the parse+ // happened to start from, or the diagnosis points at the wrong place in a deep tree.+ #[test]+ fn the_refusal_names_the_file_that_wrote_the_include() {+ let root = tree(+ "attribution",+ &[+ (+ "sub/deep.h",+ "#include \"lisdexconf.h\"\ntypedef unsigned char DEEP;\n",+ ),+ (+ "api.h",+ "#include \"sub/deep.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",+ ),+ ],+ );+ let foreign = tree("attributionforeign", &[("lisdexconf.h", GENERATED_CONFIG)]);+ let options = ParseOptions {+ extra_args: vec![format!("-I{}", foreign.display())],+ project: "lisdex".to_owned(),+ ..ParseOptions::default()+ };+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();+ let ExtractError::ForeignInclude { includer, .. } = &error else {+ panic!("{error:?}");+ };+ assert!(+ includer.ends_with("sub/deep.h"),+ "the include was written by sub/deep.h, not by the parse target: {includer:?}"+ );+ }+#[test]fn a_quoted_include_resolved_outside_every_trusted_root_is_refused() {let path = header(@@ -1166,6 +1443,7 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);let elsewhere = outside_tree("foreignquoted", "lisdexconf.h", GENERATED_CONFIG);let options = ParseOptions {extra_args: vec![format!("-I{}", elsewhere.display())],+ project: "lisdex".to_owned(),..ParseOptions::default()};let error = parse_header(&path, &options).unwrap_err();@@ -1179,25 +1457,29 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);assert!(resolved.starts_with(&elsewhere), "{resolved:?}");}- // The isolating control for the rule above: identical fixture, identical resolution, the- // ONLY difference is the bracket. If quoted-ness stops being detected this one starts- // failing too, so neither test can alibi the other.+ // The isolating control for the rule above: byte-identical fixture and resolution, the+ // only difference is that nothing marks the header as this project's. This is libpng,+ // which writes `#include "zlib.h"` with quotes for a real external dependency - treating+ // the quote as proof of ownership refused libpng outright and took the M3 gate down with+ // it. If ownership stops being required, this test fails while the one above still+ // passes, so neither can alibi the other.#[test]- fn an_angled_include_resolved_outside_every_trusted_root_is_accepted() {+ fn a_quoted_include_of_an_external_dependency_is_not_foreign() {let path = header(- "foreignangled",- "#include <lisdexconf.h>\nint parse_frame(CONFIGURED *d, unsigned long n);\n",+ "externaldep",+ "#include \"zlib.h\"\nint parse_frame(CONFIGURED *d, unsigned long n);\n",);- let elsewhere = outside_tree("foreignangled", "lisdexconf.h", GENERATED_CONFIG);+ let elsewhere = outside_tree("externaldep", "zlib.h", GENERATED_CONFIG);let options = ParseOptions {extra_args: vec![format!("-I{}", elsewhere.display())],+ project: "lisdex".to_owned(),..ParseOptions::default()};let found = parse_header(&path, &options).unwrap();assert!(found.iter().any(|item| item.name == "parse_frame"),- "an angled include is somebody else's header by construction and must still \- resolve from the system: {:?}",+ "a quoted include naming nothing this project owns is a dependency, not a \+ foreign copy of our own header: {:?}",names(&found));}@@ -1230,6 +1512,7 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);let elsewhere = outside_tree("foreignvocab", "lisdexconf.h", GENERATED_CONFIG);let options = ParseOptions {extra_args: vec![format!("-I{}", elsewhere.display())],+ project: "lisdex".to_owned(),..ParseOptions::default()};let error = parse_vocabulary(&path, &options).unwrap_err();diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rsindex b4e6fb1a..d74d7d60 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs@@ -69,6 +69,7 @@ pub struct HeaderSet {pub const PRIVATE_MARKERS: &[&str] = &["priv", "internal", "impl", "iop"];pub const PRIVATE_SUFFIXES: &[&str] = &["_p", "-p", "_int"];pub const LICENSE_STEMS: &[&str] = &["license", "licence", "copying", "copyright", "notice"];+pub const GENERATOR_SUFFIXES: &[&str] = &["in", "cmake", "prebuilt", "tmpl", "template"];#[must_use]pub fn is_private_header(path: &Path) -> bool {@@ -205,6 +206,49 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {Ok(found)}+// A tree shipping `jconfig.h.in` is telling you its build GENERATES `jconfig.h`, so a+// jconfig.h resolving from anywhere else is the wrong library's. This is the dominant B30+// shape and the one no name rule catches: pnglibconf.h and expat_config.h carry the project+// token, but jconfig.h, tiffconf.h and opj_config.h do not. Stripping the generator suffix+// yields the path the build would produce, which the ownership test already understands.+fn strip_generator(path: &Path) -> Option<PathBuf> {+ let suffix = path.extension()?.to_str()?.to_lowercase();+ if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {+ return None;+ }+ let generated = path.with_extension("");+ is_header(&generated).then_some(generated)+}++fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {+ if depth > MAX_DEPTH {+ return;+ }+ let Ok(entries) = std::fs::read_dir(dir) else {+ return;+ };+ for entry in entries.flatten() {+ let path = entry.path();+ let name = entry.file_name().to_string_lossy().to_lowercase();+ if path.is_dir() {+ if !is_excluded(&name) && !is_vendored_tree(&path, project) {+ collect_generated(&path, project, depth + 1, found);+ }+ } else if let Some(generated) = strip_generator(&path) {+ found.push(generated);+ }+ }+}++#[must_use]+pub fn generated_headers(source: &Path) -> Vec<PathBuf> {+ let mut found = Vec::new();+ collect_generated(source, &project_of(source), 0, &mut found);+ found.sort();+ found.dedup();+ found+}+fn gathered(root: &Path, project: &str) -> Result<Vec<PathBuf>, HeaderError> {let mut found = Vec::new();collect(root, project, 0, &mut found)?;@@ -291,6 +335,21 @@ mod tests {root}+ // The tree has to sit in a PROJECT-NAMED directory, because that is what project_of+ // reads. Renaming to a sibling of the fixture put it at a fixed, PID-free path in the+ // shared temp dir - so `cargo test` deleted a real /tmp/lz4, and two concurrent runs+ // fought over the same directory. Rename inside a per-process box instead.+ fn named_as_project(root: &Path, project: &str) -> PathBuf {+ let mut boxed = root.file_name().unwrap().to_os_string();+ boxed.push("-box");+ let box_dir = root.with_file_name(boxed);+ let _ = std::fs::remove_dir_all(&box_dir);+ std::fs::create_dir_all(&box_dir).unwrap();+ let renamed = box_dir.join(project);+ std::fs::rename(root, &renamed).unwrap();+ renamed+ }+fn names(set: &HeaderSet) -> Vec<String> {let mut found: Vec<String> = set.headers@@ -323,9 +382,7 @@ mod tests {"include/spng/spng.h",],);- let renamed = root.with_file_name("libjpeg-turbo-vendored");- let _ = std::fs::remove_dir_all(&renamed);- std::fs::rename(&root, &renamed).unwrap();+ let renamed = named_as_project(&root, "libjpeg-turbo-vendored");let set = discover(&renamed, None).unwrap();assert_eq!(names(&set),@@ -334,6 +391,39 @@ mod tests {);}+ #[test]+ fn generated_headers_strip_the_generator_suffix_and_ignore_everything_else() {+ let root = fixture(+ "generators",+ &[+ "src/jconfig.h.in",+ "expat_config.h.cmake",+ "scripts/pnglibconf.h.prebuilt",+ "config.h",+ "Makefile.in",+ "src/real.h",+ ],+ );+ let found: Vec<String> = generated_headers(&root)+ .iter()+ .map(|path| {+ path.strip_prefix(&root)+ .unwrap_or(path)+ .to_string_lossy()+ .into_owned()+ })+ .collect();+ assert_eq!(+ found,+ vec![+ "expat_config.h".to_owned(),+ "scripts/pnglibconf.h".to_owned(),+ "src/jconfig.h".to_owned(),+ ],+ "Makefile.in is not a header and config.h is not a generator"+ );+ }+#[test]fn a_licence_over_the_projects_own_api_excludes_nothing() {// lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their own headers.@@ -349,9 +439,7 @@ mod tests {"lz4shape",&["lib/LICENSE", "lib/lz4frame.h", "ossfuzz/x.h"],);- let renamed = root.with_file_name("lz4");- let _ = std::fs::remove_dir_all(&renamed);- std::fs::rename(&root, &renamed).unwrap();+ let renamed = named_as_project(&root, "lz4");let set = discover(&renamed, None).unwrap();assert!(names(&set).contains(&"lz4frame.h".to_owned()),@@ -370,9 +458,7 @@ mod tests {"src/spng/spng.h",],);- let renamed = root.with_file_name("libjpeg-turbo");- let _ = std::fs::remove_dir_all(&renamed);- std::fs::rename(&root, &renamed).unwrap();+ let renamed = named_as_project(&root, "libjpeg-turbo");let set = discover(&renamed, None).unwrap();assert_eq!(names(&set), vec!["turbojpeg.h"]);}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rsindex dbab0148..27272337 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs@@ -46,6 +46,17 @@ pub enum SurfaceError {declared: usize,first: String,},+ #[error(+ "{count} file(s) under {root} resolved a header this library owns to a copy outside the \+ source tree, so this ranking would describe a different build of the library. first: \+ {first}. build the target and pass its install prefix with --install, or drop the stray \+ copy off the include path"+ )]+ ForeignEnvironment {+ root: PathBuf,+ count: usize,+ first: String,+ },}#[derive(Debug, Clone)]@@ -74,7 +85,7 @@ impl SurfaceOptions {}#[must_use]- pub fn parse_options(&self, header_root: &Path) -> ParseOptions {+ pub fn parse_options(&self, header_root: &Path, tree_headers: &[PathBuf]) -> ParseOptions {let mut includes = vec![header_root.to_path_buf(), self.source.clone()];includes.extend(self.includes.iter().cloned());ParseOptions {@@ -82,6 +93,8 @@ impl SurfaceOptions {defines: self.defines.clone(),extra_args: Vec::new(),language: self.language,+ tree_headers: tree_headers.to_vec(),+ project: headers::project_of(&self.source),}}}@@ -90,6 +103,7 @@ impl SurfaceOptions {pub struct SkippedHeader {pub header: PathBuf,pub reason: String,+ pub foreign: bool,}#[derive(Debug, Clone, serde::Serialize)]@@ -98,6 +112,7 @@ pub struct Surface {pub header_root: PathBuf,pub origin: headers::HeaderOrigin,pub headers_read: usize,+ pub headers: Vec<PathBuf>,pub skipped: Vec<SkippedHeader>,pub harnesses: Vec<Harness>,pub declared: usize,@@ -140,8 +155,38 @@ pub fn library_sources(root: &Path) -> Vec<PathBuf> {found}-fn read_headers(set: &HeaderSet, options: &SurfaceOptions) -> (Vec<Declared>, Vec<SkippedHeader>) {- let parse = options.parse_options(&set.root);+// A foreign include is a statement about the ENVIRONMENT, not about one header, so it can+// never be one line in a skip list. Measured: a single foreign header in libwebp dropped the+// surface 429 -> 183 declarations, replaced the top entry point with an encoder function, and+// still exited 0 under the old per-header handling.+fn skipped_by(header: &Path, error: &extract::ExtractError) -> SkippedHeader {+ SkippedHeader {+ header: header.to_path_buf(),+ reason: error.to_string(),+ // Matched on the variant, never on the message: a reworded error must not be able+ // to silently turn a fatal environment fault back into one line of a skip list.+ foreign: matches!(error, extract::ExtractError::ForeignInclude { .. }),+ }+}++fn refuse_foreign(root: &Path, skipped: &[SkippedHeader]) -> Result<(), SurfaceError> {+ let foreign: Vec<&SkippedHeader> = skipped.iter().filter(|item| item.foreign).collect();+ match foreign.first() {+ Some(first) => Err(SurfaceError::ForeignEnvironment {+ root: root.to_path_buf(),+ count: foreign.len(),+ first: first.reason.clone(),+ }),+ None => Ok(()),+ }+}++fn read_headers(+ set: &HeaderSet,+ options: &SurfaceOptions,+ owned: &[PathBuf],+) -> (Vec<Declared>, Vec<SkippedHeader>) {+ let parse = options.parse_options(&set.root, owned);let mut declared = Vec::new();let mut skipped = Vec::new();let mut seen = BTreeSet::new();@@ -155,10 +200,7 @@ fn read_headers(set: &HeaderSet, options: &SurfaceOptions) -> (Vec<Declared>, Ve}}}- Err(error) => skipped.push(SkippedHeader {- header: header.clone(),- reason: error.to_string(),- }),+ Err(error) => skipped.push(skipped_by(header, &error)),}}(declared, skipped)@@ -172,8 +214,9 @@ fn read_sources(root: &Path,options: &SurfaceOptions,header_root: &Path,+ tree_headers: &[PathBuf],) -> (Vec<Defined>, Vec<SkippedHeader>) {- let parse = options.parse_options(header_root);+ let parse = options.parse_options(header_root, tree_headers);let mut defined = Vec::new();let mut skipped = Vec::new();let mut seen = BTreeSet::new();@@ -186,10 +229,7 @@ fn read_sources(}}}- Err(error) => skipped.push(SkippedHeader {- header: source,- reason: error.to_string(),- }),+ Err(error) => skipped.push(skipped_by(&source, &error)),}}(defined, skipped)@@ -197,9 +237,15 @@ fn read_sources(pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {let set = headers::discover(&options.source, options.install.as_deref())?;+ // Headers the build would generate count as this tree's own for the ownership test, even+ // though no such file exists to parse yet. They are appended AFTER header discovery so+ // nothing tries to read them.+ let mut owned = set.headers.clone();+ owned.extend(headers::generated_headers(&options.source));let harnesses = harness::discover(&options.source)?;- let (declared, skipped) = read_headers(&set, options);+ let (declared, skipped) = read_headers(&set, options, &owned);+ refuse_foreign(&set.root, &skipped)?;if declared.is_empty() {return Err(SurfaceError::NothingDeclared {root: set.root.clone(),@@ -224,7 +270,8 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {});}- let (defined, skipped_sources) = read_sources(&options.source, options, &set.root);+ let (defined, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);+ refuse_foreign(&set.root, &skipped_sources)?;let defined_count = defined.len();let graph = CallGraph::from(defined);@@ -260,6 +307,7 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {header_root: set.root.clone(),origin: set.origin,headers_read: set.headers.len(),+ headers: set.headers.clone(),skipped,harnesses,declared: declared.len(),diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rsindex 2df9eb6e..061016e3 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs@@ -339,7 +339,7 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {let analysis = surface::analyze(&options)?;let root = std::fs::canonicalize(&analysis.header_root).unwrap_or_else(|_| analysis.header_root.clone());- let parse = options.parse_options(&root);+ let parse = options.parse_options(&root, &analysis.headers);let bench = if args.no_verify {println!("{UNVERIFIED_BANNER}");diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.shindex 16561dc6..d008f106 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh@@ -8,7 +8,7 @@ MAGMA_SRC="${MAGMA_ROOT:-/opt/magma}"ROOT="${LISDEX_M2_ROOT:-/opt/magma-work/m2}"CACHE="$ROOT/cache"WORK="$ROOT/gate"-LISDEX="${LISDEX_BIN:-/cargo-target/debug/lisdex}"+LISDEX="${LISDEX_BIN:-${CARGO_TARGET_DIR:-/cargo-target}/debug/lisdex}"JOBS="${LISDEX_JOBS:-$(nproc)}"MAGMA_TARGETS=(libpng libtiff libxml2 sqlite3 lua)@@ -39,7 +39,10 @@ record() {SUMMARY+=("$1")}-[ -x "$LISDEX" ] || die "lisdex binary missing at $LISDEX (run: cargo build --workspace)"+# Existence was never enough: it ties the gate to nothing, so a 10-minute run could grade a+# binary from an older commit. Build what is about to be tested.+cargo build -p lisdex || die "lisdex does not build"+[ -x "$LISDEX" ] || die "lisdex binary missing at $LISDEX"[ -d "$MAGMA_SRC" ] || die "magma source missing at $MAGMA_SRC"if [ "${1:-}" = "reset" ]; thendiff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.shindex 5b2661fa..c941c29d 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh@@ -49,6 +49,16 @@ cargo build -p lisdex || {mkdir -p "$SRC" "$OUT"+# B56's lesson, applied here too: some rules have no end-to-end control that isolates them,+# so the crate's own tests ARE a gate stage. Without this, a rule can be gutted in a way no+# ranking assertion can see.+banner "0. THE SURFACE CRATE'S OWN TESTS"+if cargo test -p lisdex-surface --lib >"$OUT/unit.txt" 2>&1; then+ ok "lisdex-surface unit tests pass ($(grep -oE '[0-9]+ passed' "$OUT/unit.txt" | head -1))"+else+ bad "lisdex-surface unit tests fail: $(grep -E '^test .*FAILED|panicked' "$OUT/unit.txt" | head -1)"+fi+clone() {local name="$1" url="$2" ref="$3"if [ -d "$SRC/$name/.git" ]; then@@ -209,34 +219,69 @@ elsebad "harness awareness had no effect (cold=$cold_png warm=$warm_png)"fi-# B30. CPATH puts a directory on the compiler's default search path without lisdex-# ever being told about it, which is exactly the shape of a system-installed copy of-# the target. The two halves differ in one thing only - whether the tree still owns-# its own pnglibconf.h - so a check that refused merely because a foreign copy is-# reachable fails the second half instead of passing both.+# B30. CPATH puts a directory on the compiler's default search path without lisdex ever+# being told about it, which is exactly the shape of a system-installed copy of the target.+# Three assertions, because two were not enough: an audit found that moving pnglibconf.h out+# of the tree changes TWO things at once - the include resolves foreign AND no file of that+# name remains - so a rule with a "the tree owns something by that name" escape hatch passed+# both halves while silently reintroducing B30. Assertion 2 separates those variables.banner "B30 FOREIGN INCLUDE"+# The copy keeps the name libpng and lives OUTSIDE $SRC: the project token is derived from+# the directory, so a copy called libpng-b30 is a different project as far as ownership is+# concerned, and a fake clone sitting beside the real ones invites being mistaken for one.FOREIGN="$WORK/foreign"-B30SRC="$SRC/libpng-b30"-rm -rf "$FOREIGN" "$B30SRC"-mkdir -p "$FOREIGN"+B30ROOT="$WORK/b30"+B30SRC="$B30ROOT/libpng"+rm -rf "$FOREIGN" "$B30ROOT"+mkdir -p "$FOREIGN" "$B30ROOT"cp -r "$SRC/libpng" "$B30SRC"mv "$B30SRC/pnglibconf.h" "$FOREIGN/pnglibconf.h"+# The foreign copy must DIFFER from the tree's own. A byte-identical one produces an+# identical ranking when wrongly accepted, so the control could only ever detect location.+grep -v 'PNG_SIMPLIFIED_READ_SUPPORTED' "$FOREIGN/pnglibconf.h" >"$FOREIGN/trimmed.h"+mv "$FOREIGN/trimmed.h" "$FOREIGN/pnglibconf.h"if CPATH="$FOREIGN" "$LISDEX" surface "$B30SRC" >"$OUT/b30-foreign.txt" 2>&1; thenbad "libpng was analysed against a pnglibconf.h outside its own source tree"-elif grep -q "quoted include" "$OUT/b30-foreign.txt"; then- ok "a quoted include resolving outside the source tree is refused and named"+elif grep -q "lies outside the source tree" "$OUT/b30-foreign.txt"; then+ ok "an include resolving outside the source tree is refused and named"elsebad "libpng was refused, but not by the foreign-include check"fi-cp "$FOREIGN/pnglibconf.h" "$B30SRC/pnglibconf.h"+# libpng cannot express this one - every header it discovers sits at the root, which is also+# the quoted search path, so a shadow copy there would simply resolve. Synthesise a tree that+# OWNS a header of the same basename in a subdirectory the walk finds but the include does+# not reach. The shipped rule matches the whole sub-path and refuses; a rule that matches on+# basename alone excuses it and analyses the foreign copy. Both halves above pass either way,+# so without this the gate cannot tell those two rules apart.+# No header at the root, so discovery walks the tree and BOTH files land in the header set -+# with one at the root it stops there and the shadow is never seen, which silently disarms+# this assertion.+SHADOW="$B30ROOT/lisdex"+SHADOW_FOREIGN="$B30ROOT/shadow-foreign"+mkdir -p "$SHADOW/api" "$SHADOW/sub" "$SHADOW_FOREIGN"+printf 'typedef unsigned char LISDEX_BYTE;\n' >"$SHADOW/sub/lisdexconf.h"+printf 'typedef unsigned short LISDEX_BYTE;\n' >"$SHADOW_FOREIGN/lisdexconf.h"+printf '#include "lisdexconf.h"\nint lisdex_parse(LISDEX_BYTE *d, unsigned long n);\n' \+ >"$SHADOW/api/api.h"+if CPATH="$SHADOW_FOREIGN" "$LISDEX" surface "$SHADOW" >"$OUT/b30-shadow.txt" 2>&1; then+ bad "a same-basename header elsewhere in the tree excused the foreign copy that resolved"+elif grep -q "lies outside the source tree" "$OUT/b30-shadow.txt"; then+ ok "owning the same basename elsewhere does not excuse the copy that actually resolved"+else+ bad "the synthetic tree was refused, but not by the foreign-include check: \+$(head -1 "$OUT/b30-shadow.txt")"+fi++cp "$SRC/libpng/pnglibconf.h" "$B30SRC/pnglibconf.h"if CPATH="$FOREIGN" "$LISDEX" surface "$B30SRC" >"$OUT/b30-intree.txt" 2>&1; thenok "the same tree owning its own pnglibconf.h still analyses with the foreign copy reachable"elsebad "the foreign-include check rejects a tree that resolves its own config: \$(head -1 "$OUT/b30-intree.txt")"fi+rm -rf "$FOREIGN" "$B30ROOT"banner "RESULT"echo " passed: $PASS"