Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

fix(lisdex): B30 again - the bracket on an include is evidence of nothing

CarterPerez-dev5f3eeea3cc48
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
index 14ddea2c..ee2cae72 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
@@ -119,12 +119,18 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,
.parser(path)
.arguments(&options.argv())
.skip_function_bodies(false)
+ .detailed_preprocessing_record(true)
.parse()
.map_err(|source| ExtractError::Parse {
path: path.to_path_buf(),
reason: source.to_string(),
})?;
+ // This path tolerates every diagnostic on purpose, because a partly-parsed source
+ // still yields usable call edges. A foreign header is not that: an audit measured
+ // it moving libjpeg-turbo's #1 ranked entry point with the run still exiting 0.
+ extract::refuse_foreign_includes(&unit.get_entity(), path, options)?;
+
let mut found = Vec::new();
let mut seen = BTreeSet::new();
for child in unit.get_entity().get_children() {
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
index 12fb4583..d15712f6 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/extract.rs
@@ -23,6 +23,7 @@ impl Drop for ReentryGuard {
}
}
+pub const MIN_PROJECT_STEM: usize = 3;
pub const C_ARGS: [&str; 3] = ["-x", "c", "-std=c11"];
pub const CPP_ARGS: [&str; 3] = ["-x", "c++", "-std=c++17"];
pub const TEARDOWN_VERBS: [&str; 15] = [
@@ -53,6 +54,8 @@ pub struct ParseOptions {
pub defines: Vec<String>,
pub extra_args: Vec<String>,
pub language: Language,
+ pub tree_headers: Vec<PathBuf>,
+ pub project: String,
}
impl ParseOptions {
@@ -143,10 +146,11 @@ pub enum ExtractError {
first: String,
},
#[error(
- "{path} resolved its own quoted include {spelling:?} to {resolved}, which lies outside the source tree and every include path lisdex was given - that is a different copy of this library, so the declarations parsed here need not match the sources being built"
+ "{includer} resolved {spelling:?} to {resolved}, which lies outside the source tree and every include path lisdex was given - that is a different copy of a header this library owns, so the declarations parsed here need not match the sources being built"
)]
ForeignInclude {
path: PathBuf,
+ includer: PathBuf,
spelling: String,
resolved: PathBuf,
},
@@ -501,60 +505,137 @@ fn trusted_roots(target: &Path, options: &ParseOptions) -> Vec<PathBuf> {
.collect()
}
-// A quoted include means "my own file"; an angled one means "somebody else's". Only the
-// quoted form is evidence of the wrong library, and libclang exposes the difference nowhere
-// except the raw tokens - an angled path is punctuation, a quoted one is a single literal.
-fn is_quoted_include(entity: &Entity) -> bool {
- entity.get_range().is_some_and(|range| {
- range
- .tokenize()
- .iter()
- .any(|token| token.get_spelling().starts_with('"'))
- })
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum IncludeStyle {
+ Quoted,
+ Angled,
+ Computed,
}
-fn located_in(path: &Path, roots: &[PathBuf]) -> bool {
- roots.iter().any(|root| path.starts_with(root))
+// The bracket is NOT the signal. An audit proved it on two real libraries: brotli
+// angle-includes its own public headers (`#include <brotli/port.h>`, 30+ sites) and
+// libarchive ships `#include PLATFORM_CONFIG_H`, so a quoted-only rule let a foreign copy
+// drive the whole analysis. libclang exposes the spelling nowhere but the raw tokens - an
+// angled path is punctuation, a quoted one is one literal, and a macro is neither.
+fn include_style(entity: &Entity) -> IncludeStyle {
+ let Some(range) = entity.get_range() else {
+ return IncludeStyle::Computed;
+ };
+ let tokens = range.tokenize();
+ if tokens
+ .iter()
+ .any(|token| token.get_spelling().starts_with('"'))
+ {
+ return IncludeStyle::Quoted;
+ }
+ if tokens.iter().any(|token| token.get_spelling() == "<") {
+ return IncludeStyle::Angled;
+ }
+ IncludeStyle::Computed
}
-fn foreign_include(entity: &Entity, roots: &[PathBuf]) -> Option<(String, PathBuf)> {
+// The tree shipping a header at this very sub-path means the include names something this
+// library owns, so resolving it anywhere else is the wrong copy. This is what catches an
+// angled self-include, which no bracket test can see.
+fn tree_provides(spelling: &str, tree_headers: &[PathBuf]) -> bool {
+ if spelling.is_empty() {
+ return false;
+ }
+ let wanted = Path::new(spelling);
+ tree_headers.iter().any(|header| header.ends_with(wanted))
+}
+
+// The other half of ownership, for a header the tree does NOT ship because the build
+// generates it: libpng's pnglibconf.h and libexpat's expat_config.h both carry the project's
+// own name. This is also what keeps libpng analysable - libpng writes `#include "zlib.h"`
+// with quotes for a genuine external dependency, so the bracket says "mine" when it is not.
+// Matched against the project token AND against its leading run of letters, because trees
+// are cloned into directories carrying a version or a fork suffix - `libpng-1.6.40` tokenises
+// to `png1640`, which `pnglibconf.h` does not contain while plain `png` does.
+fn carries_project_token(spelling: &str, project: &str) -> bool {
+ // A token this short matches half the headers in existence: zlib cloned as `libz`
+ // tokenises to `z`, which would mark every header carrying a z as ours. Below the floor
+ // there is no usable name signal, so ownership falls back to what the tree ships.
+ if project.len() < MIN_PROJECT_STEM {
+ return false;
+ }
+ let lowered = spelling.to_lowercase();
+ let squashed: String = lowered
+ .chars()
+ .filter(char::is_ascii_alphanumeric)
+ .collect();
+ if squashed.contains(project) {
+ return true;
+ }
+ let stem: String = project
+ .chars()
+ .take_while(char::is_ascii_alphabetic)
+ .collect();
+ stem.len() >= MIN_PROJECT_STEM && squashed.contains(&stem)
+}
+
+// A symlink from inside the tree to a generated header elsewhere is a normal out-of-tree
+// build layout, so the raw path counts as trusted even when its target does not.
+fn located_in(raw: &Path, canonical: &Path, roots: &[PathBuf]) -> bool {
+ roots
+ .iter()
+ .any(|root| raw.starts_with(root) || canonical.starts_with(root))
+}
+
+fn foreign_include(
+ entity: &Entity,
+ roots: &[PathBuf],
+ tree_headers: &[PathBuf],
+ project: &str,
+) -> Option<(PathBuf, String, PathBuf)> {
for child in entity.get_children() {
if child.get_kind() != EntityKind::InclusionDirective {
continue;
}
- let Some(includer) = child
+ // The includer must be ours. System headers quote their own siblings, and judging
+ // those would refuse every target that includes stdio.h.
+ let Some(raw_includer) = child
.get_location()
.and_then(|location| location.get_file_location().file)
- .and_then(|file| std::fs::canonicalize(file.get_path()).ok())
+ .map(|file| file.get_path())
else {
continue;
};
- if !located_in(&includer, roots) {
+ let includer = std::fs::canonicalize(&raw_includer).unwrap_or(raw_includer.clone());
+ if !located_in(&raw_includer, &includer, roots) {
continue;
}
- let Some(resolved) = child
- .get_file()
- .and_then(|file| std::fs::canonicalize(file.get_path()).ok())
- else {
+ let Some(raw_resolved) = child.get_file().map(|file| file.get_path()) else {
continue;
};
- if located_in(&resolved, roots) || !is_quoted_include(&child) {
+ let resolved = std::fs::canonicalize(&raw_resolved).unwrap_or(raw_resolved.clone());
+ if located_in(&raw_resolved, &resolved, roots) {
continue;
}
- return Some((child.get_name().unwrap_or_default(), resolved));
+ let spelling = child.get_name().unwrap_or_default();
+ // The bracket is evidence of nothing, so it is not consulted. Ownership is: the tree
+ // ships this exact sub-path, or the name carries the project's own token, or the
+ // spelling came from a macro and cannot be judged at all.
+ let owned = include_style(&child) == IncludeStyle::Computed
+ || tree_provides(&spelling, tree_headers)
+ || carries_project_token(&spelling, project);
+ if owned {
+ return Some((includer, spelling, resolved));
+ }
}
None
}
-fn refuse_foreign_includes(
+pub(crate) fn refuse_foreign_includes(
entity: &Entity,
target: &Path,
options: &ParseOptions,
) -> Result<(), ExtractError> {
let roots = trusted_roots(target, options);
- match foreign_include(entity, &roots) {
- Some((spelling, resolved)) => Err(ExtractError::ForeignInclude {
+ match foreign_include(entity, &roots, &options.tree_headers, &options.project) {
+ Some((includer, spelling, resolved)) => Err(ExtractError::ForeignInclude {
path: target.to_path_buf(),
+ includer,
spelling,
resolved,
}),
@@ -591,6 +672,10 @@ pub fn parse_vocabulary(path: &Path, options: &ParseOptions) -> Result<Vocabular
})
.map(Diagnostic::get_text)
.collect();
+ // Ahead of the diagnostics verdict on purpose: a foreign copy is usually WHY the
+ // header does not type check, and reporting the type error instead hides it.
+ refuse_foreign_includes(&unit.get_entity(), &target, options)?;
+
if let Some(first) = failures.first() {
return Err(ExtractError::Diagnostics {
path: target.clone(),
@@ -599,8 +684,6 @@ pub fn parse_vocabulary(path: &Path, options: &ParseOptions) -> Result<Vocabular
});
}
- refuse_foreign_includes(&unit.get_entity(), &target, options)?;
-
let mut vocabulary = Vocabulary::default();
collect_vocabulary(&unit.get_entity(), &target, &mut vocabulary);
Ok(vocabulary)
@@ -636,6 +719,8 @@ pub fn parse_header(path: &Path, options: &ParseOptions) -> Result<Vec<Declared>
})
.map(Diagnostic::get_text)
.collect();
+ refuse_foreign_includes(&unit.get_entity(), &target, options)?;
+
if let Some(first) = failures.first() {
return Err(ExtractError::Diagnostics {
path: target.clone(),
@@ -644,8 +729,6 @@ pub fn parse_header(path: &Path, options: &ParseOptions) -> Result<Vec<Declared>
});
}
- refuse_foreign_includes(&unit.get_entity(), &target, options)?;
-
let mut found = Vec::new();
let mut seen = BTreeSet::new();
collect(&unit.get_entity(), &target, &mut found, &mut seen);
@@ -1157,6 +1240,200 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);
const GENERATED_CONFIG: &str = "typedef unsigned char CONFIGURED;\n";
+ fn tree(label: &str, files: &[(&str, &str)]) -> PathBuf {
+ let root = std::env::temp_dir().join(format!("lisdex-tree-{label}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&root);
+ for (relative, body) in files {
+ let path = root.join(relative);
+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+ std::fs::write(&path, body).unwrap();
+ }
+ root
+ }
+
+ // F1. brotli angle-includes its OWN public headers, so an angled include is not proof of
+ // somebody else's file. The tree shipping a header at that sub-path is what settles it.
+ #[test]
+ fn an_angled_include_of_a_header_the_tree_owns_is_refused() {
+ let root = tree(
+ "angledowned",
+ &[
+ ("inc/proj/port.h", GENERATED_CONFIG),
+ (
+ "api.h",
+ "#include <proj/port.h>\nint parse(CONFIGURED *d, unsigned long n);\n",
+ ),
+ ],
+ );
+ let foreign = tree("angledforeign", &[("proj/port.h", GENERATED_CONFIG)]);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", foreign.display())],
+ tree_headers: vec![root.join("inc/proj/port.h")],
+ ..ParseOptions::default()
+ };
+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();
+ let ExtractError::ForeignInclude { spelling, .. } = &error else {
+ panic!("an angled include of a header the tree owns is the brotli case: {error:?}");
+ };
+ assert_eq!(spelling, "proj/port.h");
+ }
+
+ // The isolating control for the rule above: byte-identical fixture, the tree simply does
+ // not own a matching header, so this is an ordinary system include and must be accepted.
+ #[test]
+ fn an_angled_include_the_tree_does_not_own_is_still_accepted() {
+ let root = tree(
+ "angledunowned",
+ &[(
+ "api.h",
+ "#include <proj/port.h>\nint parse(CONFIGURED *d, unsigned long n);\n",
+ )],
+ );
+ let foreign = tree("angledunownedforeign", &[("proj/port.h", GENERATED_CONFIG)]);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", foreign.display())],
+ ..ParseOptions::default()
+ };
+ let found = parse_header(&root.join("api.h"), &options).unwrap();
+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");
+ }
+
+ // F2. libarchive ships `#include PLATFORM_CONFIG_H`. A computed include carries neither a
+ // quote token nor an angle token, so it must not fall through to the permissive branch.
+ // The dominant B30 shape: a config the build generates, whose name carries no project
+ // token at all (jconfig.h, tiffconf.h, opj_config.h). The tree shipping jconfig.h.in is
+ // what proves the header is this build's own.
+ #[test]
+ fn a_header_the_tree_only_ships_a_generator_for_is_still_owned() {
+ let root = tree(
+ "generated",
+ &[(
+ "api.h",
+ "#include \"genconf.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",
+ )],
+ );
+ let foreign = tree("generatedforeign", &[("genconf.h", GENERATED_CONFIG)]);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", foreign.display())],
+ tree_headers: vec![root.join("src/genconf.h")],
+ ..ParseOptions::default()
+ };
+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();
+ assert!(
+ matches!(error, ExtractError::ForeignInclude { .. }),
+ "the tree generates this header, so a copy from elsewhere is the wrong one: {error:?}"
+ );
+ }
+
+ #[test]
+ fn a_project_token_too_short_to_discriminate_marks_nothing_as_owned() {
+ let path = header(
+ "shorttoken",
+ "#include \"zconf.h\"\nint parse_frame(CONFIGURED *d, unsigned long n);\n",
+ );
+ let elsewhere = outside_tree("shorttoken", "zconf.h", GENERATED_CONFIG);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", elsewhere.display())],
+ project: "z".to_owned(),
+ ..ParseOptions::default()
+ };
+ let found = parse_header(&path, &options).unwrap();
+ assert!(
+ found.iter().any(|item| item.name == "parse_frame"),
+ "a one-letter token would mark half of every include set as ours: {:?}",
+ names(&found)
+ );
+ }
+
+ #[test]
+ fn a_computed_include_resolved_outside_the_tree_is_refused() {
+ let root = tree(
+ "computed",
+ &[(
+ "api.h",
+ "#define PLATFORM_CONFIG \"conf.h\"\n#include PLATFORM_CONFIG\n\
+ int parse(CONFIGURED *d, unsigned long n);\n",
+ )],
+ );
+ let foreign = tree("computedforeign", &[("conf.h", GENERATED_CONFIG)]);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", foreign.display())],
+ ..ParseOptions::default()
+ };
+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();
+ assert!(
+ matches!(error, ExtractError::ForeignInclude { .. }),
+ "a macro-spelled include still names a file this library owns: {error:?}"
+ );
+ }
+
+ #[test]
+ fn a_computed_include_resolved_inside_the_tree_is_accepted() {
+ let root = tree(
+ "computedlocal",
+ &[
+ ("conf.h", GENERATED_CONFIG),
+ (
+ "api.h",
+ "#define PLATFORM_CONFIG \"conf.h\"\n#include PLATFORM_CONFIG\n\
+ int parse(CONFIGURED *d, unsigned long n);\n",
+ ),
+ ],
+ );
+ let found = parse_header(&root.join("api.h"), &ParseOptions::default()).unwrap();
+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");
+ }
+
+ // F6. An out-of-tree build that symlinks its generated config into the source tree is a
+ // normal layout, and the link lives inside the tree even though its target does not.
+ #[test]
+ fn a_symlink_from_inside_the_tree_to_a_generated_header_is_not_foreign() {
+ let root = tree(
+ "symlinked",
+ &[(
+ "api.h",
+ "#include \"conf.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",
+ )],
+ );
+ let built = tree("symlinkedbuild", &[("conf.h", GENERATED_CONFIG)]);
+ std::os::unix::fs::symlink(built.join("conf.h"), root.join("conf.h")).unwrap();
+ let found = parse_header(&root.join("api.h"), &ParseOptions::default()).unwrap();
+ assert!(found.iter().any(|item| item.name == "parse"), "{found:?}");
+ }
+
+ // F7. The refusal has to name the file that WROTE the include, not the file the parse
+ // happened to start from, or the diagnosis points at the wrong place in a deep tree.
+ #[test]
+ fn the_refusal_names_the_file_that_wrote_the_include() {
+ let root = tree(
+ "attribution",
+ &[
+ (
+ "sub/deep.h",
+ "#include \"lisdexconf.h\"\ntypedef unsigned char DEEP;\n",
+ ),
+ (
+ "api.h",
+ "#include \"sub/deep.h\"\nint parse(CONFIGURED *d, unsigned long n);\n",
+ ),
+ ],
+ );
+ let foreign = tree("attributionforeign", &[("lisdexconf.h", GENERATED_CONFIG)]);
+ let options = ParseOptions {
+ extra_args: vec![format!("-I{}", foreign.display())],
+ project: "lisdex".to_owned(),
+ ..ParseOptions::default()
+ };
+ let error = parse_header(&root.join("api.h"), &options).unwrap_err();
+ let ExtractError::ForeignInclude { includer, .. } = &error else {
+ panic!("{error:?}");
+ };
+ assert!(
+ includer.ends_with("sub/deep.h"),
+ "the include was written by sub/deep.h, not by the parse target: {includer:?}"
+ );
+ }
+
#[test]
fn a_quoted_include_resolved_outside_every_trusted_root_is_refused() {
let path = header(
@@ -1166,6 +1443,7 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);
let elsewhere = outside_tree("foreignquoted", "lisdexconf.h", GENERATED_CONFIG);
let options = ParseOptions {
extra_args: vec![format!("-I{}", elsewhere.display())],
+ project: "lisdex".to_owned(),
..ParseOptions::default()
};
let error = parse_header(&path, &options).unwrap_err();
@@ -1179,25 +1457,29 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);
assert!(resolved.starts_with(&elsewhere), "{resolved:?}");
}
- // The isolating control for the rule above: identical fixture, identical resolution, the
- // ONLY difference is the bracket. If quoted-ness stops being detected this one starts
- // failing too, so neither test can alibi the other.
+ // The isolating control for the rule above: byte-identical fixture and resolution, the
+ // only difference is that nothing marks the header as this project's. This is libpng,
+ // which writes `#include "zlib.h"` with quotes for a real external dependency - treating
+ // the quote as proof of ownership refused libpng outright and took the M3 gate down with
+ // it. If ownership stops being required, this test fails while the one above still
+ // passes, so neither can alibi the other.
#[test]
- fn an_angled_include_resolved_outside_every_trusted_root_is_accepted() {
+ fn a_quoted_include_of_an_external_dependency_is_not_foreign() {
let path = header(
- "foreignangled",
- "#include <lisdexconf.h>\nint parse_frame(CONFIGURED *d, unsigned long n);\n",
+ "externaldep",
+ "#include \"zlib.h\"\nint parse_frame(CONFIGURED *d, unsigned long n);\n",
);
- let elsewhere = outside_tree("foreignangled", "lisdexconf.h", GENERATED_CONFIG);
+ let elsewhere = outside_tree("externaldep", "zlib.h", GENERATED_CONFIG);
let options = ParseOptions {
extra_args: vec![format!("-I{}", elsewhere.display())],
+ project: "lisdex".to_owned(),
..ParseOptions::default()
};
let found = parse_header(&path, &options).unwrap();
assert!(
found.iter().any(|item| item.name == "parse_frame"),
- "an angled include is somebody else's header by construction and must still \
- resolve from the system: {:?}",
+ "a quoted include naming nothing this project owns is a dependency, not a \
+ foreign copy of our own header: {:?}",
names(&found)
);
}
@@ -1230,6 +1512,7 @@ void WebPFreeDecBuffer(WebPDecBuffer *buffer);
let elsewhere = outside_tree("foreignvocab", "lisdexconf.h", GENERATED_CONFIG);
let options = ParseOptions {
extra_args: vec![format!("-I{}", elsewhere.display())],
+ project: "lisdex".to_owned(),
..ParseOptions::default()
};
let error = parse_vocabulary(&path, &options).unwrap_err();
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
index b4e6fb1a..d74d7d60 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
@@ -69,6 +69,7 @@ pub struct HeaderSet {
pub const PRIVATE_MARKERS: &[&str] = &["priv", "internal", "impl", "iop"];
pub const PRIVATE_SUFFIXES: &[&str] = &["_p", "-p", "_int"];
pub const LICENSE_STEMS: &[&str] = &["license", "licence", "copying", "copyright", "notice"];
+pub const GENERATOR_SUFFIXES: &[&str] = &["in", "cmake", "prebuilt", "tmpl", "template"];
#[must_use]
pub fn is_private_header(path: &Path) -> bool {
@@ -205,6 +206,49 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {
Ok(found)
}
+// A tree shipping `jconfig.h.in` is telling you its build GENERATES `jconfig.h`, so a
+// jconfig.h resolving from anywhere else is the wrong library's. This is the dominant B30
+// shape and the one no name rule catches: pnglibconf.h and expat_config.h carry the project
+// token, but jconfig.h, tiffconf.h and opj_config.h do not. Stripping the generator suffix
+// yields the path the build would produce, which the ownership test already understands.
+fn strip_generator(path: &Path) -> Option<PathBuf> {
+ let suffix = path.extension()?.to_str()?.to_lowercase();
+ if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {
+ return None;
+ }
+ let generated = path.with_extension("");
+ is_header(&generated).then_some(generated)
+}
+
+fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {
+ if depth > MAX_DEPTH {
+ return;
+ }
+ let Ok(entries) = std::fs::read_dir(dir) else {
+ return;
+ };
+ for entry in entries.flatten() {
+ let path = entry.path();
+ let name = entry.file_name().to_string_lossy().to_lowercase();
+ if path.is_dir() {
+ if !is_excluded(&name) && !is_vendored_tree(&path, project) {
+ collect_generated(&path, project, depth + 1, found);
+ }
+ } else if let Some(generated) = strip_generator(&path) {
+ found.push(generated);
+ }
+ }
+}
+
+#[must_use]
+pub fn generated_headers(source: &Path) -> Vec<PathBuf> {
+ let mut found = Vec::new();
+ collect_generated(source, &project_of(source), 0, &mut found);
+ found.sort();
+ found.dedup();
+ found
+}
+
fn gathered(root: &Path, project: &str) -> Result<Vec<PathBuf>, HeaderError> {
let mut found = Vec::new();
collect(root, project, 0, &mut found)?;
@@ -291,6 +335,21 @@ mod tests {
root
}
+ // The tree has to sit in a PROJECT-NAMED directory, because that is what project_of
+ // reads. Renaming to a sibling of the fixture put it at a fixed, PID-free path in the
+ // shared temp dir - so `cargo test` deleted a real /tmp/lz4, and two concurrent runs
+ // fought over the same directory. Rename inside a per-process box instead.
+ fn named_as_project(root: &Path, project: &str) -> PathBuf {
+ let mut boxed = root.file_name().unwrap().to_os_string();
+ boxed.push("-box");
+ let box_dir = root.with_file_name(boxed);
+ let _ = std::fs::remove_dir_all(&box_dir);
+ std::fs::create_dir_all(&box_dir).unwrap();
+ let renamed = box_dir.join(project);
+ std::fs::rename(root, &renamed).unwrap();
+ renamed
+ }
+
fn names(set: &HeaderSet) -> Vec<String> {
let mut found: Vec<String> = set
.headers
@@ -323,9 +382,7 @@ mod tests {
"include/spng/spng.h",
],
);
- let renamed = root.with_file_name("libjpeg-turbo-vendored");
- let _ = std::fs::remove_dir_all(&renamed);
- std::fs::rename(&root, &renamed).unwrap();
+ let renamed = named_as_project(&root, "libjpeg-turbo-vendored");
let set = discover(&renamed, None).unwrap();
assert_eq!(
names(&set),
@@ -334,6 +391,39 @@ mod tests {
);
}
+ #[test]
+ fn generated_headers_strip_the_generator_suffix_and_ignore_everything_else() {
+ let root = fixture(
+ "generators",
+ &[
+ "src/jconfig.h.in",
+ "expat_config.h.cmake",
+ "scripts/pnglibconf.h.prebuilt",
+ "config.h",
+ "Makefile.in",
+ "src/real.h",
+ ],
+ );
+ let found: Vec<String> = generated_headers(&root)
+ .iter()
+ .map(|path| {
+ path.strip_prefix(&root)
+ .unwrap_or(path)
+ .to_string_lossy()
+ .into_owned()
+ })
+ .collect();
+ assert_eq!(
+ found,
+ vec![
+ "expat_config.h".to_owned(),
+ "scripts/pnglibconf.h".to_owned(),
+ "src/jconfig.h".to_owned(),
+ ],
+ "Makefile.in is not a header and config.h is not a generator"
+ );
+ }
+
#[test]
fn a_licence_over_the_projects_own_api_excludes_nothing() {
// lz4 ships lib/LICENSE and libexpat ships expat/COPYING over their own headers.
@@ -349,9 +439,7 @@ mod tests {
"lz4shape",
&["lib/LICENSE", "lib/lz4frame.h", "ossfuzz/x.h"],
);
- let renamed = root.with_file_name("lz4");
- let _ = std::fs::remove_dir_all(&renamed);
- std::fs::rename(&root, &renamed).unwrap();
+ let renamed = named_as_project(&root, "lz4");
let set = discover(&renamed, None).unwrap();
assert!(
names(&set).contains(&"lz4frame.h".to_owned()),
@@ -370,9 +458,7 @@ mod tests {
"src/spng/spng.h",
],
);
- let renamed = root.with_file_name("libjpeg-turbo");
- let _ = std::fs::remove_dir_all(&renamed);
- std::fs::rename(&root, &renamed).unwrap();
+ let renamed = named_as_project(&root, "libjpeg-turbo");
let set = discover(&renamed, None).unwrap();
assert_eq!(names(&set), vec!["turbojpeg.h"]);
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
index dbab0148..27272337 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
@@ -46,6 +46,17 @@ pub enum SurfaceError {
declared: usize,
first: String,
},
+ #[error(
+ "{count} file(s) under {root} resolved a header this library owns to a copy outside the \
+ source tree, so this ranking would describe a different build of the library. first: \
+ {first}. build the target and pass its install prefix with --install, or drop the stray \
+ copy off the include path"
+ )]
+ ForeignEnvironment {
+ root: PathBuf,
+ count: usize,
+ first: String,
+ },
}
#[derive(Debug, Clone)]
@@ -74,7 +85,7 @@ impl SurfaceOptions {
}
#[must_use]
- pub fn parse_options(&self, header_root: &Path) -> ParseOptions {
+ pub fn parse_options(&self, header_root: &Path, tree_headers: &[PathBuf]) -> ParseOptions {
let mut includes = vec![header_root.to_path_buf(), self.source.clone()];
includes.extend(self.includes.iter().cloned());
ParseOptions {
@@ -82,6 +93,8 @@ impl SurfaceOptions {
defines: self.defines.clone(),
extra_args: Vec::new(),
language: self.language,
+ tree_headers: tree_headers.to_vec(),
+ project: headers::project_of(&self.source),
}
}
}
@@ -90,6 +103,7 @@ impl SurfaceOptions {
pub struct SkippedHeader {
pub header: PathBuf,
pub reason: String,
+ pub foreign: bool,
}
#[derive(Debug, Clone, serde::Serialize)]
@@ -98,6 +112,7 @@ pub struct Surface {
pub header_root: PathBuf,
pub origin: headers::HeaderOrigin,
pub headers_read: usize,
+ pub headers: Vec<PathBuf>,
pub skipped: Vec<SkippedHeader>,
pub harnesses: Vec<Harness>,
pub declared: usize,
@@ -140,8 +155,38 @@ pub fn library_sources(root: &Path) -> Vec<PathBuf> {
found
}
-fn read_headers(set: &HeaderSet, options: &SurfaceOptions) -> (Vec<Declared>, Vec<SkippedHeader>) {
- let parse = options.parse_options(&set.root);
+// A foreign include is a statement about the ENVIRONMENT, not about one header, so it can
+// never be one line in a skip list. Measured: a single foreign header in libwebp dropped the
+// surface 429 -> 183 declarations, replaced the top entry point with an encoder function, and
+// still exited 0 under the old per-header handling.
+fn skipped_by(header: &Path, error: &extract::ExtractError) -> SkippedHeader {
+ SkippedHeader {
+ header: header.to_path_buf(),
+ reason: error.to_string(),
+ // Matched on the variant, never on the message: a reworded error must not be able
+ // to silently turn a fatal environment fault back into one line of a skip list.
+ foreign: matches!(error, extract::ExtractError::ForeignInclude { .. }),
+ }
+}
+
+fn refuse_foreign(root: &Path, skipped: &[SkippedHeader]) -> Result<(), SurfaceError> {
+ let foreign: Vec<&SkippedHeader> = skipped.iter().filter(|item| item.foreign).collect();
+ match foreign.first() {
+ Some(first) => Err(SurfaceError::ForeignEnvironment {
+ root: root.to_path_buf(),
+ count: foreign.len(),
+ first: first.reason.clone(),
+ }),
+ None => Ok(()),
+ }
+}
+
+fn read_headers(
+ set: &HeaderSet,
+ options: &SurfaceOptions,
+ owned: &[PathBuf],
+) -> (Vec<Declared>, Vec<SkippedHeader>) {
+ let parse = options.parse_options(&set.root, owned);
let mut declared = Vec::new();
let mut skipped = Vec::new();
let mut seen = BTreeSet::new();
@@ -155,10 +200,7 @@ fn read_headers(set: &HeaderSet, options: &SurfaceOptions) -> (Vec<Declared>, Ve
}
}
}
- Err(error) => skipped.push(SkippedHeader {
- header: header.clone(),
- reason: error.to_string(),
- }),
+ Err(error) => skipped.push(skipped_by(header, &error)),
}
}
(declared, skipped)
@@ -172,8 +214,9 @@ fn read_sources(
root: &Path,
options: &SurfaceOptions,
header_root: &Path,
+ tree_headers: &[PathBuf],
) -> (Vec<Defined>, Vec<SkippedHeader>) {
- let parse = options.parse_options(header_root);
+ let parse = options.parse_options(header_root, tree_headers);
let mut defined = Vec::new();
let mut skipped = Vec::new();
let mut seen = BTreeSet::new();
@@ -186,10 +229,7 @@ fn read_sources(
}
}
}
- Err(error) => skipped.push(SkippedHeader {
- header: source,
- reason: error.to_string(),
- }),
+ Err(error) => skipped.push(skipped_by(&source, &error)),
}
}
(defined, skipped)
@@ -197,9 +237,15 @@ fn read_sources(
pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {
let set = headers::discover(&options.source, options.install.as_deref())?;
+ // Headers the build would generate count as this tree's own for the ownership test, even
+ // though no such file exists to parse yet. They are appended AFTER header discovery so
+ // nothing tries to read them.
+ let mut owned = set.headers.clone();
+ owned.extend(headers::generated_headers(&options.source));
let harnesses = harness::discover(&options.source)?;
- let (declared, skipped) = read_headers(&set, options);
+ let (declared, skipped) = read_headers(&set, options, &owned);
+ refuse_foreign(&set.root, &skipped)?;
if declared.is_empty() {
return Err(SurfaceError::NothingDeclared {
root: set.root.clone(),
@@ -224,7 +270,8 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {
});
}
- let (defined, skipped_sources) = read_sources(&options.source, options, &set.root);
+ let (defined, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);
+ refuse_foreign(&set.root, &skipped_sources)?;
let defined_count = defined.len();
let graph = CallGraph::from(defined);
@@ -260,6 +307,7 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {
header_root: set.root.clone(),
origin: set.origin,
headers_read: set.headers.len(),
+ headers: set.headers.clone(),
skipped,
harnesses,
declared: declared.len(),
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
index 2df9eb6e..061016e3 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
@@ -339,7 +339,7 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
let analysis = surface::analyze(&options)?;
let root = std::fs::canonicalize(&analysis.header_root)
.unwrap_or_else(|_| analysis.header_root.clone());
- let parse = options.parse_options(&root);
+ let parse = options.parse_options(&root, &analysis.headers);
let bench = if args.no_verify {
println!("{UNVERIFIED_BANNER}");
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh
index 16561dc6..d008f106 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m2-gate.sh
@@ -8,7 +8,7 @@ MAGMA_SRC="${MAGMA_ROOT:-/opt/magma}"
ROOT="${LISDEX_M2_ROOT:-/opt/magma-work/m2}"
CACHE="$ROOT/cache"
WORK="$ROOT/gate"
-LISDEX="${LISDEX_BIN:-/cargo-target/debug/lisdex}"
+LISDEX="${LISDEX_BIN:-${CARGO_TARGET_DIR:-/cargo-target}/debug/lisdex}"
JOBS="${LISDEX_JOBS:-$(nproc)}"
MAGMA_TARGETS=(libpng libtiff libxml2 sqlite3 lua)
@@ -39,7 +39,10 @@ record() {
SUMMARY+=("$1")
}
-[ -x "$LISDEX" ] || die "lisdex binary missing at $LISDEX (run: cargo build --workspace)"
+# Existence was never enough: it ties the gate to nothing, so a 10-minute run could grade a
+# binary from an older commit. Build what is about to be tested.
+cargo build -p lisdex || die "lisdex does not build"
+[ -x "$LISDEX" ] || die "lisdex binary missing at $LISDEX"
[ -d "$MAGMA_SRC" ] || die "magma source missing at $MAGMA_SRC"
if [ "${1:-}" = "reset" ]; then
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
index 5b2661fa..c941c29d 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
@@ -49,6 +49,16 @@ cargo build -p lisdex || {
mkdir -p "$SRC" "$OUT"
+# B56's lesson, applied here too: some rules have no end-to-end control that isolates them,
+# so the crate's own tests ARE a gate stage. Without this, a rule can be gutted in a way no
+# ranking assertion can see.
+banner "0. THE SURFACE CRATE'S OWN TESTS"
+if cargo test -p lisdex-surface --lib >"$OUT/unit.txt" 2>&1; then
+ ok "lisdex-surface unit tests pass ($(grep -oE '[0-9]+ passed' "$OUT/unit.txt" | head -1))"
+else
+ bad "lisdex-surface unit tests fail: $(grep -E '^test .*FAILED|panicked' "$OUT/unit.txt" | head -1)"
+fi
+
clone() {
local name="$1" url="$2" ref="$3"
if [ -d "$SRC/$name/.git" ]; then
@@ -209,34 +219,69 @@ else
bad "harness awareness had no effect (cold=$cold_png warm=$warm_png)"
fi
-# B30. CPATH puts a directory on the compiler's default search path without lisdex
-# ever being told about it, which is exactly the shape of a system-installed copy of
-# the target. The two halves differ in one thing only - whether the tree still owns
-# its own pnglibconf.h - so a check that refused merely because a foreign copy is
-# reachable fails the second half instead of passing both.
+# B30. CPATH puts a directory on the compiler's default search path without lisdex ever
+# being told about it, which is exactly the shape of a system-installed copy of the target.
+# Three assertions, because two were not enough: an audit found that moving pnglibconf.h out
+# of the tree changes TWO things at once - the include resolves foreign AND no file of that
+# name remains - so a rule with a "the tree owns something by that name" escape hatch passed
+# both halves while silently reintroducing B30. Assertion 2 separates those variables.
banner "B30 FOREIGN INCLUDE"
+# The copy keeps the name libpng and lives OUTSIDE $SRC: the project token is derived from
+# the directory, so a copy called libpng-b30 is a different project as far as ownership is
+# concerned, and a fake clone sitting beside the real ones invites being mistaken for one.
FOREIGN="$WORK/foreign"
-B30SRC="$SRC/libpng-b30"
-rm -rf "$FOREIGN" "$B30SRC"
-mkdir -p "$FOREIGN"
+B30ROOT="$WORK/b30"
+B30SRC="$B30ROOT/libpng"
+rm -rf "$FOREIGN" "$B30ROOT"
+mkdir -p "$FOREIGN" "$B30ROOT"
cp -r "$SRC/libpng" "$B30SRC"
mv "$B30SRC/pnglibconf.h" "$FOREIGN/pnglibconf.h"
+# The foreign copy must DIFFER from the tree's own. A byte-identical one produces an
+# identical ranking when wrongly accepted, so the control could only ever detect location.
+grep -v 'PNG_SIMPLIFIED_READ_SUPPORTED' "$FOREIGN/pnglibconf.h" >"$FOREIGN/trimmed.h"
+mv "$FOREIGN/trimmed.h" "$FOREIGN/pnglibconf.h"
if CPATH="$FOREIGN" "$LISDEX" surface "$B30SRC" >"$OUT/b30-foreign.txt" 2>&1; then
bad "libpng was analysed against a pnglibconf.h outside its own source tree"
-elif grep -q "quoted include" "$OUT/b30-foreign.txt"; then
- ok "a quoted include resolving outside the source tree is refused and named"
+elif grep -q "lies outside the source tree" "$OUT/b30-foreign.txt"; then
+ ok "an include resolving outside the source tree is refused and named"
else
bad "libpng was refused, but not by the foreign-include check"
fi
-cp "$FOREIGN/pnglibconf.h" "$B30SRC/pnglibconf.h"
+# libpng cannot express this one - every header it discovers sits at the root, which is also
+# the quoted search path, so a shadow copy there would simply resolve. Synthesise a tree that
+# OWNS a header of the same basename in a subdirectory the walk finds but the include does
+# not reach. The shipped rule matches the whole sub-path and refuses; a rule that matches on
+# basename alone excuses it and analyses the foreign copy. Both halves above pass either way,
+# so without this the gate cannot tell those two rules apart.
+# No header at the root, so discovery walks the tree and BOTH files land in the header set -
+# with one at the root it stops there and the shadow is never seen, which silently disarms
+# this assertion.
+SHADOW="$B30ROOT/lisdex"
+SHADOW_FOREIGN="$B30ROOT/shadow-foreign"
+mkdir -p "$SHADOW/api" "$SHADOW/sub" "$SHADOW_FOREIGN"
+printf 'typedef unsigned char LISDEX_BYTE;\n' >"$SHADOW/sub/lisdexconf.h"
+printf 'typedef unsigned short LISDEX_BYTE;\n' >"$SHADOW_FOREIGN/lisdexconf.h"
+printf '#include "lisdexconf.h"\nint lisdex_parse(LISDEX_BYTE *d, unsigned long n);\n' \
+ >"$SHADOW/api/api.h"
+if CPATH="$SHADOW_FOREIGN" "$LISDEX" surface "$SHADOW" >"$OUT/b30-shadow.txt" 2>&1; then
+ bad "a same-basename header elsewhere in the tree excused the foreign copy that resolved"
+elif grep -q "lies outside the source tree" "$OUT/b30-shadow.txt"; then
+ ok "owning the same basename elsewhere does not excuse the copy that actually resolved"
+else
+ bad "the synthetic tree was refused, but not by the foreign-include check: \
+$(head -1 "$OUT/b30-shadow.txt")"
+fi
+
+cp "$SRC/libpng/pnglibconf.h" "$B30SRC/pnglibconf.h"
if CPATH="$FOREIGN" "$LISDEX" surface "$B30SRC" >"$OUT/b30-intree.txt" 2>&1; then
ok "the same tree owning its own pnglibconf.h still analyses with the foreign copy reachable"
else
bad "the foreign-include check rejects a tree that resolves its own config: \
$(head -1 "$OUT/b30-intree.txt")"
fi
+rm -rf "$FOREIGN" "$B30ROOT"
banner "RESULT"
echo " passed: $PASS"