Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): B26 - configure-only recovery, and a generated header cmake's own convention hid
CarterPerez-dev1fe4aca12cb0
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rsindex 8c7415c3..5b0d0463 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs@@ -22,6 +22,7 @@ const PREPASS_VARIANT_SUFFIX: &str = "-prepass";const LABEL_PREPASS: &str = "prepass";const LABEL_INSTRUMENTED: &str = "instrumented";const LABEL_CAPTURE: &str = "capture";+const CONFIGURE_ONLY_LABEL: &str = "configure-only";const STEP_CLEAN: &str = "clean";#[derive(Debug, thiserror::Error)]@@ -41,6 +42,10 @@ pub enum BuildError {Replay(#[from] replay::ReplayError),#[error("artifact error: {0}")]Artifact(#[from] artifact::ArtifactError),+ #[error(+ "no build system under {tree} could run its configure step, so the headers it generates were not produced: {refusals}"+ )]+ NotConfigured { tree: PathBuf, refusals: String },}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -628,6 +633,77 @@ fn run_tier(}}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Configured {+ pub system: BuildSystem,+ pub build_dir: PathBuf,+}++// The configure step alone, for a tree whose PUBLIC headers are generated by its build.+// Compiling the library is not needed to read its API and costs minutes; a cmake configure+// costs seconds. Measured on a fresh clone: libtiff 1.97s, openjpeg 1.33s, and both emit+// the header their own public headers include.+pub fn configure_only(+ detection: &Detection,+ layout: &BuildLayout,+ instrumentation: &Instrumentation,+ options: &BuildOptions,+) -> Result<Configured, BuildError> {+ layout.prepare().map_err(|source| BuildError::Io {+ path: layout.work.clone(),+ source,+ })?;++ let mut refusals = Vec::new();+ for system in &detection.candidates {+ let plan_options = PlanOptions {+ jobs: options.jobs,+ generator: Generator::default(),+ bootstrap: plan::BootstrapPolicy::Auto,+ variant: options.variant.clone(),+ };+ let Ok(mut planned) =+ plan::plan(detection, *system, layout, instrumentation, &plan_options)+ else {+ continue;+ };+ let Some(stop) = planned+ .steps+ .iter()+ .position(|step| step.name == plan::STEP_CONFIGURE)+ else {+ continue;+ };+ planned.steps.truncate(stop + 1);++ let (_, failure) = execute(+ layout,+ &planned,+ Tier::FlagInjection,+ CONFIGURE_ONLY_LABEL,+ &[],+ options.step_timeout,+ )?;+ match failure {+ None => {+ return Ok(Configured {+ system: *system,+ build_dir: planned.build_dir,+ });+ }+ Some(reason) => refusals.push(format!("{}: {reason}", system.as_str())),+ }+ }+ Err(BuildError::NotConfigured {+ tree: layout.source.clone(),+ refusals: if refusals.is_empty() {+ "no candidate build system plans a configure step".to_owned()+ } else {+ refusals.join("; ")+ },+ })+}+pub fn build(detection: &Detection,layout: &BuildLayout,diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rsindex e1db00e6..74b562ac 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs@@ -214,13 +214,25 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {// shape and the one no name rule catches: pnglibconf.h and expat_config.h carry the project// token, but jconfig.h, tiffconf.h and opj_config.h do not. Stripping the generator suffix// yields the path the build would produce, which the ownership test already understands.+// Stacked on purpose. cmake's own convention is `<name>.h.cmake.in`, two generator+// suffixes deep, and stripping only one leaves `opj_config.h.cmake`, which is not a header+// - so openjpeg's generated config header was invisible both to B26's detection and to the+// OWNED set the foreign-include refusal is built on.fn strip_generator(path: &Path) -> Option<PathBuf> {- let suffix = path.extension()?.to_str()?.to_lowercase();- if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {- return None;+ let mut generated = path.to_path_buf();+ let mut stripped = false;+ while let Some(suffix) = generated+ .extension()+ .and_then(|suffix| suffix.to_str())+ .map(str::to_lowercase)+ {+ if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {+ break;+ }+ generated = generated.with_extension("");+ stripped = true;}- let generated = path.with_extension("");- is_header(&generated).then_some(generated)+ (stripped && is_header(&generated)).then_some(generated)}fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {@@ -243,6 +255,25 @@ fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<Pa}}+// A header the build GENERATES and that nothing has generated yet. The template is the+// tree's own statement that the file is produced rather than shipped, so this is a+// structural fact about the checkout and never a guess read off a compiler diagnostic.+#[must_use]+pub fn missing_generated(source: &Path, includes: &[PathBuf]) -> Vec<PathBuf> {+ generated_headers(source)+ .into_iter()+ .filter(|generated| {+ if generated.is_file() {+ return false;+ }+ let Some(name) = generated.file_name() else {+ return false;+ };+ !includes.iter().any(|dir| dir.join(name).is_file())+ })+ .collect()+}+#[must_use]pub fn generated_headers(source: &Path) -> Vec<PathBuf> {let mut found = Vec::new();@@ -473,6 +504,34 @@ mod tests {);}+ #[test]+ fn a_generated_header_counts_as_missing_only_until_something_produces_it() {+ let root = fixture(+ "missinggen",+ &[+ "lib/opj_config.h.cmake.in",+ "lib/already.h.in",+ "lib/already.h",+ ],+ );+ let missing: Vec<String> = missing_generated(&root, &[])+ .iter()+ .filter_map(|path| path.file_name())+ .map(|name| name.to_string_lossy().into_owned())+ .collect();+ assert_eq!(+ missing,+ vec!["opj_config.h".to_owned()],+ "already.h exists in the tree, so nothing needs to generate it"+ );++ let elsewhere = fixture("missinggendir", &["opj_config.h"]);+ assert!(+ missing_generated(&root, std::slice::from_ref(&elsewhere)).is_empty(),+ "a header supplied on the include path is not missing either"+ );+ }+#[test]fn generated_headers_strip_the_generator_suffix_and_ignore_everything_else() {let root = fixture(@@ -484,6 +543,10 @@ mod tests {"config.h","Makefile.in","src/real.h",+ // cmake's own convention, two generator suffixes deep. Stripping one+ // leaves opj_config.h.cmake, which is not a header, so openjpeg's+ // generated config header used to be invisible here.+ "lib/opj_config.h.cmake.in",],);let found: Vec<String> = generated_headers(&root)@@ -499,6 +562,7 @@ mod tests {found,vec!["expat_config.h".to_owned(),+ "lib/opj_config.h".to_owned(),"scripts/pnglibconf.h".to_owned(),"src/jconfig.h".to_owned(),],diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rsindex c93cde1b..0966ae0f 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs@@ -41,6 +41,7 @@ use lisdex_report::bundle::{self, Reproducer};use lisdex_report::{json, markdown};use lisdex_surface::eval;use lisdex_surface::extract::{self, Language};+use lisdex_surface::headers;use lisdex_surface::surface::{self, SurfaceOptions};use lisdex_triage::classify::classify;@@ -70,6 +71,8 @@ const DEFAULT_SURFACE_TOP: usize = 20;const MAX_SKIPPED_SHOWN: usize = 10;const CANDIDATES_DIR: &str = "candidates";const ORACLE_WORK_DIR: &str = "oracle";+const CONFIGURE_SCRATCH: &str = "surface-configure";+const CONFIGURE_VARIANT: &str = "configure";const NO_SEEDS: &str = "--seeds is required: the harness oracles run the candidate against a \valid corpus, and without one lisdex cannot tell a working harness from \one that reads nothing. Pass --no-verify to emit an unchecked harness \@@ -908,7 +911,7 @@ fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {|| source.display().to_string(),|name| name.to_string_lossy().into_owned(),);- let analysis = match surface::analyze(&options) {+ let analysis = match analyze_configuring_if_needed(&options) {Ok(analysis) => analysis,Err(err) => {println!(" {label:<12} NOT ANALYSED {err}");@@ -965,6 +968,98 @@ fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {Ok(())}+// B26. A tree whose PUBLIC headers include a header its build generates cannot be parsed+// on a fresh clone. Compiling it to find out what it declares costs minutes; running only+// the configure step costs seconds, and the tree itself says which headers are generated+// because it ships their templates. So: try the analysis first, and only when it fails+// AND the tree declares a generated header that nothing has produced, configure and retry+// once. A tree that parses on the first attempt pays nothing.+fn analyze_configuring_if_needed(+ options: &SurfaceOptions,+) -> Result<surface::Surface, Box<dyn Error>> {+ let refusal = match surface::analyze(options) {+ Ok(analysis) => return Ok(analysis),+ Err(refusal) => refusal,+ };++ let missing = headers::missing_generated(&options.source, &options.includes);+ if missing.is_empty() {+ return Err(refusal.into());+ }+ let names: Vec<String> = missing+ .iter()+ .filter_map(|path| path.file_name())+ .map(|name| name.to_string_lossy().into_owned())+ .collect();+ eprintln!(+ "this tree generates {} at build time; running its configure step only",+ names.join(", ")+ );++ let detected = detect::detect(&options.source)?;+ let work = lisdex_core::scratch::unclaimed_path(CONFIGURE_SCRATCH);+ let layout = BuildLayout::validated(&detected.root, &work)?;+ let instrumentation = Instrumentation::uninstrumented();+ let build_options = ladder::BuildOptions::new(+ CONFIGURE_VARIANT,+ std::thread::available_parallelism().map_or(1, std::num::NonZero::get),+ );+ let configured =+ match ladder::configure_only(&detected, &layout, &instrumentation, &build_options) {+ Ok(configured) => configured,+ Err(err) => {+ eprintln!("configure did not run: {err}");+ return Err(refusal.into());+ }+ };++ let mut retry = options.clone();+ retry+ .includes+ .extend(generated_include_dirs(&configured.build_dir, &names));+ match surface::analyze(&retry) {+ Ok(analysis) => {+ eprintln!(+ "configured with {} and re-read the headers",+ configured.system.as_str()+ );+ Ok(analysis)+ }+ // The ORIGINAL refusal, not the retry's. It is the one that names the remedy, and+ // reporting a second failure against a build directory the user never asked for+ // would point them at a path that is about to be deleted.+ Err(_) => Err(refusal.into()),+ }+}++// cmake writes a generated header beside the source directory that declared it, so the+// build tree mirrors the source tree and the header can sit several levels down.+fn generated_include_dirs(build_dir: &Path, names: &[String]) -> Vec<PathBuf> {+ let mut found = vec![build_dir.to_path_buf()];+ let mut stack = vec![build_dir.to_path_buf()];+ while let Some(dir) = stack.pop() {+ let Ok(entries) = std::fs::read_dir(&dir) else {+ continue;+ };+ for entry in entries.flatten() {+ let path = entry.path();+ if path.is_dir() {+ stack.push(path);+ } else if path+ .file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| names.iter().any(|wanted| wanted == name))+ && let Some(parent) = path.parent()+ {+ found.push(parent.to_path_buf());+ }+ }+ }+ found.sort();+ found.dedup();+ found+}+fn parse_arm(spec: &str) -> Result<(String, PathBuf), Box<dyn Error>> {let (name, path) = spec.split_once('=')@@ -1072,7 +1167,7 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {options.oss_fuzz.clone_from(&args.oss_fuzz);options.library_symbols = library_symbols(&args.libraries)?;- let analysis = surface::analyze(&options)?;+ let analysis = analyze_configuring_if_needed(&options)?;report_ladder(&analysis.ladder);let root = std::fs::canonicalize(&analysis.header_root).unwrap_or_else(|_| analysis.header_root.clone());@@ -1480,7 +1575,7 @@ fn surface(args: &SurfaceArgs) -> Result<(), Box<dyn Error>> {options.oss_fuzz.clone_from(&args.oss_fuzz);options.library_symbols = library_symbols(&args.libraries)?;- let analysis = surface::analyze(&options)?;+ let analysis = analyze_configuring_if_needed(&options)?;if args.json {println!("{}", serde_json::to_string_pretty(&analysis)?);diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.shindex 1ac4fa36..6ad62b5f 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh@@ -173,26 +173,48 @@ for target in libpng libarchive; dofidone-banner "1b. targets needing a build-generated config header say so, loudly"-# libtiff needs tiffconf.h and openjpeg needs opj_config.h, both produced by-# their build. Analysing without one must fail with an actionable diagnosis-# rather than silently reporting an empty or partial surface.+banner "1b. targets needing a build-generated config header analyse point-and-run (B26)"+# libtiff needs tiffconf.h and openjpeg needs opj_config.h, both produced by their build.+# This stage used to assert they FAILED with an actionable diagnosis; B26 made them work+# with no flags by running the configure step ONLY when the tree ships a template for a+# header nothing has generated. Measured: libtiff 6 -> 223 declarations, openjpeg 13 -> 89,+# ~3.8s each including configure.#-# B68. This asserted the message named `--install`, and `lisdex build` produces NO install-# prefix - the ladder never runs an install step, so the remedy named a thing the tool cannot-# make. The generated header is written into the BUILD tree and `--include` is what reads it,-# so the assertion is now the flag that works AND the absence of the one that never did.+# B68's finding still holds and is still asserted below: `lisdex build` produces NO install+# prefix, so the fallback diagnosis must name `--include` and never "install prefix".for target in libtiff openjpeg; doif analyse "$target"; then- bad "$target unexpectedly succeeded with no build; the gate assumption is stale"- elif grep -q -- "--include" "$OUT/$target.err" \- && ! grep -q -- "install prefix" "$OUT/$target.err"; then- ok "$target refused with a diagnosis naming --include, a flag that resolves it"+ DECLARED="$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['declared'])" \+ "$OUT/$target.json" 2>/dev/null || echo 0)"+ if [ "${DECLARED:-0}" -ge 50 ]; then+ ok "$target analysed with no flags and no prior build ($DECLARED declarations)"+ else+ bad "$target analysed but reported only $DECLARED declarations, so the config header did not land"+ fielse- bad "$target failed without telling the user what to do: $(head -1 "$OUT/$target.err")"+ bad "$target still needs a manual build: $(head -1 "$OUT/$target.err")"fidone+# The refusal is still the FLOOR. A tree that declares a generated header but whose build+# cannot be configured must fall back to the actionable diagnosis rather than to a partial+# surface - otherwise the recovery above would be hiding the failure it replaced.+UNCONFIGURABLE="$WORK/b26-unconfigurable"+rm -rf "$UNCONFIGURABLE"+mkdir -p "$UNCONFIGURABLE/include"+printf '#cmakedefine HAVE_THING 1\n' >"$UNCONFIGURABLE/include/gen_config.h.cmake.in"+printf '#include "gen_config.h"\nint gen_parse(const unsigned char *d, unsigned long n);\n' \+ >"$UNCONFIGURABLE/include/gen.h"+if "$LISDEX" surface "$UNCONFIGURABLE" >"$OUT/b26-unconfigurable.err" 2>&1; then+ bad "a tree with an ungeneratable config header was analysed anyway"+elif grep -q -- "--include" "$OUT/b26-unconfigurable.err" \+ && ! grep -q -- "install prefix" "$OUT/b26-unconfigurable.err"; then+ ok "a tree whose configure cannot run still refuses with a diagnosis naming --include"+else+ bad "the fallback diagnosis was lost: $(head -1 "$OUT/b26-unconfigurable.err")"+fi+rm -rf "$UNCONFIGURABLE"+banner "2. NEGATIVE CONTROLS - a ranker that name-matches must fail here"require_ranked_outside "$OUT/libpng.json" png_sig_cmp 10require_ranked_outside "$OUT/libarchive.json" archive_read_support_format_zip 20