Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

feat(lisdex): B26 - configure-only recovery, and a generated header cmake's own convention hid

CarterPerez-dev1fe4aca12cb0
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
index 8c7415c3..5b0d0463 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
@@ -22,6 +22,7 @@ const PREPASS_VARIANT_SUFFIX: &str = "-prepass";
const LABEL_PREPASS: &str = "prepass";
const LABEL_INSTRUMENTED: &str = "instrumented";
const LABEL_CAPTURE: &str = "capture";
+const CONFIGURE_ONLY_LABEL: &str = "configure-only";
const STEP_CLEAN: &str = "clean";
#[derive(Debug, thiserror::Error)]
@@ -41,6 +42,10 @@ pub enum BuildError {
Replay(#[from] replay::ReplayError),
#[error("artifact error: {0}")]
Artifact(#[from] artifact::ArtifactError),
+ #[error(
+ "no build system under {tree} could run its configure step, so the headers it generates were not produced: {refusals}"
+ )]
+ NotConfigured { tree: PathBuf, refusals: String },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -628,6 +633,77 @@ fn run_tier(
}
}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Configured {
+ pub system: BuildSystem,
+ pub build_dir: PathBuf,
+}
+
+// The configure step alone, for a tree whose PUBLIC headers are generated by its build.
+// Compiling the library is not needed to read its API and costs minutes; a cmake configure
+// costs seconds. Measured on a fresh clone: libtiff 1.97s, openjpeg 1.33s, and both emit
+// the header their own public headers include.
+pub fn configure_only(
+ detection: &Detection,
+ layout: &BuildLayout,
+ instrumentation: &Instrumentation,
+ options: &BuildOptions,
+) -> Result<Configured, BuildError> {
+ layout.prepare().map_err(|source| BuildError::Io {
+ path: layout.work.clone(),
+ source,
+ })?;
+
+ let mut refusals = Vec::new();
+ for system in &detection.candidates {
+ let plan_options = PlanOptions {
+ jobs: options.jobs,
+ generator: Generator::default(),
+ bootstrap: plan::BootstrapPolicy::Auto,
+ variant: options.variant.clone(),
+ };
+ let Ok(mut planned) =
+ plan::plan(detection, *system, layout, instrumentation, &plan_options)
+ else {
+ continue;
+ };
+ let Some(stop) = planned
+ .steps
+ .iter()
+ .position(|step| step.name == plan::STEP_CONFIGURE)
+ else {
+ continue;
+ };
+ planned.steps.truncate(stop + 1);
+
+ let (_, failure) = execute(
+ layout,
+ &planned,
+ Tier::FlagInjection,
+ CONFIGURE_ONLY_LABEL,
+ &[],
+ options.step_timeout,
+ )?;
+ match failure {
+ None => {
+ return Ok(Configured {
+ system: *system,
+ build_dir: planned.build_dir,
+ });
+ }
+ Some(reason) => refusals.push(format!("{}: {reason}", system.as_str())),
+ }
+ }
+ Err(BuildError::NotConfigured {
+ tree: layout.source.clone(),
+ refusals: if refusals.is_empty() {
+ "no candidate build system plans a configure step".to_owned()
+ } else {
+ refusals.join("; ")
+ },
+ })
+}
+
pub fn build(
detection: &Detection,
layout: &BuildLayout,
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
index e1db00e6..74b562ac 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/headers.rs
@@ -214,13 +214,25 @@ fn collect_shallow(root: &Path) -> Result<Vec<PathBuf>, HeaderError> {
// shape and the one no name rule catches: pnglibconf.h and expat_config.h carry the project
// token, but jconfig.h, tiffconf.h and opj_config.h do not. Stripping the generator suffix
// yields the path the build would produce, which the ownership test already understands.
+// Stacked on purpose. cmake's own convention is `<name>.h.cmake.in`, two generator
+// suffixes deep, and stripping only one leaves `opj_config.h.cmake`, which is not a header
+// - so openjpeg's generated config header was invisible both to B26's detection and to the
+// OWNED set the foreign-include refusal is built on.
fn strip_generator(path: &Path) -> Option<PathBuf> {
- let suffix = path.extension()?.to_str()?.to_lowercase();
- if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {
- return None;
+ let mut generated = path.to_path_buf();
+ let mut stripped = false;
+ while let Some(suffix) = generated
+ .extension()
+ .and_then(|suffix| suffix.to_str())
+ .map(str::to_lowercase)
+ {
+ if !GENERATOR_SUFFIXES.contains(&suffix.as_str()) {
+ break;
+ }
+ generated = generated.with_extension("");
+ stripped = true;
}
- let generated = path.with_extension("");
- is_header(&generated).then_some(generated)
+ (stripped && is_header(&generated)).then_some(generated)
}
fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<PathBuf>) {
@@ -243,6 +255,25 @@ fn collect_generated(dir: &Path, project: &str, depth: usize, found: &mut Vec<Pa
}
}
+// A header the build GENERATES and that nothing has generated yet. The template is the
+// tree's own statement that the file is produced rather than shipped, so this is a
+// structural fact about the checkout and never a guess read off a compiler diagnostic.
+#[must_use]
+pub fn missing_generated(source: &Path, includes: &[PathBuf]) -> Vec<PathBuf> {
+ generated_headers(source)
+ .into_iter()
+ .filter(|generated| {
+ if generated.is_file() {
+ return false;
+ }
+ let Some(name) = generated.file_name() else {
+ return false;
+ };
+ !includes.iter().any(|dir| dir.join(name).is_file())
+ })
+ .collect()
+}
+
#[must_use]
pub fn generated_headers(source: &Path) -> Vec<PathBuf> {
let mut found = Vec::new();
@@ -473,6 +504,34 @@ mod tests {
);
}
+ #[test]
+ fn a_generated_header_counts_as_missing_only_until_something_produces_it() {
+ let root = fixture(
+ "missinggen",
+ &[
+ "lib/opj_config.h.cmake.in",
+ "lib/already.h.in",
+ "lib/already.h",
+ ],
+ );
+ let missing: Vec<String> = missing_generated(&root, &[])
+ .iter()
+ .filter_map(|path| path.file_name())
+ .map(|name| name.to_string_lossy().into_owned())
+ .collect();
+ assert_eq!(
+ missing,
+ vec!["opj_config.h".to_owned()],
+ "already.h exists in the tree, so nothing needs to generate it"
+ );
+
+ let elsewhere = fixture("missinggendir", &["opj_config.h"]);
+ assert!(
+ missing_generated(&root, std::slice::from_ref(&elsewhere)).is_empty(),
+ "a header supplied on the include path is not missing either"
+ );
+ }
+
#[test]
fn generated_headers_strip_the_generator_suffix_and_ignore_everything_else() {
let root = fixture(
@@ -484,6 +543,10 @@ mod tests {
"config.h",
"Makefile.in",
"src/real.h",
+ // cmake's own convention, two generator suffixes deep. Stripping one
+ // leaves opj_config.h.cmake, which is not a header, so openjpeg's
+ // generated config header used to be invisible here.
+ "lib/opj_config.h.cmake.in",
],
);
let found: Vec<String> = generated_headers(&root)
@@ -499,6 +562,7 @@ mod tests {
found,
vec![
"expat_config.h".to_owned(),
+ "lib/opj_config.h".to_owned(),
"scripts/pnglibconf.h".to_owned(),
"src/jconfig.h".to_owned(),
],
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
index c93cde1b..0966ae0f 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
@@ -41,6 +41,7 @@ use lisdex_report::bundle::{self, Reproducer};
use lisdex_report::{json, markdown};
use lisdex_surface::eval;
use lisdex_surface::extract::{self, Language};
+use lisdex_surface::headers;
use lisdex_surface::surface::{self, SurfaceOptions};
use lisdex_triage::classify::classify;
@@ -70,6 +71,8 @@ const DEFAULT_SURFACE_TOP: usize = 20;
const MAX_SKIPPED_SHOWN: usize = 10;
const CANDIDATES_DIR: &str = "candidates";
const ORACLE_WORK_DIR: &str = "oracle";
+const CONFIGURE_SCRATCH: &str = "surface-configure";
+const CONFIGURE_VARIANT: &str = "configure";
const NO_SEEDS: &str = "--seeds is required: the harness oracles run the candidate against a \
valid corpus, and without one lisdex cannot tell a working harness from \
one that reads nothing. Pass --no-verify to emit an unchecked harness \
@@ -908,7 +911,7 @@ fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {
|| source.display().to_string(),
|name| name.to_string_lossy().into_owned(),
);
- let analysis = match surface::analyze(&options) {
+ let analysis = match analyze_configuring_if_needed(&options) {
Ok(analysis) => analysis,
Err(err) => {
println!(" {label:<12} NOT ANALYSED {err}");
@@ -965,6 +968,98 @@ fn rank_eval(args: &RankEvalArgs) -> Result<(), Box<dyn Error>> {
Ok(())
}
+// B26. A tree whose PUBLIC headers include a header its build generates cannot be parsed
+// on a fresh clone. Compiling it to find out what it declares costs minutes; running only
+// the configure step costs seconds, and the tree itself says which headers are generated
+// because it ships their templates. So: try the analysis first, and only when it fails
+// AND the tree declares a generated header that nothing has produced, configure and retry
+// once. A tree that parses on the first attempt pays nothing.
+fn analyze_configuring_if_needed(
+ options: &SurfaceOptions,
+) -> Result<surface::Surface, Box<dyn Error>> {
+ let refusal = match surface::analyze(options) {
+ Ok(analysis) => return Ok(analysis),
+ Err(refusal) => refusal,
+ };
+
+ let missing = headers::missing_generated(&options.source, &options.includes);
+ if missing.is_empty() {
+ return Err(refusal.into());
+ }
+ let names: Vec<String> = missing
+ .iter()
+ .filter_map(|path| path.file_name())
+ .map(|name| name.to_string_lossy().into_owned())
+ .collect();
+ eprintln!(
+ "this tree generates {} at build time; running its configure step only",
+ names.join(", ")
+ );
+
+ let detected = detect::detect(&options.source)?;
+ let work = lisdex_core::scratch::unclaimed_path(CONFIGURE_SCRATCH);
+ let layout = BuildLayout::validated(&detected.root, &work)?;
+ let instrumentation = Instrumentation::uninstrumented();
+ let build_options = ladder::BuildOptions::new(
+ CONFIGURE_VARIANT,
+ std::thread::available_parallelism().map_or(1, std::num::NonZero::get),
+ );
+ let configured =
+ match ladder::configure_only(&detected, &layout, &instrumentation, &build_options) {
+ Ok(configured) => configured,
+ Err(err) => {
+ eprintln!("configure did not run: {err}");
+ return Err(refusal.into());
+ }
+ };
+
+ let mut retry = options.clone();
+ retry
+ .includes
+ .extend(generated_include_dirs(&configured.build_dir, &names));
+ match surface::analyze(&retry) {
+ Ok(analysis) => {
+ eprintln!(
+ "configured with {} and re-read the headers",
+ configured.system.as_str()
+ );
+ Ok(analysis)
+ }
+ // The ORIGINAL refusal, not the retry's. It is the one that names the remedy, and
+ // reporting a second failure against a build directory the user never asked for
+ // would point them at a path that is about to be deleted.
+ Err(_) => Err(refusal.into()),
+ }
+}
+
+// cmake writes a generated header beside the source directory that declared it, so the
+// build tree mirrors the source tree and the header can sit several levels down.
+fn generated_include_dirs(build_dir: &Path, names: &[String]) -> Vec<PathBuf> {
+ let mut found = vec![build_dir.to_path_buf()];
+ let mut stack = vec![build_dir.to_path_buf()];
+ while let Some(dir) = stack.pop() {
+ let Ok(entries) = std::fs::read_dir(&dir) else {
+ continue;
+ };
+ for entry in entries.flatten() {
+ let path = entry.path();
+ if path.is_dir() {
+ stack.push(path);
+ } else if path
+ .file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| names.iter().any(|wanted| wanted == name))
+ && let Some(parent) = path.parent()
+ {
+ found.push(parent.to_path_buf());
+ }
+ }
+ }
+ found.sort();
+ found.dedup();
+ found
+}
+
fn parse_arm(spec: &str) -> Result<(String, PathBuf), Box<dyn Error>> {
let (name, path) = spec
.split_once('=')
@@ -1072,7 +1167,7 @@ fn harness(args: &HarnessArgs) -> Result<(), Box<dyn Error>> {
options.oss_fuzz.clone_from(&args.oss_fuzz);
options.library_symbols = library_symbols(&args.libraries)?;
- let analysis = surface::analyze(&options)?;
+ let analysis = analyze_configuring_if_needed(&options)?;
report_ladder(&analysis.ladder);
let root = std::fs::canonicalize(&analysis.header_root)
.unwrap_or_else(|_| analysis.header_root.clone());
@@ -1480,7 +1575,7 @@ fn surface(args: &SurfaceArgs) -> Result<(), Box<dyn Error>> {
options.oss_fuzz.clone_from(&args.oss_fuzz);
options.library_symbols = library_symbols(&args.libraries)?;
- let analysis = surface::analyze(&options)?;
+ let analysis = analyze_configuring_if_needed(&options)?;
if args.json {
println!("{}", serde_json::to_string_pretty(&analysis)?);
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
index 1ac4fa36..6ad62b5f 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
@@ -173,26 +173,48 @@ for target in libpng libarchive; do
fi
done
-banner "1b. targets needing a build-generated config header say so, loudly"
-# libtiff needs tiffconf.h and openjpeg needs opj_config.h, both produced by
-# their build. Analysing without one must fail with an actionable diagnosis
-# rather than silently reporting an empty or partial surface.
+banner "1b. targets needing a build-generated config header analyse point-and-run (B26)"
+# libtiff needs tiffconf.h and openjpeg needs opj_config.h, both produced by their build.
+# This stage used to assert they FAILED with an actionable diagnosis; B26 made them work
+# with no flags by running the configure step ONLY when the tree ships a template for a
+# header nothing has generated. Measured: libtiff 6 -> 223 declarations, openjpeg 13 -> 89,
+# ~3.8s each including configure.
#
-# B68. This asserted the message named `--install`, and `lisdex build` produces NO install
-# prefix - the ladder never runs an install step, so the remedy named a thing the tool cannot
-# make. The generated header is written into the BUILD tree and `--include` is what reads it,
-# so the assertion is now the flag that works AND the absence of the one that never did.
+# B68's finding still holds and is still asserted below: `lisdex build` produces NO install
+# prefix, so the fallback diagnosis must name `--include` and never "install prefix".
for target in libtiff openjpeg; do
if analyse "$target"; then
- bad "$target unexpectedly succeeded with no build; the gate assumption is stale"
- elif grep -q -- "--include" "$OUT/$target.err" \
- && ! grep -q -- "install prefix" "$OUT/$target.err"; then
- ok "$target refused with a diagnosis naming --include, a flag that resolves it"
+ DECLARED="$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['declared'])" \
+ "$OUT/$target.json" 2>/dev/null || echo 0)"
+ if [ "${DECLARED:-0}" -ge 50 ]; then
+ ok "$target analysed with no flags and no prior build ($DECLARED declarations)"
+ else
+ bad "$target analysed but reported only $DECLARED declarations, so the config header did not land"
+ fi
else
- bad "$target failed without telling the user what to do: $(head -1 "$OUT/$target.err")"
+ bad "$target still needs a manual build: $(head -1 "$OUT/$target.err")"
fi
done
+# The refusal is still the FLOOR. A tree that declares a generated header but whose build
+# cannot be configured must fall back to the actionable diagnosis rather than to a partial
+# surface - otherwise the recovery above would be hiding the failure it replaced.
+UNCONFIGURABLE="$WORK/b26-unconfigurable"
+rm -rf "$UNCONFIGURABLE"
+mkdir -p "$UNCONFIGURABLE/include"
+printf '#cmakedefine HAVE_THING 1\n' >"$UNCONFIGURABLE/include/gen_config.h.cmake.in"
+printf '#include "gen_config.h"\nint gen_parse(const unsigned char *d, unsigned long n);\n' \
+ >"$UNCONFIGURABLE/include/gen.h"
+if "$LISDEX" surface "$UNCONFIGURABLE" >"$OUT/b26-unconfigurable.err" 2>&1; then
+ bad "a tree with an ungeneratable config header was analysed anyway"
+elif grep -q -- "--include" "$OUT/b26-unconfigurable.err" \
+ && ! grep -q -- "install prefix" "$OUT/b26-unconfigurable.err"; then
+ ok "a tree whose configure cannot run still refuses with a diagnosis naming --include"
+else
+ bad "the fallback diagnosis was lost: $(head -1 "$OUT/b26-unconfigurable.err")"
+fi
+rm -rf "$UNCONFIGURABLE"
+
banner "2. NEGATIVE CONTROLS - a ranker that name-matches must fail here"
require_ranked_outside "$OUT/libpng.json" png_sig_cmp 10
require_ranked_outside "$OUT/libarchive.json" archive_read_support_format_zip 20