Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(lisdex): stream nm listings instead of capping them, and find nested build roots

CarterPerez-dev0c3e9d6f7346
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/verify.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/verify.rs
index 5bb5939e..dfd13e50 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/verify.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/verify.rs
@@ -163,35 +163,39 @@ impl Requirements {
}
}
+pub fn count_symbol_line(counts: &mut SymbolCounts, line: &str) {
+ let trimmed = line.trim_end();
+ if trimmed.ends_with(MEMBER_HEADER_SUFFIX) {
+ return;
+ }
+ let Some(symbol) = trimmed.split_whitespace().last() else {
+ return;
+ };
+ if symbol.contains(PATTERN_ASAN) {
+ counts.asan += 1;
+ }
+ if symbol.contains(PATTERN_UBSAN) {
+ counts.ubsan += 1;
+ }
+ if symbol.contains(PATTERN_MSAN) {
+ counts.msan += 1;
+ }
+ if symbol.contains(PATTERN_AFL) || symbol.contains(PATTERN_COVERAGE) {
+ counts.afl += 1;
+ }
+ if symbol.contains(PATTERN_CMPLOG) {
+ counts.cmplog += 1;
+ }
+ if symbol.contains(PATTERN_MAGMA) {
+ counts.magma += 1;
+ }
+}
+
#[must_use]
pub fn scan_symbols(text: &str) -> SymbolCounts {
let mut counts = SymbolCounts::default();
for line in text.lines() {
- let trimmed = line.trim_end();
- if trimmed.ends_with(MEMBER_HEADER_SUFFIX) {
- continue;
- }
- let Some(symbol) = trimmed.split_whitespace().last() else {
- continue;
- };
- if symbol.contains(PATTERN_ASAN) {
- counts.asan += 1;
- }
- if symbol.contains(PATTERN_UBSAN) {
- counts.ubsan += 1;
- }
- if symbol.contains(PATTERN_MSAN) {
- counts.msan += 1;
- }
- if symbol.contains(PATTERN_AFL) || symbol.contains(PATTERN_COVERAGE) {
- counts.afl += 1;
- }
- if symbol.contains(PATTERN_CMPLOG) {
- counts.cmplog += 1;
- }
- if symbol.contains(PATTERN_MAGMA) {
- counts.magma += 1;
- }
+ count_symbol_line(&mut counts, line);
}
counts
}
@@ -254,34 +258,54 @@ pub fn scan_defined_functions(text: &str) -> BTreeSet<String> {
found
}
+pub fn collect_defined_line(found: &mut BTreeSet<String>, line: &str) {
+ let trimmed = line.trim_end();
+ if trimmed.ends_with(MEMBER_HEADER_SUFFIX) {
+ return;
+ }
+ let fields: Vec<&str> = trimmed.split_whitespace().collect();
+ if fields.len() < SYMBOL_FIELDS {
+ return;
+ }
+ let kind = fields[fields.len() - SYMBOL_FIELDS];
+ let name = fields[fields.len() - 1];
+ if TEXT_SYMBOL_TYPES.contains(&kind) {
+ found.insert(name.to_owned());
+ }
+}
+
pub fn defined_functions(path: &Path) -> Result<BTreeSet<String>, VerifyError> {
- let execution = process::run(&defined_spec(path), Duration::from_secs(NM_TIMEOUT_SECS))?;
+ let (execution, found) = process::run_folding(
+ &defined_spec(path),
+ Duration::from_secs(NM_TIMEOUT_SECS),
+ BTreeSet::new(),
+ collect_defined_line,
+ )?;
if !execution.success {
return Err(VerifyError::Unreadable {
nm: LLVM_NM.to_owned(),
path: path.to_path_buf(),
- output: execution.tail(10),
+ output: execution.diagnosis(10),
});
}
- if execution.truncated {
- return Err(VerifyError::Truncated {
+ Ok(found)
+}
+
+pub fn inspect(path: &Path) -> Result<SymbolCounts, VerifyError> {
+ let (execution, counts) = process::run_folding(
+ &nm_spec(path),
+ Duration::from_secs(NM_TIMEOUT_SECS),
+ SymbolCounts::default(),
+ count_symbol_line,
+ )?;
+ if !execution.success {
+ return Err(VerifyError::Unreadable {
nm: LLVM_NM.to_owned(),
path: path.to_path_buf(),
- captured: process::MAX_CAPTURED_BYTES,
+ output: execution.diagnosis(10),
});
}
- Ok(scan_defined_functions(&execution.output))
-}
-
-pub fn inspect(path: &Path) -> Result<SymbolCounts, VerifyError> {
- let execution = process::run(&nm_spec(path), Duration::from_secs(NM_TIMEOUT_SECS))?;
- scan_listing(
- path,
- execution.success,
- execution.truncated,
- &execution.output,
- &execution.tail(10),
- )
+ Ok(counts)
}
#[cfg(test)]
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
index 033a3fec..a5592e4a 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
@@ -1,7 +1,7 @@
// ©AngelaMos | 2026
// process.rs
-use std::io::Read;
+use std::io::{BufRead, Read};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
@@ -214,6 +214,126 @@ pub fn drain(
(captured, handle)
}
+pub fn fold_lines<T, F>(
+ stream: impl Read + Send + 'static,
+ init: T,
+ mut fold: F,
+) -> std::thread::JoinHandle<T>
+where
+ T: Send + 'static,
+ F: FnMut(&mut T, &str) + Send + 'static,
+{
+ std::thread::spawn(move || {
+ let mut accumulator = init;
+ let reader = std::io::BufReader::new(stream);
+ for line in reader.lines() {
+ let Ok(line) = line else { break };
+ fold(&mut accumulator, &line);
+ }
+ accumulator
+ })
+}
+
+pub fn run_folding<T, F>(
+ spec: &CommandSpec,
+ timeout: Duration,
+ init: T,
+ fold: F,
+) -> Result<(Execution, T), ProcessError>
+where
+ T: Send + 'static,
+ F: FnMut(&mut T, &str) + Send + 'static,
+{
+ use std::os::unix::process::CommandExt;
+
+ let started = Instant::now();
+ let mut command = Command::new(&spec.program);
+ command.args(&spec.args);
+ if let Some(cwd) = &spec.cwd {
+ command.current_dir(cwd);
+ }
+ for name in scrubbed_names(spec) {
+ command.env_remove(name);
+ }
+ for (key, value) in &spec.envs {
+ command.env(key, value);
+ }
+
+ let mut child = command
+ .process_group(0)
+ .stdin(Stdio::null())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .map_err(|source| ProcessError::Launch {
+ program: spec.program.clone(),
+ source,
+ })?;
+
+ let folded = child
+ .stdout
+ .take()
+ .map(|stream| fold_lines(stream, init, fold));
+ let stderr = child
+ .stderr
+ .take()
+ .map(|stream| drain(stream, MAX_CAPTURED_BYTES));
+
+ let mut timed_out = false;
+ let status = loop {
+ if let Some(status) = child.try_wait().map_err(|source| ProcessError::Io {
+ program: spec.program.clone(),
+ source,
+ })? {
+ break status;
+ }
+ if started.elapsed() >= timeout {
+ terminate(&mut child);
+ timed_out = true;
+ break child.wait().map_err(|source| ProcessError::Io {
+ program: spec.program.clone(),
+ source,
+ })?;
+ }
+ std::thread::sleep(Duration::from_millis(POLL_MS));
+ };
+
+ let accumulator = match folded {
+ Some(handle) => handle.join().map_err(|_| ProcessError::ReaderPanicked {
+ program: spec.program.clone(),
+ })?,
+ None => {
+ return Err(ProcessError::ReaderPanicked {
+ program: spec.program.clone(),
+ });
+ }
+ };
+
+ let deadline = Instant::now() + Duration::from_millis(DRAIN_GRACE_MS);
+ let mut diagnostic = Vec::new();
+ if let Some((buffer, handle)) = stderr {
+ while !handle.is_finished() && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(POLL_MS));
+ }
+ diagnostic = take_captured(&buffer);
+ }
+ let stderr_text = String::from_utf8_lossy(&diagnostic).into_owned();
+
+ Ok((
+ Execution {
+ command_line: spec.command_line(),
+ code: status.code(),
+ success: status.success() && !timed_out,
+ timed_out,
+ truncated: false,
+ output: stderr_text.clone(),
+ stderr: stderr_text,
+ duration: started.elapsed(),
+ },
+ accumulator,
+ ))
+}
+
#[must_use]
pub fn take_captured(captured: &Captured) -> Vec<u8> {
captured
@@ -542,4 +662,68 @@ mod tests {
let run = run(&spec, Duration::from_secs(10)).unwrap();
assert_eq!(run.diagnosis(2), "ld: cannot find -lz");
}
+
+ #[test]
+ fn capturing_output_larger_than_the_limit_truncates_and_says_so() {
+ let spec = CommandSpec::new("seq").args(["1", "200000"]);
+ let run = run(&spec, Duration::from_mins(1)).unwrap();
+ assert!(run.success);
+ assert!(
+ run.truncated,
+ "seq 1 200000 emits more than the capture limit, so the capped path must flag it"
+ );
+ assert!(run.output.len() <= MAX_CAPTURED_BYTES + TRUNCATION_NOTICE.len());
+ }
+
+ #[test]
+ fn folding_reads_every_line_of_output_larger_than_the_limit() {
+ let spec = CommandSpec::new("seq").args(["1", "200000"]);
+ let (execution, counted) = run_folding(
+ &spec,
+ Duration::from_mins(1),
+ 0usize,
+ |total: &mut usize, _line: &str| *total += 1,
+ )
+ .unwrap();
+ assert!(execution.success);
+ assert!(!execution.truncated);
+ assert_eq!(
+ counted, 200_000,
+ "the folding path must see every line, not the first {MAX_CAPTURED_BYTES} bytes of them"
+ );
+ }
+
+ #[test]
+ fn folding_reports_a_failing_command_and_keeps_its_stderr() {
+ let spec = CommandSpec::new("sh").args([
+ "-c",
+ "printf 'a\\nb\\n'; printf 'llvm-nm: error: unreadable\\n' >&2; exit 3",
+ ]);
+ let (execution, counted) = run_folding(
+ &spec,
+ Duration::from_secs(10),
+ 0usize,
+ |total: &mut usize, _line: &str| *total += 1,
+ )
+ .unwrap();
+ assert!(!execution.success);
+ assert_eq!(execution.code, Some(3));
+ assert_eq!(counted, 2);
+ assert_eq!(execution.diagnosis(1), "llvm-nm: error: unreadable");
+ }
+
+ #[test]
+ fn folding_kills_a_command_that_outruns_its_timeout() {
+ let spec = CommandSpec::new("sh").args(["-c", "printf 'one\\n'; sleep 30"]);
+ let (execution, counted) = run_folding(
+ &spec,
+ Duration::from_millis(500),
+ 0usize,
+ |total: &mut usize, _line: &str| *total += 1,
+ )
+ .unwrap();
+ assert!(execution.timed_out);
+ assert!(!execution.success);
+ assert_eq!(counted, 1);
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-ingest/src/detect.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-ingest/src/detect.rs
index 598a04f0..682d145d 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-ingest/src/detect.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-ingest/src/detect.rs
@@ -17,6 +17,29 @@ pub const SCONS_SCRIPT: &str = "SConstruct";
pub const WAF_SCRIPT: &str = "wscript";
pub const MAKEFILE: [&str; 3] = ["GNUmakefile", "Makefile", "makefile"];
+pub const VENDORED_DIRS: [&str; 18] = [
+ "contrib",
+ "third_party",
+ "thirdparty",
+ "3rdparty",
+ "external",
+ "externals",
+ "deps",
+ "dependencies",
+ "vendor",
+ "subprojects",
+ "debian",
+ "test",
+ "tests",
+ "example",
+ "examples",
+ "doc",
+ "docs",
+ "node_modules",
+];
+
+pub const NESTED_SEARCH_MAX_DEPTH: usize = 4;
+
pub const PRECEDENCE: [BuildSystem; 7] = [
BuildSystem::Meson,
BuildSystem::Bazel,
@@ -268,6 +291,85 @@ fn rank(evidence: &[Evidence]) -> Vec<BuildSystem> {
.collect()
}
+fn is_searchable_dir(name: &str) -> bool {
+ !name.starts_with('.')
+ && !VENDORED_DIRS
+ .iter()
+ .any(|skipped| name.eq_ignore_ascii_case(skipped))
+}
+
+fn subdirectories(dir: &Path) -> Result<Vec<PathBuf>, DetectError> {
+ let entries = std::fs::read_dir(dir).map_err(|source| DetectError::Io {
+ path: dir.to_path_buf(),
+ source,
+ })?;
+
+ let mut found = Vec::new();
+ for entry in entries {
+ let entry = entry.map_err(|source| DetectError::Io {
+ path: dir.to_path_buf(),
+ source,
+ })?;
+ let file_type = entry.file_type().map_err(|source| DetectError::Io {
+ path: entry.path(),
+ source,
+ })?;
+ if !file_type.is_dir() {
+ continue;
+ }
+ if !is_searchable_dir(&entry.file_name().to_string_lossy()) {
+ continue;
+ }
+ found.push(entry.path());
+ }
+ found.sort();
+ Ok(found)
+}
+
+fn supported_precedence(evidence: &[Evidence]) -> Option<usize> {
+ rank(evidence)
+ .into_iter()
+ .filter(|system| system.is_supported())
+ .filter_map(|system| PRECEDENCE.iter().position(|known| *known == system))
+ .min()
+}
+
+fn nested_build_root(root: &Path) -> Result<Option<(PathBuf, Vec<Evidence>)>, DetectError> {
+ let mut frontier = vec![root.to_path_buf()];
+ for _ in 0..NESTED_SEARCH_MAX_DEPTH {
+ let mut next = Vec::new();
+ let mut qualifying: Vec<(usize, PathBuf, Vec<Evidence>)> = Vec::new();
+ for dir in &frontier {
+ for child in subdirectories(dir)? {
+ let evidence = scan(&child)?;
+ match supported_precedence(&evidence) {
+ Some(precedence) => qualifying.push((precedence, child, evidence)),
+ None => next.push(child),
+ }
+ }
+ }
+ if let Some(best) = qualifying
+ .iter()
+ .map(|(precedence, _, _)| *precedence)
+ .min()
+ {
+ let mut tied = qualifying
+ .into_iter()
+ .filter(|(precedence, _, _)| *precedence == best);
+ let winner = tied.next();
+ if tied.next().is_some() {
+ return Ok(None);
+ }
+ return Ok(winner.map(|(_, path, evidence)| (path, evidence)));
+ }
+ if next.is_empty() {
+ break;
+ }
+ frontier = next;
+ }
+ Ok(None)
+}
+
fn sole_subdirectory(root: &Path) -> Result<Option<PathBuf>, DetectError> {
let entries = std::fs::read_dir(root).map_err(|source| DetectError::Io {
path: root.to_path_buf(),
@@ -304,11 +406,19 @@ pub fn detect(root: &Path) -> Result<Detection, DetectError> {
}
let evidence = scan(root)?;
- if evidence.is_empty()
- && let Some(nested) = sole_subdirectory(root)?
- {
- let nested_evidence = scan(&nested)?;
- if !nested_evidence.is_empty() {
+ if evidence.is_empty() {
+ if let Some(nested) = sole_subdirectory(root)? {
+ let nested_evidence = scan(&nested)?;
+ if !nested_evidence.is_empty() {
+ let candidates = rank(&nested_evidence);
+ return Ok(Detection {
+ root: nested,
+ evidence: nested_evidence,
+ candidates,
+ });
+ }
+ }
+ if let Some((nested, nested_evidence)) = nested_build_root(root)? {
let candidates = rank(&nested_evidence);
return Ok(Detection {
root: nested,
@@ -549,4 +659,98 @@ mod tests {
assert!(BuildSystem::Make.is_supported());
assert!(!BuildSystem::Bazel.is_supported());
}
+
+ #[test]
+ fn a_build_root_nested_under_many_subdirectories_is_found() {
+ let fixture = Fixture::new("nested-many")
+ .file("Project/CMake/CMakeLists.txt")
+ .file("Project/GNU/Library/configure.ac")
+ .file("Source/MediaInfo/File__Analyze.cpp")
+ .dir("Release");
+ let found = fixture.detect();
+ assert_eq!(found.primary(), Some(BuildSystem::CMake));
+ assert_eq!(found.root, fixture.root.join("Project/CMake"));
+ }
+
+ #[test]
+ fn a_shallower_build_root_outranks_a_deeper_vendored_copy() {
+ let fixture = Fixture::new("nested-shallower")
+ .file("Project/CMake/CMakeLists.txt")
+ .file("Project/zlib/projects/GNU/configure.ac")
+ .file("Project/zlib/projects/GNU/Makefile.am");
+ let found = fixture.detect();
+ assert_eq!(found.root, fixture.root.join("Project/CMake"));
+ }
+
+ #[test]
+ fn a_marker_inside_a_vendored_directory_is_not_a_build_root() {
+ let found = Fixture::new("nested-vendored")
+ .file("contrib/zlib/CMakeLists.txt")
+ .file("third_party/brotli/configure.ac")
+ .file("tests/harness/Makefile.am")
+ .detect();
+ assert!(found.candidates.is_empty());
+ assert_eq!(found.primary(), None);
+ }
+
+ #[test]
+ fn a_tree_with_no_markers_at_any_depth_stays_empty() {
+ let found = Fixture::new("nested-none")
+ .file("Source/MediaInfo/File__Analyze.cpp")
+ .file("Project/notes.txt")
+ .dir("Release")
+ .detect();
+ assert!(found.evidence.is_empty());
+ assert!(found.candidates.is_empty());
+ }
+
+ #[test]
+ fn a_root_marker_is_never_overridden_by_a_nested_one() {
+ let fixture = Fixture::new("nested-root-wins")
+ .file(CONFIGURE_AC[0])
+ .file("Project/CMake/CMakeLists.txt")
+ .dir("Source");
+ let found = fixture.detect();
+ assert_eq!(found.primary(), Some(BuildSystem::Autotools));
+ assert_eq!(found.root, fixture.root);
+ }
+
+ #[test]
+ fn a_build_root_deeper_than_the_search_bound_is_not_found() {
+ let found = Fixture::new("nested-too-deep")
+ .file("a/b/c/d/e/CMakeLists.txt")
+ .dir("Source")
+ .detect();
+ assert!(found.candidates.is_empty());
+ }
+
+ #[test]
+ fn an_unsupported_nested_build_system_is_not_a_build_root() {
+ let found = Fixture::new("nested-unsupported")
+ .file("Project/bazel/WORKSPACE")
+ .dir("Source")
+ .detect();
+ assert!(found.candidates.is_empty());
+ }
+
+ #[test]
+ fn two_nested_build_roots_tied_at_the_same_depth_are_refused() {
+ let found = Fixture::new("nested-ambiguous")
+ .file("Project/one/CMakeLists.txt")
+ .file("Project/two/CMakeLists.txt")
+ .dir("Source")
+ .detect();
+ assert!(found.candidates.is_empty());
+ }
+
+ #[test]
+ fn a_stronger_build_system_breaks_a_same_depth_tie() {
+ let fixture = Fixture::new("nested-tiebreak")
+ .file("Project/cmake/CMakeLists.txt")
+ .file("Project/autotools/configure.ac")
+ .dir("Source");
+ let found = fixture.detect();
+ assert_eq!(found.primary(), Some(BuildSystem::CMake));
+ assert_eq!(found.root, fixture.root.join("Project/cmake"));
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/harnesses/mediainfolib_buffer.cpp b/PROJECTS/advanced/zero-day-vulnerability-scanner/harnesses/mediainfolib_buffer.cpp
new file mode 100644
index 00000000..5f7c46f0
--- /dev/null
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/harnesses/mediainfolib_buffer.cpp
@@ -0,0 +1,29 @@
+// ©AngelaMos | 2026
+// mediainfolib_buffer.cpp
+
+#include <cstddef>
+#include <cstdint>
+
+#include "MediaInfo/MediaInfo.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+ if (size == 0) {
+ return 0;
+ }
+
+ MediaInfoLib::MediaInfo handle;
+
+ handle.Option(__T("Complete"), __T("1"));
+ handle.Option(__T("ParseSpeed"), __T("1.0"));
+
+ if (handle.Open_Buffer_Init(static_cast<ZenLib::int64u>(size), 0) == 0) {
+ return 0;
+ }
+
+ handle.Open_Buffer_Continue(reinterpret_cast<const ZenLib::int8u *>(data), size);
+ handle.Open_Buffer_Finalize();
+ handle.Inform();
+ handle.Close();
+
+ return 0;
+}