Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): B24 - sqlite3 builds, and the row named one blocker where there were two
CarterPerez-devdb05dd538e31
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rsindex 8b12b318..bee8c1d3 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs@@ -72,7 +72,8 @@ fn is_excluded_file(name: &str) -> bool {.any(|prefix| name.starts_with(prefix))}-fn is_elf(path: &Path) -> bool {+#[must_use]+pub fn is_elf(path: &Path) -> bool {let Ok(mut file) = std::fs::File::open(path) else {return false;};diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rsindex b5068fe2..090baddb 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs@@ -610,7 +610,10 @@ fn attempt_compile_commands_replay(let report = replay::replay(&entries, instrumentation)?;steps.push(StepOutcome {name: "replay".to_owned(),- command_line: format!("replay {} compile commands", report.attempted),+ command_line: format!(+ "replay {} compile commands, {} skipped, {} libtool stamp(s) refreshed",+ report.attempted, report.skipped, report.restamped+ ),success: report.succeeded(),timed_out: false,log: None,diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rsindex 91ba4078..3867ec61 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs@@ -15,6 +15,13 @@ pub const REPLAY_TIMEOUT_SECS: u64 = 600;const LAUNCHERS: [&str; 4] = ["ccache", "distcc", "sccache", "icecc"];const FRONTEND_ONLY_ARGS: [&str; 2] = ["-cc1", "-cc1as"];const CXX_EXTENSIONS: [&str; 6] = ["cpp", "cxx", "cc", "c++", "C", "CPP"];+const OUTPUT_FLAG: &str = "-o";+const LIBTOOL_STAMP_EXTENSION: &str = "lo";+const LIBTOOL_OBJECT_DIR: &str = ".libs";++// A file the library links INTO, as opposed to a program the build runs. `libtiff.so.6.0.2`+// is why every dot-separated part is checked rather than only the last one.+const LIBRARY_EXTENSIONS: [&str; 7] = ["o", "obj", "a", "so", "lo", "la", "dylib"];#[derive(Debug, thiserror::Error)]pub enum ReplayError {@@ -48,6 +55,7 @@ pub struct CompileCommand {pub struct ReplayReport {pub attempted: usize,pub skipped: usize,+ pub restamped: usize,pub failures: Vec<String>,}@@ -133,6 +141,45 @@ pub fn is_cxx(file: &str) -> bool {.is_some_and(|ext| CXX_EXTENSIONS.iter().any(|known| *known == ext))}+#[must_use]+pub fn output_of(entry: &CompileCommand, argv: &[String]) -> Option<PathBuf> {+ let named = entry.output.clone().or_else(|| {+ argv.iter()+ .position(|arg| arg == OUTPUT_FLAG)+ .and_then(|at| argv.get(at + 1))+ .cloned()+ })?;+ let path = PathBuf::from(named);+ Some(if path.is_absolute() {+ path+ } else {+ Path::new(&entry.directory).join(path)+ })+}++#[must_use]+pub fn names_a_library(path: &Path) -> bool {+ path.file_name()+ .and_then(|name| name.to_str())+ .is_some_and(|name| {+ name.split('.')+ .skip(1)+ .any(|part| LIBRARY_EXTENSIONS.contains(&part))+ })+}++// The uninstrumented prepass built this path into a program, and the build itself RUNS it:+// sqlite3's `mksqlite3h.tcl` execs `$PWD/mksourceid` to stamp a source id into sqlite3.h.+// Replaying that command with LTO instrumentation replaced the ELF with LLVM bitcode, which+// nothing can exec, so the relink died on a codegen step 500 seconds into the build. A host+// tool is evidence of itself: the prepass left an executable ELF whose name is not a library.+#[must_use]+pub fn is_a_host_tool(path: &Path) -> bool {+ !names_a_library(path)+ && lisdex_core::process::is_executable(path)+ && crate::artifact::is_elf(path)+}+#[must_use]pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Option<CommandSpec> {let argv = strip_launchers(argv(entry));@@ -142,6 +189,9 @@ pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Opt{return None;}+ if output_of(entry, &argv).is_some_and(|output| is_a_host_tool(&output)) {+ return None;+ }let (_, rest) = argv.split_first()?;let compiler = if is_cxx(&entry.file) {&instrumentation.toolchain.cxx@@ -157,6 +207,38 @@ pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Opt)}+// Replay overwrites the object in place, and under libtool the archive does NOT depend on+// that object: it depends on the `.lo` stamp beside it. Measured on sqlite3, where replay+// wrote an instrumented sqlite3.o at 03:44 while .libs/libsqlite3.a still held the+// uninstrumented ELF from 03:43, and the relink re-archived nothing because the stamp had+// not moved - so the library reported zero sanitizer symbols and the tier was rejected.+// Refreshing the stamp makes the relink re-run `llvm-ar` over what replay actually produced.+#[must_use]+pub fn libtool_stamp_of(output: &Path) -> Option<PathBuf> {+ let stem = output.file_stem()?;+ let beside = output.with_extension(LIBTOOL_STAMP_EXTENSION);+ if beside.is_file() {+ return Some(beside);+ }+ // A PIC object lives in `.libs/`, one level below the stamp libtool keeps for it.+ let parent = output.parent()?;+ if parent.file_name()? != LIBTOOL_OBJECT_DIR {+ return None;+ }+ let above = parent+ .parent()?+ .join(stem)+ .with_extension(LIBTOOL_STAMP_EXTENSION);+ above.is_file().then_some(above)+}++// Rewriting the bytes is what moves the modification time; there is no portable touch in+// std, and a libtool stamp is a couple of hundred bytes of text.+fn refresh(stamp: &Path) -> std::io::Result<()> {+ let contents = std::fs::read(stamp)?;+ std::fs::write(stamp, contents)+}+pub fn load(path: &Path) -> Result<Vec<CompileCommand>, ReplayError> {let text = std::fs::read_to_string(path).map_err(|source| ReplayError::Io {path: path.to_path_buf(),@@ -181,7 +263,15 @@ pub fn replay(};report.attempted += 1;let execution = process::run(&spec, timeout)?;- if !execution.success {+ if execution.success {+ if let Some(stamp) = output_of(entry, &argv(entry))+ .as_deref()+ .and_then(libtool_stamp_of)+ && refresh(&stamp).is_ok()+ {+ report.restamped += 1;+ }+ } else {report.failures.push(format!("{}: {}", entry.file, execution.tail(20)));@@ -369,8 +459,154 @@ mod tests {let report = ReplayReport {attempted: 3,skipped: 0,+ restamped: 0,failures: Vec::new(),};assert!(report.succeeded());}++ fn host_tool_fixture(label: &str, name: &str, executable: bool, elf: bool) -> PathBuf {+ use std::os::unix::fs::PermissionsExt;+ let dir = lisdex_core::scratch::path(&format!("replay-{label}"));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let path = dir.join(name);+ let body: &[u8] = if elf {+ b"\x7fELF\x02\x01\x01"+ } else {+ b"BC\xc0\xde"+ };+ std::fs::write(&path, body).unwrap();+ let mode = if executable { 0o755 } else { 0o644 };+ std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).unwrap();+ path+ }++ fn command_writing(output: &Path) -> CompileCommand {+ CompileCommand {+ directory: output.parent().unwrap().display().to_string(),+ file: "tool/mksourceid.c".to_owned(),+ command: Some(format!("cc -O2 -o {} mksourceid.c", output.display())),+ arguments: None,+ output: None,+ }+ }++ #[test]+ fn a_program_the_build_runs_is_left_as_the_prepass_built_it() {+ let tool = host_tool_fixture("hosttool", "mksourceid", true, true);+ assert!(is_a_host_tool(&tool));+ assert!(+ rewrite(&command_writing(&tool), &Instrumentation::default()).is_none(),+ "sqlite3 execs $PWD/mksourceid from mksqlite3h.tcl, and replaying it as LTO \+ bitcode killed the relink 500 seconds into the build"+ );+ }++ #[test]+ fn a_shared_library_is_still_instrumented_even_though_it_is_executable() {+ for (index, name) in ["libtiff.so.6.0.2", "libz.so", "libarchive.a", "parse.o"]+ .into_iter()+ .enumerate()+ {+ let artifact = host_tool_fixture(&format!("libartifact{index}"), name, true, true);+ assert!(+ !is_a_host_tool(&artifact),+ "{name} is what the fuzzer needs instrumented, and shared objects carry the \+ executable bit too"+ );+ assert!(rewrite(&command_writing(&artifact), &Instrumentation::default()).is_some());+ }+ }++ #[test]+ fn a_path_the_prepass_never_built_is_replayed_normally() {+ let dir = lisdex_core::scratch::path("replay-absent");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let absent = dir.join("mksourceid");+ assert!(!is_a_host_tool(&absent));+ assert!(rewrite(&command_writing(&absent), &Instrumentation::default()).is_some());+ }++ #[test]+ fn a_non_elf_file_of_the_same_name_is_not_mistaken_for_a_program() {+ let bitcode = host_tool_fixture("bitcodetool", "mksourceid", true, false);+ assert!(+ !is_a_host_tool(&bitcode),+ "an executable bit on LLVM bitcode is not a runnable program, and treating it as \+ one would skip the very command that has to be replayed to repair it"+ );+ }++ #[test]+ fn the_stamp_an_archive_depends_on_is_found_beside_the_object() {+ let dir = lisdex_core::scratch::path("replay-stamp");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let object = dir.join("sqlite3.o");+ let stamp = dir.join("sqlite3.lo");+ std::fs::write(&object, b"BC\xc0\xde").unwrap();+ std::fs::write(&stamp, b"# libtool object\npic_object=None\n").unwrap();+ assert_eq!(libtool_stamp_of(&object).as_deref(), Some(stamp.as_path()));+ }++ #[test]+ fn a_pic_object_finds_the_stamp_one_directory_above_it() {+ let dir = lisdex_core::scratch::path("replay-picstamp");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(dir.join(LIBTOOL_OBJECT_DIR)).unwrap();+ let object = dir.join(LIBTOOL_OBJECT_DIR).join("sqlite3.o");+ let stamp = dir.join("sqlite3.lo");+ std::fs::write(&object, b"BC\xc0\xde").unwrap();+ std::fs::write(&stamp, b"# libtool object\n").unwrap();+ assert_eq!(+ libtool_stamp_of(&object).as_deref(),+ Some(stamp.as_path()),+ "libtool keeps the stamp beside the build directory, not beside the PIC object"+ );+ }++ #[test]+ fn a_build_with_no_libtool_has_no_stamp_to_refresh() {+ let dir = lisdex_core::scratch::path("replay-nostamp");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let object = dir.join("png.o");+ std::fs::write(&object, b"BC\xc0\xde").unwrap();+ assert!(+ libtool_stamp_of(&object).is_none(),+ "cmake and plain make targets have no .lo, and inventing one would create a file \+ their makefiles never asked for"+ );+ }++ #[test]+ fn refreshing_a_stamp_moves_it_ahead_of_the_object_replay_rewrote() {+ let dir = lisdex_core::scratch::path("replay-refresh");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let stamp = dir.join("sqlite3.lo");+ let body = b"# libtool object\nnon_pic_object='sqlite3.o'\n";+ std::fs::write(&stamp, body).unwrap();+ let before = std::fs::metadata(&stamp).unwrap().modified().unwrap();+ std::thread::sleep(std::time::Duration::from_millis(1100));+ refresh(&stamp).unwrap();+ let after = std::fs::metadata(&stamp).unwrap().modified().unwrap();+ assert!(after > before, "make compares stamps by modification time");+ assert_eq!(+ std::fs::read(&stamp).unwrap(),+ body,+ "the stamp names the object libtool archives, so rewriting it must not alter it"+ );+ }++ #[test]+ fn the_output_is_read_from_the_database_field_before_the_argv() {+ let tool = host_tool_fixture("outputfield", "lemon", true, true);+ let mut entry = command_writing(&tool);+ entry.command = Some("cc -O2 -o something-else.o lemon.c".to_owned());+ entry.output = Some(tool.display().to_string());+ assert!(rewrite(&entry, &Instrumentation::default()).is_none());+ }}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rsindex cd5dd134..1e88d005 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs@@ -148,7 +148,8 @@ impl Execution {}}-fn is_executable(path: &Path) -> bool {+#[must_use]+pub fn is_executable(path: &Path) -> bool {use std::os::unix::fs::PermissionsExt;std::fs::metadata(path).is_ok_and(|meta| meta.is_file() && meta.permissions().mode() & EXECUTABLE_BITS != 0)