Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

feat(lisdex): B24 - sqlite3 builds, and the row named one blocker where there were two

CarterPerez-devdb05dd538e31
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs
index 8b12b318..bee8c1d3 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/artifact.rs
@@ -72,7 +72,8 @@ fn is_excluded_file(name: &str) -> bool {
.any(|prefix| name.starts_with(prefix))
}
-fn is_elf(path: &Path) -> bool {
+#[must_use]
+pub fn is_elf(path: &Path) -> bool {
let Ok(mut file) = std::fs::File::open(path) else {
return false;
};
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
index b5068fe2..090baddb 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/ladder.rs
@@ -610,7 +610,10 @@ fn attempt_compile_commands_replay(
let report = replay::replay(&entries, instrumentation)?;
steps.push(StepOutcome {
name: "replay".to_owned(),
- command_line: format!("replay {} compile commands", report.attempted),
+ command_line: format!(
+ "replay {} compile commands, {} skipped, {} libtool stamp(s) refreshed",
+ report.attempted, report.skipped, report.restamped
+ ),
success: report.succeeded(),
timed_out: false,
log: None,
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs
index 91ba4078..3867ec61 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-build/src/replay.rs
@@ -15,6 +15,13 @@ pub const REPLAY_TIMEOUT_SECS: u64 = 600;
const LAUNCHERS: [&str; 4] = ["ccache", "distcc", "sccache", "icecc"];
const FRONTEND_ONLY_ARGS: [&str; 2] = ["-cc1", "-cc1as"];
const CXX_EXTENSIONS: [&str; 6] = ["cpp", "cxx", "cc", "c++", "C", "CPP"];
+const OUTPUT_FLAG: &str = "-o";
+const LIBTOOL_STAMP_EXTENSION: &str = "lo";
+const LIBTOOL_OBJECT_DIR: &str = ".libs";
+
+// A file the library links INTO, as opposed to a program the build runs. `libtiff.so.6.0.2`
+// is why every dot-separated part is checked rather than only the last one.
+const LIBRARY_EXTENSIONS: [&str; 7] = ["o", "obj", "a", "so", "lo", "la", "dylib"];
#[derive(Debug, thiserror::Error)]
pub enum ReplayError {
@@ -48,6 +55,7 @@ pub struct CompileCommand {
pub struct ReplayReport {
pub attempted: usize,
pub skipped: usize,
+ pub restamped: usize,
pub failures: Vec<String>,
}
@@ -133,6 +141,45 @@ pub fn is_cxx(file: &str) -> bool {
.is_some_and(|ext| CXX_EXTENSIONS.iter().any(|known| *known == ext))
}
+#[must_use]
+pub fn output_of(entry: &CompileCommand, argv: &[String]) -> Option<PathBuf> {
+ let named = entry.output.clone().or_else(|| {
+ argv.iter()
+ .position(|arg| arg == OUTPUT_FLAG)
+ .and_then(|at| argv.get(at + 1))
+ .cloned()
+ })?;
+ let path = PathBuf::from(named);
+ Some(if path.is_absolute() {
+ path
+ } else {
+ Path::new(&entry.directory).join(path)
+ })
+}
+
+#[must_use]
+pub fn names_a_library(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| {
+ name.split('.')
+ .skip(1)
+ .any(|part| LIBRARY_EXTENSIONS.contains(&part))
+ })
+}
+
+// The uninstrumented prepass built this path into a program, and the build itself RUNS it:
+// sqlite3's `mksqlite3h.tcl` execs `$PWD/mksourceid` to stamp a source id into sqlite3.h.
+// Replaying that command with LTO instrumentation replaced the ELF with LLVM bitcode, which
+// nothing can exec, so the relink died on a codegen step 500 seconds into the build. A host
+// tool is evidence of itself: the prepass left an executable ELF whose name is not a library.
+#[must_use]
+pub fn is_a_host_tool(path: &Path) -> bool {
+ !names_a_library(path)
+ && lisdex_core::process::is_executable(path)
+ && crate::artifact::is_elf(path)
+}
+
#[must_use]
pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Option<CommandSpec> {
let argv = strip_launchers(argv(entry));
@@ -142,6 +189,9 @@ pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Opt
{
return None;
}
+ if output_of(entry, &argv).is_some_and(|output| is_a_host_tool(&output)) {
+ return None;
+ }
let (_, rest) = argv.split_first()?;
let compiler = if is_cxx(&entry.file) {
&instrumentation.toolchain.cxx
@@ -157,6 +207,38 @@ pub fn rewrite(entry: &CompileCommand, instrumentation: &Instrumentation) -> Opt
)
}
+// Replay overwrites the object in place, and under libtool the archive does NOT depend on
+// that object: it depends on the `.lo` stamp beside it. Measured on sqlite3, where replay
+// wrote an instrumented sqlite3.o at 03:44 while .libs/libsqlite3.a still held the
+// uninstrumented ELF from 03:43, and the relink re-archived nothing because the stamp had
+// not moved - so the library reported zero sanitizer symbols and the tier was rejected.
+// Refreshing the stamp makes the relink re-run `llvm-ar` over what replay actually produced.
+#[must_use]
+pub fn libtool_stamp_of(output: &Path) -> Option<PathBuf> {
+ let stem = output.file_stem()?;
+ let beside = output.with_extension(LIBTOOL_STAMP_EXTENSION);
+ if beside.is_file() {
+ return Some(beside);
+ }
+ // A PIC object lives in `.libs/`, one level below the stamp libtool keeps for it.
+ let parent = output.parent()?;
+ if parent.file_name()? != LIBTOOL_OBJECT_DIR {
+ return None;
+ }
+ let above = parent
+ .parent()?
+ .join(stem)
+ .with_extension(LIBTOOL_STAMP_EXTENSION);
+ above.is_file().then_some(above)
+}
+
+// Rewriting the bytes is what moves the modification time; there is no portable touch in
+// std, and a libtool stamp is a couple of hundred bytes of text.
+fn refresh(stamp: &Path) -> std::io::Result<()> {
+ let contents = std::fs::read(stamp)?;
+ std::fs::write(stamp, contents)
+}
+
pub fn load(path: &Path) -> Result<Vec<CompileCommand>, ReplayError> {
let text = std::fs::read_to_string(path).map_err(|source| ReplayError::Io {
path: path.to_path_buf(),
@@ -181,7 +263,15 @@ pub fn replay(
};
report.attempted += 1;
let execution = process::run(&spec, timeout)?;
- if !execution.success {
+ if execution.success {
+ if let Some(stamp) = output_of(entry, &argv(entry))
+ .as_deref()
+ .and_then(libtool_stamp_of)
+ && refresh(&stamp).is_ok()
+ {
+ report.restamped += 1;
+ }
+ } else {
report
.failures
.push(format!("{}: {}", entry.file, execution.tail(20)));
@@ -369,8 +459,154 @@ mod tests {
let report = ReplayReport {
attempted: 3,
skipped: 0,
+ restamped: 0,
failures: Vec::new(),
};
assert!(report.succeeded());
}
+
+ fn host_tool_fixture(label: &str, name: &str, executable: bool, elf: bool) -> PathBuf {
+ use std::os::unix::fs::PermissionsExt;
+ let dir = lisdex_core::scratch::path(&format!("replay-{label}"));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join(name);
+ let body: &[u8] = if elf {
+ b"\x7fELF\x02\x01\x01"
+ } else {
+ b"BC\xc0\xde"
+ };
+ std::fs::write(&path, body).unwrap();
+ let mode = if executable { 0o755 } else { 0o644 };
+ std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).unwrap();
+ path
+ }
+
+ fn command_writing(output: &Path) -> CompileCommand {
+ CompileCommand {
+ directory: output.parent().unwrap().display().to_string(),
+ file: "tool/mksourceid.c".to_owned(),
+ command: Some(format!("cc -O2 -o {} mksourceid.c", output.display())),
+ arguments: None,
+ output: None,
+ }
+ }
+
+ #[test]
+ fn a_program_the_build_runs_is_left_as_the_prepass_built_it() {
+ let tool = host_tool_fixture("hosttool", "mksourceid", true, true);
+ assert!(is_a_host_tool(&tool));
+ assert!(
+ rewrite(&command_writing(&tool), &Instrumentation::default()).is_none(),
+ "sqlite3 execs $PWD/mksourceid from mksqlite3h.tcl, and replaying it as LTO \
+ bitcode killed the relink 500 seconds into the build"
+ );
+ }
+
+ #[test]
+ fn a_shared_library_is_still_instrumented_even_though_it_is_executable() {
+ for (index, name) in ["libtiff.so.6.0.2", "libz.so", "libarchive.a", "parse.o"]
+ .into_iter()
+ .enumerate()
+ {
+ let artifact = host_tool_fixture(&format!("libartifact{index}"), name, true, true);
+ assert!(
+ !is_a_host_tool(&artifact),
+ "{name} is what the fuzzer needs instrumented, and shared objects carry the \
+ executable bit too"
+ );
+ assert!(rewrite(&command_writing(&artifact), &Instrumentation::default()).is_some());
+ }
+ }
+
+ #[test]
+ fn a_path_the_prepass_never_built_is_replayed_normally() {
+ let dir = lisdex_core::scratch::path("replay-absent");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let absent = dir.join("mksourceid");
+ assert!(!is_a_host_tool(&absent));
+ assert!(rewrite(&command_writing(&absent), &Instrumentation::default()).is_some());
+ }
+
+ #[test]
+ fn a_non_elf_file_of_the_same_name_is_not_mistaken_for_a_program() {
+ let bitcode = host_tool_fixture("bitcodetool", "mksourceid", true, false);
+ assert!(
+ !is_a_host_tool(&bitcode),
+ "an executable bit on LLVM bitcode is not a runnable program, and treating it as \
+ one would skip the very command that has to be replayed to repair it"
+ );
+ }
+
+ #[test]
+ fn the_stamp_an_archive_depends_on_is_found_beside_the_object() {
+ let dir = lisdex_core::scratch::path("replay-stamp");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let object = dir.join("sqlite3.o");
+ let stamp = dir.join("sqlite3.lo");
+ std::fs::write(&object, b"BC\xc0\xde").unwrap();
+ std::fs::write(&stamp, b"# libtool object\npic_object=None\n").unwrap();
+ assert_eq!(libtool_stamp_of(&object).as_deref(), Some(stamp.as_path()));
+ }
+
+ #[test]
+ fn a_pic_object_finds_the_stamp_one_directory_above_it() {
+ let dir = lisdex_core::scratch::path("replay-picstamp");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(dir.join(LIBTOOL_OBJECT_DIR)).unwrap();
+ let object = dir.join(LIBTOOL_OBJECT_DIR).join("sqlite3.o");
+ let stamp = dir.join("sqlite3.lo");
+ std::fs::write(&object, b"BC\xc0\xde").unwrap();
+ std::fs::write(&stamp, b"# libtool object\n").unwrap();
+ assert_eq!(
+ libtool_stamp_of(&object).as_deref(),
+ Some(stamp.as_path()),
+ "libtool keeps the stamp beside the build directory, not beside the PIC object"
+ );
+ }
+
+ #[test]
+ fn a_build_with_no_libtool_has_no_stamp_to_refresh() {
+ let dir = lisdex_core::scratch::path("replay-nostamp");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let object = dir.join("png.o");
+ std::fs::write(&object, b"BC\xc0\xde").unwrap();
+ assert!(
+ libtool_stamp_of(&object).is_none(),
+ "cmake and plain make targets have no .lo, and inventing one would create a file \
+ their makefiles never asked for"
+ );
+ }
+
+ #[test]
+ fn refreshing_a_stamp_moves_it_ahead_of_the_object_replay_rewrote() {
+ let dir = lisdex_core::scratch::path("replay-refresh");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let stamp = dir.join("sqlite3.lo");
+ let body = b"# libtool object\nnon_pic_object='sqlite3.o'\n";
+ std::fs::write(&stamp, body).unwrap();
+ let before = std::fs::metadata(&stamp).unwrap().modified().unwrap();
+ std::thread::sleep(std::time::Duration::from_millis(1100));
+ refresh(&stamp).unwrap();
+ let after = std::fs::metadata(&stamp).unwrap().modified().unwrap();
+ assert!(after > before, "make compares stamps by modification time");
+ assert_eq!(
+ std::fs::read(&stamp).unwrap(),
+ body,
+ "the stamp names the object libtool archives, so rewriting it must not alter it"
+ );
+ }
+
+ #[test]
+ fn the_output_is_read_from_the_database_field_before_the_argv() {
+ let tool = host_tool_fixture("outputfield", "lemon", true, true);
+ let mut entry = command_writing(&tool);
+ entry.command = Some("cc -O2 -o something-else.o lemon.c".to_owned());
+ entry.output = Some(tool.display().to_string());
+ assert!(rewrite(&entry, &Instrumentation::default()).is_none());
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
index cd5dd134..1e88d005 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-core/src/process.rs
@@ -148,7 +148,8 @@ impl Execution {
}
}
-fn is_executable(path: &Path) -> bool {
+#[must_use]
+pub fn is_executable(path: &Path) -> bool {
use std::os::unix::fs::PermissionsExt;
std::fs::metadata(path)
.is_ok_and(|meta| meta.is_file() && meta.permissions().mode() & EXECUTABLE_BITS != 0)