Building 70 Projects ranging from beginner to advanced so anyone can β learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company π
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): B25 - indirect dispatch resolved, and archive_read_next_header goes 147 of 159 to 6
CarterPerez-dev3167154b1b65
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rsindex 9db8120f..e1f2caff 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs@@ -4,11 +4,12 @@use std::collections::{BTreeMap, BTreeSet, VecDeque};use std::path::{Path, PathBuf};-use clang::{Entity, EntityKind, Linkage};+use clang::{Entity, EntityKind, Linkage, TypeKind};use crate::extract::{self, ExtractError, ParseOptions};pub const BASE_COMPLEXITY: u32 = 1;+pub const SIGNATURE_SEPARATOR: &str = ", ";// C gives a name meaning only within a linkage. An external name is one function across the// whole program, so the name alone identifies it. An internal name is one function per@@ -50,9 +51,33 @@ pub struct Defined {pub nested_loops: u32,pub max_nesting: u32,pub calls: Vec<Symbol>,+ pub indirect: Vec<String>,pub exported: bool,}+// The two halves of an indirect call are never in the same translation unit, and one half is+// often in no FUNCTION at all: libarchive names its targets in a file-scope designated+// initializer, `archive_read_vtable = { .archive_read_next_header = _archive_read_next_header }`,+// which is how C dispatch tables are usually written. Scanning only bodies found the format+// readers and missed every public entry point, so the addresses are collected per source and+// resolved once the whole tree has been read.+#[derive(Debug, Clone, Default)]+pub struct Sources {+ pub defined: Vec<Defined>,+ pub taken: Vec<(String, Symbol)>,+}++impl Sources {+ pub fn absorb(&mut self, other: Self, seen: &mut BTreeSet<Symbol>) {+ for item in other.defined {+ if seen.insert(item.symbol()) {+ self.defined.push(item);+ }+ }+ self.taken.extend(other.taken);+ }+}+impl Defined {#[must_use]pub fn danger(&self) -> u32 {@@ -106,6 +131,8 @@ struct Metrics {nested_loops: u32,max_nesting: u32,calls: BTreeSet<Symbol>,+ indirect: BTreeSet<String>,+ taken: BTreeSet<(String, Symbol)>,}fn origin_of(entity: &Entity) -> Option<PathBuf> {@@ -124,9 +151,42 @@ fn callee(target: &Entity) -> Option<Symbol> {}}-fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {+// Canonical, because a dispatch table is written in the library's own typedefs while the+// function it points at is usually written in the underlying types. openjpeg declares the+// same signature as OPJ_BOOL(...) in one place and int(...) in another, and matching the+// spelling instead of the type would connect neither.+fn signature_of(kind: &clang::Type) -> Option<String> {+ let mut target = kind.get_canonical_type();+ if target.get_kind() == TypeKind::Pointer {+ target = target.get_pointee_type()?.get_canonical_type();+ }+ if target.get_kind() != TypeKind::FunctionPrototype {+ return None;+ }+ let result = target+ .get_result_type()?+ .get_canonical_type()+ .get_display_name();+ let arguments: Vec<String> = target+ .get_argument_types()?+ .iter()+ .map(|argument| argument.get_canonical_type().get_display_name())+ .collect();+ Some(format!("{result}({})", arguments.join(SIGNATURE_SEPARATOR)))+}++// A call whose callee clang cannot resolve to a function is a call through a pointer. The+// canonical libarchive dispatch, `(a->format->read_header)(a, entry)`, resolves to NOTHING -+// not to the FieldDecl - so the signature has to come off the callee EXPRESSION's own type.+fn indirect_signature(entity: &Entity) -> Option<String> {+ let callee = entity.get_children().into_iter().next()?;+ signature_of(&callee.get_type()?)+}++fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32, in_callee: bool) {let children = entity.get_children();let parent_is_if = entity.get_kind() == EntityKind::IfStmt;+ let parent_is_call = entity.get_kind() == EntityKind::CallExpr;let last = children.len().saturating_sub(1);for (position, child) in children.iter().enumerate() {let kind = child.get_kind();@@ -140,12 +200,33 @@ fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {metrics.nested_loops += 1;}}- if kind == EntityKind::CallExpr+ if kind == EntityKind::CallExpr {+ match child.get_reference() {+ Some(target)+ if target.get_kind() == EntityKind::FunctionDecl+ && let Some(symbol) = callee(&target) =>+ {+ metrics.calls.insert(symbol);+ }+ _ => {+ if let Some(signature) = indirect_signature(child) {+ metrics.indirect.insert(signature);+ }+ }+ }+ }+ // Reached outside a callee position, a reference to a function is its ADDRESS, which+ // is the only evidence that some dispatch table may later call it. The flag has to+ // travel the whole callee subtree: the callee sits under an implicit decay cast, so a+ // check one level deep counts every ordinary call as an address taken.+ if kind == EntityKind::DeclRefExpr+ && !in_callee&& let Some(target) = child.get_reference()&& target.get_kind() == EntityKind::FunctionDecl&& let Some(symbol) = callee(&target)+ && let Some(signature) = target.get_type().as_ref().and_then(signature_of){- metrics.calls.insert(symbol);+ metrics.taken.insert((signature, symbol));}let chained_else = parent_is_if && kind == EntityKind::IfStmt && position == last;let next_depth = if is_nesting(kind) && !chained_else {@@ -163,11 +244,12 @@ fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {} else {loop_depth},+ in_callee || (parent_is_call && position == 0),);}}-pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>, ExtractError> {+pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Sources, ExtractError> {if !path.is_file() {return Err(ExtractError::NotAFile(path.to_path_buf()));}@@ -188,7 +270,7 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,// it moving libjpeg-turbo's #1 ranked entry point with the run still exiting 0.extract::refuse_foreign_includes(&unit.get_entity(), path, options)?;- let mut found = Vec::new();+ let mut found = Sources::default();let mut seen = BTreeSet::new();for child in unit.get_entity().get_children() {collect(&child, &mut found, &mut seen);@@ -197,8 +279,25 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,})}-fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String>) {+// A file-scope initializer cannot call anything, so every function it names is an address+// being taken. Walking it needs no callee bookkeeping for that reason.+fn initializer_targets(entity: &Entity, taken: &mut Vec<(String, Symbol)>) {+ for child in entity.get_children() {+ if child.get_kind() == EntityKind::DeclRefExpr+ && let Some(target) = child.get_reference()+ && target.get_kind() == EntityKind::FunctionDecl+ && let Some(symbol) = callee(&target)+ && let Some(signature) = target.get_type().as_ref().and_then(signature_of)+ {+ taken.push((signature, symbol));+ }+ initializer_targets(&child, taken);+ }+}++fn collect(entity: &Entity, found: &mut Sources, seen: &mut BTreeSet<String>) {match entity.get_kind() {+ EntityKind::VarDecl => initializer_targets(entity, &mut found.taken),EntityKind::FunctionDecl => {if !entity.is_definition() {return;@@ -213,8 +312,9 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<Stringreturn;}let mut metrics = Metrics::default();- walk(entity, &mut metrics, 0, 0);- found.push(Defined {+ walk(entity, &mut metrics, 0, 0, false);+ found.taken.extend(metrics.taken);+ found.defined.push(Defined {name,origin,own: BASE_COMPLEXITY + metrics.decisions,@@ -222,6 +322,7 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<Stringnested_loops: metrics.nested_loops,max_nesting: metrics.max_nesting,calls: metrics.calls.into_iter().collect(),+ indirect: metrics.indirect.into_iter().collect(),exported: entity.get_linkage() != Some(Linkage::Internal),});}@@ -237,19 +338,40 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String#[derive(Debug, Clone, Default, PartialEq, Eq)]pub struct CallGraph {nodes: BTreeMap<Symbol, Defined>,+ dispatch: BTreeMap<String, BTreeSet<Symbol>>,}-impl From<Vec<Defined>> for CallGraph {- fn from(defined: Vec<Defined>) -> Self {+impl From<Sources> for CallGraph {+ // Resolution has to happen here rather than per translation unit, because the two halves+ // are never in the same one: libarchive dispatches through `a->format->read_header` in+ // archive_read.c while every target's address is taken in its own format reader, passed+ // as an argument to __archive_read_register_format. That is also why the row's suggested+ // "resolve the field's assignments within the TU" cannot work.+ fn from(sources: Sources) -> Self {+ let mut dispatch: BTreeMap<String, BTreeSet<Symbol>> = BTreeMap::new();+ for (signature, symbol) in sources.taken {+ dispatch.entry(signature).or_default().insert(symbol);+ }Self {- nodes: defined+ nodes: sources+ .defined.into_iter().map(|item| (item.symbol(), item)).collect(),+ dispatch,}}}+impl From<Vec<Defined>> for CallGraph {+ fn from(defined: Vec<Defined>) -> Self {+ Self::from(Sources {+ defined,+ taken: Vec::new(),+ })+ }+}+impl CallGraph {// Every by-name query on this graph carries a name a public header declared, and a header// declares each name once. Resolving it by name rather than by key keeps that query@@ -292,10 +414,25 @@ impl CallGraph {queue.push_back(call.clone());}}+ for signature in &node.indirect {+ let Some(targets) = self.dispatch.get(signature) else {+ continue;+ };+ for target in targets {+ if !visited.contains(target) {+ queue.push_back(target.clone());+ }+ }+ }}visited}+ #[must_use]+ pub fn dispatch_targets(&self, signature: &str) -> usize {+ self.dispatch.get(signature).map_or(0, BTreeSet::len)+ }+#[must_use]pub fn reaches(&self, reached: &BTreeSet<Symbol>, name: &str) -> bool {self.resolve(name)@@ -354,7 +491,9 @@ mod tests {}fn defined(label: &str, body: &str) -> Vec<Defined> {- parse_source(&source(label, body), &ParseOptions::default()).unwrap()+ parse_source(&source(label, body), &ParseOptions::default())+ .unwrap()+ .defined}fn find<'a>(found: &'a [Defined], name: &str) -> &'a Defined {@@ -519,6 +658,7 @@ mod tests {nested_loops: 0,max_nesting: 0,calls: Vec::new(),+ indirect: Vec::new(),exported: true,};let looping = Defined {@@ -574,6 +714,7 @@ mod tests {nested_loops: 0,max_nesting: 0,calls,+ indirect: Vec::new(),exported: true,}}@@ -587,10 +728,18 @@ mod tests {nested_loops: 0,max_nesting: 0,calls,+ indirect: Vec::new(),exported: false,}}+ fn dispatcher(name: &str, danger: u32, signature: &str) -> Defined {+ Defined {+ indirect: vec![signature.to_owned()],+ ..node(name, danger, Vec::new())+ }+ }+#[test]fn accumulated_complexity_sums_the_whole_reachable_call_tree() {let graph = CallGraph::from(vec![@@ -734,8 +883,10 @@ mod tests {.unwrap();let options = ParseOptions::default();- let mut all = parse_source(&small, &options).unwrap();- all.extend(parse_source(&large, &options).unwrap());+ let mut all = Sources::default();+ let mut seen = BTreeSet::new();+ all.absorb(parse_source(&small, &options).unwrap(), &mut seen);+ all.absorb(parse_source(&large, &options).unwrap(), &mut seen);let graph = CallGraph::from(all);assert_eq!(graph.accumulated_danger("entry_a"), 2);@@ -746,4 +897,86 @@ mod tests {one-line body and reported this as 2");}++ #[test]+ fn a_call_through_a_pointer_reaches_every_target_of_that_signature() {+ let signature = "int(char *)".to_owned();+ let graph = CallGraph::from(Sources {+ defined: vec![+ dispatcher("entry", 1, &signature),+ node("handler_one", 10, Vec::new()),+ node("handler_two", 20, Vec::new()),+ node("unrelated", 40, Vec::new()),+ ],+ taken: vec![+ (signature.clone(), Symbol::exported("handler_one")),+ (signature, Symbol::exported("handler_two")),+ ],+ });+ assert_eq!(+ graph.accumulated_danger("entry"),+ 31,+ "a dispatch table's targets are the only functions an indirect call can reach, \+ and the ones whose address is never taken are not among them"+ );+ }++ // Two ways to be wrong are covered here, because an empty dispatch table would satisfy a+ // simpler version of this test no matter what the code did. `wrong_signature` IS+ // registered, so the assertion can only hold if the signature is actually consulted, and+ // `never_registered` is not, so it can only hold if registration is required too.+ #[test]+ fn an_indirect_call_reaches_only_targets_registered_under_its_own_signature() {+ let graph = CallGraph::from(Sources {+ defined: vec![+ dispatcher("entry", 1, "int(char *)"),+ node("registered", 10, Vec::new()),+ node("wrong_signature", 90, Vec::new()),+ node("never_registered", 500, Vec::new()),+ ],+ taken: vec![+ ("int(char *)".to_owned(), Symbol::exported("registered")),+ (+ "void(double)".to_owned(),+ Symbol::exported("wrong_signature"),+ ),+ ],+ });+ assert_eq!(+ graph.accumulated_danger("entry"),+ 11,+ "matching a shared signature alone, or ignoring the signature and taking every \+ registered function, is the over-approximation Fuzz Introspector accepts"+ );+ }++ #[test]+ fn a_dispatch_table_written_at_file_scope_is_read() {+ let dir = lisdex_core::scratch::path("complexity-vtable");+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ let path = dir.join("unit.c");+ std::fs::write(+ &path,+ "struct vtable { int (*next_header)(int); };\n\+ static int deep(int a) {\n\+ if (a) { return 1; }\n\+ if (a > 1) { return 2; }\n\+ if (a > 2) { return 3; }\n\+ return 0;\n\+ }\n\+ static const struct vtable table = { .next_header = deep };\n\+ int entry(struct vtable *v, int a) { return (v->next_header)(a); }\n",+ )+ .unwrap();++ let graph = CallGraph::from(parse_source(&path, &ParseOptions::default()).unwrap());+ assert_eq!(+ graph.accumulated_danger("entry"),+ 6,+ "entry's own 1 plus deep's 5; libarchive names every public entry point's target \+ in a file-scope designated initializer, so a scan of function bodies alone \+ reports this as 1"+ );+ }}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rsindex c356ee21..23c2d026 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs@@ -4,7 +4,7 @@use std::collections::BTreeSet;use std::path::{Path, PathBuf};-use crate::complexity::{self, CallGraph, Defined};+use crate::complexity::{self, CallGraph, Sources};use crate::extract::{self, Declared, Language, ParseOptions};use crate::harness::{self, Harness};use crate::headers::{self, HeaderError, HeaderSet};@@ -258,24 +258,18 @@ fn read_sources(options: &SurfaceOptions,header_root: &Path,tree_headers: &[PathBuf],-) -> (Vec<Defined>, Vec<SkippedHeader>) {+) -> (Sources, Vec<SkippedHeader>) {let parse = options.parse_options(header_root, tree_headers);- let mut defined = Vec::new();+ let mut sources = Sources::default();let mut skipped = Vec::new();let mut seen = BTreeSet::new();for source in library_sources(root) {match complexity::parse_source(&source, &parse) {- Ok(found) => {- for item in found {- if seen.insert(item.symbol()) {- defined.push(item);- }- }- }+ Ok(found) => sources.absorb(found, &mut seen),Err(error) => skipped.push(skipped_by(&source, &error)),}}- (defined, skipped)+ (sources, skipped)}pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {@@ -347,10 +341,10 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {&declared.iter().map(|item| item.name.clone()).collect(),)?;- let (defined, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);+ let (sources, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);refuse_foreign(&set.root, &skipped_sources)?;- let defined_count = defined.len();- let graph = CallGraph::from(defined);+ let defined_count = sources.defined.len();+ let graph = CallGraph::from(sources);let seeds: Vec<String> = if options.ignore_harnesses {Vec::new()diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.shindex f73d2493..d737405f 100755--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh@@ -239,6 +239,13 @@ require_ranked_within "$OUT/libarchive.json" archive_read_open_memory 10# in-memory sibling needs no callback synthesis at all and ranks 1. 30 is the# measured position, kept tight so a real regression past it still fails.require_ranked_within "$OUT/libarchive.json" archive_read_open 30+# B25's own severity criterion, kept as the thing that must not come back: these three land+# in the BOTTOM 20% of the ranking whenever the call graph drops indirect dispatch, because+# every one of them reaches the format readers only through a->format->read_header. Measured+# at 6, 26 and 27 of 159 with dispatch resolved, against 147, 132 and 154 without it.+require_ranked_within "$OUT/libarchive.json" archive_read_next_header 20+require_ranked_within "$OUT/libarchive.json" archive_read_data 40+require_ranked_within "$OUT/libarchive.json" archive_read_data_block 40banner "5. the harness-aware mode changes the answer"analyse libpng --frontier