Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can β€” learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company πŸ‘‡

feat(lisdex): B25 - indirect dispatch resolved, and archive_read_next_header goes 147 of 159 to 6

CarterPerez-dev3167154b1b65
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
index 9db8120f..e1f2caff 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/complexity.rs
@@ -4,11 +4,12 @@
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::path::{Path, PathBuf};
-use clang::{Entity, EntityKind, Linkage};
+use clang::{Entity, EntityKind, Linkage, TypeKind};
use crate::extract::{self, ExtractError, ParseOptions};
pub const BASE_COMPLEXITY: u32 = 1;
+pub const SIGNATURE_SEPARATOR: &str = ", ";
// C gives a name meaning only within a linkage. An external name is one function across the
// whole program, so the name alone identifies it. An internal name is one function per
@@ -50,9 +51,33 @@ pub struct Defined {
pub nested_loops: u32,
pub max_nesting: u32,
pub calls: Vec<Symbol>,
+ pub indirect: Vec<String>,
pub exported: bool,
}
+// The two halves of an indirect call are never in the same translation unit, and one half is
+// often in no FUNCTION at all: libarchive names its targets in a file-scope designated
+// initializer, `archive_read_vtable = { .archive_read_next_header = _archive_read_next_header }`,
+// which is how C dispatch tables are usually written. Scanning only bodies found the format
+// readers and missed every public entry point, so the addresses are collected per source and
+// resolved once the whole tree has been read.
+#[derive(Debug, Clone, Default)]
+pub struct Sources {
+ pub defined: Vec<Defined>,
+ pub taken: Vec<(String, Symbol)>,
+}
+
+impl Sources {
+ pub fn absorb(&mut self, other: Self, seen: &mut BTreeSet<Symbol>) {
+ for item in other.defined {
+ if seen.insert(item.symbol()) {
+ self.defined.push(item);
+ }
+ }
+ self.taken.extend(other.taken);
+ }
+}
+
impl Defined {
#[must_use]
pub fn danger(&self) -> u32 {
@@ -106,6 +131,8 @@ struct Metrics {
nested_loops: u32,
max_nesting: u32,
calls: BTreeSet<Symbol>,
+ indirect: BTreeSet<String>,
+ taken: BTreeSet<(String, Symbol)>,
}
fn origin_of(entity: &Entity) -> Option<PathBuf> {
@@ -124,9 +151,42 @@ fn callee(target: &Entity) -> Option<Symbol> {
}
}
-fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {
+// Canonical, because a dispatch table is written in the library's own typedefs while the
+// function it points at is usually written in the underlying types. openjpeg declares the
+// same signature as OPJ_BOOL(...) in one place and int(...) in another, and matching the
+// spelling instead of the type would connect neither.
+fn signature_of(kind: &clang::Type) -> Option<String> {
+ let mut target = kind.get_canonical_type();
+ if target.get_kind() == TypeKind::Pointer {
+ target = target.get_pointee_type()?.get_canonical_type();
+ }
+ if target.get_kind() != TypeKind::FunctionPrototype {
+ return None;
+ }
+ let result = target
+ .get_result_type()?
+ .get_canonical_type()
+ .get_display_name();
+ let arguments: Vec<String> = target
+ .get_argument_types()?
+ .iter()
+ .map(|argument| argument.get_canonical_type().get_display_name())
+ .collect();
+ Some(format!("{result}({})", arguments.join(SIGNATURE_SEPARATOR)))
+}
+
+// A call whose callee clang cannot resolve to a function is a call through a pointer. The
+// canonical libarchive dispatch, `(a->format->read_header)(a, entry)`, resolves to NOTHING -
+// not to the FieldDecl - so the signature has to come off the callee EXPRESSION's own type.
+fn indirect_signature(entity: &Entity) -> Option<String> {
+ let callee = entity.get_children().into_iter().next()?;
+ signature_of(&callee.get_type()?)
+}
+
+fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32, in_callee: bool) {
let children = entity.get_children();
let parent_is_if = entity.get_kind() == EntityKind::IfStmt;
+ let parent_is_call = entity.get_kind() == EntityKind::CallExpr;
let last = children.len().saturating_sub(1);
for (position, child) in children.iter().enumerate() {
let kind = child.get_kind();
@@ -140,12 +200,33 @@ fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {
metrics.nested_loops += 1;
}
}
- if kind == EntityKind::CallExpr
+ if kind == EntityKind::CallExpr {
+ match child.get_reference() {
+ Some(target)
+ if target.get_kind() == EntityKind::FunctionDecl
+ && let Some(symbol) = callee(&target) =>
+ {
+ metrics.calls.insert(symbol);
+ }
+ _ => {
+ if let Some(signature) = indirect_signature(child) {
+ metrics.indirect.insert(signature);
+ }
+ }
+ }
+ }
+ // Reached outside a callee position, a reference to a function is its ADDRESS, which
+ // is the only evidence that some dispatch table may later call it. The flag has to
+ // travel the whole callee subtree: the callee sits under an implicit decay cast, so a
+ // check one level deep counts every ordinary call as an address taken.
+ if kind == EntityKind::DeclRefExpr
+ && !in_callee
&& let Some(target) = child.get_reference()
&& target.get_kind() == EntityKind::FunctionDecl
&& let Some(symbol) = callee(&target)
+ && let Some(signature) = target.get_type().as_ref().and_then(signature_of)
{
- metrics.calls.insert(symbol);
+ metrics.taken.insert((signature, symbol));
}
let chained_else = parent_is_if && kind == EntityKind::IfStmt && position == last;
let next_depth = if is_nesting(kind) && !chained_else {
@@ -163,11 +244,12 @@ fn walk(entity: &Entity, metrics: &mut Metrics, depth: u32, loop_depth: u32) {
} else {
loop_depth
},
+ in_callee || (parent_is_call && position == 0),
);
}
}
-pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>, ExtractError> {
+pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Sources, ExtractError> {
if !path.is_file() {
return Err(ExtractError::NotAFile(path.to_path_buf()));
}
@@ -188,7 +270,7 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,
// it moving libjpeg-turbo's #1 ranked entry point with the run still exiting 0.
extract::refuse_foreign_includes(&unit.get_entity(), path, options)?;
- let mut found = Vec::new();
+ let mut found = Sources::default();
let mut seen = BTreeSet::new();
for child in unit.get_entity().get_children() {
collect(&child, &mut found, &mut seen);
@@ -197,8 +279,25 @@ pub fn parse_source(path: &Path, options: &ParseOptions) -> Result<Vec<Defined>,
})
}
-fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String>) {
+// A file-scope initializer cannot call anything, so every function it names is an address
+// being taken. Walking it needs no callee bookkeeping for that reason.
+fn initializer_targets(entity: &Entity, taken: &mut Vec<(String, Symbol)>) {
+ for child in entity.get_children() {
+ if child.get_kind() == EntityKind::DeclRefExpr
+ && let Some(target) = child.get_reference()
+ && target.get_kind() == EntityKind::FunctionDecl
+ && let Some(symbol) = callee(&target)
+ && let Some(signature) = target.get_type().as_ref().and_then(signature_of)
+ {
+ taken.push((signature, symbol));
+ }
+ initializer_targets(&child, taken);
+ }
+}
+
+fn collect(entity: &Entity, found: &mut Sources, seen: &mut BTreeSet<String>) {
match entity.get_kind() {
+ EntityKind::VarDecl => initializer_targets(entity, &mut found.taken),
EntityKind::FunctionDecl => {
if !entity.is_definition() {
return;
@@ -213,8 +312,9 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String
return;
}
let mut metrics = Metrics::default();
- walk(entity, &mut metrics, 0, 0);
- found.push(Defined {
+ walk(entity, &mut metrics, 0, 0, false);
+ found.taken.extend(metrics.taken);
+ found.defined.push(Defined {
name,
origin,
own: BASE_COMPLEXITY + metrics.decisions,
@@ -222,6 +322,7 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String
nested_loops: metrics.nested_loops,
max_nesting: metrics.max_nesting,
calls: metrics.calls.into_iter().collect(),
+ indirect: metrics.indirect.into_iter().collect(),
exported: entity.get_linkage() != Some(Linkage::Internal),
});
}
@@ -237,19 +338,40 @@ fn collect(entity: &Entity, found: &mut Vec<Defined>, seen: &mut BTreeSet<String
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CallGraph {
nodes: BTreeMap<Symbol, Defined>,
+ dispatch: BTreeMap<String, BTreeSet<Symbol>>,
}
-impl From<Vec<Defined>> for CallGraph {
- fn from(defined: Vec<Defined>) -> Self {
+impl From<Sources> for CallGraph {
+ // Resolution has to happen here rather than per translation unit, because the two halves
+ // are never in the same one: libarchive dispatches through `a->format->read_header` in
+ // archive_read.c while every target's address is taken in its own format reader, passed
+ // as an argument to __archive_read_register_format. That is also why the row's suggested
+ // "resolve the field's assignments within the TU" cannot work.
+ fn from(sources: Sources) -> Self {
+ let mut dispatch: BTreeMap<String, BTreeSet<Symbol>> = BTreeMap::new();
+ for (signature, symbol) in sources.taken {
+ dispatch.entry(signature).or_default().insert(symbol);
+ }
Self {
- nodes: defined
+ nodes: sources
+ .defined
.into_iter()
.map(|item| (item.symbol(), item))
.collect(),
+ dispatch,
}
}
}
+impl From<Vec<Defined>> for CallGraph {
+ fn from(defined: Vec<Defined>) -> Self {
+ Self::from(Sources {
+ defined,
+ taken: Vec::new(),
+ })
+ }
+}
+
impl CallGraph {
// Every by-name query on this graph carries a name a public header declared, and a header
// declares each name once. Resolving it by name rather than by key keeps that query
@@ -292,10 +414,25 @@ impl CallGraph {
queue.push_back(call.clone());
}
}
+ for signature in &node.indirect {
+ let Some(targets) = self.dispatch.get(signature) else {
+ continue;
+ };
+ for target in targets {
+ if !visited.contains(target) {
+ queue.push_back(target.clone());
+ }
+ }
+ }
}
visited
}
+ #[must_use]
+ pub fn dispatch_targets(&self, signature: &str) -> usize {
+ self.dispatch.get(signature).map_or(0, BTreeSet::len)
+ }
+
#[must_use]
pub fn reaches(&self, reached: &BTreeSet<Symbol>, name: &str) -> bool {
self.resolve(name)
@@ -354,7 +491,9 @@ mod tests {
}
fn defined(label: &str, body: &str) -> Vec<Defined> {
- parse_source(&source(label, body), &ParseOptions::default()).unwrap()
+ parse_source(&source(label, body), &ParseOptions::default())
+ .unwrap()
+ .defined
}
fn find<'a>(found: &'a [Defined], name: &str) -> &'a Defined {
@@ -519,6 +658,7 @@ mod tests {
nested_loops: 0,
max_nesting: 0,
calls: Vec::new(),
+ indirect: Vec::new(),
exported: true,
};
let looping = Defined {
@@ -574,6 +714,7 @@ mod tests {
nested_loops: 0,
max_nesting: 0,
calls,
+ indirect: Vec::new(),
exported: true,
}
}
@@ -587,10 +728,18 @@ mod tests {
nested_loops: 0,
max_nesting: 0,
calls,
+ indirect: Vec::new(),
exported: false,
}
}
+ fn dispatcher(name: &str, danger: u32, signature: &str) -> Defined {
+ Defined {
+ indirect: vec![signature.to_owned()],
+ ..node(name, danger, Vec::new())
+ }
+ }
+
#[test]
fn accumulated_complexity_sums_the_whole_reachable_call_tree() {
let graph = CallGraph::from(vec![
@@ -734,8 +883,10 @@ mod tests {
.unwrap();
let options = ParseOptions::default();
- let mut all = parse_source(&small, &options).unwrap();
- all.extend(parse_source(&large, &options).unwrap());
+ let mut all = Sources::default();
+ let mut seen = BTreeSet::new();
+ all.absorb(parse_source(&small, &options).unwrap(), &mut seen);
+ all.absorb(parse_source(&large, &options).unwrap(), &mut seen);
let graph = CallGraph::from(all);
assert_eq!(graph.accumulated_danger("entry_a"), 2);
@@ -746,4 +897,86 @@ mod tests {
one-line body and reported this as 2"
);
}
+
+ #[test]
+ fn a_call_through_a_pointer_reaches_every_target_of_that_signature() {
+ let signature = "int(char *)".to_owned();
+ let graph = CallGraph::from(Sources {
+ defined: vec![
+ dispatcher("entry", 1, &signature),
+ node("handler_one", 10, Vec::new()),
+ node("handler_two", 20, Vec::new()),
+ node("unrelated", 40, Vec::new()),
+ ],
+ taken: vec![
+ (signature.clone(), Symbol::exported("handler_one")),
+ (signature, Symbol::exported("handler_two")),
+ ],
+ });
+ assert_eq!(
+ graph.accumulated_danger("entry"),
+ 31,
+ "a dispatch table's targets are the only functions an indirect call can reach, \
+ and the ones whose address is never taken are not among them"
+ );
+ }
+
+ // Two ways to be wrong are covered here, because an empty dispatch table would satisfy a
+ // simpler version of this test no matter what the code did. `wrong_signature` IS
+ // registered, so the assertion can only hold if the signature is actually consulted, and
+ // `never_registered` is not, so it can only hold if registration is required too.
+ #[test]
+ fn an_indirect_call_reaches_only_targets_registered_under_its_own_signature() {
+ let graph = CallGraph::from(Sources {
+ defined: vec![
+ dispatcher("entry", 1, "int(char *)"),
+ node("registered", 10, Vec::new()),
+ node("wrong_signature", 90, Vec::new()),
+ node("never_registered", 500, Vec::new()),
+ ],
+ taken: vec![
+ ("int(char *)".to_owned(), Symbol::exported("registered")),
+ (
+ "void(double)".to_owned(),
+ Symbol::exported("wrong_signature"),
+ ),
+ ],
+ });
+ assert_eq!(
+ graph.accumulated_danger("entry"),
+ 11,
+ "matching a shared signature alone, or ignoring the signature and taking every \
+ registered function, is the over-approximation Fuzz Introspector accepts"
+ );
+ }
+
+ #[test]
+ fn a_dispatch_table_written_at_file_scope_is_read() {
+ let dir = lisdex_core::scratch::path("complexity-vtable");
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("unit.c");
+ std::fs::write(
+ &path,
+ "struct vtable { int (*next_header)(int); };\n\
+ static int deep(int a) {\n\
+ if (a) { return 1; }\n\
+ if (a > 1) { return 2; }\n\
+ if (a > 2) { return 3; }\n\
+ return 0;\n\
+ }\n\
+ static const struct vtable table = { .next_header = deep };\n\
+ int entry(struct vtable *v, int a) { return (v->next_header)(a); }\n",
+ )
+ .unwrap();
+
+ let graph = CallGraph::from(parse_source(&path, &ParseOptions::default()).unwrap());
+ assert_eq!(
+ graph.accumulated_danger("entry"),
+ 6,
+ "entry's own 1 plus deep's 5; libarchive names every public entry point's target \
+ in a file-scope designated initializer, so a scan of function bodies alone \
+ reports this as 1"
+ );
+ }
}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
index c356ee21..23c2d026 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-surface/src/surface.rs
@@ -4,7 +4,7 @@
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
-use crate::complexity::{self, CallGraph, Defined};
+use crate::complexity::{self, CallGraph, Sources};
use crate::extract::{self, Declared, Language, ParseOptions};
use crate::harness::{self, Harness};
use crate::headers::{self, HeaderError, HeaderSet};
@@ -258,24 +258,18 @@ fn read_sources(
options: &SurfaceOptions,
header_root: &Path,
tree_headers: &[PathBuf],
-) -> (Vec<Defined>, Vec<SkippedHeader>) {
+) -> (Sources, Vec<SkippedHeader>) {
let parse = options.parse_options(header_root, tree_headers);
- let mut defined = Vec::new();
+ let mut sources = Sources::default();
let mut skipped = Vec::new();
let mut seen = BTreeSet::new();
for source in library_sources(root) {
match complexity::parse_source(&source, &parse) {
- Ok(found) => {
- for item in found {
- if seen.insert(item.symbol()) {
- defined.push(item);
- }
- }
- }
+ Ok(found) => sources.absorb(found, &mut seen),
Err(error) => skipped.push(skipped_by(&source, &error)),
}
}
- (defined, skipped)
+ (sources, skipped)
}
pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {
@@ -347,10 +341,10 @@ pub fn analyze(options: &SurfaceOptions) -> Result<Surface, SurfaceError> {
&declared.iter().map(|item| item.name.clone()).collect(),
)?;
- let (defined, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);
+ let (sources, skipped_sources) = read_sources(&options.source, options, &set.root, &owned);
refuse_foreign(&set.root, &skipped_sources)?;
- let defined_count = defined.len();
- let graph = CallGraph::from(defined);
+ let defined_count = sources.defined.len();
+ let graph = CallGraph::from(sources);
let seeds: Vec<String> = if options.ignore_harnesses {
Vec::new()
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
index f73d2493..d737405f 100755
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m3-gate.sh
@@ -239,6 +239,13 @@ require_ranked_within "$OUT/libarchive.json" archive_read_open_memory 10
# in-memory sibling needs no callback synthesis at all and ranks 1. 30 is the
# measured position, kept tight so a real regression past it still fails.
require_ranked_within "$OUT/libarchive.json" archive_read_open 30
+# B25's own severity criterion, kept as the thing that must not come back: these three land
+# in the BOTTOM 20% of the ranking whenever the call graph drops indirect dispatch, because
+# every one of them reaches the format readers only through a->format->read_header. Measured
+# at 6, 26 and 27 of 159 with dispatch resolved, against 147, 132 and 154 without it.
+require_ranked_within "$OUT/libarchive.json" archive_read_next_header 20
+require_ranked_within "$OUT/libarchive.json" archive_read_data 40
+require_ranked_within "$OUT/libarchive.json" archive_read_data_block 40
banner "5. the harness-aware mode changes the answer"
analyse libpng --frontier