Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): M6.1 - dictionaries chosen from the corpus, capped where afl-fuzz would have died
CarterPerez-dev11ed2c849a8d
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rsnew file mode 100644index 00000000..e1bd56f6--- /dev/null+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rs@@ -0,0 +1,391 @@+// ©AngelaMos | 2026+// dictionary.rs++use std::collections::BTreeMap;+use std::path::{Path, PathBuf};++pub const MAX_DICTIONARIES: usize = 4;+pub const DEFAULT_DICTIONARY_DIR: &str = "/AFLplusplus/dictionaries";++const RIFF_FORM_OFFSET: usize = 8;+const RIFF_FORM_LEN: usize = 4;+const MP4_BRAND_OFFSET: usize = 4;++#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]+pub enum Format {+ Png,+ Jpeg,+ Jpeg2000,+ Gif,+ Tiff,+ Webp,+ Bmp,+ Psd,+ Dds,+ Icns,+ OpenExr,+ Pdf,+ Zip,+ Bzip2,+ Ogg,+ Wav,+ Webm,+ Mp4,+ Pcap,+}++impl Format {+ #[must_use]+ pub fn as_str(self) -> &'static str {+ match self {+ Self::Png => "png",+ Self::Jpeg => "jpeg",+ Self::Jpeg2000 => "jpeg2000",+ Self::Gif => "gif",+ Self::Tiff => "tiff",+ Self::Webp => "webp",+ Self::Bmp => "bmp",+ Self::Psd => "psd",+ Self::Dds => "dds",+ Self::Icns => "icns",+ Self::OpenExr => "openexr",+ Self::Pdf => "pdf",+ Self::Zip => "zip",+ Self::Bzip2 => "bz2",+ Self::Ogg => "ogg",+ Self::Wav => "wav",+ Self::Webm => "webm",+ Self::Mp4 => "mp4",+ Self::Pcap => "pcap",+ }+ }++ #[must_use]+ pub fn dictionary_file(self) -> String {+ format!("{}.dict", self.as_str())+ }+}++fn starts_with(bytes: &[u8], magic: &[u8]) -> bool {+ bytes.len() >= magic.len() && &bytes[..magic.len()] == magic+}++fn riff_form(bytes: &[u8]) -> Option<&[u8]> {+ if !starts_with(bytes, b"RIFF") {+ return None;+ }+ bytes.get(RIFF_FORM_OFFSET..RIFF_FORM_OFFSET + RIFF_FORM_LEN)+}++#[must_use]+pub fn detect(bytes: &[u8]) -> Option<Format> {+ if starts_with(bytes, b"\x89PNG\r\n\x1a\n") {+ return Some(Format::Png);+ }+ if starts_with(bytes, b"\xff\xd8\xff") {+ return Some(Format::Jpeg);+ }+ if starts_with(bytes, b"\x00\x00\x00\x0cjP \r\n\x87\n") {+ return Some(Format::Jpeg2000);+ }+ if starts_with(bytes, b"GIF87a") || starts_with(bytes, b"GIF89a") {+ return Some(Format::Gif);+ }+ if starts_with(bytes, b"II\x2a\x00") || starts_with(bytes, b"MM\x00\x2a") {+ return Some(Format::Tiff);+ }+ if let Some(form) = riff_form(bytes) {+ return match form {+ b"WEBP" => Some(Format::Webp),+ b"WAVE" => Some(Format::Wav),+ _ => None,+ };+ }+ if starts_with(bytes, b"BM") {+ return Some(Format::Bmp);+ }+ if starts_with(bytes, b"8BPS") {+ return Some(Format::Psd);+ }+ if starts_with(bytes, b"DDS ") {+ return Some(Format::Dds);+ }+ if starts_with(bytes, b"icns") {+ return Some(Format::Icns);+ }+ if starts_with(bytes, b"\x76\x2f\x31\x01") {+ return Some(Format::OpenExr);+ }+ if starts_with(bytes, b"%PDF-") {+ return Some(Format::Pdf);+ }+ if starts_with(bytes, b"PK\x03\x04") {+ return Some(Format::Zip);+ }+ if starts_with(bytes, b"BZh") {+ return Some(Format::Bzip2);+ }+ if starts_with(bytes, b"OggS") {+ return Some(Format::Ogg);+ }+ if starts_with(bytes, b"\x1a\x45\xdf\xa3") {+ return Some(Format::Webm);+ }+ if bytes+ .get(MP4_BRAND_OFFSET..MP4_BRAND_OFFSET + RIFF_FORM_LEN)+ .is_some_and(|brand| brand == b"ftyp")+ {+ return Some(Format::Mp4);+ }+ if starts_with(bytes, b"\xd4\xc3\xb2\xa1")+ || starts_with(bytes, b"\xa1\xb2\xc3\xd4")+ || starts_with(bytes, b"\x0a\x0d\x0d\x0a")+ {+ return Some(Format::Pcap);+ }+ None+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Selection {+ dictionaries: Vec<PathBuf>,+ formats: Vec<Format>,+ seeds_read: usize,+ seeds_recognised: usize,+ formats_over_cap: usize,+}++impl Selection {+ #[must_use]+ pub fn dictionaries(&self) -> &[PathBuf] {+ &self.dictionaries+ }++ #[must_use]+ pub fn formats(&self) -> &[Format] {+ &self.formats+ }++ #[must_use]+ pub fn seeds_read(&self) -> usize {+ self.seeds_read+ }++ #[must_use]+ pub fn seeds_recognised(&self) -> usize {+ self.seeds_recognised+ }++ #[must_use]+ pub fn formats_over_cap(&self) -> usize {+ self.formats_over_cap+ }++ #[must_use]+ pub fn is_empty(&self) -> bool {+ self.dictionaries.is_empty()+ }+}++fn census(seed_dir: &Path) -> std::io::Result<(usize, usize, BTreeMap<Format, usize>)> {+ let mut counts: BTreeMap<Format, usize> = BTreeMap::new();+ let mut read = 0usize;+ let mut recognised = 0usize;+ for entry in std::fs::read_dir(seed_dir)? {+ let path = entry?.path();+ if !path.is_file() {+ continue;+ }+ read += 1;+ let Ok(bytes) = std::fs::read(&path) else {+ continue;+ };+ if let Some(format) = detect(&bytes) {+ recognised += 1;+ *counts.entry(format).or_insert(0) += 1;+ }+ }+ Ok((read, recognised, counts))+}++pub fn select(seed_dir: &Path, dictionary_dir: &Path) -> std::io::Result<Selection> {+ let (seeds_read, seeds_recognised, counts) = census(seed_dir)?;++ let mut ranked: Vec<(Format, usize)> = counts.into_iter().collect();+ ranked.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));++ let mut formats = Vec::new();+ let mut dictionaries = Vec::new();+ let mut available = 0usize;+ for (format, _) in ranked {+ let path = dictionary_dir.join(format.dictionary_file());+ if !path.is_file() {+ continue;+ }+ available += 1;+ if dictionaries.len() < MAX_DICTIONARIES {+ formats.push(format);+ dictionaries.push(path);+ }+ }++ Ok(Selection {+ dictionaries,+ formats,+ seeds_read,+ seeds_recognised,+ formats_over_cap: available.saturating_sub(MAX_DICTIONARIES),+ })+}++#[cfg(test)]+mod tests {+ use super::*;++ const PNG: &[u8] = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR";+ const GIF: &[u8] = b"GIF89a\x01\x00";+ const TIFF_LE: &[u8] = b"II\x2a\x00\x08\x00\x00\x00";+ const TIFF_BE: &[u8] = b"MM\x00\x2a\x00\x00\x00\x08";++ fn scratch(name: &str) -> PathBuf {+ let dir = std::env::temp_dir().join(format!("lisdex-dict-test-{name}"));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).expect("scratch dir");+ dir+ }++ fn write(dir: &Path, name: &str, bytes: &[u8]) {+ std::fs::write(dir.join(name), bytes).expect("fixture write");+ }++ #[test]+ fn the_container_formats_are_told_apart_by_their_magic() {+ assert_eq!(detect(PNG), Some(Format::Png));+ assert_eq!(detect(b"\xff\xd8\xff\xe0"), Some(Format::Jpeg));+ assert_eq!(detect(GIF), Some(Format::Gif));+ assert_eq!(detect(b"GIF87a\x01\x00"), Some(Format::Gif));+ assert_eq!(detect(TIFF_LE), Some(Format::Tiff));+ assert_eq!(+ detect(TIFF_BE),+ Some(Format::Tiff),+ "TIFF is byte-order tagged and both orders are real files; recognising only one would send half a corpus to no dictionary"+ );+ assert_eq!(detect(b"%PDF-1.7"), Some(Format::Pdf));+ assert_eq!(detect(b"BZh9"), Some(Format::Bzip2));+ }++ #[test]+ fn the_two_riff_formats_are_separated_by_their_form_type() {+ let webp = b"RIFF\x24\x00\x00\x00WEBPVP8 ";+ let wav = b"RIFF\x24\x00\x00\x00WAVEfmt ";+ assert_eq!(detect(webp), Some(Format::Webp));+ assert_eq!(+ detect(wav),+ Some(Format::Wav),+ "WEBP and WAV share the RIFF magic and differ only at offset 8, so keying on RIFF alone would hand a WAV corpus the webp dictionary"+ );+ assert_eq!(+ detect(b"RIFF\x24\x00\x00\x00AVI "),+ None,+ "a RIFF form lisdex has no dictionary for is unrecognised rather than guessed at"+ );+ }++ #[test]+ fn mp4_is_recognised_by_a_brand_that_does_not_sit_at_offset_zero() {+ assert_eq!(+ detect(b"\x00\x00\x00\x18ftypmp42"),+ Some(Format::Mp4),+ "the ftyp box follows a length field, so a detector that only reads offset 0 misses every mp4"+ );+ }++ #[test]+ fn a_short_or_unknown_input_is_refused_rather_than_misread() {+ assert_eq!(detect(b""), None);+ assert_eq!(detect(b"\x89PN"), None);+ assert_eq!(detect(b"RIFF"), None);+ assert_eq!(detect(b"not a container at all"), None);+ }++ #[test]+ fn the_selection_never_exceeds_the_four_afl_accepts_and_reports_what_it_dropped() {+ let seeds = scratch("cap-seeds");+ let dicts = scratch("cap-dicts");+ for (name, bytes) in [+ ("a.png", PNG),+ ("b.gif", GIF),+ ("c.tif", TIFF_LE),+ ("d.pdf", b"%PDF-1.7 x" as &[u8]),+ ("e.bz2", b"BZh9abc"),+ ("f.zip", b"PK\x03\x04zzzz"),+ ] {+ write(&seeds, name, bytes);+ }+ for format in ["png", "gif", "tiff", "pdf", "bz2", "zip"] {+ write(&dicts, &format!("{format}.dict"), b"\"token\"\n");+ }++ let selection = select(&seeds, &dicts).expect("selection must run");+ assert_eq!(+ selection.dictionaries().len(),+ MAX_DICTIONARIES,+ "afl-fuzz FATALs on a fifth -x, so the cap has to hold here rather than at the command line"+ );+ assert_eq!(+ selection.formats_over_cap(),+ 2,+ "the two formats that did not fit are counted so the operator learns the corpus is broader than the campaign covers"+ );+ assert_eq!(selection.seeds_read(), 6);+ assert_eq!(selection.seeds_recognised(), 6);+ }++ #[test]+ fn the_commonest_format_in_the_corpus_is_selected_first() {+ let seeds = scratch("rank-seeds");+ let dicts = scratch("rank-dicts");+ write(&seeds, "one.gif", GIF);+ for index in 0..5 {+ write(&seeds, &format!("{index}.png"), PNG);+ }+ write(&dicts, "png.dict", b"\"IHDR\"\n");+ write(&dicts, "gif.dict", b"\"GIF89a\"\n");++ let selection = select(&seeds, &dicts).unwrap();+ assert_eq!(+ selection.formats().first(),+ Some(&Format::Png),+ "a corpus that is mostly PNG with one stray GIF is a PNG campaign, and with only four slots the order decides what survives the cap"+ );+ }++ #[test]+ fn a_format_with_no_shipped_dictionary_selects_nothing_rather_than_a_missing_path() {+ let seeds = scratch("missing-seeds");+ let dicts = scratch("missing-dicts");+ write(&seeds, "a.png", PNG);+ let selection = select(&seeds, &dicts).unwrap();+ assert!(+ selection.is_empty(),+ "passing -x with a path that does not exist makes afl-fuzz fail to start, so an absent dictionary must drop out of the selection"+ );+ assert_eq!(selection.seeds_recognised(), 1);+ }++ #[test]+ fn a_corpus_of_unrecognised_files_yields_no_dictionary_and_says_so() {+ let seeds = scratch("unknown-seeds");+ let dicts = scratch("unknown-dicts");+ write(&seeds, "a.bin", b"whatever this is");+ write(&dicts, "png.dict", b"\"IHDR\"\n");+ let selection = select(&seeds, &dicts).unwrap();+ assert!(selection.is_empty());+ assert_eq!(selection.seeds_read(), 1);+ assert_eq!(+ selection.seeds_recognised(),+ 0,+ "the two counts differ here, which is how an operator tells an unrecognised corpus from an empty one"+ );+ }+}diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rsindex 675522ea..7dd0a5d9 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs@@ -1,6 +1,7 @@// ©AngelaMos | 2026// lib.rs+pub mod dictionary;pub mod minimize;pub mod plot;pub mod runner;diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rsindex 58e3c246..0b2acd1e 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs@@ -14,6 +14,7 @@ use lisdex_core::config::{use lisdex_core::crash::Crash;use lisdex_core::report::SanitizerReport;+use crate::dictionary;use crate::stats::{FuzzerStats, STATS_FILE_NAME, StatsError};pub const AFL_FUZZ_BIN: &str = "afl-fuzz";@@ -33,6 +34,7 @@ const ARG_MEMORY: &str = "-m";const ARG_CMPLOG: &str = "-c";const ARG_DURATION: &str = "-V";const ARG_SKIP_DETERMINISTIC: &str = "-d";+const ARG_DICTIONARY: &str = "-x";const ARG_SEPARATOR: &str = "--";const ENV_SKIP_CPUFREQ: &str = "AFL_SKIP_CPUFREQ";const ENV_NO_AFFINITY: &str = "AFL_NO_AFFINITY";@@ -75,6 +77,7 @@ pub struct AflOptions {pub duration: Option<Duration>,pub bench_until_crash: bool,pub skip_deterministic: bool,+ pub dictionaries: Vec<PathBuf>,}#[derive(Debug, Clone, PartialEq, Eq)]@@ -110,6 +113,11 @@ impl AflOptions {args.push(ARG_SKIP_DETERMINISTIC.to_owned());}+ for dictionary in self.dictionaries.iter().take(dictionary::MAX_DICTIONARIES) {+ args.push(ARG_DICTIONARY.to_owned());+ args.push(dictionary.display().to_string());+ }+args.push(ARG_SEPARATOR.to_owned());args.push(self.target.display().to_string());@@ -330,9 +338,48 @@ mod tests {duration: Some(Duration::from_mins(1)),bench_until_crash: true,skip_deterministic: true,+ dictionaries: Vec::new(),}}+ #[test]+ fn each_dictionary_becomes_its_own_flag() {+ let mut options = options();+ options.dictionaries = vec![PathBuf::from("/d/png.dict"), PathBuf::from("/d/exif.dict")];+ let args = options.invocation().args;+ assert_eq!(+ args.windows(2)+ .filter(|pair| pair[0] == ARG_DICTIONARY)+ .count(),+ 2+ );+ assert!(args.contains(&"/d/png.dict".to_owned()));+ assert!(args.contains(&"/d/exif.dict".to_owned()));+ }++ #[test]+ fn a_fifth_dictionary_never_reaches_the_command_line() {+ let mut options = options();+ options.dictionaries = (0..6)+ .map(|index| PathBuf::from(format!("/d/{index}.dict")))+ .collect();+ let args = options.invocation().args;+ assert_eq!(+ args.iter().filter(|arg| *arg == ARG_DICTIONARY).count(),+ dictionary::MAX_DICTIONARIES,+ "afl-fuzz FATALs on the fifth -x, so a caller that assembled too many must be truncated here rather than killing the campaign at startup"+ );+ }++ #[test]+ fn no_dictionary_means_no_flag_at_all() {+ let args = options().invocation().args;+ assert!(+ !args.contains(&ARG_DICTIONARY.to_owned()),+ "an empty -x would be a startup error, so the flag has to be absent rather than present and blank"+ );+ }+fn scratch(name: &str) -> PathBuf {let dir = std::env::temp_dir().join(format!("lisdex-runner-{}-{name}", std::process::id()));let _ = std::fs::remove_dir_all(&dir);diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rsindex 1ec00f7d..cd5b776e 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs@@ -21,6 +21,7 @@ use lisdex_core::crash::Crash;use lisdex_core::dedup;use lisdex_core::report::SanitizerReport;use lisdex_core::store::Store;+use lisdex_fuzz::dictionary;use lisdex_fuzz::minimize::{self, Outcome, TminInvocation};use lisdex_fuzz::plot;use lisdex_fuzz::runner::{self, AflOptions};@@ -273,6 +274,16 @@ struct FuzzArgs {help = "skip afl-fuzz deterministic stages")]skip_deterministic: bool,+ #[arg(+ long,+ help = "dictionary file, repeatable; overrides the automatic selection made from the seed corpus"+ )]+ dict: Vec<PathBuf>,+ #[arg(+ long,+ help = "directory of format dictionaries to select from (defaults to the ones AFL++ ships)"+ )]+ dict_dir: Option<PathBuf>,}#[derive(clap::Args)]@@ -1210,6 +1221,8 @@ fn fuzz(args: &FuzzArgs) -> Result<(), Box<dyn Error>> {store.update_campaign_status(campaign_id, status)?;}+ let dictionaries = campaign_dictionaries(args);+let options = AflOptions {target: args.target.clone(),cmplog: args.cmplog.clone(),@@ -1218,6 +1231,7 @@ fn fuzz(args: &FuzzArgs) -> Result<(), Box<dyn Error>> {duration: args.duration.map(Duration::from_secs),bench_until_crash: args.until_crash,skip_deterministic: args.skip_deterministic,+ dictionaries,};println!("campaign {} target {label}", campaign_id.0);@@ -1323,6 +1337,68 @@ fn crash_line(crash: &Crash) -> String {)}+fn campaign_dictionaries(args: &FuzzArgs) -> Vec<PathBuf> {+ if !args.dict.is_empty() {+ let chosen: Vec<PathBuf> = args+ .dict+ .iter()+ .take(dictionary::MAX_DICTIONARIES)+ .cloned()+ .collect();+ if args.dict.len() > chosen.len() {+ eprintln!(+ "warning: {} dictionaries given but {} accepts at most {}; the rest are dropped",+ args.dict.len(),+ runner::AFL_FUZZ_BIN,+ dictionary::MAX_DICTIONARIES+ );+ }+ for path in &chosen {+ println!("dictionary {}", path.display());+ }+ return chosen;+ }++ let dir = args+ .dict_dir+ .clone()+ .unwrap_or_else(|| PathBuf::from(dictionary::DEFAULT_DICTIONARY_DIR));+ let selection = match dictionary::select(&args.seeds, &dir) {+ Ok(selection) => selection,+ Err(err) => {+ eprintln!("warning: no dictionary selected: {err}");+ return Vec::new();+ }+ };+ if selection.is_empty() {+ println!(+ "dictionary none ({} of {} seeds recognised)",+ selection.seeds_recognised(),+ selection.seeds_read()+ );+ return Vec::new();+ }+ let names: Vec<&str> = selection+ .formats()+ .iter()+ .map(|format| format.as_str())+ .collect();+ println!(+ "dictionary {} ({} of {} seeds recognised)",+ names.join(", "),+ selection.seeds_recognised(),+ selection.seeds_read()+ );+ if selection.formats_over_cap() > 0 {+ eprintln!(+ "warning: {} further format(s) in this corpus have dictionaries that did not fit under the {}-file limit",+ selection.formats_over_cap(),+ dictionary::MAX_DICTIONARIES+ );+ }+ selection.dictionaries().to_vec()+}+fn finish_stream(err: &std::io::Error) -> Result<(), Box<dyn Error>> {if err.kind() == std::io::ErrorKind::BrokenPipe {return Ok(());