Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡

feat(lisdex): M6.1 - dictionaries chosen from the corpus, capped where afl-fuzz would have died

CarterPerez-dev11ed2c849a8d
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rs
new file mode 100644
index 00000000..e1bd56f6
--- /dev/null
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/dictionary.rs
@@ -0,0 +1,391 @@
+// ©AngelaMos | 2026
+// dictionary.rs
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+pub const MAX_DICTIONARIES: usize = 4;
+pub const DEFAULT_DICTIONARY_DIR: &str = "/AFLplusplus/dictionaries";
+
+const RIFF_FORM_OFFSET: usize = 8;
+const RIFF_FORM_LEN: usize = 4;
+const MP4_BRAND_OFFSET: usize = 4;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Format {
+ Png,
+ Jpeg,
+ Jpeg2000,
+ Gif,
+ Tiff,
+ Webp,
+ Bmp,
+ Psd,
+ Dds,
+ Icns,
+ OpenExr,
+ Pdf,
+ Zip,
+ Bzip2,
+ Ogg,
+ Wav,
+ Webm,
+ Mp4,
+ Pcap,
+}
+
+impl Format {
+ #[must_use]
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Self::Png => "png",
+ Self::Jpeg => "jpeg",
+ Self::Jpeg2000 => "jpeg2000",
+ Self::Gif => "gif",
+ Self::Tiff => "tiff",
+ Self::Webp => "webp",
+ Self::Bmp => "bmp",
+ Self::Psd => "psd",
+ Self::Dds => "dds",
+ Self::Icns => "icns",
+ Self::OpenExr => "openexr",
+ Self::Pdf => "pdf",
+ Self::Zip => "zip",
+ Self::Bzip2 => "bz2",
+ Self::Ogg => "ogg",
+ Self::Wav => "wav",
+ Self::Webm => "webm",
+ Self::Mp4 => "mp4",
+ Self::Pcap => "pcap",
+ }
+ }
+
+ #[must_use]
+ pub fn dictionary_file(self) -> String {
+ format!("{}.dict", self.as_str())
+ }
+}
+
+fn starts_with(bytes: &[u8], magic: &[u8]) -> bool {
+ bytes.len() >= magic.len() && &bytes[..magic.len()] == magic
+}
+
+fn riff_form(bytes: &[u8]) -> Option<&[u8]> {
+ if !starts_with(bytes, b"RIFF") {
+ return None;
+ }
+ bytes.get(RIFF_FORM_OFFSET..RIFF_FORM_OFFSET + RIFF_FORM_LEN)
+}
+
+#[must_use]
+pub fn detect(bytes: &[u8]) -> Option<Format> {
+ if starts_with(bytes, b"\x89PNG\r\n\x1a\n") {
+ return Some(Format::Png);
+ }
+ if starts_with(bytes, b"\xff\xd8\xff") {
+ return Some(Format::Jpeg);
+ }
+ if starts_with(bytes, b"\x00\x00\x00\x0cjP \r\n\x87\n") {
+ return Some(Format::Jpeg2000);
+ }
+ if starts_with(bytes, b"GIF87a") || starts_with(bytes, b"GIF89a") {
+ return Some(Format::Gif);
+ }
+ if starts_with(bytes, b"II\x2a\x00") || starts_with(bytes, b"MM\x00\x2a") {
+ return Some(Format::Tiff);
+ }
+ if let Some(form) = riff_form(bytes) {
+ return match form {
+ b"WEBP" => Some(Format::Webp),
+ b"WAVE" => Some(Format::Wav),
+ _ => None,
+ };
+ }
+ if starts_with(bytes, b"BM") {
+ return Some(Format::Bmp);
+ }
+ if starts_with(bytes, b"8BPS") {
+ return Some(Format::Psd);
+ }
+ if starts_with(bytes, b"DDS ") {
+ return Some(Format::Dds);
+ }
+ if starts_with(bytes, b"icns") {
+ return Some(Format::Icns);
+ }
+ if starts_with(bytes, b"\x76\x2f\x31\x01") {
+ return Some(Format::OpenExr);
+ }
+ if starts_with(bytes, b"%PDF-") {
+ return Some(Format::Pdf);
+ }
+ if starts_with(bytes, b"PK\x03\x04") {
+ return Some(Format::Zip);
+ }
+ if starts_with(bytes, b"BZh") {
+ return Some(Format::Bzip2);
+ }
+ if starts_with(bytes, b"OggS") {
+ return Some(Format::Ogg);
+ }
+ if starts_with(bytes, b"\x1a\x45\xdf\xa3") {
+ return Some(Format::Webm);
+ }
+ if bytes
+ .get(MP4_BRAND_OFFSET..MP4_BRAND_OFFSET + RIFF_FORM_LEN)
+ .is_some_and(|brand| brand == b"ftyp")
+ {
+ return Some(Format::Mp4);
+ }
+ if starts_with(bytes, b"\xd4\xc3\xb2\xa1")
+ || starts_with(bytes, b"\xa1\xb2\xc3\xd4")
+ || starts_with(bytes, b"\x0a\x0d\x0d\x0a")
+ {
+ return Some(Format::Pcap);
+ }
+ None
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Selection {
+ dictionaries: Vec<PathBuf>,
+ formats: Vec<Format>,
+ seeds_read: usize,
+ seeds_recognised: usize,
+ formats_over_cap: usize,
+}
+
+impl Selection {
+ #[must_use]
+ pub fn dictionaries(&self) -> &[PathBuf] {
+ &self.dictionaries
+ }
+
+ #[must_use]
+ pub fn formats(&self) -> &[Format] {
+ &self.formats
+ }
+
+ #[must_use]
+ pub fn seeds_read(&self) -> usize {
+ self.seeds_read
+ }
+
+ #[must_use]
+ pub fn seeds_recognised(&self) -> usize {
+ self.seeds_recognised
+ }
+
+ #[must_use]
+ pub fn formats_over_cap(&self) -> usize {
+ self.formats_over_cap
+ }
+
+ #[must_use]
+ pub fn is_empty(&self) -> bool {
+ self.dictionaries.is_empty()
+ }
+}
+
+fn census(seed_dir: &Path) -> std::io::Result<(usize, usize, BTreeMap<Format, usize>)> {
+ let mut counts: BTreeMap<Format, usize> = BTreeMap::new();
+ let mut read = 0usize;
+ let mut recognised = 0usize;
+ for entry in std::fs::read_dir(seed_dir)? {
+ let path = entry?.path();
+ if !path.is_file() {
+ continue;
+ }
+ read += 1;
+ let Ok(bytes) = std::fs::read(&path) else {
+ continue;
+ };
+ if let Some(format) = detect(&bytes) {
+ recognised += 1;
+ *counts.entry(format).or_insert(0) += 1;
+ }
+ }
+ Ok((read, recognised, counts))
+}
+
+pub fn select(seed_dir: &Path, dictionary_dir: &Path) -> std::io::Result<Selection> {
+ let (seeds_read, seeds_recognised, counts) = census(seed_dir)?;
+
+ let mut ranked: Vec<(Format, usize)> = counts.into_iter().collect();
+ ranked.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
+
+ let mut formats = Vec::new();
+ let mut dictionaries = Vec::new();
+ let mut available = 0usize;
+ for (format, _) in ranked {
+ let path = dictionary_dir.join(format.dictionary_file());
+ if !path.is_file() {
+ continue;
+ }
+ available += 1;
+ if dictionaries.len() < MAX_DICTIONARIES {
+ formats.push(format);
+ dictionaries.push(path);
+ }
+ }
+
+ Ok(Selection {
+ dictionaries,
+ formats,
+ seeds_read,
+ seeds_recognised,
+ formats_over_cap: available.saturating_sub(MAX_DICTIONARIES),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const PNG: &[u8] = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR";
+ const GIF: &[u8] = b"GIF89a\x01\x00";
+ const TIFF_LE: &[u8] = b"II\x2a\x00\x08\x00\x00\x00";
+ const TIFF_BE: &[u8] = b"MM\x00\x2a\x00\x00\x00\x08";
+
+ fn scratch(name: &str) -> PathBuf {
+ let dir = std::env::temp_dir().join(format!("lisdex-dict-test-{name}"));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).expect("scratch dir");
+ dir
+ }
+
+ fn write(dir: &Path, name: &str, bytes: &[u8]) {
+ std::fs::write(dir.join(name), bytes).expect("fixture write");
+ }
+
+ #[test]
+ fn the_container_formats_are_told_apart_by_their_magic() {
+ assert_eq!(detect(PNG), Some(Format::Png));
+ assert_eq!(detect(b"\xff\xd8\xff\xe0"), Some(Format::Jpeg));
+ assert_eq!(detect(GIF), Some(Format::Gif));
+ assert_eq!(detect(b"GIF87a\x01\x00"), Some(Format::Gif));
+ assert_eq!(detect(TIFF_LE), Some(Format::Tiff));
+ assert_eq!(
+ detect(TIFF_BE),
+ Some(Format::Tiff),
+ "TIFF is byte-order tagged and both orders are real files; recognising only one would send half a corpus to no dictionary"
+ );
+ assert_eq!(detect(b"%PDF-1.7"), Some(Format::Pdf));
+ assert_eq!(detect(b"BZh9"), Some(Format::Bzip2));
+ }
+
+ #[test]
+ fn the_two_riff_formats_are_separated_by_their_form_type() {
+ let webp = b"RIFF\x24\x00\x00\x00WEBPVP8 ";
+ let wav = b"RIFF\x24\x00\x00\x00WAVEfmt ";
+ assert_eq!(detect(webp), Some(Format::Webp));
+ assert_eq!(
+ detect(wav),
+ Some(Format::Wav),
+ "WEBP and WAV share the RIFF magic and differ only at offset 8, so keying on RIFF alone would hand a WAV corpus the webp dictionary"
+ );
+ assert_eq!(
+ detect(b"RIFF\x24\x00\x00\x00AVI "),
+ None,
+ "a RIFF form lisdex has no dictionary for is unrecognised rather than guessed at"
+ );
+ }
+
+ #[test]
+ fn mp4_is_recognised_by_a_brand_that_does_not_sit_at_offset_zero() {
+ assert_eq!(
+ detect(b"\x00\x00\x00\x18ftypmp42"),
+ Some(Format::Mp4),
+ "the ftyp box follows a length field, so a detector that only reads offset 0 misses every mp4"
+ );
+ }
+
+ #[test]
+ fn a_short_or_unknown_input_is_refused_rather_than_misread() {
+ assert_eq!(detect(b""), None);
+ assert_eq!(detect(b"\x89PN"), None);
+ assert_eq!(detect(b"RIFF"), None);
+ assert_eq!(detect(b"not a container at all"), None);
+ }
+
+ #[test]
+ fn the_selection_never_exceeds_the_four_afl_accepts_and_reports_what_it_dropped() {
+ let seeds = scratch("cap-seeds");
+ let dicts = scratch("cap-dicts");
+ for (name, bytes) in [
+ ("a.png", PNG),
+ ("b.gif", GIF),
+ ("c.tif", TIFF_LE),
+ ("d.pdf", b"%PDF-1.7 x" as &[u8]),
+ ("e.bz2", b"BZh9abc"),
+ ("f.zip", b"PK\x03\x04zzzz"),
+ ] {
+ write(&seeds, name, bytes);
+ }
+ for format in ["png", "gif", "tiff", "pdf", "bz2", "zip"] {
+ write(&dicts, &format!("{format}.dict"), b"\"token\"\n");
+ }
+
+ let selection = select(&seeds, &dicts).expect("selection must run");
+ assert_eq!(
+ selection.dictionaries().len(),
+ MAX_DICTIONARIES,
+ "afl-fuzz FATALs on a fifth -x, so the cap has to hold here rather than at the command line"
+ );
+ assert_eq!(
+ selection.formats_over_cap(),
+ 2,
+ "the two formats that did not fit are counted so the operator learns the corpus is broader than the campaign covers"
+ );
+ assert_eq!(selection.seeds_read(), 6);
+ assert_eq!(selection.seeds_recognised(), 6);
+ }
+
+ #[test]
+ fn the_commonest_format_in_the_corpus_is_selected_first() {
+ let seeds = scratch("rank-seeds");
+ let dicts = scratch("rank-dicts");
+ write(&seeds, "one.gif", GIF);
+ for index in 0..5 {
+ write(&seeds, &format!("{index}.png"), PNG);
+ }
+ write(&dicts, "png.dict", b"\"IHDR\"\n");
+ write(&dicts, "gif.dict", b"\"GIF89a\"\n");
+
+ let selection = select(&seeds, &dicts).unwrap();
+ assert_eq!(
+ selection.formats().first(),
+ Some(&Format::Png),
+ "a corpus that is mostly PNG with one stray GIF is a PNG campaign, and with only four slots the order decides what survives the cap"
+ );
+ }
+
+ #[test]
+ fn a_format_with_no_shipped_dictionary_selects_nothing_rather_than_a_missing_path() {
+ let seeds = scratch("missing-seeds");
+ let dicts = scratch("missing-dicts");
+ write(&seeds, "a.png", PNG);
+ let selection = select(&seeds, &dicts).unwrap();
+ assert!(
+ selection.is_empty(),
+ "passing -x with a path that does not exist makes afl-fuzz fail to start, so an absent dictionary must drop out of the selection"
+ );
+ assert_eq!(selection.seeds_recognised(), 1);
+ }
+
+ #[test]
+ fn a_corpus_of_unrecognised_files_yields_no_dictionary_and_says_so() {
+ let seeds = scratch("unknown-seeds");
+ let dicts = scratch("unknown-dicts");
+ write(&seeds, "a.bin", b"whatever this is");
+ write(&dicts, "png.dict", b"\"IHDR\"\n");
+ let selection = select(&seeds, &dicts).unwrap();
+ assert!(selection.is_empty());
+ assert_eq!(selection.seeds_read(), 1);
+ assert_eq!(
+ selection.seeds_recognised(),
+ 0,
+ "the two counts differ here, which is how an operator tells an unrecognised corpus from an empty one"
+ );
+ }
+}
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs
index 675522ea..7dd0a5d9 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/lib.rs
@@ -1,6 +1,7 @@
// ©AngelaMos | 2026
// lib.rs
+pub mod dictionary;
pub mod minimize;
pub mod plot;
pub mod runner;
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs
index 58e3c246..0b2acd1e 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex-fuzz/src/runner.rs
@@ -14,6 +14,7 @@ use lisdex_core::config::{
use lisdex_core::crash::Crash;
use lisdex_core::report::SanitizerReport;
+use crate::dictionary;
use crate::stats::{FuzzerStats, STATS_FILE_NAME, StatsError};
pub const AFL_FUZZ_BIN: &str = "afl-fuzz";
@@ -33,6 +34,7 @@ const ARG_MEMORY: &str = "-m";
const ARG_CMPLOG: &str = "-c";
const ARG_DURATION: &str = "-V";
const ARG_SKIP_DETERMINISTIC: &str = "-d";
+const ARG_DICTIONARY: &str = "-x";
const ARG_SEPARATOR: &str = "--";
const ENV_SKIP_CPUFREQ: &str = "AFL_SKIP_CPUFREQ";
const ENV_NO_AFFINITY: &str = "AFL_NO_AFFINITY";
@@ -75,6 +77,7 @@ pub struct AflOptions {
pub duration: Option<Duration>,
pub bench_until_crash: bool,
pub skip_deterministic: bool,
+ pub dictionaries: Vec<PathBuf>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -110,6 +113,11 @@ impl AflOptions {
args.push(ARG_SKIP_DETERMINISTIC.to_owned());
}
+ for dictionary in self.dictionaries.iter().take(dictionary::MAX_DICTIONARIES) {
+ args.push(ARG_DICTIONARY.to_owned());
+ args.push(dictionary.display().to_string());
+ }
+
args.push(ARG_SEPARATOR.to_owned());
args.push(self.target.display().to_string());
@@ -330,9 +338,48 @@ mod tests {
duration: Some(Duration::from_mins(1)),
bench_until_crash: true,
skip_deterministic: true,
+ dictionaries: Vec::new(),
}
}
+ #[test]
+ fn each_dictionary_becomes_its_own_flag() {
+ let mut options = options();
+ options.dictionaries = vec![PathBuf::from("/d/png.dict"), PathBuf::from("/d/exif.dict")];
+ let args = options.invocation().args;
+ assert_eq!(
+ args.windows(2)
+ .filter(|pair| pair[0] == ARG_DICTIONARY)
+ .count(),
+ 2
+ );
+ assert!(args.contains(&"/d/png.dict".to_owned()));
+ assert!(args.contains(&"/d/exif.dict".to_owned()));
+ }
+
+ #[test]
+ fn a_fifth_dictionary_never_reaches_the_command_line() {
+ let mut options = options();
+ options.dictionaries = (0..6)
+ .map(|index| PathBuf::from(format!("/d/{index}.dict")))
+ .collect();
+ let args = options.invocation().args;
+ assert_eq!(
+ args.iter().filter(|arg| *arg == ARG_DICTIONARY).count(),
+ dictionary::MAX_DICTIONARIES,
+ "afl-fuzz FATALs on the fifth -x, so a caller that assembled too many must be truncated here rather than killing the campaign at startup"
+ );
+ }
+
+ #[test]
+ fn no_dictionary_means_no_flag_at_all() {
+ let args = options().invocation().args;
+ assert!(
+ !args.contains(&ARG_DICTIONARY.to_owned()),
+ "an empty -x would be a startup error, so the flag has to be absent rather than present and blank"
+ );
+ }
+
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("lisdex-runner-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
index 1ec00f7d..cd5b776e 100644
--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/crates/lisdex/src/main.rs
@@ -21,6 +21,7 @@ use lisdex_core::crash::Crash;
use lisdex_core::dedup;
use lisdex_core::report::SanitizerReport;
use lisdex_core::store::Store;
+use lisdex_fuzz::dictionary;
use lisdex_fuzz::minimize::{self, Outcome, TminInvocation};
use lisdex_fuzz::plot;
use lisdex_fuzz::runner::{self, AflOptions};
@@ -273,6 +274,16 @@ struct FuzzArgs {
help = "skip afl-fuzz deterministic stages"
)]
skip_deterministic: bool,
+ #[arg(
+ long,
+ help = "dictionary file, repeatable; overrides the automatic selection made from the seed corpus"
+ )]
+ dict: Vec<PathBuf>,
+ #[arg(
+ long,
+ help = "directory of format dictionaries to select from (defaults to the ones AFL++ ships)"
+ )]
+ dict_dir: Option<PathBuf>,
}
#[derive(clap::Args)]
@@ -1210,6 +1221,8 @@ fn fuzz(args: &FuzzArgs) -> Result<(), Box<dyn Error>> {
store.update_campaign_status(campaign_id, status)?;
}
+ let dictionaries = campaign_dictionaries(args);
+
let options = AflOptions {
target: args.target.clone(),
cmplog: args.cmplog.clone(),
@@ -1218,6 +1231,7 @@ fn fuzz(args: &FuzzArgs) -> Result<(), Box<dyn Error>> {
duration: args.duration.map(Duration::from_secs),
bench_until_crash: args.until_crash,
skip_deterministic: args.skip_deterministic,
+ dictionaries,
};
println!("campaign {} target {label}", campaign_id.0);
@@ -1323,6 +1337,68 @@ fn crash_line(crash: &Crash) -> String {
)
}
+fn campaign_dictionaries(args: &FuzzArgs) -> Vec<PathBuf> {
+ if !args.dict.is_empty() {
+ let chosen: Vec<PathBuf> = args
+ .dict
+ .iter()
+ .take(dictionary::MAX_DICTIONARIES)
+ .cloned()
+ .collect();
+ if args.dict.len() > chosen.len() {
+ eprintln!(
+ "warning: {} dictionaries given but {} accepts at most {}; the rest are dropped",
+ args.dict.len(),
+ runner::AFL_FUZZ_BIN,
+ dictionary::MAX_DICTIONARIES
+ );
+ }
+ for path in &chosen {
+ println!("dictionary {}", path.display());
+ }
+ return chosen;
+ }
+
+ let dir = args
+ .dict_dir
+ .clone()
+ .unwrap_or_else(|| PathBuf::from(dictionary::DEFAULT_DICTIONARY_DIR));
+ let selection = match dictionary::select(&args.seeds, &dir) {
+ Ok(selection) => selection,
+ Err(err) => {
+ eprintln!("warning: no dictionary selected: {err}");
+ return Vec::new();
+ }
+ };
+ if selection.is_empty() {
+ println!(
+ "dictionary none ({} of {} seeds recognised)",
+ selection.seeds_recognised(),
+ selection.seeds_read()
+ );
+ return Vec::new();
+ }
+ let names: Vec<&str> = selection
+ .formats()
+ .iter()
+ .map(|format| format.as_str())
+ .collect();
+ println!(
+ "dictionary {} ({} of {} seeds recognised)",
+ names.join(", "),
+ selection.seeds_recognised(),
+ selection.seeds_read()
+ );
+ if selection.formats_over_cap() > 0 {
+ eprintln!(
+ "warning: {} further format(s) in this corpus have dictionaries that did not fit under the {}-file limit",
+ selection.formats_over_cap(),
+ dictionary::MAX_DICTIONARIES
+ );
+ }
+ selection.dictionaries().to_vec()
+}
+
fn finish_stream(err: &std::io::Error) -> Result<(), Box<dyn Error>> {
if err.kind() == std::io::ErrorKind::BrokenPipe {
return Ok(());