Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(lisdex): M7 exit gate - 22 checks, and its first run failed on a defect in the gate
CarterPerez-dev3317eddc53b7
diff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/justfile b/PROJECTS/advanced/zero-day-vulnerability-scanner/justfileindex 6d3d2682..b9fbfe68 100644--- a/PROJECTS/advanced/zero-day-vulnerability-scanner/justfile+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/justfile@@ -75,3 +75,9 @@ m4-gate:m4-gate-reset:docker compose -f dev.compose.yml exec toolchain bash scripts/m4-gate.sh reset++m7-gate:+ docker compose -f dev.compose.yml exec toolchain bash scripts/m7-gate.sh++m7-gate-reset:+ docker compose -f dev.compose.yml exec toolchain bash scripts/m7-gate.sh resetdiff --git a/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m7-gate.sh b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m7-gate.shnew file mode 100755index 00000000..4f57cc98--- /dev/null+++ b/PROJECTS/advanced/zero-day-vulnerability-scanner/scripts/m7-gate.sh@@ -0,0 +1,379 @@+#!/usr/bin/env bash+# ©AngelaMos | 2026+# m7-gate.sh++set -euo pipefail++WORK="${MAGMA_WORK:-/opt/magma-work}"+REPO="$WORK/libpng/repo"+OUT_ROOT="$WORK/out/libpng"+SEEDS="$WORK/libpng/corpus/libpng_read_fuzzer"+GATE="$WORK/m7"+FIXTURES="/work/crates/lisdex-triage/tests/fixtures"+FINDINGS="$GATE/findings"+DB="$GATE/lisdex.sqlite"+FIXTURE_DB="$GATE/fixtures.sqlite"+REPORTS="$GATE/reports"+MIN_OUT="$GATE/minimized"+GATE_LOG="$GATE/stage.log"+LISDEX_BIN="${CARGO_TARGET_DIR:-/cargo-target}/debug/lisdex"+TARGET="$OUT_ROOT/afl/lisdex_libpng_read"+CMPLOG="$OUT_ROOT/cmplog/lisdex_libpng_read"+GATE_DURATION="${LISDEX_M7_DURATION:-600}"+CANARY_PREFIX="PNG"+REPRODUCE_TIMEOUT=10++MIN_SEVERITY_TIERS=3+FIXTURE_COUNT=19+IDENTITY_FRAMES=4++FAILURES=0++log() { printf '\n[m7] %s\n' "$1"; }+die() { printf '\n[m7] FAIL: %s\n' "$1" >&2; exit 1; }+pass() { printf ' PASS %s\n' "$1"; }+fail() { printf ' FAIL %s\n' "$1"; FAILURES=$((FAILURES + 1)); }++if [ "${1:-}" = "reset" ]; then+ log "resetting $GATE"+ rm -rf "$GATE"+fi++[ -d "$REPO" ] || die "magma libpng not fetched; run: just magma-libpng"+[ -x "$TARGET" ] || die "magma libpng not built; run: just magma-libpng"+[ -x "$CMPLOG" ] || die "cmplog target missing at $CMPLOG"+[ -d "$SEEDS" ] || die "seed corpus missing at $SEEDS"+[ -d "$FIXTURES" ] || die "captured report fixtures missing at $FIXTURES"++mkdir -p "$GATE" "$REPORTS"++log "stage 1/8: build lisdex"+cargo build -p lisdex+[ -x "$LISDEX_BIN" ] || die "lisdex binary not found at $LISDEX_BIN"++log "stage 2/8: the triage layer answers for itself"+# The end-to-end checks below cannot isolate one mechanism from another. Anchoring, the+# double-free override, the UBSan routing and the null-deref boundary all feed the same+# printed verdict, so gutting any single one leaves the others able to keep a corpus-wide+# assertion green. Each owns tests that no other mechanism can satisfy.+for crate in lisdex-triage lisdex-core lisdex-fuzz; do+ if cargo test -p "$crate" > "$GATE_LOG" 2>&1; then+ pass "$crate: $(grep -m1 'test result' "$GATE_LOG")"+ else+ tail -20 "$GATE_LOG" >&2+ fail "$crate's own tests do not pass - a triage mechanism is not doing its job"+ fi+done++log "stage 3/8: assemble the real-error corpus as a campaign the shipped binary reads"+rm -f "$FIXTURE_DB"+sqlite3 "$FIXTURE_DB" <<'SQL'+CREATE TABLE campaigns (id INTEGER PRIMARY KEY, target TEXT NOT NULL, status TEXT NOT NULL);+CREATE TABLE crashes (+ id INTEGER PRIMARY KEY,+ campaign_id INTEGER NOT NULL REFERENCES campaigns(id),+ input BLOB NOT NULL,+ sanitizer_report TEXT NOT NULL,+ stack_hash TEXT+);+INSERT INTO campaigns (id, target, status) VALUES (1, 'captured-reports', 'completed');+SQL+for fixture in "$FIXTURES"/*.txt; do+ sqlite3 "$FIXTURE_DB" \+ "INSERT INTO crashes (campaign_id, input, sanitizer_report, stack_hash)+ VALUES (1, zeroblob(1), CAST(readfile('$fixture') AS TEXT), NULL);"+done+LOADED="$(sqlite3 "$FIXTURE_DB" 'SELECT COUNT(*) FROM crashes;')"+if [ "$LOADED" -eq "$FIXTURE_COUNT" ]; then+ pass "$LOADED captured sanitizer reports loaded, none synthesised"+else+ fail "expected $FIXTURE_COUNT captured reports, loaded $LOADED"+fi++"$LISDEX_BIN" crashes --db "$FIXTURE_DB" --campaign 1 > "$GATE/fixtures.log" 2>&1 \+ || die "lisdex could not read the captured-report corpus"++log "stage 4/8: CHECK 2 - severity matches hand-labeled ground truth on REAL sanitizer errors"+TIERS="$(grep -oE 'severity [A-Z_]+' "$GATE/fixtures.log" | awk '{print $2}' | sort -u)"+TIER_COUNT="$(printf '%s\n' "$TIERS" | grep -c .)"+if [ "$TIER_COUNT" -ge "$MIN_SEVERITY_TIERS" ]; then+ pass "the corpus separates $TIER_COUNT severity tiers: $(printf '%s' "$TIERS" | tr '\n' ' ')"+else+ fail "only $TIER_COUNT severity tier(s) across the whole corpus - a layer that answers the \+same thing for every crash cannot be shown to discriminate"+fi+for tier in EXPLOITABLE NOT_EXPLOITABLE PROBABLY_EXPLOITABLE; do+ if printf '%s\n' "$TIERS" | grep -qx "$tier"; then+ pass "$tier is exercised by a real captured report"+ else+ fail "$tier never appears - the gate cannot tell a working classifier from a constant"+ fi+done++DFREE_LINE="$(grep -m1 'class double-free' "$GATE/fixtures.log" || true)"+if printf '%s' "$DFREE_LINE" | grep -q 'severity PROBABLY_EXPLOITABLE'; then+ pass "the double-free override fired: libcasr says NOT_EXPLOITABLE, lisdex publishes \+PROBABLY_EXPLOITABLE"+else+ fail "double-free is not overridden: ${DFREE_LINE:-no double-free report classified}"+fi++# D18: libcasr stamps NOT_EXPLOITABLE on every UBSan finding without consulting its own taxonomy,+# so a verdict of safe would come from a path incapable of returning anything else. The library+# refuses that in its own tests; this asserts the SHIPPED BINARY publishes the refusal.+UBSAN_FIXTURES="$(find "$FIXTURES" -name 'ub_*.txt' | wc -l)"+UNDEFINED_COUNT="$(grep -c 'severity UNDEFINED' "$GATE/fixtures.log" || true)"+if [ "$UBSAN_FIXTURES" -ge 1 ] && [ "${UNDEFINED_COUNT:-0}" -eq "$UBSAN_FIXTURES" ]; then+ pass "all $UBSAN_FIXTURES undefined-behaviour reports publish as UNDEFINED, so none was \+stamped safe by a taxonomy that cannot say anything else (doc 07 D18)"+else+ fail "$UNDEFINED_COUNT of $UBSAN_FIXTURES undefined-behaviour reports published as UNDEFINED; \+the rest were given a tier by the path that hardcodes NOT_EXPLOITABLE"+fi++NON_SECURITY="$(grep -c '/non-security' "$GATE/fixtures.log" || true)"+if [ "${NON_SECURITY:-0}" -ge 1 ] && [ "${NON_SECURITY:-0}" -lt "$FIXTURE_COUNT" ]; then+ pass "the null-deref boundary splits the corpus: $NON_SECURITY of $LOADED filed non-security"+else+ fail "the boundary put ${NON_SECURITY:-0} of $LOADED on the non-security side - a boundary \+that classifies everything the same way is not a boundary"+fi++log "stage 5/8: CHECK 3 - an unresolvable-symbol crash is FLAGGED, never keyed on an offset"+UNKEYED_FIXTURES="$(grep -c 'stack_hash none' "$GATE/fixtures.log" || true)"+if [ "$UNKEYED_FIXTURES" -eq "$LOADED" ]; then+ pass "all $LOADED captures are unsymbolized and ALL are refused a key, not keyed on a \+module offset"+else+ fail "$UNKEYED_FIXTURES of $LOADED unsymbolized captures were refused a key; the rest were \+keyed on something that is not a symbol"+fi++log "stage 6/8: a real campaign against magma libpng, with canary ground truth"+rm -rf "$FINDINGS" "$DB"+"$LISDEX_BIN" fuzz \+ --target "$TARGET" \+ --cmplog "$CMPLOG" \+ --seeds "$SEEDS" \+ --findings "$FINDINGS" \+ --db "$DB" \+ --duration "$GATE_DURATION" \+ --label m7-gate | tee "$GATE/fuzz.log"++CRASH_ROWS="$(sqlite3 "$DB" 'SELECT COUNT(*) FROM crashes;')"+if [ "${CRASH_ROWS:-0}" -ge 2 ]; then+ pass "campaign persisted $CRASH_ROWS crashes"+else+ die "the campaign found ${CRASH_ROWS:-0} crashes in ${GATE_DURATION}s; raise \+LISDEX_M7_DURATION - there is nothing to check identity against"+fi++"$LISDEX_BIN" crashes --db "$DB" --campaign 1 > "$GATE/campaign.log" 2>&1 \+ || die "lisdex could not read the campaign it just wrote"++CAMPAIGN_UNKEYED="$(grep -c 'stack_hash none' "$GATE/campaign.log" || true)"+if [ "${CAMPAIGN_UNKEYED:-0}" -eq 0 ]; then+ pass "and every symbolized campaign crash DID get a key - a classifier that always \+refuses would fail here"+else+ fail "$CAMPAIGN_UNKEYED symbolized campaign crashes were refused a key"+fi++# D38: the canary signals with kill(pid, condition * 11), so the report is a synthetic SEGV+# whose fault address is the reporting process id. Every canary bug therefore classifies+# identically, which is why CHECK 2 above runs on captured sanitizer errors instead. This is+# the measurement behind that rule, re-taken on every gate run rather than trusted.+CANARY_TIERS="$(grep -oE 'severity [A-Z_]+' "$GATE/campaign.log" | awk '{print $2}' | sort -u \+ | grep -c . || true)"+if [ "${CANARY_TIERS:-0}" -eq 1 ]; then+ pass "and the SAME severity code collapses to 1 tier on canary crashes, so the two \+corpora are not interchangeable (doc 07 D38)"+else+ fail "canary crashes produced $CANARY_TIERS severity tiers; D38 says they must produce \+exactly 1, so either the corpus or the claim has changed and needs re-censusing"+fi++log "stage 7/8: CHECK 1 - dedup collapses to the canary bug count, zero merges and zero splits"+grep -n "MAGMA_LOG(\"$CANARY_PREFIX" "$REPO"/*.c \+ | sed 's|^.*/||' \+ | sed -E "s/^([^:]+):([0-9]+):.*MAGMA_LOG\(\"($CANARY_PREFIX[0-9]+)\".*/\1:\2 \3/" \+ | grep -E "^[^ ]+ $CANARY_PREFIX[0-9]+$" | sort -u > "$GATE/ground-truth.txt"+GROUND_TRUTH_SITES="$(wc -l < "$GATE/ground-truth.txt")"+if [ "$GROUND_TRUTH_SITES" -ge 1 ]; then+ pass "canary ground truth derived from the patched sources: $GROUND_TRUTH_SITES sites"+else+ die "no MAGMA_LOG canary sites found in $REPO - there is no ground truth to compare against"+fi++rm -f "$GATE/pairs.txt"+sqlite3 "$DB" 'SELECT id FROM crashes ORDER BY id;' > "$GATE/ids.txt"+STORED_VS_RECOMPUTED_MISMATCH=0+while read -r id; do+ sqlite3 "$DB" "SELECT sanitizer_report FROM crashes WHERE id = $id;" > "$REPORTS/$id.txt"+ bug=""+ while read -r site name; do+ if grep -qF "$site:" "$REPORTS/$id.txt"; then+ bug="$name"+ break+ fi+ done < "$GATE/ground-truth.txt"+ key="$(sqlite3 "$DB" "SELECT COALESCE(substr(stack_hash, 1, 16), 'none') FROM crashes WHERE id = $id;")"+ [ -n "$bug" ] && printf '%s %s\n' "$key" "$bug" >> "$GATE/pairs.txt"+done < "$GATE/ids.txt"++# B90: the persisted column is a cache and the shipped read path recomputes from the retained+# report. On a campaign written by this same binary the two must agree exactly; if they do not,+# the write path and the read path have drifted apart.+sqlite3 "$DB" "SELECT COALESCE(substr(stack_hash, 1, 16), 'none') FROM crashes ORDER BY id;" \+ | sort | uniq -c > "$GATE/stored-keys.txt"+grep -oE 'stack_hash [0-9a-f]{16}|stack_hash none' "$GATE/campaign.log" \+ | awk '{print $2}' | sort | uniq -c > "$GATE/recomputed-keys.txt"+if diff -q "$GATE/stored-keys.txt" "$GATE/recomputed-keys.txt" > /dev/null; then+ pass "the persisted key and the key recomputed on read agree on all $CRASH_ROWS rows"+else+ diff "$GATE/stored-keys.txt" "$GATE/recomputed-keys.txt" >&2 || true+ fail "the stored key and the recomputed key disagree - the write and read paths have drifted"+fi++LABELED="$(wc -l < "$GATE/pairs.txt")"+if [ "$LABELED" -ge 2 ]; then+ pass "$LABELED of $CRASH_ROWS crashes attributed to a canary bug"+else+ die "only $LABELED crashes could be attributed to a canary; there is no identity to check"+fi++MERGES="$(awk '{print $1, $2}' "$GATE/pairs.txt" | sort -u | awk '{print $1}' | uniq -d | grep -c . || true)"+SPLITS="$(awk '{print $2, $1}' "$GATE/pairs.txt" | sort -u | awk '{print $1}' | uniq -d | grep -c . || true)"+DISTINCT_KEYS="$(awk '{print $1}' "$GATE/pairs.txt" | sort -u | grep -c .)"+DISTINCT_BUGS="$(awk '{print $2}' "$GATE/pairs.txt" | sort -u | grep -c .)"++printf '\n key bug crashes\n'+sort "$GATE/pairs.txt" | uniq -c | awk '{printf " %-18s %-10s %s\n", $2, $3, $1}'+printf '\n'++if [ "${MERGES:-0}" -eq 0 ]; then+ pass "zero FALSE MERGES: no dedup key holds more than one true bug"+else+ fail "$MERGES dedup key(s) hold more than one true bug - distinct bugs were deleted"+fi+if [ "${SPLITS:-0}" -eq 0 ]; then+ pass "zero FALSE SPLITS: no true bug is spread across more than one key"+else+ fail "$SPLITS true bug(s) are spread across more than one key - the count is inflated"+fi+if [ "$DISTINCT_KEYS" -eq "$DISTINCT_BUGS" ]; then+ pass "$CRASH_ROWS AFL crashes collapse to $DISTINCT_KEYS keys against $DISTINCT_BUGS true \+bugs, one to one"+else+ fail "$DISTINCT_KEYS keys against $DISTINCT_BUGS true bugs"+fi+if [ "$DISTINCT_BUGS" -ge 2 ]; then+ pass "and the corpus holds $DISTINCT_BUGS DISTINCT bugs, so a dedup layer that merges \+everything would be caught here"+else+ fail "only $DISTINCT_BUGS distinct bug in the corpus; a single-bug corpus cannot detect a \+false merge, so this check proves nothing - raise LISDEX_M7_DURATION"+fi++log "stage 8/8: CHECK 4 - a minimized PoC still proves the SAME bug"+rm -rf "$MIN_OUT"+"$LISDEX_BIN" minimize \+ --db "$DB" \+ --campaign 1 \+ --target "$TARGET" \+ --out "$MIN_OUT" | tee "$GATE/minimize.log"++VERIFIED="$(grep -c 'identity verified' "$GATE/minimize.log" || true)"+MIN_FILES="$(find "$MIN_OUT" -name '*.min' | wc -l)"+if [ "${VERIFIED:-0}" -eq "$DISTINCT_KEYS" ]; then+ pass "every one of the $DISTINCT_KEYS bugs produced a reproducer whose identity was verified"+else+ fail "$VERIFIED of $DISTINCT_KEYS bugs produced a verified reproducer"+fi+if [ "$MIN_FILES" -eq "${VERIFIED:-0}" ]; then+ pass "exactly $MIN_FILES minimized files on disk, one per verified bug and none for a \+rejected reduction"+else+ fail "$MIN_FILES minimized files on disk against $VERIFIED verified reductions - a \+rejected reduction was left behind as if it were a reproducer"+fi++# The minimizer reports its own verdict from its own stack hash. This re-establishes identity+# without that mechanism: the raw module offsets of the top frames, which no lisdex code parses,+# filters or hashes. In-process symbolization would name the canary bug directly but was measured+# at 84 s per replay, and it is not needed - stage 7 already attributed each ORIGINAL to a canary+# bug from the source line, so equal raw frames carry that name to the reduction.+frames_of() {+ ASAN_OPTIONS=symbolize=0:detect_leaks=0 timeout "$REPRODUCE_TIMEOUT" \+ "$TARGET" "$1" > /dev/null 2> "$GATE/replay.txt" || true+ grep -oE '\+0x[0-9a-f]+\)' "$GATE/replay.txt" | head -"$IDENTITY_FRAMES"+}++INDEPENDENT_OK=0+INDEPENDENT_BAD=0+: > "$GATE/signatures.txt"+for minimized in "$MIN_OUT"/*.min; do+ [ -e "$minimized" ] || continue+ original="${minimized%.min}.orig"+ [ -e "$original" ] || continue+ stem="$(basename "$minimized" .min)"+ bug="$(awk -v key="$stem" '$1 == key { print $2; exit }' "$GATE/pairs.txt")"+ frames_of "$original" > "$GATE/frames-orig.txt"+ frames_of "$minimized" > "$GATE/frames-min.txt"+ ORIGINAL_FRAMES="$(grep -c . "$GATE/frames-orig.txt" || true)"+ ORIGINAL_BYTES="$(wc -c < "$original")"+ MINIMIZED_BYTES="$(wc -c < "$minimized")"+ printf '%s %s\n' "${bug:-unattributed}" \+ "$(tr '\n' ' ' < "$GATE/frames-orig.txt")" >> "$GATE/signatures.txt"++ if [ -z "$bug" ]; then+ INDEPENDENT_BAD=$((INDEPENDENT_BAD + 1))+ fail "$stem: no canary bug was attributed to this key in stage 7"+ elif [ "${ORIGINAL_FRAMES:-0}" -lt "$IDENTITY_FRAMES" ]; then+ # Two inputs that both fail to crash produce two empty frame lists, and an equality+ # test on those reads as "identity preserved". The comparison is only meaningful once+ # the original is shown to actually crash with frames to compare.+ INDEPENDENT_BAD=$((INDEPENDENT_BAD + 1))+ fail "$stem: the ORIGINAL replayed only ${ORIGINAL_FRAMES:-0} frames, so an equality \+test against the reduction would compare nothing with nothing"+ elif diff -q "$GATE/frames-orig.txt" "$GATE/frames-min.txt" > /dev/null; then+ INDEPENDENT_OK=$((INDEPENDENT_OK + 1))+ pass "$stem: $ORIGINAL_BYTES -> $MINIMIZED_BYTES bytes, still faults at the identical \+top $IDENTITY_FRAMES module offsets, and stage 7 attributed those to $bug"+ else+ INDEPENDENT_BAD=$((INDEPENDENT_BAD + 1))+ diff "$GATE/frames-orig.txt" "$GATE/frames-min.txt" >&2 || true+ fail "$stem: the reduction faults somewhere else, so it is not a reproducer for $bug"+ fi+done+if [ "$INDEPENDENT_OK" -ge 1 ] && [ "$INDEPENDENT_BAD" -eq 0 ]; then+ pass "$INDEPENDENT_OK reproducer(s) re-verified from raw crash addresses, independently of \+the stack hash the minimizer checked itself against"+else+ fail "independent re-check: $INDEPENDENT_OK held, $INDEPENDENT_BAD did not"+fi++# The window has to be wide enough to tell these bugs apart or the equality test above passes for+# a PoC that migrated. Measured on this corpus: every canary crash shares its top two frames,+# libc kill and magma_log, and only diverges below them.+SIGNED_BUGS="$(awk '{print $1}' "$GATE/signatures.txt" | sort -u | grep -c . || true)"+DISTINCT_SIGNATURES="$(cut -d' ' -f2- "$GATE/signatures.txt" | sort -u | grep -c . || true)"+if [ "${SIGNED_BUGS:-0}" -ge 2 ] && [ "$DISTINCT_SIGNATURES" -eq "$SIGNED_BUGS" ]; then+ pass "the $IDENTITY_FRAMES-frame window gives all $SIGNED_BUGS bugs distinct signatures, so \+it can detect a reduction that migrated to another bug"+else+ fail "$SIGNED_BUGS bug(s) produced $DISTINCT_SIGNATURES distinct $IDENTITY_FRAMES-frame \+signature(s); the window cannot separate the bugs it is being asked to hold apart"+fi++log "RESULT"+if [ "$FAILURES" -eq 0 ]; then+ printf '\n[m7] M7 EXIT GATE PASSED\n'+ printf ' CHECK 1 dedup is one to one against canary ground truth\n'+ printf ' CHECK 2 severity is hand-labeled on real errors, never on canaries\n'+ printf ' CHECK 3 an unsymbolized crash is flagged, never keyed on an offset\n'+ printf ' CHECK 4 every reproducer still proves its own bug, re-checked independently\n'+else+ printf '\n[m7] M7 EXIT GATE FAILED: %s check(s)\n' "$FAILURES"+ exit 1+fi