The world's first agentic reverse engineer.
add corpus build scripts for Debian packages and cross-compilation
Amruth5870fdc2f7f8
diff --git a/.gitignore b/.gitignoreindex 5a58de4..8d1290e 100644--- a/.gitignore+++ b/.gitignore@@ -223,4 +223,6 @@ internal-testing/.DS_Store-docs/\ No newline at end of file+docs/++corpus/\ No newline at end of filediff --git a/tools/xda/scripts/Dockerfile.corpus b/tools/xda/scripts/Dockerfile.corpusnew file mode 100644index 0000000..456210c--- /dev/null+++ b/tools/xda/scripts/Dockerfile.corpus@@ -0,0 +1,34 @@+# build: docker build --platform linux/arm64 -t xda-corpus -f scripts/Dockerfile.corpus .+# run: docker run --platform linux/arm64 -v $(pwd)/corpus:/out xda-corpus++FROM --platform=linux/arm64 debian:bookworm++RUN apt-get update && apt-get install -y --no-install-recommends \+ # Real-world C/C++ binaries+ coreutils binutils findutils diffutils grep gawk sed \+ tar gzip bzip2 xz-utils \+ curl wget \+ openssh-client openssh-server \+ openssl \+ sqlite3 \+ nginx-core \+ redis-server \+ lua5.4 \+ vim-tiny \+ tmux \+ git \+ # Debug+ && echo "deb http://deb.debian.org/debian-debug bookworm-debug main" >> /etc/apt/sources.list \+ && apt-get update \+ && apt-get install -y --no-install-recommends \+ coreutils-dbgsym binutils-dbgsym findutils-dbgsym \+ tar-dbgsym grep-dbgsym gawk-dbgsym sed-dbgsym \+ curl-dbgsym openssh-client-dbgsym \+ libssl3-dbgsym libsqlite3-0-dbgsym \+ redis-server-dbgsym lua5.4-dbgsym \+ && rm -rf /var/lib/apt/lists/*++COPY scripts/fetch_debian_packages.sh /extract.sh+RUN chmod +x /extract.sh++CMD ["/extract.sh"]\ No newline at end of filediff --git a/tools/xda/scripts/build_corpus.sh b/tools/xda/scripts/build_corpus.shnew file mode 100755index 0000000..edef77c--- /dev/null+++ b/tools/xda/scripts/build_corpus.sh@@ -0,0 +1,96 @@+#!/usr/bin/env bash+# tools/xda/scripts/build_corpus.sh+# Cross-compiles C projects at multiple optimization levels for a target architecture.+# Usage: ./build_corpus.sh <arch> <output_dir>+# Example: ./build_corpus.sh aarch64 ./corpus/aarch64++set -euo pipefail++ARCH="${1:?Usage: build_corpus.sh <arch> <output_dir>}"+OUT="${2:?Usage: build_corpus.sh <arch> <output_dir>}"++case "$ARCH" in+ x86_64) CC="gcc"; STRIP="strip" ;;+ aarch64) CC="aarch64-linux-gnu-gcc"; STRIP="aarch64-linux-gnu-strip" ;;+ arm) CC="arm-linux-gnueabihf-gcc"; STRIP="arm-linux-gnueabihf-strip" ;;+ mips) CC="mips-linux-gnu-gcc"; STRIP="mips-linux-gnu-strip" ;;+ riscv64) CC="riscv64-linux-gnu-gcc"; STRIP="riscv64-linux-gnu-strip" ;;+ *) echo "Unknown arch: $ARCH"; exit 1 ;;+esac++# Verify compiler exists+command -v "$CC" >/dev/null 2>&1 || { echo "$CC not found. Install cross-compiler."; exit 1; }++mkdir -p "$OUT"++# Source files to compile — create a set of diverse test programs+SOURCES_DIR=$(mktemp -d)+trap 'rm -rf "$SOURCES_DIR"' EXIT++# Generate diverse source files+cat > "$SOURCES_DIR/algorithms.c" << 'CSRC'+#include <stdlib.h>+#include <string.h>++void bubble_sort(int *arr, int n) {+ for (int i = 0; i < n - 1; i++)+ for (int j = 0; j < n - i - 1; j++)+ if (arr[j] > arr[j + 1]) { int t = arr[j]; arr[j] = arr[j + 1]; arr[j + 1] = t; }+}+int binary_search(int *arr, int n, int target) {+ int lo = 0, hi = n - 1;+ while (lo <= hi) { int mid = (lo + hi) / 2; if (arr[mid] == target) return mid; if (arr[mid] < target) lo = mid + 1; else hi = mid - 1; }+ return -1;+}+void *mempool_alloc(size_t size) { static char pool[4096]; static size_t offset = 0; if (offset + size > 4096) return NULL; void *p = &pool[offset]; offset += size; return p; }+struct node { int val; struct node *next; };+struct node *list_insert(struct node *head, int val) { struct node *n = malloc(sizeof(*n)); n->val = val; n->next = head; return n; }+int list_sum(struct node *head) { int s = 0; while (head) { s += head->val; head = head->next; } return s; }+unsigned crc32_byte(unsigned crc, unsigned char b) { crc ^= b; for (int i = 0; i < 8; i++) crc = (crc >> 1) ^ (0xEDB88320 & -(crc & 1)); return crc; }+int main() { int arr[] = {5,3,1,4,2}; bubble_sort(arr, 5); return binary_search(arr, 5, 3); }+CSRC++cat > "$SOURCES_DIR/string_ops.c" << 'CSRC'+#include <string.h>+#include <ctype.h>+int str_count_char(const char *s, char c) { int n = 0; while (*s) { if (*s == c) n++; s++; } return n; }+void str_reverse(char *s) { int len = strlen(s); for (int i = 0; i < len / 2; i++) { char t = s[i]; s[i] = s[len - 1 - i]; s[len - 1 - i] = t; } }+void str_to_upper(char *s) { while (*s) { *s = toupper(*s); s++; } }+int str_is_palindrome(const char *s) { int l = 0, r = strlen(s) - 1; while (l < r) { if (s[l++] != s[r--]) return 0; } return 1; }+char *str_find_substr(const char *haystack, const char *needle) { int nlen = strlen(needle); if (!nlen) return (char*)haystack; for (; *haystack; haystack++) { if (!strncmp(haystack, needle, nlen)) return (char*)haystack; } return 0; }+int main() { char buf[] = "hello"; str_reverse(buf); return str_is_palindrome(buf); }+CSRC++cat > "$SOURCES_DIR/state_machine.c" << 'CSRC'+enum state { IDLE, RUNNING, PAUSED, ERROR, DONE };+enum event { START, PAUSE, RESUME, FAIL, FINISH };+enum state transition(enum state s, enum event e) {+ switch (s) {+ case IDLE: return e == START ? RUNNING : s;+ case RUNNING: switch (e) { case PAUSE: return PAUSED; case FAIL: return ERROR; case FINISH: return DONE; default: return s; }+ case PAUSED: return e == RESUME ? RUNNING : (e == FAIL ? ERROR : s);+ case ERROR: return s;+ case DONE: return s;+ }+ return s;+}+int run_machine(enum event *events, int n) { enum state s = IDLE; for (int i = 0; i < n; i++) s = transition(s, events[i]); return s; }+int main() { enum event seq[] = {START, PAUSE, RESUME, FINISH}; return run_machine(seq, 4); }+CSRC++# Compile each source at each optimization level+for src in "$SOURCES_DIR"/*.c; do+ name=$(basename "$src" .c)+ for opt in O0 O1 O2 O3 Os; do+ out_debug="${OUT}/${name}_${opt}_debug"+ out_stripped="${OUT}/${name}_${opt}_stripped"++ "$CC" -g "-${opt}" -o "$out_debug" "$src" 2>/dev/null || continue+ cp "$out_debug" "$out_stripped"+ "$STRIP" "$out_stripped"++ echo "Built: ${name}_${opt}"+ done+done++echo "Corpus built in $OUT"\ No newline at end of filediff --git a/tools/xda/scripts/fetch_debian_pkgs.sh b/tools/xda/scripts/fetch_debian_pkgs.shnew file mode 100644index 0000000..d91b1bd--- /dev/null+++ b/tools/xda/scripts/fetch_debian_pkgs.sh@@ -0,0 +1,45 @@+#!/usr/bin/env bash+# tools/xda/scripts/fetch_debian_packages.sh+# Pairs stripped binaries with their debug info from /usr/lib/debug+# Output: /out/pairs/<binary_name>/binary and /out/pairs/<binary_name>/debug++set -euo pipefail++OUT_DIR="/out/pairs"+mkdir -p "$OUT_DIR"++count=0++# Find all ELF binaries in standard paths+for bin in /usr/bin/* /usr/sbin/* /usr/lib/*/lib*.so*; do+ [ -f "$bin" ] || continue++ # Check if it's actually ELF+ file_type=$(file -b "$bin" 2>/dev/null) || continue+ echo "$file_type" | grep -q "ELF" || continue++ # Look for corresponding debug info+ # Debian stores debug files in /usr/lib/debug/<original-path>.debug+ debug_path="/usr/lib/debug${bin}.debug"+ if [ ! -f "$debug_path" ]; then+ # Also check build-id based paths+ build_id=$(readelf -n "$bin" 2>/dev/null | grep "Build ID" | awk '{print $3}') || continue+ if [ -n "$build_id" ]; then+ prefix="${build_id:0:2}"+ suffix="${build_id:2}"+ debug_path="/usr/lib/debug/.build-id/${prefix}/${suffix}.debug"+ fi+ fi++ [ -f "$debug_path" ] || continue++ name=$(basename "$bin")+ pair_dir="${OUT_DIR}/${name}"+ mkdir -p "$pair_dir"+ cp "$bin" "${pair_dir}/binary"+ cp "$debug_path" "${pair_dir}/debug"++ count=$((count + 1))+done++echo "Extracted $count binary/debug pairs to $OUT_DIR"