Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

60.9 KB
Raw Blame
#!/bin/bash
# shellcheck disable=SC2154,SC2034  # Variables defined in reconftw.cfg and runtime globals shared across modules
# reconFTW - Utility functions module
# Contains: deleteOutScoped, cleanup, rotate_logs, sanitization, validation,
#           getElapsedTime, retry, disk check, rate limiting, caching
# This file is sourced by reconftw.sh - do not execute directly
[[ -z "${SCRIPTPATH:-}" ]] && {
    echo "Error: This module must be sourced by reconftw.sh" >&2
    exit 1
}

###############################################################################################################
########################################## OPTIONS & MGMT #####################################################
###############################################################################################################

function deleteOutScoped() {
    if [[ -s "$1" ]]; then
        while IFS= read -r outscoped; do
            [[ -z "$outscoped" ]] && continue
            # Escape regex metacharacters including / to prevent sed delimiter injection
            local escaped
            escaped=$(printf '%s' "$outscoped" | sed 's/[.[\*^$()+?{|/\\]/\\&/g')
            if grep -q "^[*]" <<<"$outscoped"; then
                escaped=$(printf '%s' "${outscoped:1}" | sed 's/[.[\*^$()+?{|/\\]/\\&/g')
                sed_i "/${escaped}$/d" "$2"
            else
                sed_i "/${escaped}/d" "$2"
            fi
        done <"$1"
    fi
}

# Callers (apileaks, jschecks) run sequentially (OSINT phase then Web phase).
# If future changes run callers in parallel, add file locking around the anew+p1radup section.
function merge_scoped_urls_into_url_extract() {
    local source_file="$1"
    local source_label="${2:-url source}"
    local raw_file=".tmp/${source_label}_urls_raw.txt"
    local scoped_file=".tmp/${source_label}_urls_scoped.txt"
    local normalized_file=".tmp/${source_label}_urls_normalized.txt"
    local query_file=".tmp/${source_label}_urls_query.txt"
    local plain_file=".tmp/${source_label}_urls_plain.txt"
    local domain_regex
    local added_count=0

    [[ ! -s "$source_file" ]] && return 0

    trap 'rm -f "$raw_file" "$scoped_file" "$normalized_file" "$query_file" "$plain_file" 2>/dev/null' RETURN

    ensure_dirs .tmp webs || return 1
    domain_regex=$(domain_match_regex "$domain")
    : >"$raw_file"
    : >"$scoped_file"
    : >"$normalized_file"
    : >"$query_file"
    : >"$plain_file"

    grep -aEo 'https?://[^ ]+' "$source_file" \
        | sed -E 's/[[:space:]]+$//' \
        | sed -E 's/[),;]$//' \
        | sed 's/"$//' \
        | sed "s/'$//" \
        | sort -u >"$raw_file" 2>/dev/null || true

    if [[ -s "$raw_file" ]]; then
        awk -v re="$domain_regex" -F/ '
            /^https?:\/\// {
                h=$3
                sub(/:.*/, "", h)
                if (h ~ re) print
            }
        ' "$raw_file" >"$scoped_file" 2>/dev/null || true
    fi

    if [[ -n "${outOfScope_file:-}" ]] && [[ -s "$outOfScope_file" ]] && [[ -s "$scoped_file" ]]; then
        deleteOutScoped "$outOfScope_file" "$scoped_file" || true
    fi

    if [[ "${INSCOPE:-false}" == true ]] && [[ -s "$scoped_file" ]]; then
        if ! check_inscope "$scoped_file" 2>>"$LOGFILE" >/dev/null; then
            print_warnf "check_inscope command failed."
        fi
    fi

    [[ ! -s "$scoped_file" ]] && return 0

    awk '/\?/{print > q} !/\?/{print > p}' q="$query_file" p="$plain_file" "$scoped_file"
    if command -v urless >/dev/null 2>&1 && [[ -s "$query_file" ]]; then
        urless <"$query_file" >"$normalized_file" 2>>"$LOGFILE" || true
    else
        cat "$query_file" >"$normalized_file" 2>/dev/null || true
    fi
    [[ -s "$plain_file" ]] && cat "$plain_file" >>"$normalized_file"
    sort -u "$normalized_file" -o "$normalized_file" 2>/dev/null || true

    if [[ -s "$normalized_file" ]]; then
        if ! added_count=$(anew webs/url_extract.txt <"$normalized_file" | sed '/^$/d' | wc -l | tr -d ' '); then
            added_count=0
        fi
        [[ "$added_count" =~ ^[0-9]+$ ]] || added_count=0

        if [[ "$added_count" -gt 0 ]]; then
            if [[ "${ASSET_STORE:-false}" == "true" ]]; then
                append_assets_from_file url value webs/url_extract.txt
            fi
            if command -v p1radup >/dev/null 2>&1; then
                p1radup -i webs/url_extract.txt -o webs/url_extract_nodupes.txt -s 2>>"$LOGFILE" >/dev/null || true
            else
                sort -u webs/url_extract.txt >webs/url_extract_nodupes.txt 2>>"$LOGFILE" || true
            fi
            notification "${source_label}: ${added_count} scoped URLs merged" "info"
        fi
    fi
}

function cleanup_on_exit() {
    local exit_code="${1:-130}"
    printf "\n%b[%s] Interrupted. Cleaning up...%b\n" "$bred" "$(date +'%Y-%m-%d %H:%M:%S')" "$reset"
    
    # Save checkpoint before cleanup if enabled
    if [[ "${CHECKPOINT_ENABLED:-false}" == "true" ]] && [[ -n "${CHECKPOINT_DIR:-}" ]]; then
        echo "interrupted=$(date -Iseconds)" >> "$CHECKPOINT_DIR/scan_info.txt" 2>/dev/null || true
    fi
    
    # Clean temporary chunk files
    rm -rf -- "${dir:-.}/.tmp/chunks" 2>/dev/null
    
    # Kill any background processes we spawned (safely)
    local pids
    pids=$(jobs -p 2>/dev/null) || true
    if [[ -n "$pids" ]]; then
        echo "$pids" | while read -r pid; do
            [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true
        done
    fi
    
    # Kill tracked interactsh process if running
    if [[ -n "${INTERACTSH_PID:-}" ]]; then
        kill "$INTERACTSH_PID" 2>/dev/null || true
    fi
    
    # Log the interruption
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] Interrupted by signal (exit code: $exit_code)" >>"${LOGFILE:-/dev/null}"

    exit "$exit_code"
}

# Remove stale .inprogress_<fn> sentinels on EXIT (D-02 clean-exit-only).
# Gated on _RECON_CLEAN_EXIT: only sweeps when the workflow's end() set the
# flag (clean traversal). SIGINT/SIGTERM via cleanup_on_exit does NOT set the
# flag, so the sentinel survives and the next run's resume banner fires
# (CR-01 fix; closes SC1 indicator gap from 01-VERIFICATION.md).
function _cleanup_inprogress() {
    [[ "${_RECON_CLEAN_EXIT:-false}" == "true" ]] || return 0
    [[ -n "${called_fn_dir:-}" ]] && rm -f "${called_fn_dir}"/.inprogress_* 2>/dev/null
    return 0
}

function rotate_logs() {
    local log_dir="$1"
    local max_logs="${2:-10}"
    local max_age_days="${3:-30}"

    [[ ! -d "$log_dir" ]] && return 0

    # Delete logs older than max_age_days
    find "$log_dir" -name "*.txt" -type f -mtime +"${max_age_days}" -delete 2>/dev/null || true

    # If still too many, keep only the newest max_logs
    local count
    count=$(find "$log_dir" -name "*.txt" -type f 2>/dev/null | wc -l)
    if [[ $count -gt $max_logs ]]; then
        find "$log_dir" -name "*.txt" -type f -printf '%T@ %p\n' 2>/dev/null \
            | sort -n | head -n $((count - max_logs)) | cut -d' ' -f2- \
            | xargs -r rm -f -- 2>/dev/null || true
    fi
}

# sanitize_interlace_input() — canonical definition in lib/validation.sh

function validate_config() {
    local warnings=0
    local errors=0

    # Check conflicting configs
    if [[ ${VULNS_GENERAL:-false} == true && ${SUBDOMAINS_GENERAL:-false} == false ]]; then
        print_warnf "VULNS_GENERAL=true but SUBDOMAINS_GENERAL=false -- vulnerability scans need subdomain data"
        warnings=$((warnings + 1))
    fi

    # Legacy/no-op config knobs (warn-only; runtime uses safe defaults)
    if [[ -n "${PERMUTATIONS_ENGINE:-}" && "${PERMUTATIONS_ENGINE}" != "gotator" ]]; then
        print_warnf "PERMUTATIONS_ENGINE is deprecated/ignored (got '%s'); gotator is always used" "$PERMUTATIONS_ENGINE"
        warnings=$((warnings + 1))
    fi
    for deprecated_noop in NUCLEI_FLAGS NUCLEI_FLAGS_JS BRUTESPRAY_THREADS METAFINDER_LIMIT RESOLVE_DOMAINS_THREADS INTRUSIVE FARADAY_SERVER FARADAY_USER FARADAY_PASS; do
        if [[ -n "${!deprecated_noop:-}" ]]; then
            print_warnf "%s is deprecated/removed; set it in config has no effect anymore" "$deprecated_noop"
            warnings=$((warnings + 1))
        fi
    done
    if [[ -n "${PERMUTATIONS_WORDLIST_MODE:-}" ]]; then
        case "${PERMUTATIONS_WORDLIST_MODE}" in
            auto|full|short) : ;;
            *)
                print_warnf "PERMUTATIONS_WORDLIST_MODE invalid: '%s' (use auto|full|short)" "$PERMUTATIONS_WORDLIST_MODE"
                warnings=$((warnings + 1))
                ;;
        esac
    fi
    if [[ -n "${DNS_RESOLVER:-}" ]]; then
        case "${DNS_RESOLVER}" in
            auto|puredns|dnsx) : ;;
            *)
                print_warnf "DNS_RESOLVER invalid: '%s' (use auto|puredns|dnsx)" "$DNS_RESOLVER"
                warnings=$((warnings + 1))
                ;;
        esac
    fi
    for deprecated in CORS OPEN_REDIRECT PROTO_POLLUTION FAVICON; do
        if [[ -n "${!deprecated:-}" ]]; then
            print_warnf "%s flag is deprecated/removed; set it in config has no effect anymore" "$deprecated"
            warnings=$((warnings + 1))
        fi
    done
    if [[ -n "${WEBPROBESIMPLE:-}" ]]; then
        print_warnf "WEBPROBESIMPLE is deprecated/ignored; use WEBPROBEFULL + WEBPROBE_PORTS"
        warnings=$((warnings + 1))
    fi
    if [[ -n "${HTTPX_FLAGS:-}" ]]; then
        print_warnf "HTTPX_FLAGS is deprecated/ignored for web probing; use WEBPROBE_PORTS"
        warnings=$((warnings + 1))
    fi

    if [[ -n "${MONITOR_MIN_SEVERITY:-}" ]]; then
        case "${MONITOR_MIN_SEVERITY,,}" in
            critical|high|medium|low|info) : ;;
            *)
                print_warnf "MONITOR_MIN_SEVERITY invalid: '%s' (use critical|high|medium|low|info)" "$MONITOR_MIN_SEVERITY"
                warnings=$((warnings + 1))
                ;;
        esac
    fi

    # Validate numeric thread/rate variables
    for var in FFUF_THREADS HTTPX_THREADS DALFOX_THREADS KATANA_THREADS HTTPX_RATELIMIT NUCLEI_RATELIMIT FFUF_RATELIMIT DNSX_THREADS DNSX_RATE_LIMIT PERMUTATIONS_SHORT_THRESHOLD AXIOM_FLEET_COUNT; do
        if [[ -n "${!var:-}" && ! "${!var}" =~ ^[0-9]+$ ]]; then
            print_errorf "%s must be numeric, got: %s" "$var" "${!var}"
            errors=$((errors + 1))
        fi
    done

    # Check Axiom tool availability if enabled
    if [[ ${AXIOM:-false} == true ]] && ! command -v axiom-scan >/dev/null 2>&1; then
        print_warnf "AXIOM=true but axiom-scan not found in PATH"
        warnings=$((warnings + 1))
    fi

    if [[ $errors -gt 0 ]]; then
        print_errorf "Configuration has %d error(s). Please fix before running." "$errors"
        return $E_CONFIG
    fi
    if [[ $warnings -gt 0 ]]; then
        print_notice INFO "config" "Configuration has ${warnings} warning(s)."
    fi
    return 0
}

# Auto-tune concurrency knobs according to host resources and PERF_PROFILE.
# Usage: apply_performance_profile
function apply_performance_profile() {
    # shellcheck disable=SC2034  # Thread/rate vars are consumed by sourced modules at runtime
    local profile="${PERF_PROFILE:-balanced}"
    local cores mem_gb

    cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
    if [[ "$OSTYPE" == "darwin"* ]]; then
        mem_gb=$(( $(sysctl -n hw.memsize 2>/dev/null || echo 8589934592) / 1024 / 1024 / 1024 ))
    else
        mem_gb=$(( $(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 8388608) / 1024 / 1024 ))
    fi
    [[ "$cores" =~ ^[0-9]+$ ]] || cores=4
    [[ "$mem_gb" =~ ^[0-9]+$ ]] || mem_gb=8

    case "$profile" in
        low)
            PARALLEL_MAX_JOBS=${PARALLEL_MAX_JOBS:-2}
            FFUF_THREADS=$((cores * 4))
            HTTPX_THREADS=$((cores * 5))
            HTTPX_UNCOMMONPORTS_THREADS=$((cores * 10))
            KATANA_THREADS=$((cores * 3))
            DALFOX_THREADS=$((cores * 20))
            ;;
        max)
            PARALLEL_MAX_JOBS=${PARALLEL_MAX_JOBS:-$((cores > 2 ? cores - 1 : 2))}
            FFUF_THREADS=$((cores * 14))
            HTTPX_THREADS=$((cores * 16))
            HTTPX_UNCOMMONPORTS_THREADS=$((cores * 28))
            KATANA_THREADS=$((cores * 6))
            DALFOX_THREADS=$((cores * 60))
            ;;
        *)
            PARALLEL_MAX_JOBS=${PARALLEL_MAX_JOBS:-$((cores > 1 ? cores / 2 : 1))}
            FFUF_THREADS=$((cores * 10))
            HTTPX_THREADS=$((cores * 12))
            HTTPX_UNCOMMONPORTS_THREADS=$((cores * 25))
            KATANA_THREADS=$((cores * 5))
            DALFOX_THREADS=$((cores * 50))
            ;;
    esac

    # Clamp aggressive defaults for low-memory hosts.
    if [[ "$mem_gb" -lt 6 ]]; then
        ((PARALLEL_MAX_JOBS > 2)) && PARALLEL_MAX_JOBS=2
        ((FFUF_THREADS > 24)) && FFUF_THREADS=24
        ((HTTPX_THREADS > 40)) && HTTPX_THREADS=40
        ((HTTPX_UNCOMMONPORTS_THREADS > 80)) && HTTPX_UNCOMMONPORTS_THREADS=80
    fi

    ((PARALLEL_MAX_JOBS < 1)) && PARALLEL_MAX_JOBS=1
    PERF_PROFILE_INFO="PERF_PROFILE=${profile} | cores=${cores} mem=${mem_gb}GB | jobs=${PARALLEL_MAX_JOBS} ffuf=${FFUF_THREADS} httpx=${HTTPX_THREADS}"
}

function getElapsedTime {
    # Sets $runtime (for backward compat) and also prints to stdout
    runtime=""
    local T=$(($2 - $1))
    local D=$((T / 60 / 60 / 24))
    local H=$((T / 60 / 60 % 24))
    local M=$((T / 60 % 60))
    local S=$((T % 60))
    ((D > 0)) && runtime="${runtime}${D} days, "
    ((H > 0)) && runtime="${runtime}${H} hours, "
    ((M > 0)) && runtime="${runtime}${M} minutes, "
    runtime="${runtime}${S} seconds."
}

# Retry with exponential backoff
# Usage: retry_with_backoff <max_attempts> <command> [args...]
# Example: retry_with_backoff 3 curl -s "https://example.com"
function retry_with_backoff() {
    local max_attempts="$1"
    shift
    local attempt=1
    local delay=1
    local max_delay=60

    while [ $attempt -le "$max_attempts" ]; do
        if "$@"; then
            return 0
        fi

        if [ $attempt -lt "$max_attempts" ]; then
            printf "%b[%s] Attempt %d/%d failed. Retrying in %d seconds...%b\n" \
                "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$attempt" "$max_attempts" "$delay" "$reset" >>"$LOGFILE"
            sleep "$delay"

            # Exponential backoff with max cap
            delay=$((delay * 2))
            if [ $delay -gt $max_delay ]; then
                delay=$max_delay
            fi
        fi

        attempt=$((attempt + 1))
    done

    printf "%b[%s] Command failed after %d attempts: %s%b\n" \
        "$bred" "$(date +'%Y-%m-%d %H:%M:%S')" "$max_attempts" "$*" "$reset" >>"$LOGFILE"
    return 1
}

# Lightweight log helper for non-fatal explanations
# Usage: log_note "message" [function] [line]
function log_note() {
    local msg="$1"
    local fn="${2:-main}"
    local ln="${3:-0}"
    local ts
    ts="$(date +'%Y-%m-%d %H:%M:%S')"
    echo "[$ts] NOTE @ ${fn}:${ln} :: ${msg}" >>"${LOGFILE:-/dev/null}"
}

# Explain common non-fatal ERRs (e.g., anew -q with no new lines)
# Usage: explain_err <rc> <cmd> <func> <line>
function explain_err() {
    local rc="$1"
    local cmd="$2"
    local fn="$3"
    local ln="$4"

    [[ $rc -ne 1 ]] && return 0

    # Detect 'anew -q <target>' and emit a helpful note
    if [[ $cmd =~ \banew[[:space:]]+-q[[:space:]]+([^[:space:]]+) ]]; then
        local target="${BASH_REMATCH[1]}"

        # Strip quotes
        target="${target%\"}"
        target="${target#\"}"
        target="${target%\'}"
        target="${target#\'}"

        # Resolve variable references like $WAF_LIST or ${WAF_LIST}
        if [[ $target =~ ^\\$\\{?([A-Za-z_][A-Za-z0-9_]*)\\}?$ ]]; then
            local var="${BASH_REMATCH[1]}"
            target="${!var}"
        fi

        if [[ -z "$target" ]]; then
            log_note "anew returned no new lines (target unresolved)" "$fn" "$ln"
        elif [[ ! -e "$target" ]]; then
            log_note "anew target missing: $target (likely upstream produced no data)" "$fn" "$ln"
        elif [[ ! -s "$target" ]]; then
            log_note "anew target empty: $target (no new lines to add)" "$fn" "$ln"
        else
            log_note "anew returned no new lines for $target" "$fn" "$ln"
        fi
        return 0
    fi

    # Generic note for wc -l failures (often from missing input in pipelines)
    if [[ $cmd =~ \bwc[[:space:]]+-l\b ]]; then
        log_note "wc -l failed (likely upstream pipeline had no input or a missing file)" "$fn" "$ln"
        return 0
    fi
}

# Check available disk space
# Usage: check_disk_space <required_gb> <path>
# Returns 0 if enough space, 1 otherwise
function check_disk_space() {
    local required_gb="$1"
    local check_path="${2:-.}"

    # Get available space in GB (portable across macOS/Linux).
    local available_gb
    available_gb=$(df -Pk "$check_path" 2>/dev/null | awk 'NR==2 {print int($4 / 1024 / 1024)}')

    if [ -z "$available_gb" ] || [ "$available_gb" -lt "$required_gb" ]; then
        DISK_SPACE_INFO="Disk space LOW: required ${required_gb}GB, available ${available_gb:-0}GB at ${check_path}"
        return 1
    fi

    DISK_SPACE_INFO="Disk space OK: ${available_gb}GB available at ${check_path}"
    return 0
}

# Mid-run disk-full check: thin wrapper around check_disk_space using MIN_DISK_SPACE_GB (D-07/D-08).
function _check_disk_mid_run() {
    check_disk_space "${MIN_DISK_SPACE_GB:-5}" "${dir:-.}"
    return $?
}

# Hard-abort the run on disk-full mid-run detection (D-09). EXIT trap (Plan 01-01) clears .inprogress_* on the way out.
function _abort_disk_full() {
    _print_error "disk_full: aborting (${DISK_SPACE_INFO:-disk space exhausted})"
    log_json "ERROR" "${FUNCNAME[1]:-main}" "Disk space exhausted" "reason=disk_full" "info=${DISK_SPACE_INFO:-unknown}"
    exit 1
}

# Show progress bar for operations
# Usage: progress_bar <current> <total> <message>
function progress_bar() {
    local current=$1
    local total=$2
    local message="${3:-Processing}"
    local width=50

    # Calculate percentage
    local percent=$((current * 100 / total))
    local filled=$((width * current / total))
    local empty=$((width - filled))

    # Build progress bar
    local bar=""
    for ((i = 0; i < filled; i++)); do bar="${bar}█"; done
    for ((i = 0; i < empty; i++)); do bar="${bar}░"; done

    # Print progress (using \r to overwrite the line)
    printf "\r%b[%s] %s [%s] %d%% (%d/%d)%b" \
        "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$message" "$bar" "$percent" "$current" "$total" "$reset"

    # Print newline when complete
    if [ "$current" -eq "$total" ]; then
        printf "\n"
    fi
}

# Execute command in dry-run mode if enabled
# Usage: run_command <command> [args...]
# _proxychains_bin — echo the proxychains binary to use, or fail.
# PROXYCHAINS_BIN wins when set; otherwise proxychains4 is preferred because the
# proxychains-ng fork is what ships as `proxychains4` on Debian/Ubuntu and as
# `proxychains-ng` (binary `proxychains4`) on Homebrew.
function _proxychains_bin() {
    if [[ -n "${PROXYCHAINS_BIN:-}" ]]; then
        command -v "$PROXYCHAINS_BIN" 2>/dev/null && return 0
        return 1
    fi
    command -v proxychains4 2>/dev/null && return 0
    command -v proxychains 2>/dev/null && return 0
    return 1
}

# _proxychains_applies <tool-name> — false for tools proxychains cannot carry.
#
# proxychains hooks libc connect() and covers TCP only. Three classes break
# under it, and breaking them silently is worse than not proxying them:
#
#   * mass-DNS resolvers (massdns, puredns, dnsx, shuffledns) send raw UDP/53
#     from their own socket code. Under proxychains they either fall back to the
#     system resolver or return nothing at all — the resolution step then
#     reports zero hosts and the whole run looks empty for no visible reason.
#   * naabu/nmap SYN scanning uses raw sockets, which never reach connect().
#   * statically-linked Go binaries do not resolve libc symbols, so the LD_PRELOAD
#     hook is a no-op; they leak DIRECTLY while appearing proxied. The
#     ProjectDiscovery tools are the common case here, which is why the honest
#     answer for them is their own -proxy flag, not this wrapper.
#
# PROXYCHAINS_EXCLUDE lets an operator extend the list without editing code.
function _proxychains_applies() {
    local _tool="${1:-}"
    [[ -z "$_tool" ]] && return 1

    # Narrowing IFS here is REQUIRED, not stylistic. reconftw.sh:8 sets
    # IFS=$'\n\t' process-wide, so an unquoted ${PROXYCHAINS_EXCLUDE} does NOT
    # split on spaces — and reconftw.cfg documents this variable as
    # space-separated. PROXYCHAINS_EXCLUDE="httpx nuclei" therefore arrived as
    # the SINGLE word "httpx nuclei", matched no tool name, and both tools were
    # proxied anyway: the operator's exclusion silently did nothing. Splitting
    # on space/tab/newline accepts every separator a user might reach for.
    local -a _extra=()
    if [[ -n "${PROXYCHAINS_EXCLUDE:-}" ]]; then
        local _oifs="$IFS"
        IFS=$' \t\n'
        # shellcheck disable=SC2206 # deliberate word splitting, IFS narrowed above
        _extra=(${PROXYCHAINS_EXCLUDE})
        IFS="$_oifs"
    fi

    local _skip
    for _skip in massdns puredns dnsx shuffledns naabu nmap masscan \
        "${_extra[@]}"; do
        [[ "$_tool" == "$_skip" ]] && return 1
    done
    return 0
}

function run_command() {
    if [[ "${DRY_RUN:-false}" == "true" ]]; then
        # Extract tool name (first word, strip path)
        local tool_name="${1##*/}"
        local full_cmd="$*"
        local redacted_cmd="$full_cmd"

        if declare -F redact_secrets >/dev/null 2>&1; then
            redacted_cmd=$(redact_secrets "$redacted_cmd")
        fi
        redacted_cmd=$(echo "$redacted_cmd" \
            | sed -E 's/(-token-string[[:space:]]+)[^[:space:]]+/\1[REDACTED]/g' \
            | sed -E 's/([?&]apiKey=)[^&[:space:]]+/\1[REDACTED]/g' \
            | sed -E 's/(Authorization:[[:space:]]*Bearer[[:space:]])[^[:space:]]+/\1[REDACTED]/g')

        # Track command for module summary
        if declare -F ui_dryrun_track >/dev/null 2>&1; then
            ui_dryrun_track "$tool_name" "$redacted_cmd"
        else
            # Fallback to old behavior if ui_dryrun_track not available
            printf "%b[DRY-RUN] Would execute: %s%b\n" "$yellow" "$redacted_cmd" "$reset"
        fi
        return 0
    fi

    local cmd_name="${1##*/}"
    if [[ "$cmd_name" == axiom-* ]]; then
        _run_axiom_command_with_detection "$@"
        return $?
    fi

    # PROXYCHAINS (#1041): route this tool's traffic through the configured
    # chain. Applied HERE because run_command is the single gate every external
    # tool goes through, so one place covers all of them without touching each
    # tool's own (mutually incompatible) proxy flags.
    if [[ "${PROXYCHAINS:-false}" == "true" ]] && _proxychains_applies "$cmd_name"; then
        local _pc
        if _pc=$(_proxychains_bin); then
            local -a _pc_argv=("$_pc" -q)
            [[ -n "${PROXYCHAINS_CONF:-}" ]] && _pc_argv+=(-f "$PROXYCHAINS_CONF")
            set -- "${_pc_argv[@]}" "$@"
        else
            # Fail LOUD, not silently direct: the whole point of enabling this is
            # that the operator does not want these packets leaving un-proxied.
            notification "PROXYCHAINS=true but no proxychains binary was found — ${cmd_name} would run UN-PROXIED; skipping it" warn
            return 1
        fi
    fi

    if [[ "${ADAPTIVE_RATE_LIMIT:-false}" == "true" ]]; then
        run_with_adaptive_rate "$@"
    else
        if [[ -n "${DEBUG_LOG:-}" ]]; then
            if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
                "$@" 2> >(tee -a "$DEBUG_LOG" >&2)
            else
                "$@" 2>>"$DEBUG_LOG"
            fi
        else
            "$@"
        fi
    fi
}

# Cross-platform sed_i wrapper
# Usage: sed_i 's/old/new/g' file.txt
# Works on both macOS (BSD sed) and Linux (GNU sed)
function sed_i() {
    if [[ $# -lt 2 ]]; then
        echo "Usage: sed_i 'pattern' file" >&2
        return 1
    fi

    local pattern="$1"
    local file="$2"

    # Check if we're using GNU sed or BSD sed
    if sed --version >/dev/null 2>&1; then
        # GNU sed (Linux or installed via brew on macOS)
        sed -i "$pattern" "$file"
    else
        # BSD sed (default macOS)
        sed -i '' "$pattern" "$file"
    fi
}

###############################################################################################################
#################################### RATE LIMITING ADAPTATIVO #################################################
###############################################################################################################

# Adaptive rate limiting - detects rate limit errors and adjusts automatically
# Global variables for rate limiting
ADAPTIVE_RATE_LIMIT=${ADAPTIVE_RATE_LIMIT:-false}
CURRENT_RATE_LIMIT=${NUCLEI_RATELIMIT:-150}
MIN_RATE_LIMIT=10
MAX_RATE_LIMIT=500
RATE_LIMIT_BACKOFF_FACTOR=0.5
RATE_LIMIT_INCREASE_FACTOR=1.2
PARALLEL_PRESSURE_LEVEL=${PARALLEL_PRESSURE_LEVEL:-normal}
AXIOM_RUNTIME_DISABLED=${AXIOM_RUNTIME_DISABLED:-false}
AXIOM_RUNTIME_DISABLE_REASON=${AXIOM_RUNTIME_DISABLE_REASON:-}
AXIOM_RUNTIME_DISABLE_CMD=${AXIOM_RUNTIME_DISABLE_CMD:-}
AXIOM_RUNTIME_DISABLE_NOTIFIED=${AXIOM_RUNTIME_DISABLE_NOTIFIED:-false}

# Detect rate limit errors in command output
# Usage: detect_rate_limit_error <output_file_or_string>
function detect_rate_limit_error() {
    local output="$1"

    # Check for common rate limit indicators
    if grep -qiE "(429|too many requests|rate limit|limit exceeded|503|service unavailable)" <<<"$output"; then
        return 0 # Rate limit detected
    fi
    return 1 # No rate limit detected
}

axiom_runtime_enabled() {
    [[ ${AXIOM:-false} == true ]] && [[ ${AXIOM_RUNTIME_DISABLED:-false} != true ]]
}

axiom_transport_error_detected() {
    local output="$1"
    [[ -z "$output" ]] && return 1
    grep -qiE \
        "(Unable to reach any instance selected|No instance supplied or instance not found|No axiom instances running|No axiom instances selected|Could not resolve hostname|Temporary failure in name resolution|Name or service not known|Connection timed out|i/o timeout|context deadline exceeded|dial tcp|connection refused|kex_exchange_identification|Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED)" \
        <<<"$output"
}

axiom_disable_runtime() {
    local reason="${1:-transport error}"
    local cmd="${2:-axiom command}"

    if [[ ${AXIOM_RUNTIME_DISABLED:-false} == true ]]; then
        return 0
    fi

    AXIOM=false
    AXIOM_RUNTIME_DISABLED=true
    AXIOM_RUNTIME_DISABLE_REASON="$reason"
    AXIOM_RUNTIME_DISABLE_CMD="$cmd"

    if [[ ${AXIOM_RUNTIME_DISABLE_NOTIFIED:-false} != true ]]; then
        AXIOM_RUNTIME_DISABLE_NOTIFIED=true
        if declare -F notification >/dev/null 2>&1; then
            notification "Axiom disabled for this run (${reason}). Falling back to local execution." warn
        fi
    fi

    log_note "axiom runtime disabled: reason='${reason}' cmd='${cmd}'" "${FUNCNAME[0]}" "${LINENO}"
}

_handle_axiom_command_result() {
    local rc="${1:-0}"
    local cmd="${2:-}"
    local err_file="${3:-}"
    local err_out=""

    if [[ -n "$err_file" && -f "$err_file" ]]; then
        err_out=$(cat "$err_file")
    fi

    if [[ "$rc" -ne 0 ]] && axiom_transport_error_detected "$err_out"; then
        axiom_disable_runtime "transport/connectivity failure" "$cmd"
    fi
}

_run_axiom_command_with_detection() {
    local -a cmd=("$@")
    local axiom_err_tmp
    local rc
    axiom_err_tmp=$(mktemp)

    if [[ -n "${DEBUG_LOG:-}" ]]; then
        if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
            "${cmd[@]}" 2> >(tee -a "$axiom_err_tmp" "$DEBUG_LOG" >&2)
        else
            "${cmd[@]}" 2> >(tee -a "$axiom_err_tmp" "$DEBUG_LOG" >/dev/null)
        fi
    else
        "${cmd[@]}" 2> >(tee -a "$axiom_err_tmp" >&2)
    fi
    rc=$?

    _handle_axiom_command_result "$rc" "${cmd[*]}" "$axiom_err_tmp"
    rm -f "$axiom_err_tmp" 2>/dev/null || true
    return "$rc"
}

# Adjust rate limit based on errors
# Usage: adjust_rate_limit <increase|decrease>
function adjust_rate_limit() {
    local action="$1"

    [[ "$ADAPTIVE_RATE_LIMIT" != "true" ]] && return 0

    local new_limit
    case "$action" in
        decrease)
            new_limit=$(awk "BEGIN {printf \"%.0f\", $CURRENT_RATE_LIMIT * $RATE_LIMIT_BACKOFF_FACTOR}")
        if [ "$new_limit" -lt "$MIN_RATE_LIMIT" ]; then
            new_limit=$MIN_RATE_LIMIT
        fi
        CURRENT_RATE_LIMIT=$new_limit
        PARALLEL_PRESSURE_LEVEL="high"
        printf "%b[%s] Rate limit decreased to %d req/s due to errors%b\n" \
            "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$CURRENT_RATE_LIMIT" "$reset" | tee -a "$LOGFILE"
        ;;
        increase)
            new_limit=$(awk "BEGIN {printf \"%.0f\", $CURRENT_RATE_LIMIT * $RATE_LIMIT_INCREASE_FACTOR}")
        if [ "$new_limit" -gt "$MAX_RATE_LIMIT" ]; then
            new_limit=$MAX_RATE_LIMIT
        fi
        CURRENT_RATE_LIMIT=$new_limit
        PARALLEL_PRESSURE_LEVEL="normal"
        printf "%b[%s] Rate limit increased to %d req/s%b\n" \
            "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$CURRENT_RATE_LIMIT" "$reset" >>"$LOGFILE"
        ;;
    esac

    # Update tool-specific rate limits
    NUCLEI_RATELIMIT=$CURRENT_RATE_LIMIT
    # shellcheck disable=SC2034  # Used by external tools
    FFUF_RATELIMIT=$CURRENT_RATE_LIMIT
    # shellcheck disable=SC2034  # Used by external tools
    HTTPX_RATELIMIT=$CURRENT_RATE_LIMIT
}

# Execute command with adaptive rate limiting
# Usage: run_with_adaptive_rate <command> [args...]
function run_with_adaptive_rate() {
    [[ "$ADAPTIVE_RATE_LIMIT" != "true" ]] && {
        "$@"
        return $?
    }

    local max_retries=3
    local retry=0
    local temp_output
    temp_output=$(mktemp)

    while [ $retry -lt $max_retries ]; do
        if "$@" 2>&1 | tee "$temp_output"; then
            # Command succeeded, check if we can increase rate
            if [ $retry -eq 0 ]; then
                adjust_rate_limit increase
            fi
            rm -f "$temp_output"
            return 0
        fi

        # Check if failure was due to rate limiting
        if detect_rate_limit_error "$(cat "$temp_output")"; then
            adjust_rate_limit decrease
            retry=$((retry + 1))
            if [ $retry -lt $max_retries ]; then
                printf "%b[%s] Retrying with lower rate limit (%d/%d)...%b\n" \
                    "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$retry" "$max_retries" "$reset" | tee -a "$LOGFILE"
                sleep 2
            fi
        else
            # Non rate-limit error, don't retry
            rm -f "$temp_output"
            return 1
        fi
    done

    rm -f "$temp_output"
    return 1
}

###############################################################################################################
####################################### SECURITY ################################################
###############################################################################################################

# Sanitize domain input to prevent command injection.
# Accepts bare domains, URLs (strips scheme/userinfo/path/query/fragment/port),
# and IPv4 addresses (validated inline — octets > 255 rejected).
# Usage: sanitize_domain <domain_or_url>
# Returns: sanitized domain (lowercase) or IPv4
function sanitize_domain() {
    local input_domain="$1"
    local working="$input_domain"

    # 1) Strip scheme (scheme://) — RFC 3986 scheme charset
    if [[ "$working" =~ ^[a-zA-Z][a-zA-Z0-9+.-]*:// ]]; then
        working="${working#*://}"
    fi

    # 2) Cut at first / ? # (path / query / fragment).
    #    Must happen BEFORE userinfo strip: an @ inside a path is not userinfo.
    working="${working%%[/?#]*}"

    # 3) Strip userinfo (user:pass@host)
    if [[ "$working" == *@* ]]; then
        working="${working##*@}"
    fi

    # 4) Strip :port (bare IPv6 literals are not supported elsewhere in reconFTW)
    working="${working%%:*}"

    # 5) IPv4 post-normalization: redirect to inline octet validation.
    if [[ "$working" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
        local IFS_bak="$IFS"
        IFS='.'
        local -a _octs=($working)
        IFS="$IFS_bak"
        local _o
        for _o in "${_octs[@]}"; do
            if (( _o > 255 )); then
                print_errorf "Invalid IP octet in '%s'" "$input_domain"
                return 1
            fi
        done
        echo "$working"
        return 0
    fi

    # 6) Hardening: char whitelist + lowercase + trim leading/trailing dots/hyphens
    local sanitized
    sanitized=$(echo "$working" | tr -cd 'a-zA-Z0-9.-')
    sanitized=$(echo "$sanitized" | tr '[:upper:]' '[:lower:]')
    sanitized=$(echo "$sanitized" | sed 's/^[.-]*//; s/[.-]*$//')

    # Check if domain is empty after sanitization
    if [[ -z "$sanitized" ]]; then
        print_errorf "Invalid domain after sanitization: '%s'" "$input_domain"
        return 1
    fi

    # Check for basic domain format (at least one dot for TLD)
    if [[ ! "$sanitized" =~ \. ]]; then
        # Important: sanitize_domain is commonly used in command substitution (domain=$(sanitize_domain ...)).
        # Emit warnings to stderr so they don't pollute the captured value.
        print_warnf "Domain '%s' has no TLD, may be invalid" "$sanitized" >&2
    fi

    # Log only when the tr whitelist actually removed characters beyond case/trim.
    # URL normalization (scheme/userinfo/path/port strip) is silent — that's the expected path.
    local working_trivial
    working_trivial=$(echo "$working" | tr '[:upper:]' '[:lower:]' | sed 's/^[.-]*//; s/[.-]*$//')
    if [[ "$working_trivial" != "$sanitized" ]]; then
        print_notice INFO "sanitize_domain" "Domain sanitized from '${input_domain}' to '${sanitized}'" >&2
    fi

    echo "$sanitized"
    return 0
}

# Validate and sanitize IP/CIDR input
# Usage: sanitize_ip <ip_or_cidr>
function sanitize_ip() {
    local input="$1"
    local sanitized

    # Allow digits, dots, and slash for CIDR
    sanitized=$(echo "$input" | tr -cd '0-9./,')

    if [[ -z "$sanitized" ]]; then
        print_errorf "Invalid IP/CIDR after sanitization: '%s'" "$input"
        return 1
    fi

    echo "$sanitized"
    return 0
}

# Sanitize a single entry from a -l list file.
# Detects IP/CIDR vs domain and applies the appropriate sanitizer.
# Outputs the sanitized value; returns 1 if the entry is invalid.
# Usage: domain=$(_sanitize_list_entry "$raw") || continue
_sanitize_list_entry() {
    local raw="$1"
    if [[ "$raw" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/[0-9]+)?$ ]]; then
        sanitize_ip "$raw"
    else
        sanitize_domain "$raw"
    fi
}

###############################################################################################################
####################################### SECURITY CHECKS #######################################################
###############################################################################################################

# Check and warn about insecure permissions on sensitive files
# Usage: check_secrets_permissions
function check_secrets_permissions() {
    local sensitive_files=(
        "${SCRIPTPATH}/secrets.cfg"
        "${SCRIPTPATH}/.github_tokens"
        "${SCRIPTPATH}/.gitlab_tokens"
        "${HOME}/.config/notify/provider-config.yaml"
    )
    
    local warnings=0
    
    for file in "${sensitive_files[@]}"; do
        if [[ -f "$file" ]]; then
            local perms
            if [[ "$OSTYPE" == "darwin"* ]]; then
                perms=$(stat -f "%OLp" "$file" 2>/dev/null)
            else
                perms=$(stat -c "%a" "$file" 2>/dev/null)
            fi
            
            # Check if file is readable by group or others
            if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
                print_warnf "Insecure permissions (%s) on sensitive file: %s" "$perms" "$file"
                print_notice WARN "config" "Recommended: chmod 600 ${file}"
                ((warnings++))
            fi
        fi
    done
    
    return $warnings
}

###############################################################################################################
####################################### DEEP/AXIOM HELPERS ####################################################
###############################################################################################################

# Check if we should run in DEEP mode based on flag or item count
# Usage: should_run_deep <count> [limit]
# Returns: 0 if should run deep, 1 otherwise
function should_run_deep() {
    local count="${1:-0}"
    local limit="${2:-$DEEP_LIMIT}"

    [[ "$DEEP" == true ]] && return 0
    [[ "$count" -le "$limit" ]] && return 0
    return 1
}

# Check if we should run in DEEP mode with custom limit
# Usage: should_run_deep2 <count>
# Returns: 0 if should run deep, 1 otherwise
function should_run_deep2() {
    local count="${1:-0}"
    should_run_deep "$count" "${DEEP_LIMIT2:-500}"
}

###############################################################################################################
####################################### CACHÉ DE RECURSOS #####################################################
###############################################################################################################

CACHE_DIR="${SCRIPTPATH}/.cache"
CACHE_MAX_AGE_DAYS="${CACHE_MAX_AGE_DAYS:-30}" # 1 month default, user-configurable

# Initialize cache directory
function cache_init() {
    mkdir -p "$CACHE_DIR"/{wordlists,resolvers,tools}
}

# Resolve TTL per cache type.
# Usage: cache_max_age_for_type <wordlists|resolvers|tools>
function cache_max_age_for_type() {
    case "${1:-tools}" in
        resolvers) echo "${CACHE_MAX_AGE_DAYS_RESOLVERS:-${CACHE_MAX_AGE_DAYS:-30}}" ;;
        wordlists) echo "${CACHE_MAX_AGE_DAYS_WORDLISTS:-${CACHE_MAX_AGE_DAYS:-30}}" ;;
        tools|*) echo "${CACHE_MAX_AGE_DAYS_TOOLS:-${CACHE_MAX_AGE_DAYS:-30}}" ;;
    esac
}

# Check if cached file is still valid (less than 30 days old)
# Usage: cache_is_valid <cache_file>
# Returns: 0 if valid, 1 if expired or missing
function cache_is_valid() {
    local cache_file="$1"
    local cache_type="${2:-tools}"
    local max_age_days
    max_age_days=$(cache_max_age_for_type "$cache_type")

    [[ ! -f "$cache_file" ]] && return 1
    [[ "${CACHE_REFRESH:-false}" == "true" ]] && return 1

    # Get file modification time in seconds since epoch
    local file_mtime
    if [[ "$(uname -s)" == "Darwin" ]]; then
        # macOS: stat -f %m returns modification time
        file_mtime=$(stat -f "%m" "$cache_file" 2>/dev/null)
    else
        # Linux: stat -c %Y returns modification time
        file_mtime=$(stat -c "%Y" "$cache_file" 2>/dev/null)
    fi
    
    # Fallback if stat failed
    if [[ -z "$file_mtime" ]] || ! [[ "$file_mtime" =~ ^[0-9]+$ ]]; then
        return 1
    fi

    local current_time
    current_time=$(date +%s)
    local file_age_seconds=$((current_time - file_mtime))
    local file_age_days=$((file_age_seconds / 86400))

    if [ "$file_age_days" -lt "$max_age_days" ]; then
        printf "%b[%s] Using cached file (age: %d days): %s%b\n" \
            "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$file_age_days" "$(basename "$cache_file")" "$reset" >>"$LOGFILE"
        return 0
    else
        printf "%b[%s] Cache expired (age: %d days): %s%b\n" \
            "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$file_age_days" "$(basename "$cache_file")" "$reset" >>"$LOGFILE"
        return 1
    fi
}

# Download file with caching support
# Usage: cached_download <url> <destination> [cache_name]
# If cache exists and is valid, copies from cache instead of downloading
function cached_download() {
    cached_download_typed "$1" "$2" "${3:-$(basename "$1")}" "tools"
}

# Download file with typed cache support
# Usage: cached_download_typed <url> <destination> [cache_name] [cache_type]
function cached_download_typed() {
    local url="$1"
    local destination="$2"
    local cache_name="${3:-$(basename "$url")}"
    local cache_type="${4:-tools}"
    local cache_file="$CACHE_DIR/$cache_type/$cache_name"
    local -a curl_cmd

    cache_init

    # Check if we have valid cached version
    if cache_is_valid "$cache_file" "$cache_type"; then
        cp "$cache_file" "$destination"
        return 0
    fi

    mkdir -p "$(dirname "$cache_file")"
    # Download fresh copy
    if _ui_human_output_enabled; then
        printf "%b[%s] Downloading: %s%b\n" \
            "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$(basename "$url")" "$reset"
    elif [[ -n "${LOGFILE:-}" ]]; then
        printf "[%s] Downloading: %s\n" \
            "$(date +'%Y-%m-%d %H:%M:%S')" "$(basename "$url")" >>"$LOGFILE" 2>/dev/null || true
    fi

    curl_cmd=(curl -sL "$url" -o "$destination")
    if [[ "$cache_type" == "resolvers" ]]; then
        curl_cmd=(
            curl -fsSL
            --connect-timeout "${RESOLVER_DOWNLOAD_CONNECT_TIMEOUT:-10}"
            --max-time "${RESOLVER_DOWNLOAD_MAX_TIME:-120}"
            --retry "${RESOLVER_DOWNLOAD_RETRY:-2}"
            --retry-delay "${RESOLVER_DOWNLOAD_RETRY_DELAY:-2}"
            --retry-connrefused
            "$url" -o "$destination"
        )
    fi

    if run_command "${curl_cmd[@]}"; then
        # Save to cache for future use
        cp "$destination" "$cache_file"
        printf "%b[%s] Cached for future use: %s%b\n" \
            "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$cache_name" "$reset" >>"$LOGFILE"
        return 0
    else
        printf "%b[%s] Download failed: %s%b\n" \
            "$bred" "$(date +'%Y-%m-%d %H:%M:%S')" "$url" "$reset" >&2
        return 1
    fi
}

# Clear old cache files
# Usage: cache_clean [max_age_days]
function cache_clean() {
    local max_age="${1:-${CACHE_MAX_AGE_DAYS:-30}}"

    [[ ! -d "$CACHE_DIR" ]] && return 0

    local cleaned=0
    local current_time
    current_time=$(date +%s)
    
    while IFS= read -r -d '' file; do
        local file_mtime
        if [[ "$(uname -s)" == "Darwin" ]]; then
            file_mtime=$(stat -f "%m" "$file" 2>/dev/null)
        else
            file_mtime=$(stat -c "%Y" "$file" 2>/dev/null)
        fi
        
        # Skip if stat failed
        [[ -z "$file_mtime" ]] || ! [[ "$file_mtime" =~ ^[0-9]+$ ]] && continue

        local file_age_seconds=$((current_time - file_mtime))
        local file_age_days=$((file_age_seconds / 86400))

        if [ $file_age_days -gt $max_age ]; then
            rm -f "$file"
            cleaned=$((cleaned + 1))
        fi
    done < <(find "$CACHE_DIR" -type f -print0 2>/dev/null)

    if [ $cleaned -gt 0 ]; then
        printf "%b[%s] Cleaned %d expired cache files%b\n" \
            "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$cleaned" "$reset"
    fi
}

###############################################################################################################
####################################### WORDLIST HELPERS ######################################################
###############################################################################################################

# Ensure a plaintext wordlist exists; if missing, try to expand a sibling .gz file.
# Usage: ensure_wordlist_file <path>
function ensure_wordlist_file() {
    local file="$1"
    local gz_file="${file}.gz"

    # Prefer existing plaintext.
    [[ -s "$file" ]] && return 0
    [[ ! -s "$gz_file" ]] && return 1

    command -v gzip >/dev/null 2>&1 || return 1

    mkdir -p "$(dirname "$file")" 2>/dev/null || true

    local tmp
    if command -v mktemp >/dev/null 2>&1; then
        tmp="$(mktemp "${file}.tmp.XXXXXX" 2>/dev/null || true)"
    fi
    [[ -z "${tmp:-}" ]] && tmp="${file}.tmp.$$"

    if gzip -dc "$gz_file" >"$tmp" 2>/dev/null; then
        mv -f "$tmp" "$file"
        return 0
    fi

    rm -f "$tmp" 2>/dev/null || true
    return 1
}

###############################################################################################################
####################################### CHECKPOINT SYSTEM #####################################################
###############################################################################################################

# Checkpoint system for scan recovery
# Allows resuming scans from the last completed phase after interruption
CHECKPOINT_DIR=""
CHECKPOINT_ENABLED=${CHECKPOINT_ENABLED:-true}

# Initialize checkpoint system for a scan
# Usage: checkpoint_init
function checkpoint_init() {
    [[ "$CHECKPOINT_ENABLED" != "true" ]] && return 0
    [[ -z "${dir:-}" ]] && return 1

    CHECKPOINT_DIR="${dir}/.checkpoints"
    mkdir -p "$CHECKPOINT_DIR"

    # Save scan metadata
    {
        echo "domain=${domain:-unknown}"
        echo "started=$(date -Iseconds)"
        echo "mode=${MODE:-unknown}"
        echo "deep=${DEEP:-false}"
    } >"$CHECKPOINT_DIR/scan_info.txt"

    printf "%b[%s] Checkpoint system initialized%b\n" \
        "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$reset" >>"${LOGFILE:-/dev/null}"
}

# Save a checkpoint after completing a phase
# Usage: checkpoint_save <phase_name>
function checkpoint_save() {
    [[ "$CHECKPOINT_ENABLED" != "true" ]] && return 0
    [[ -z "$CHECKPOINT_DIR" ]] && return 1

    local phase="$1"
    local checkpoint_file="$CHECKPOINT_DIR/${phase}.done"

    # Save phase completion with timestamp
    {
        echo "phase=$phase"
        echo "completed=$(date -Iseconds)"
        echo "status=completed"
    } >"$checkpoint_file"

    printf "%b[%s] Checkpoint saved: %s%b\n" \
        "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$phase" "$reset" >>"${LOGFILE:-/dev/null}"
}

# Check if a phase checkpoint exists (already completed)
# Usage: checkpoint_exists <phase_name>
# Returns: 0 if checkpoint exists, 1 if not
function checkpoint_exists() {
    [[ "$CHECKPOINT_ENABLED" != "true" ]] && return 1
    [[ -z "$CHECKPOINT_DIR" ]] && return 1

    local phase="$1"
    [[ -f "$CHECKPOINT_DIR/${phase}.done" ]]
}

# List completed checkpoints
# Usage: checkpoint_list
function checkpoint_list() {
    [[ -z "$CHECKPOINT_DIR" ]] && return 1
    [[ ! -d "$CHECKPOINT_DIR" ]] && return 1

    printf "%b[%s] Completed phases:%b\n" "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$reset"
    for f in "$CHECKPOINT_DIR"/*.done; do
        [[ -f "$f" ]] || continue
        local phase
        phase=$(basename "$f" .done)
        local completed
        completed=$(grep "^completed=" "$f" 2>/dev/null | cut -d= -f2)
        printf "  - %s (completed: %s)\n" "$phase" "${completed:-unknown}"
    done
}

# Clear all checkpoints (for fresh scan)
# Usage: checkpoint_clear
function checkpoint_clear() {
    [[ -z "$CHECKPOINT_DIR" ]] && return 1

    rm -rf "$CHECKPOINT_DIR"
    mkdir -p "$CHECKPOINT_DIR"

    printf "%b[%s] Checkpoints cleared%b\n" \
        "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$reset"
}

###############################################################################################################
####################################### CIRCUIT BREAKER #######################################################
###############################################################################################################

# Circuit breaker pattern for unreliable external tools
# Prevents repeated calls to tools that are consistently failing
declare -A CIRCUIT_BREAKER_FAILURES
declare -A CIRCUIT_BREAKER_STATE
CIRCUIT_BREAKER_THRESHOLD=${CIRCUIT_BREAKER_THRESHOLD:-3}
CIRCUIT_BREAKER_TIMEOUT=${CIRCUIT_BREAKER_TIMEOUT:-300}  # 5 minutes

# Check if a tool's circuit breaker is open (should skip)
# Usage: circuit_breaker_is_open <tool_name>
# Returns: 0 if open (skip tool), 1 if closed (run tool)
function circuit_breaker_is_open() {
    local tool="$1"
    local state="${CIRCUIT_BREAKER_STATE[$tool]:-closed}"
    local failures="${CIRCUIT_BREAKER_FAILURES[$tool]:-0}"

    if [[ "$state" == "open" ]]; then
        # Check if timeout has passed
        local opened_at="${CIRCUIT_BREAKER_STATE[${tool}_opened]:-0}"
        local now
        now=$(date +%s)
        if (( now - opened_at > CIRCUIT_BREAKER_TIMEOUT )); then
            # Reset to half-open (allow one retry)
            CIRCUIT_BREAKER_STATE[$tool]="half-open"
            return 1  # Allow retry
        fi
        printf "%b[%s] Circuit breaker OPEN for %s (skipping)%b\n" \
            "$yellow" "$(date +'%Y-%m-%d %H:%M:%S')" "$tool" "$reset" >>"${LOGFILE:-/dev/null}"
        return 0  # Skip
    fi

    return 1  # Run tool
}

# Record a tool failure
# Usage: circuit_breaker_record_failure <tool_name>
function circuit_breaker_record_failure() {
    local tool="$1"
    local failures="${CIRCUIT_BREAKER_FAILURES[$tool]:-0}"
    failures=$((failures + 1))
    CIRCUIT_BREAKER_FAILURES[$tool]=$failures

    if (( failures >= CIRCUIT_BREAKER_THRESHOLD )); then
        CIRCUIT_BREAKER_STATE[$tool]="open"
        CIRCUIT_BREAKER_STATE[${tool}_opened]=$(date +%s)
        printf "%b[%s] Circuit breaker OPENED for %s after %d failures%b\n" \
            "$bred" "$(date +'%Y-%m-%d %H:%M:%S')" "$tool" "$failures" "$reset"
    fi
}

# Record a tool success (reset failure count)
# Usage: circuit_breaker_record_success <tool_name>
function circuit_breaker_record_success() {
    local tool="$1"
    CIRCUIT_BREAKER_FAILURES[$tool]=0
    CIRCUIT_BREAKER_STATE[$tool]="closed"
}

###############################################################################################################
########################################## DNS RESOLVER AUTO-DETECTION ########################################
###############################################################################################################

# Get the primary local IP address (cross-platform: macOS + Linux).
# Uses the default route to find the correct interface/IP regardless of naming.
_get_local_ip() {
    local ip=""
    if [[ "$(uname)" == "Darwin" ]]; then
        # Get the interface used for the default route (not hardcoded to en0/en1)
        local iface
        iface=$(route -n get default 2>/dev/null | awk '/interface:/{print $2}')
        [[ -n "$iface" ]] && ip=$(ipconfig getifaddr "$iface" 2>/dev/null)
    else
        # Use ip route to find the src IP for outbound traffic (works on all Linux)
        ip=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}')
        # Fallback to hostname -I if ip route fails
        [[ -z "$ip" ]] && ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    fi
    echo "$ip"
}

# Cached resolver selection for this run when DNS_RESOLVER=auto (set by init_dns_resolver()).
DNS_RESOLVER_SELECTED="${DNS_RESOLVER_SELECTED:-}"

# Return 0 if the IP looks like a publicly routable IPv4 address.
# Conservative: anything unknown/unroutable returns false (so we default to dnsx).
_ip_is_public_ipv4() {
    local ip="$1"
    [[ -z "$ip" ]] && return 1

    # Split by dots using IFS (avoids BASH_REMATCH portability issues)
    local old_ifs="$IFS"
    IFS='.'
    local -a parts=($ip)
    IFS="$old_ifs"
    [[ ${#parts[@]} -ne 4 ]] && return 1

    local o1="${parts[0]}" o2="${parts[1]}" o3="${parts[2]}" o4="${parts[3]}"

    # Validate each octet is a number 0-255
    local o
    for o in "$o1" "$o2" "$o3" "$o4"; do
        [[ "$o" =~ ^[0-9]+$ ]] || return 1
        [[ "$o" -ge 0 ]] && [[ "$o" -le 255 ]] || return 1
    done

    # Non-routable ranges (using [[ -eq ]] to avoid (( )) ERR trap issues)
    [[ "$o1" -eq 0 ]] && return 1
    [[ "$o1" -eq 10 ]] && return 1
    [[ "$o1" -eq 127 ]] && return 1
    [[ "$o1" -eq 169 ]] && [[ "$o2" -eq 254 ]] && return 1  # link-local
    [[ "$o1" -eq 172 ]] && [[ "$o2" -ge 16 ]] && [[ "$o2" -le 31 ]] && return 1
    [[ "$o1" -eq 192 ]] && [[ "$o2" -eq 168 ]] && return 1
    [[ "$o1" -eq 100 ]] && [[ "$o2" -ge 64 ]] && [[ "$o2" -le 127 ]] && return 1  # CGNAT
    [[ "$o1" -ge 224 ]] && return 1  # multicast/reserved/broadcast

    return 0
}

# Check if a cloud metadata endpoint is reachable (AWS, GCP, Azure, DO, Hetzner, Oracle).
# Returns 0 if running on a cloud VPS, 1 otherwise.
_is_cloud_vps() {
    [[ "${DRY_RUN:-false}" == "true" ]] && return 1

    # Probe for the PRESENCE of a metadata service, not for its willingness to
    # serve an unauthenticated read (issue #1050).
    #
    # The previous `curl -sf` could not tell those apart. IMDSv2 has been the
    # default on new EC2 instances since 2019 and answers an unauthenticated GET
    # with 401; `-f` turns any >=400 status into exit 22, so a perfectly live
    # metadata service was read as "no metadata service at all". Every modern
    # AWS box was therefore misclassified as a home/NAT network — which is not
    # cosmetic: _can_use_puredns() uses this to pick the resolver, so
    # DNS_RESOLVER=auto silently downgraded puredns to dnsx on exactly the hosts
    # puredns exists to be fast on.
    #
    # %{http_code} is '000' only when no HTTP response was received at all
    # (connection refused, timeout, no route) — which is what a non-cloud host
    # gives for link-local 169.254.169.254. Any real status (200, 401, 403, 404)
    # proves something is listening there, and that is the whole question.
    local _code
    _code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' \
        http://169.254.169.254/ 2>/dev/null)
    [[ -n "$_code" ]] && [[ "$_code" != "000" ]] && return 0
    return 1
}

# Determine if puredns is safe to use (public network / cloud VPS).
# Returns 0 (true = puredns safe) or 1 (false = use dnsx).
# Priority:
#   1. Local IP is public                       → puredns (bare-metal / dedicated)
#   2. Cloud metadata endpoint responds          → puredns (cloud VPS with 1:1 NAT)
#   3. None of the above                         → dnsx   (home/office/unknown)
# Note: external IP check (curl ifconfig.me) is intentionally NOT used because
# home networks also have public IPs from the ISP, making it indistinguishable
# from cloud VPS. For rare VPS without metadata, use DNS_RESOLVER=puredns in config.
_can_use_puredns() {
    local ip="$1"
    _ip_is_public_ipv4 "$ip" && return 0
    _is_cloud_vps && return 0
    return 1
}

# Initialize and cache DNS resolver selection (evaluate once per run).
# Must be called after config is loaded.
init_dns_resolver() {
    local mode="${DNS_RESOLVER:-auto}"
    local ip
    ip=$(_get_local_ip)

    # Expose NAT detection and local IP for UI/logging.
    RECON_LOCAL_IP="${ip:-}"
    RECON_BEHIND_NAT="yes"
    if _can_use_puredns "$ip"; then
        RECON_BEHIND_NAT="no"
    fi
    export RECON_LOCAL_IP RECON_BEHIND_NAT

    DNS_RESOLVER_SELECTED=""
    case "$mode" in
        puredns|dnsx)
            DNS_RESOLVER_SELECTED="$mode"
            ;;
        auto|"")
            if [[ "$RECON_BEHIND_NAT" == "no" ]]; then
                DNS_RESOLVER_SELECTED="puredns"
            else
                DNS_RESOLVER_SELECTED="dnsx"
            fi
            ;;
        *)
            print_warnf "DNS_RESOLVER invalid: '%s' (use auto|puredns|dnsx). Defaulting to auto." "$mode"
            if [[ "$RECON_BEHIND_NAT" == "no" ]]; then
                DNS_RESOLVER_SELECTED="puredns"
            else
                DNS_RESOLVER_SELECTED="dnsx"
            fi
            ;;
    esac

    export DNS_RESOLVER_SELECTED
    printf "[%s] DNS resolver selected: %s (DNS_RESOLVER=%s, local_ip=%s, behind_nat=%s)\n" \
        "$(date +'%Y-%m-%d %H:%M:%S')" "$DNS_RESOLVER_SELECTED" "${DNS_RESOLVER:-auto}" "${ip:-}" "$RECON_BEHIND_NAT" >>"${LOGFILE:-/dev/null}"
}

# Select DNS resolver based on DNS_RESOLVER config and NAT detection.
# Returns "puredns" or "dnsx".
_select_dns_resolver() {
    local mode="${DNS_RESOLVER:-auto}"
    case "$mode" in
        puredns) echo "puredns" ;;
        dnsx)    echo "dnsx" ;;
        auto|"")
            if [[ -n "${DNS_RESOLVER_SELECTED:-}" ]]; then
                echo "$DNS_RESOLVER_SELECTED"
            elif _is_behind_nat; then
                echo "dnsx"
            else
                echo "puredns"
            fi
            ;;
        *)
            # Invalid values behave like auto (safe default behind NAT/unknown networks).
            if [[ -n "${DNS_RESOLVER_SELECTED:-}" ]]; then
                echo "$DNS_RESOLVER_SELECTED"
            elif _is_behind_nat; then
                echo "dnsx"
            else
                echo "puredns"
            fi
            ;;
    esac
}

# Return 0 when timeout should be enforced, 1 when disabled.
# Usage: _dns_timeout_enabled <timeout_value>
_dns_timeout_enabled() {
    case "${1:-0}" in
        "" | 0 | 0s | 0m | 0h | 0d) return 1 ;;
        *) return 0 ;;
    esac
}

# Ensure resolver files required by the selected resolver mode are present.
# Usage: _ensure_dns_resolver_files <dnsx|puredns>
_ensure_dns_resolver_files() {
    local resolver_mode="$1"

    case "$resolver_mode" in
        dnsx)
            if [[ ! -s "$resolvers_trusted" ]]; then
                print_errorf "Missing required trusted resolvers file for dnsx: %s" "$resolvers_trusted"
                return 1
            fi
            ;;
        puredns)
            if [[ ! -s "$resolvers" ]]; then
                print_errorf "Missing required resolvers file for puredns: %s" "$resolvers"
                return 1
            fi
            if [[ ! -s "$resolvers_trusted" ]]; then
                print_errorf "Missing required trusted resolvers file for puredns: %s" "$resolvers_trusted"
                return 1
            fi
            ;;
        *)
            print_errorf "Unsupported DNS resolver mode: %s" "$resolver_mode"
            return 1
            ;;
    esac

    return 0
}

# Execute DNS command with heartbeat and optional hard-timeout.
# Usage: _run_dns_with_heartbeat <label> <timeout_value> <command...>
_run_dns_with_heartbeat() {
    local label="$1"
    local timeout_value="$2"
    shift 2

    local heartbeat_interval="${DNS_HEARTBEAT_INTERVAL_SECONDS:-20}"
    [[ "$heartbeat_interval" =~ ^[0-9]+$ ]] || heartbeat_interval=20

    if _dns_timeout_enabled "$timeout_value"; then
        if [[ -n "${TIMEOUT_CMD:-}" ]]; then
            run_with_heartbeat "$label" "$heartbeat_interval" "$TIMEOUT_CMD" -k 10s "$timeout_value" "$@"
        else
            warn_once "dns-timeout-command-missing" "DNS timeout requested but timeout command is unavailable; continuing without hard timeout."
            run_with_heartbeat "$label" "$heartbeat_interval" "$@"
        fi
    else
        run_with_heartbeat "$label" "$heartbeat_interval" "$@"
    fi
}

# Resolve a list of domains using the auto-selected resolver.
# Usage: _resolve_domains <input_file> <output_file>
# Replaces all direct puredns resolve calls for consistent NAT-safe behavior.
_resolve_domains() {
    local input_file="$1"
    local output_file="$2"
    local resolver
    resolver=$(_select_dns_resolver)

    _ensure_dns_resolver_files "$resolver" || return 1

    if [[ "$resolver" == "dnsx" ]]; then
        local raw_output_file="${output_file}.dnsx.raw"
        if ! _run_dns_with_heartbeat "dns resolve (${resolver})" "${DNS_RESOLVE_TIMEOUT:-0}" \
            dnsx -l "$input_file" -silent -retry 2 \
            -t "${DNSX_THREADS:-25}" -rl "${DNSX_RATE_LIMIT:-100}" \
            -r "$resolvers_trusted" -wt 5 -o "$raw_output_file"; then
            rm -f "$raw_output_file" 2>/dev/null || true
            return 1
        fi
        if [[ -s "$raw_output_file" ]]; then
            cut -d' ' -f1 "$raw_output_file" | sort -u >"$output_file"
        else
            : >"$output_file"
        fi
        rm -f "$raw_output_file" 2>/dev/null || true
    else
        _run_dns_with_heartbeat "dns resolve (${resolver})" "${DNS_RESOLVE_TIMEOUT:-0}" \
            puredns resolve "$input_file" -w "$output_file" \
            -r "$resolvers" --resolvers-trusted "$resolvers_trusted" \
            -l "$PUREDNS_PUBLIC_LIMIT" --rate-limit-trusted "$PUREDNS_TRUSTED_LIMIT" \
            --wildcard-tests "$PUREDNS_WILDCARDTEST_LIMIT" \
            --wildcard-batch "$PUREDNS_WILDCARDBATCH_LIMIT" \
            || return 1
    fi
}

# Bruteforce subdomains using the auto-selected resolver.
# Usage: _bruteforce_domains <wordlist> <target_domain> <output_file>
#
# puredns bruteforce: wordlist + domain → massdns raw UDP → wildcard filter → trusted validation
# dnsx equivalent:    -d domain -w wordlist → Go net resolver → wt threshold filter
#
# dnsx -d domain -w wordlist generates word.domain combinations and resolves them.
# -wt 5 filters wildcards where >5 subdomains resolve to the same IP (basic heuristic,
# less robust than puredns's wildcard detection but safe for home routers).
_bruteforce_domains() {
    local wordlist="$1"
    local target_domain="$2"
    local output_file="$3"
    local resolver
    resolver=$(_select_dns_resolver)

    _ensure_dns_resolver_files "$resolver" || return 1

    if [[ "$resolver" == "dnsx" ]]; then
        local raw_output_file="${output_file}.dnsx.raw"
        if ! _run_dns_with_heartbeat "dns bruteforce (${resolver})" "${DNS_BRUTE_TIMEOUT:-0}" \
            dnsx -d "$target_domain" -w "$wordlist" -silent -retry 2 \
            -t "${DNSX_THREADS:-25}" -rl "${DNSX_RATE_LIMIT:-100}" \
            -r "$resolvers_trusted" -wt 5 -o "$raw_output_file"; then
            rm -f "$raw_output_file" 2>/dev/null || true
            return 1
        fi
        if [[ -s "$raw_output_file" ]]; then
            cut -d' ' -f1 "$raw_output_file" | sort -u >"$output_file"
        else
            : >"$output_file"
        fi
        rm -f "$raw_output_file" 2>/dev/null || true
    else
        _run_dns_with_heartbeat "dns bruteforce (${resolver})" "${DNS_BRUTE_TIMEOUT:-0}" \
            puredns bruteforce "$wordlist" "$target_domain" \
            -w "$output_file" -r "$resolvers" --resolvers-trusted "$resolvers_trusted" \
            -l "$PUREDNS_PUBLIC_LIMIT" --rate-limit-trusted "$PUREDNS_TRUSTED_LIMIT" \
            --wildcard-tests "$PUREDNS_WILDCARDTEST_LIMIT" \
            --wildcard-batch "$PUREDNS_WILDCARDBATCH_LIMIT" \
            || return 1
    fi
}