Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

86.6 KB
Raw Blame
#!/bin/bash
# reconFTW - Core framework module
# Contains: UI wrappers, banner, version check, tools check, logging,
#           output/notification, lifecycle (start_func/end_func), plugins, assets
# This file is sourced by reconftw.sh - do not execute directly

# shellcheck disable=SC2154  # Variables defined in reconftw.cfg

[[ -z "${SCRIPTPATH:-}" ]] && {
    echo "Error: This module must be sourced by reconftw.sh" >&2
    exit 1
}

# pt_header kept as no-op for backward compatibility (used by help())
pt_header() { :; }

# Ensure a safe default for early log redirections
# If LOGFILE is unset or empty, send logs to /dev/null until later initialization
: "${LOGFILE:=/dev/null}"

# List of environment-variable names whose VALUES should be redacted from logs
# and dry-run previews. Values resolved via ${!var} indirection from the
# current shell scope.
REDACT_VARS=(
    "SHODAN_API_KEY"
    "WHOISXML_API"
    "PDCP_API_KEY"
    "GITHUB_TOKEN"
    "GH_TOKEN"
    "GITLAB_TOKEN"
    "DISCORD_WEBHOOK_URL"
    "SLACK_WEBHOOK_URL"
    "SLACK_BOT_TOKEN"
    "TELEGRAM_BOT_TOKEN"
    "slack_auth"
    "telegram_key"
    "telegram_api_key"
    "discord_url"
    "discord_webhook_url"
    "XSS_SERVER"
    "COLLAB_SERVER"
)

# Values (not names) registered at runtime from config files or CLI parsing.
# Used when a secret lives in a local var that `${!var}` can't reach from
# inside redact_secrets, e.g. telegram_key parsed inside sendToNotify().
REGISTERED_SECRETS=()

# register_secret()
# Description: Adds a raw secret VALUE to REGISTERED_SECRETS so later redaction
# passes will replace it with [REDACTED]. Safe to call repeatedly (deduped).
# Arguments: $1 - secret value
function register_secret() {
    local value="$1"
    [[ -z "$value" || ${#value} -le 4 ]] && return 0

    local existing
    for existing in "${REGISTERED_SECRETS[@]}"; do
        [[ "$existing" == "$value" ]] && return 0
    done
    REGISTERED_SECRETS+=("$value")
}

# redact_secrets()
# Description: Redacts sensitive values from a string. Consults both the
# REDACT_VARS list (env-var-name lookups) and REGISTERED_SECRETS (raw values
# from config files).
# Arguments: $1 - String to redact
# Returns: Redacted string via stdout
function redact_secrets() {
    local text="$1"
    local redacted="$text"
    local var value secret

    for var in "${REDACT_VARS[@]}"; do
        value="${!var:-}"
        if [[ -n "$value" && ${#value} -gt 4 ]]; then
            redacted="${redacted//$value/[REDACTED]}"
        fi
    done

    for secret in "${REGISTERED_SECRETS[@]}"; do
        if [[ -n "$secret" && ${#secret} -gt 4 ]]; then
            redacted="${redacted//$secret/[REDACTED]}"
        fi
    done

    echo "$redacted"
}

# _trace_redact_stream()
# Description: Reads xtrace lines from stdin, runs each through redact_secrets()
# and appends the scrubbed line to $LOGFILE. Used as the sink for
# BASH_XTRACEFD so SHOW_COMMANDS=true never writes raw secrets to disk.
_trace_redact_stream() {
    local line
    while IFS= read -r line; do
        if declare -F redact_secrets >/dev/null 2>&1; then
            line=$(redact_secrets "$line")
        fi
        printf '%s\n' "$line"
    done >>"$LOGFILE"
}

enable_command_trace() {
    # Enable bash xtrace to the current LOGFILE when SHOW_COMMANDS=true.
    # xtrace output is routed through _trace_redact_stream so registered
    # secrets (REDACT_VARS + REGISTERED_SECRETS) are replaced with [REDACTED]
    # before the line reaches disk.
    if [[ ${SHOW_COMMANDS:-false} != true ]]; then
        return
    fi
    [[ -z ${LOGFILE:-} ]] && return

    _print_status WARN "Command tracing enabled" "Logs routed through redact_secrets"

    # Close any previous trace descriptor (native bash {varname} redirect syntax)
    if [[ -n ${TRACE_FD:-} ]]; then
        exec {TRACE_FD}>&- 2>/dev/null || true
    fi

    # Open a new descriptor against a redaction filter (process substitution).
    # Each xtrace line passes through _trace_redact_stream before hitting LOGFILE.
    if ! exec {TRACE_FD}> >(_trace_redact_stream); then
        return
    fi
    export BASH_XTRACEFD=$TRACE_FD
    export PS4='+ ${BASH_SOURCE##*/}:${LINENO}: '
    set -x
}

function banner_grabber() {
    local banner_file="${SCRIPTPATH}/banners.txt"

    # Check if the banner file exists
    if [[ ! -f $banner_file ]]; then
        echo "Banner file not found: $banner_file" >&2
        return 1
    fi

    # Source the banner file
    # shellcheck source=/dev/null
    source "$banner_file"

    # Collect all banner variable names
    mapfile -t banner_vars < <(compgen -A variable | grep '^banner[0-9]\+$')

    # Check if any banners are available
    if [[ ${#banner_vars[@]} -eq 0 ]]; then
        echo "No banners found in $banner_file" >&2
        return 1
    fi

    # Select a random banner
    local rand_index=$((RANDOM % ${#banner_vars[@]}))
    local banner_var="${banner_vars[$rand_index]}"
    local banner_code="${!banner_var}"

    # Output the banner code
    printf "%b\n" "$banner_code"
}

function banner_grabber_compact() {
    local banner_file="${SCRIPTPATH}/banners.txt"

    if [[ ! -f $banner_file ]]; then
        echo "Banner file not found: $banner_file" >&2
        return 1
    fi

    # shellcheck source=/dev/null
    source "$banner_file"

    local -a compact_vars=()
    local v code line_count
    for v in $(compgen -A variable | grep '^banner[0-9]\+$'); do
        code="${!v}"
        line_count=$(printf "%s\n" "$code" | wc -l | tr -d ' ')
        if [[ "$line_count" =~ ^[0-9]+$ ]] && ((line_count <= 4)); then
            compact_vars+=("$v")
        fi
    done

    if [[ ${#compact_vars[@]} -eq 0 ]]; then
        return 1
    fi

    local rand_index=$((RANDOM % ${#compact_vars[@]}))
    local banner_var="${compact_vars[$rand_index]}"
    printf "%b\n" "${!banner_var}"
}

function banner() {
    local banner_code
    local term_lines=0
    if [[ -t 1 ]] && command -v tput >/dev/null 2>&1; then
        term_lines=$(tput lines 2>/dev/null || echo 0)
    fi

    if [[ "$term_lines" =~ ^[0-9]+$ ]] && ((term_lines > 0 && term_lines < 28)); then
        banner_code=$(banner_grabber_compact || banner_grabber || true)
    else
        banner_code=$(banner_grabber || true)
    fi

    if [[ -n "$banner_code" ]]; then
        printf "\n%b%s%b\n" "$bgreen" "$banner_code" "$reset"
    else
        printf "\n%bFailed to load banner.%b\n" "$bgreen" "$reset"
    fi
}

###############################################################################################################
################################################### TOOLS #####################################################
###############################################################################################################

function check_version() {

    # Check if git is installed
    if ! command -v git >/dev/null 2>&1; then
        _print_error "Git is not installed. Cannot check for updates"
        return 1
    fi

    # Check if current directory is a git repository
    if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
        _print_error "Current directory is not a git repository. Cannot check for updates"
        return 1
    fi

    # Fetch updates with a bounded timeout when available.
    # Do not run an unbounded fallback fetch after a timeout, otherwise callers
    # wrapped by `timeout` (tests/CI) can be killed before main logic runs.
    if [[ -n $TIMEOUT_CMD ]]; then
        if ! $TIMEOUT_CMD 10 git fetch >/dev/null 2>&1; then
            _print_error "Unable to check updates (git fetch timed out)"
            return 1
        fi
    elif ! git fetch >/dev/null 2>&1; then
        _print_error "Unable to check updates"
        return 1
    fi

    # Get current branch name
    local BRANCH
    BRANCH=$(git rev-parse --abbrev-ref HEAD)

    # Get upstream branch
    local UPSTREAM
    UPSTREAM=$(git rev-parse --abbrev-ref --symbolic-full-name "@{u}" 2>/dev/null)
    if [[ -z $UPSTREAM ]]; then
        _print_error "No upstream branch set for '${BRANCH}'. Cannot check for updates"
        return 1
    fi

    # Get local and remote commit hashes
    local LOCAL REMOTE
    LOCAL=$(git rev-parse HEAD)
    REMOTE=$(git rev-parse "$UPSTREAM")

    # Compare local and remote hashes
    if [[ $LOCAL != "$REMOTE" ]]; then
        _print_status WARN "New version available" "Run ./install.sh to update"
    fi
}

resolve_cloud_enum_runtime() {
    local -n runtime_cmd_ref="$1"
    local cloud_enum_python cloud_enum_script cloud_enum_requirements

    runtime_cmd_ref=()
    cloud_enum_python="${tools}/cloud_enum/venv/bin/python3"
    cloud_enum_script="${tools}/cloud_enum/cloud_enum.py"
    cloud_enum_requirements="${tools}/cloud_enum/requirements.txt"

    if [[ -x "$cloud_enum_python" ]] && [[ -f "$cloud_enum_script" ]]; then
        runtime_cmd_ref=("$cloud_enum_python" "$cloud_enum_script")
        return 0
    fi

    if command -v uv >/dev/null 2>&1 \
        && [[ -f "$cloud_enum_script" ]] \
        && [[ -f "$cloud_enum_requirements" ]]; then
        runtime_cmd_ref=(
            uv run
            --directory "${tools}/cloud_enum"
            --with-requirements "$cloud_enum_requirements"
            cloud_enum.py
        )
        return 0
    fi

    return 1
}

resolve_cloud_enum_execution_settings() {
    local caller="$1"
    local -n profile_ref="$2"
    local -n threads_ref="$3"
    local -n quickscan_ref="$4"
    local -n mutations_ref="$5"
    local -n brute_ref="$6"
    local fuzz_file script_file

    fuzz_file="${tools}/cloud_enum/enum_tools/fuzz.txt"
    script_file="${tools}/cloud_enum/cloud_enum.py"

    profile_ref=$(printf "%s" "${CLOUD_ENUM_S3_PROFILE:-optimized}" | tr '[:upper:]' '[:lower:]')
    threads_ref="${CLOUD_ENUM_S3_THREADS:-20}"
    if [[ ! "$threads_ref" =~ ^[0-9]+$ ]] || [[ "$threads_ref" -le 0 ]]; then
        _print_msg WARN "${caller}: invalid CLOUD_ENUM_S3_THREADS (${threads_ref}); using default 20."
        threads_ref=20
    fi

    case "$profile_ref" in
        optimized)
            quickscan_ref=true
            ;;
        exhaustive)
            quickscan_ref=false
            ;;
        *)
            _print_msg WARN "${caller}: invalid CLOUD_ENUM_S3_PROFILE (${profile_ref}); using optimized."
            profile_ref="optimized"
            quickscan_ref=true
            ;;
    esac

    if [[ "$quickscan_ref" != true ]] && [[ ! -r "$fuzz_file" ]]; then
        _print_msg WARN "${caller}: cloud_enum exhaustive profile requested but mutations file not found (${fuzz_file}); using optimized."
        profile_ref="optimized"
        quickscan_ref=true
    fi

    if [[ "$quickscan_ref" == true ]]; then
        if [[ -r "$fuzz_file" ]]; then
            mutations_ref="$fuzz_file"
            brute_ref="$fuzz_file"
        elif [[ -r "$script_file" ]]; then
            # cloud_enum validates -m/-b paths even in quickscan mode.
            mutations_ref="$script_file"
            brute_ref="$script_file"
        else
            _print_msg WARN "${caller}: cloud_enum support file not found (${fuzz_file} or ${script_file})."
            return 1
        fi
    else
        mutations_ref="$fuzz_file"
        brute_ref="$fuzz_file"
    fi

    return 0
}

build_cloud_enum_command() {
    local -n cmd_ref="$1"
    local caller="$2"
    local target_domain="$3"
    local output_json_path="${4:-}"
    local company_name cloud_enum_profile cloud_enum_mutations cloud_enum_brute cloud_enum_quickscan cloud_enum_threads
    local -a cloud_enum_runtime_cmd

    cmd_ref=()

    if ! resolve_cloud_enum_runtime cloud_enum_runtime_cmd; then
        _print_msg WARN "${caller}: cloud_enum runtime not found (checked ${tools}/cloud_enum local runtime and uv run)."
        return 1
    fi

    if ! resolve_cloud_enum_execution_settings "$caller" \
        cloud_enum_profile \
        cloud_enum_threads \
        cloud_enum_quickscan \
        cloud_enum_mutations \
        cloud_enum_brute; then
        return 1
    fi

    company_name=$(unfurl format %r <<<"$target_domain")
    cmd_ref=(
        env PYTHONWARNINGS=ignore
        "${cloud_enum_runtime_cmd[@]}"
        -k "$company_name"
        -k "$target_domain"
        -k "${target_domain%%.*}"
        -t "$cloud_enum_threads"
        -m "$cloud_enum_mutations"
        -b "$cloud_enum_brute"
    )

    if [[ -f "$resolvers" ]]; then
        cmd_ref+=(-nsf "$resolvers")
    fi
    if [[ "$cloud_enum_quickscan" == true ]]; then
        cmd_ref+=(-qs)
    fi
    if [[ -n "$output_json_path" ]]; then
        cmd_ref+=(-f json -l "$output_json_path")
    fi

    return 0
}

format_pending_tools_message() {
    local -a missing=("$@")
    local joined=""

    if (( ${#missing[@]} == 1 )); then
        printf "Pending tool: %s" "${missing[0]}"
        return 0
    fi

    printf -v joined "%s, " "${missing[@]}"
    joined="${joined%, }"
    printf "Pending tools: %s" "$joined"
}

function tools_installed() {
    local all_installed=true
    local missing_tools=()

    # Some vendored wordlists are stored compressed to keep the repo small.
    # Expand them on-demand before checking tool/file presence.
    ensure_wordlist_file "${subs_wordlist:-}" || true

    # Check environment variables
    local env_vars=("GOPATH" "GOROOT" "PATH")
    for var in "${env_vars[@]}"; do
        if [[ -z ${!var} ]]; then
            _print_status FAIL "${var} variable"
            all_installed=false
            missing_tools+=("$var environment variable")
        fi
    done

    # Define tools and their paths/commands
    declare -A tools_files=(
        ["dorks_hunter"]="${tools}/dorks_hunter/dorks_hunter.py"
        ["dorks_hunter_python"]="${tools}/dorks_hunter/venv/bin/python3"
        ["testssl.sh"]="${tools}/testssl.sh/testssl.sh"
        ["CMSeeK"]="${tools}/CMSeeK/cmseek.py"
        ["CMSeeK_python"]="${tools}/CMSeeK/venv/bin/python3"
        ["OneListForAll"]="$fuzz_wordlist"
        ["lfi_wordlist"]="$lfi_wordlist"
        ["ssti_wordlist"]="$ssti_wordlist"
        ["subs_wordlist"]="$subs_wordlist"
        ["subs_wordlist_big"]="$subs_wordlist_big"
        ["resolvers"]="$resolvers"
        ["resolvers_trusted"]="$resolvers_trusted"
        ["getjswords"]="${tools}/getjswords.py"
        ["JSA"]="${tools}/JSA/jsa.py"
        ["JSA_python"]="${tools}/JSA/venv/bin/python3"
        ["nmap-parse-output"]="${tools}/ultimate-nmap-parser/ultimate-nmap-parser.sh"
        ["regulator"]="${tools}/regulator/main.py"
        ["regulator_python"]="${tools}/regulator/venv/bin/python3"
        ["nomore403"]="${tools}/nomore403/nomore403"
        ["ffufPostprocessing"]="${tools}/ffufPostprocessing/ffufPostprocessing"
        ["spoofy"]="${tools}/Spoofy/spoofy.py"
        ["spoofy_python"]="${tools}/Spoofy/venv/bin/python3"
        ["swaggerspy"]="${tools}/SwaggerSpy/swaggerspy.py"
        ["swaggerspy_python"]="${tools}/SwaggerSpy/venv/bin/python3"
        ["LeakSearch"]="${tools}/LeakSearch/LeakSearch.py"
        ["LeakSearch_python"]="${tools}/LeakSearch/venv/bin/python3"
        ["msftrecon"]="${tools}/msftrecon/msftrecon/msftrecon.py"
        ["msftrecon_python"]="${tools}/msftrecon/venv/bin/python3"
        ["Scopify"]="${tools}/Scopify/scopify.py"
        ["Scopify_python"]="${tools}/Scopify/venv/bin/python3"
        ["EmailHarvester"]="${tools}/EmailHarvester/EmailHarvester.py"
        ["EmailHarvester_python"]="${tools}/EmailHarvester/venv/bin/python3"
        ["reconftw_ai"]="${tools}/reconftw_ai/reconftw_ai.py"
        ["reconftw_ai_python"]="${tools}/reconftw_ai/venv/bin/python3"
        ["ghleaks"]="${tools}/ghleaks/ghleaks"
        ["SSTImap"]="${tools}/SSTImap/sstimap.py"
        ["SSTImap_python"]="${tools}/SSTImap/venv/bin/python3"
    )

    declare -A tools_folders=(
        ["nuclei-templates"]="${NUCLEI_TEMPLATES_PATH}"
    )

    declare -A tools_commands=(
        ["python3"]="python3"
        ["curl"]="curl"
        ["wget"]="wget"
        ["zip"]="zip"
        ["gzip"]="gzip"
        ["nmap"]="nmap"
        ["dig"]="dig"
        ["timeout"]="${TIMEOUT_CMD:-timeout}"
        ["brutespray"]="brutespray"
        ["xnLinkFinder"]="xnLinkFinder"
        ["xnldorker"]="xnldorker"
        ["urlfinder"]="urlfinder"
        ["github-endpoints"]="github-endpoints"
        ["github-subdomains"]="github-subdomains"
        ["gitlab-subdomains"]="gitlab-subdomains"
        ["katana"]="katana"
        ["wafw00f"]="wafw00f"
        ["dnsvalidator"]="dnsvalidator"
        ["whois"]="whois"
        ["dnsx"]="dnsx"
        ["gotator"]="gotator"
        ["Nuclei"]="nuclei"
        ["gf"]="gf"
        ["postleaksNg"]="postleaksNg"
        ["Gxss"]="Gxss"
        ["subjs"]="subjs"
        ["ffuf"]="ffuf"
        ["Massdns"]="massdns"
        ["qsreplace"]="qsreplace"
        ["interlace"]="interlace"
        ["Anew"]="anew"
        ["unfurl"]="unfurl"
        ["crlfuzz"]="crlfuzz"
        ["Httpx"]="httpx"
        ["jq"]="jq"
        ["notify"]="notify"
        ["dalfox"]="dalfox"
        ["puredns"]="puredns"
        ["analyticsrelationships"]="analyticsrelationships"
        ["mapcidr"]="mapcidr"
        ["cdncheck"]="cdncheck"
        ["interactsh-client"]="interactsh-client"
        ["tlsx"]="tlsx"
        ["smap"]="smap"
	    ["gitdorks_go"]="gitdorks_go"
	    ["smugglex"]="smugglex"
	    ["dsieve"]="dsieve"
        ["inscope"]="inscope"
        ["enumerepo"]="enumerepo"
        ["Web-Cache-Vulnerability-Scanner"]="Web-Cache-Vulnerability-Scanner"
        ["subfinder"]="subfinder"
        ["ghauri"]="ghauri"
        ["hakip2host"]="hakip2host"
        ["crt"]="crt"
        ["titus"]="titus"
        ["trufflehog"]="trufflehog"
        ["s3scanner"]="s3scanner"
        ["mantra"]="mantra"
        ["nmapurls"]="nmapurls"
	    ["naabu"]="naabu"
	    ["porch-pirate"]="porch-pirate"
	    ["shortscan"]="shortscan"
        ["cewler"]="cewler"
        ["hakoriginfinder"]="hakoriginfinder"
        ["sourcemapper"]="sourcemapper"
        ["jsluice"]="jsluice"
        ["commix"]="commix"
        ["waymore"]="waymore"
        ["urless"]="urless"
        ["dnstake"]="dnstake"
        ["cent"]="cent"
        ["csprecon"]="csprecon"
        ["VhostFinder"]="VhostFinder"
        ["grpcurl"]="grpcurl"
        ["arjun"]="arjun"
        ["gqlspection"]="gqlspection"
        ["toxicache"]="toxicache"
        ["favirecon"]="favirecon"
        ["TInjA"]="TInjA"
        ["second-order"]="second-order"
        ["exifray"]="exifray"
        ["fray"]="fray"
        ["sj"]="sj"
    )

    # Check for tool files
    for tool in "${!tools_files[@]}"; do
        if [[ ! -f ${tools_files[$tool]} ]]; then
            #			printf "%b [*] %s\t\t[NO]%b\n" "$bred" "$tool" "$reset"
            all_installed=false
            missing_tools+=("$tool")
        fi
    done

    # Check for tool folders
    for folder in "${!tools_folders[@]}"; do
        if [[ ! -d ${tools_folders[$folder]} ]]; then
            # printf "%b [*] %s\t\t[NO]%b\n" "$bred" "$folder" "$reset"
            all_installed=false
            missing_tools+=("$folder") # Correctly pushing the folder name
        fi
    done

    # Check for tool commands
    for tool in "${!tools_commands[@]}"; do
        if ! command -v "${tools_commands[$tool]}" >/dev/null 2>&1; then
            #			printf "%b [*] %s\t\t[NO]%b\n" "$bred" "$tool" "$reset"
            all_installed=false
            missing_tools+=("$tool")
        fi
    done

    local -a cloud_enum_runtime_cmd=()
    if ! resolve_cloud_enum_runtime cloud_enum_runtime_cmd; then
        _print_msg WARN "cloud_enum runtime not available; cloud enumeration steps will be skipped."
    fi

    if [[ $all_installed == true ]]; then
        : # Tools check OK, no output
    else
        _print_msg WARN "$(format_pending_tools_message "${missing_tools[@]}")"

        mark_missing_tools_warn_once "${missing_tools[@]}"
    fi

    if [[ $CHECK_TOOLS_OR_EXIT == true && $all_installed != true ]]; then
        exit 2
    fi
}

# Mark warn_once keys for tools already reported in preflight to avoid duplicate warnings in runtime.
# Usage: mark_missing_tools_warn_once "${missing_tools[@]}"
mark_missing_tools_warn_once() {
    local missing=("$@")
    [[ ${#missing[@]} -eq 0 ]] && return 0

    # dnstake is reported in preflight as "Pending tool", so suppress duplicate subtakeover warning.
    if [[ " ${missing[*]} " == *" dnstake "* ]]; then
        if ! declare -p WARN_ONCE_KEYS >/dev/null 2>&1; then
            declare -gA WARN_ONCE_KEYS=()
        fi
        WARN_ONCE_KEYS["missing-tool-dnstake"]=1
    fi
}

###############################################################################################################
#################################### CRITICAL DEPENDENCIES CHECK ##############################################
###############################################################################################################

# Check critical dependencies required for basic operation
# This performs a quick check of essential tools before starting
function check_critical_dependencies() {
    local critical_tools=(
        "bash:Bash shell"
        "python3:Python 3"
        "curl:Curl"
        "git:Git"
        "jq:JQ JSON processor"
    )

    local missing_critical=()

    for item in "${critical_tools[@]}"; do
        local tool="${item%%:*}"
        if ! command -v "$tool" >/dev/null 2>&1; then
            missing_critical+=("$tool")
        fi
    done

    if (( ${#missing_critical[@]} > 0 )); then
        _print_msg WARN "Missing critical dependencies: ${missing_critical[*]}"
        return 1
    fi

    return 0
}


###############################################################################################################
####################################### LOGGING ##################################################
###############################################################################################################

# Structured JSON logging (optional, controlled by STRUCTURED_LOGGING config)
STRUCTURED_LOGGING=${STRUCTURED_LOGGING:-false}
STRUCTURED_LOG_FILE=""

# Initialize structured logging
function log_init() {
    [[ "$STRUCTURED_LOGGING" != "true" ]] && return 0

    STRUCTURED_LOG_FILE="${dir}/.log/structured_$(date +%Y%m%d_%H%M%S).jsonl"
    mkdir -p "$(dirname "$STRUCTURED_LOG_FILE")"

    _print_status OK "Structured logging enabled" "$STRUCTURED_LOG_FILE"
}

# Log structured JSON event
# Usage: log_json <level> <function> <message> [key1=val1] [key2=val2] ...
function log_json() {
    [[ "$STRUCTURED_LOGGING" != "true" ]] && return 0
    [[ -z "$STRUCTURED_LOG_FILE" ]] && return 0

    local level="$1"
    local function="$2"
    local message="$3"
    shift 3

    # Build JSON object with all key-value pairs in a single jq invocation
    local -a jq_args=(
        --arg ts "$(date -Iseconds)"
        --arg lvl "$level"
        --arg fn "$function"
        --arg msg "$message"
        --arg domain "${domain:-N/A}"
    )
    local jq_expr='{timestamp:$ts, level:$lvl, function:$fn, message:$msg, domain:$domain'
    local idx=0
    for kv in "$@"; do
        if [[ "$kv" =~ ^([^=]+)=(.+)$ ]]; then
            jq_args+=(--arg "k${idx}" "${BASH_REMATCH[1]}" --arg "v${idx}" "${BASH_REMATCH[2]}")
            jq_expr+=", (\$k${idx}): \$v${idx}"
            ((idx++))
        fi
    done
    jq_expr+='}'

    jq -n "${jq_args[@]}" "$jq_expr" >>"$STRUCTURED_LOG_FILE"
}

# Convenience wrappers for different log levels
function log_info() {
    log_json "INFO" "${FUNCNAME[1]:-main}" "$@"
}

function log_warn() {
    log_json "WARN" "${FUNCNAME[1]:-main}" "$@"
}

function log_error() {
    log_json "ERROR" "${FUNCNAME[1]:-main}" "$@"
}

function log_success() {
    log_json "SUCCESS" "${FUNCNAME[1]:-main}" "$@"
}

###############################################################################################################
############################################ PROGRESS / ETA ###################################################
###############################################################################################################

_PROGRESS_TOTAL_STEPS=0
_PROGRESS_CURRENT_STEP=0
_PROGRESS_START_TIME=0

# Initialize progress tracking
# Usage: progress_init <total_steps> [total_est_seconds]
function progress_init() {
    _PROGRESS_TOTAL_STEPS=${1:-0}
    _PROGRESS_CURRENT_STEP=0
    _PROGRESS_START_TIME=$(date +%s)
    _PROGRESS_TOTAL_EST=${2:-0}
}

# Adjust total progress steps at runtime (for conditional stage skipping)
# Usage: progress_adjust_total <delta_steps>
function progress_adjust_total() {
    local delta="${1:-0}"
    _PROGRESS_TOTAL_STEPS=$((_PROGRESS_TOTAL_STEPS + delta))
    if [[ $_PROGRESS_TOTAL_STEPS -lt 1 ]]; then
        _PROGRESS_TOTAL_STEPS=1
    fi
}

# Module-level progress tracker
# Usage: progress_module "OSINT"
# Shows: [3/7] 43% | OSINT complete | OK:8 WARN:0 FAIL:0 SKIP:2
_PROGRESS_MODULE_CURRENT=0
_PROGRESS_MODULE_TOTAL=0

function progress_module_init() {
    _PROGRESS_MODULE_TOTAL=${1:-0}
    _PROGRESS_MODULE_CURRENT=0
}

function progress_module() {
    [[ "${OUTPUT_VERBOSITY:-1}" -lt 1 ]] && return 0
    ((_PROGRESS_MODULE_CURRENT++)) || true
    local module_name="${1:-}"
    local pct=0
    if [[ $_PROGRESS_MODULE_TOTAL -gt 0 ]]; then
        pct=$((_PROGRESS_MODULE_CURRENT * 100 / _PROGRESS_MODULE_TOTAL))
    fi
    local counters=""
    if declare -F ui_counts_summary >/dev/null 2>&1; then
        counters=$(ui_counts_summary)
    fi
    printf "  %b[%d/%d] %d%% │ %s complete │ %s%b\n\n" \
        "${bblue:-}" "$_PROGRESS_MODULE_CURRENT" "$_PROGRESS_MODULE_TOTAL" "$pct" \
        "$module_name" "$counters" "${reset:-}"
}

###############################################################################################################
######################################### INCREMENTAL MODE ####################################################
###############################################################################################################

# Incremental mode - only scan new assets since last run
# This mode compares current results with previous runs and only processes new findings

INCREMENTAL_MODE=${INCREMENTAL_MODE:-false}
INCREMENTAL_DIR=".incremental"

# Initialize incremental mode structure
function incremental_init() {
    [[ "$INCREMENTAL_MODE" != "true" ]] && return 0

    mkdir -p "$INCREMENTAL_DIR"/{subdomains,webs,hosts,vulns,previous}

    print_notice INFO "incremental" "Incremental mode enabled - will only process new findings"
}

# Save current state for next incremental run
# Usage: incremental_save <type> <file>
function incremental_save() {
    [[ "$INCREMENTAL_MODE" != "true" ]] && return 0

    local type="$1"
    local file="$2"

    if [[ -s "$file" ]]; then
        cp "$file" "$INCREMENTAL_DIR/previous/${type}_$(date +%Y%m%d_%H%M%S).txt"
        cp "$file" "$INCREMENTAL_DIR/previous/${type}_latest.txt"
        printf "%b[%s] Saved %s state for incremental mode%b\n" \
            "$bgreen" "$(date +'%Y-%m-%d %H:%M:%S')" "$type" "$reset" >>"$LOGFILE"
    fi
}

# Get new items compared to previous run
# Usage: incremental_diff <type> <current_file> <output_file>
# Returns: 0 if new items found, 1 if no new items
function incremental_diff() {
    [[ "$INCREMENTAL_MODE" != "true" ]] && {
        cp "$2" "$3" 2>/dev/null
        return 0
    }

    local type="$1"
    local current_file="$2"
    local output_file="$3"
    local previous_file="$INCREMENTAL_DIR/previous/${type}_latest.txt"

    if [[ ! -f "$previous_file" ]]; then
        # First run, all items are new
        cp "$current_file" "$output_file" 2>/dev/null
        local count
        count=$(wc -l <"$output_file" 2>/dev/null || echo 0)
        print_notice INFO "incremental" "First run for ${type}: ${count} items total"
        return 0
    fi

    # Compare and get only new items
    if [[ -s "$current_file" ]]; then
        comm -13 <(sort -u "$previous_file") <(sort -u "$current_file") | sed '/^$/d' >"$output_file"
        local new_count total_count previous_count
        new_count=$(wc -l <"$output_file" 2>/dev/null || echo 0)
        total_count=$(wc -l <"$current_file" 2>/dev/null || echo 0)
        previous_count=$(wc -l <"$previous_file" 2>/dev/null || echo 0)

        print_notice INFO "incremental" "${type}: ${new_count} new (previous: ${previous_count}, total: ${total_count})"

        if [[ $new_count -eq 0 ]]; then
            return 1 # No new items
        fi
        return 0 # New items found
    else
        touch "$output_file"
        return 1
    fi
}

# Generate incremental report
function incremental_report() {
    [[ "$INCREMENTAL_MODE" != "true" ]] && return 0

    local report_file
    local new_subs new_webs
    local subs_diff webs_diff
    report_file="$INCREMENTAL_DIR/incremental_report_$(date +%Y%m%d_%H%M%S).txt"

    subs_diff="$(mktemp 2>/dev/null || echo /tmp/incremental_subs.$$)"
    webs_diff="$(mktemp 2>/dev/null || echo /tmp/incremental_webs.$$)"
    : >"$subs_diff"
    : >"$webs_diff"

    if [[ -f "$INCREMENTAL_DIR/previous/subdomains_latest.txt" ]]; then
        comm -13 <(sort -u "$INCREMENTAL_DIR/previous/subdomains_latest.txt" 2>/dev/null || true) \
            <(sort -u "subdomains/subdomains.txt" 2>/dev/null || true) >"$subs_diff"
        new_subs=$(wc -l <"$subs_diff" | tr -d ' ')
    else
        new_subs=0
    fi

    if [[ -f "$INCREMENTAL_DIR/previous/webs_latest.txt" ]]; then
        comm -13 <(sort -u "$INCREMENTAL_DIR/previous/webs_latest.txt" 2>/dev/null || true) \
            <(sort -u "webs/webs.txt" 2>/dev/null || true) >"$webs_diff"
        new_webs=$(wc -l <"$webs_diff" | tr -d ' ')
    else
        new_webs=0
    fi

    {
        echo "==============================================="
        echo "Incremental Scan Report"
        echo "Domain: $domain"
        echo "Date: $(date +'%Y-%m-%d %H:%M:%S')"
        echo "==============================================="
        echo ""
        echo "New Subdomains: $new_subs"
        if [[ $new_subs -gt 0 ]]; then
            echo "---"
            head -20 "$subs_diff"
            [[ $new_subs -gt 20 ]] && echo "... and $((new_subs - 20)) more"
            echo ""
        fi
        echo "New Webs: $new_webs"
        if [[ $new_webs -gt 0 ]]; then
            echo "---"
            head -20 "$webs_diff"
            [[ $new_webs -gt 20 ]] && echo "... and $((new_webs - 20)) more"
            echo ""
        fi
        echo "==============================================="
        echo "Full report saved to: $report_file"
        echo "==============================================="
    } >"$report_file"

    rm -f "$subs_diff" "$webs_diff" 2>/dev/null || true
    _print_status INFO "incremental_report" "subs ${new_subs}, webs ${new_webs} -> ${report_file}"
}

# Hash helper for alert fingerprinting.
_hash_string() {
    local input="$1"
    if command -v sha256sum >/dev/null 2>&1; then
        printf '%s' "$input" | sha256sum | awk '{print $1}'
    else
        printf '%s' "$input" | shasum -a 256 | awk '{print $1}'
    fi
}

_monitor_severity_rank() {
    case "${1,,}" in
        critical) echo 5 ;;
        high) echo 4 ;;
        medium) echo 3 ;;
        low) echo 2 ;;
        info) echo 1 ;;
        *) echo 0 ;;
    esac
}

_monitor_selected_severities() {
    local min_sev="${MONITOR_MIN_SEVERITY:-high}"
    local min_rank
    min_rank=$(_monitor_severity_rank "$min_sev")
    if [[ "$min_rank" -eq 0 ]]; then
        log_note "monitor: invalid MONITOR_MIN_SEVERITY='${min_sev}', defaulting to high" "${FUNCNAME[0]}" "${LINENO}"
        min_rank=4
    fi

    local sev rank
    for sev in critical high medium low info; do
        rank=$(_monitor_severity_rank "$sev")
        if ((rank >= min_rank)); then
            printf '%s\n' "$sev"
        fi
    done
}

_monitor_mark_alert_seen() {
    local category="$1"
    local value="$2"
    local seen_file="${ALERT_SEEN_FILE:-.incremental/alerts_seen.hashes}"
    local fp
    fp=$(_hash_string "${category}|${value}")
    mkdir -p "$(dirname "$seen_file")"
    touch "$seen_file"

    if [[ "${ALERT_SUPPRESSION:-true}" == "true" ]] && grep -Fxq "$fp" "$seen_file"; then
        return 1
    fi
    echo "$fp" >>"$seen_file"
    return 0
}

_monitor_alert_summary() {
    local snap="$1"
    local critical_u=0 high_u=0 medium_u=0 low_u=0 info_u=0
    local critical_s=0 high_s=0 medium_s=0 low_s=0 info_s=0
    local nuclei_u=0 nuclei_s=0 subs_u=0 webs_u=0 subs_s=0 webs_s=0
    local line

    local sev
    while IFS= read -r sev; do
        [[ -z "$sev" ]] && continue
        local sev_u=0 sev_s=0
        local sev_file="$snap/${sev}_new.txt"
        if [[ -s "$sev_file" ]]; then
            while IFS= read -r line; do
                [[ -z "$line" ]] && continue
                if _monitor_mark_alert_seen "$sev" "$line"; then
                    ((sev_u++))
                else
                    ((sev_s++))
                fi
            done <"$sev_file"
        fi
        case "$sev" in
            critical) critical_u=$sev_u; critical_s=$sev_s ;;
            high) high_u=$sev_u; high_s=$sev_s ;;
            medium) medium_u=$sev_u; medium_s=$sev_s ;;
            low) low_u=$sev_u; low_s=$sev_s ;;
            info) info_u=$sev_u; info_s=$sev_s ;;
        esac
        nuclei_u=$((nuclei_u + sev_u))
        nuclei_s=$((nuclei_s + sev_s))
    done < <(_monitor_selected_severities)

    if [[ -s "$snap/subdomains_new.txt" ]]; then
        while IFS= read -r line; do
            [[ -z "$line" ]] && continue
            if _monitor_mark_alert_seen "subdomain" "$line"; then ((subs_u++)); else ((subs_s++)); fi
        done <"$snap/subdomains_new.txt"
    fi
    if [[ -s "$snap/webs_new.txt" ]]; then
        while IFS= read -r line; do
            [[ -z "$line" ]] && continue
            if _monitor_mark_alert_seen "web" "$line"; then ((webs_u++)); else ((webs_s++)); fi
        done <"$snap/webs_new.txt"
    fi

    if command -v jq >/dev/null 2>&1; then
        jq -n \
            --arg ts "$(date -Iseconds)" \
            --arg cycle "${MONITOR_CYCLE:-0}" \
            --arg min_sev "${MONITOR_MIN_SEVERITY:-high}" \
            --argjson nuclei_new "$nuclei_u" \
            --argjson critical_new "$critical_u" \
            --argjson high_new "$high_u" \
            --argjson medium_new "$medium_u" \
            --argjson low_new "$low_u" \
            --argjson info_new "$info_u" \
            --argjson subdomains_new "$subs_u" \
            --argjson webs_new "$webs_u" \
            --argjson nuclei_suppressed "$nuclei_s" \
            --argjson critical_suppressed "$critical_s" \
            --argjson high_suppressed "$high_s" \
            --argjson medium_suppressed "$medium_s" \
            --argjson low_suppressed "$low_s" \
            --argjson info_suppressed "$info_s" \
            --argjson subdomains_suppressed "$subs_s" \
            --argjson webs_suppressed "$webs_s" \
            '{
                timestamp:$ts,
                cycle:($cycle|tonumber),
                monitor_min_severity:$min_sev,
                alerts:{
                    nuclei_new:$nuclei_new,
                    critical_new:$critical_new,
                    high_new:$high_new,
                    medium_new:$medium_new,
                    low_new:$low_new,
                    info_new:$info_new,
                    subdomains_new:$subdomains_new,
                    webs_new:$webs_new
                },
                suppressed:{
                    nuclei:$nuclei_suppressed,
                    critical:$critical_suppressed,
                    high:$high_suppressed,
                    medium:$medium_suppressed,
                    low:$low_suppressed,
                    info:$info_suppressed,
                    subdomains:$subdomains_suppressed,
                    webs:$webs_suppressed
                }
            }' >"$snap/alerts.json"
    fi

    printf '%s %s %s %s %s %s %s %s\n' \
        "$nuclei_u" "$subs_u" "$webs_u" "$nuclei_s" \
        "$subs_s" "$webs_s" "$critical_u" "$high_u"
}

# Monitor snapshots and delta tracking.
# Stores historical snapshots under .incremental/history/<timestamp>/ and computes deltas vs latest.
function monitor_snapshot() {
    [[ "${MONITOR_MODE:-false}" != "true" ]] && return 0

    ensure_dirs .incremental/history || return 1
    local ts snap prev_link prev_dir
    ts=$(date +%Y%m%d_%H%M%S)
    snap=".incremental/history/${ts}"
    # Second-resolution timestamps collide whenever two cycles land in the same
    # second. `mkdir -p` then SUCCEEDS on the existing directory, this cycle's
    # copies overwrite the previous snapshot, and `latest` still points at it —
    # so the delta below is computed against the very files just overwritten.
    # Every count comes out zero, the baseline needed to detect the next change
    # is gone, and nothing reports an error. Disambiguate before creating.
    local _snap_seq=1
    while [[ -e "$snap" ]]; do
        _snap_seq=$((_snap_seq + 1))
        snap=".incremental/history/${ts}-${_snap_seq}"
    done
    mkdir -p "$snap"

    # Snapshot key artifacts for this cycle.
    local src
    for src in \
        "subdomains/subdomains.txt" \
        "webs/webs_all.txt" \
        "report/report.json" \
        "hotlist.txt"; do
        if [[ -s "$src" ]]; then
            cp "$src" "$snap/$(basename "$src")"
        fi
    done
    local -a monitor_severities=()
    mapfile -t monitor_severities < <(_monitor_selected_severities)
    local sev
    for sev in "${monitor_severities[@]}"; do
        if [[ -s "nuclei_output/${sev}.txt" ]]; then
            cp "nuclei_output/${sev}.txt" "$snap/${sev}.txt"
        fi
    done

    prev_link=".incremental/history/latest"
    prev_dir=""
    if [[ -L "$prev_link" || -d "$prev_link" ]]; then
        prev_dir=$(cd "$prev_link" 2>/dev/null && pwd -P || true)
    fi

    local subs_new=0 webs_new=0
    local critical_new=0 high_new=0 medium_new=0 low_new=0 info_new=0
    local nuclei_new_total=0 delta_total=0
    if [[ -n "$prev_dir" ]]; then
        if [[ -s "$snap/subdomains.txt" ]] && [[ -s "$prev_dir/subdomains.txt" ]]; then
            comm -13 <(sort -u "$prev_dir/subdomains.txt") <(sort -u "$snap/subdomains.txt") >"$snap/subdomains_new.txt"
            subs_new=$(wc -l <"$snap/subdomains_new.txt" 2>/dev/null | tr -d ' ')
        fi
        if [[ -s "$snap/webs_all.txt" ]] && [[ -s "$prev_dir/webs_all.txt" ]]; then
            comm -13 <(sort -u "$prev_dir/webs_all.txt") <(sort -u "$snap/webs_all.txt") >"$snap/webs_new.txt"
            webs_new=$(wc -l <"$snap/webs_new.txt" 2>/dev/null | tr -d ' ')
        fi
        for sev in "${monitor_severities[@]}"; do
            local sev_new_file="$snap/${sev}_new.txt"
            local sev_count=0
            if [[ -s "$snap/${sev}.txt" ]] && [[ -s "$prev_dir/${sev}.txt" ]]; then
                comm -13 <(sort -u "$prev_dir/${sev}.txt") <(sort -u "$snap/${sev}.txt") >"$sev_new_file"
                sev_count=$(wc -l <"$sev_new_file" 2>/dev/null | tr -d ' ')
            fi
            case "$sev" in
                critical) critical_new=$sev_count ;;
                high) high_new=$sev_count ;;
                medium) medium_new=$sev_count ;;
                low) low_new=$sev_count ;;
                info) info_new=$sev_count ;;
            esac
            nuclei_new_total=$((nuclei_new_total + sev_count))
        done
    else
        # Baseline cycle: don't alert as "new", just record baseline.
        subs_new=0
        webs_new=0
        critical_new=0
        high_new=0
        medium_new=0
        low_new=0
        info_new=0
        nuclei_new_total=0
    fi

    subs_new=${subs_new:-0}
    webs_new=${webs_new:-0}
    critical_new=${critical_new:-0}
    high_new=${high_new:-0}
    medium_new=${medium_new:-0}
    low_new=${low_new:-0}
    info_new=${info_new:-0}
    nuclei_new_total=${nuclei_new_total:-0}
    delta_total=$((subs_new + webs_new + nuclei_new_total))
    if command -v jq >/dev/null 2>&1; then
        jq -n \
            --arg ts "$(date -Iseconds)" \
            --arg cycle "${MONITOR_CYCLE:-0}" \
            --arg baseline "$( [[ -z "$prev_dir" ]] && echo true || echo false )" \
            --arg min_sev "${MONITOR_MIN_SEVERITY:-high}" \
            --argjson subs_new "$subs_new" \
            --argjson webs_new "$webs_new" \
            --argjson critical_new "$critical_new" \
            --argjson high_new "$high_new" \
            --argjson medium_new "$medium_new" \
            --argjson low_new "$low_new" \
            --argjson info_new "$info_new" \
            --argjson nuclei_new_total "$nuclei_new_total" \
            --argjson total_new "$delta_total" \
            '{
                timestamp:$ts,
                cycle:($cycle|tonumber),
                baseline:($baseline=="true"),
                monitor_min_severity:$min_sev,
                deltas:{
                    subdomains_new:$subs_new,
                    webs_new:$webs_new,
                    high_findings_new:$high_new,
                    critical_findings_new:$critical_new,
                    medium_findings_new:$medium_new,
                    low_findings_new:$low_new,
                    info_findings_new:$info_new,
                    nuclei_findings_new_total:$nuclei_new_total,
                    total_new:$total_new
                }
            }' >"$snap/delta.json"
    fi

    if [[ -z "$prev_dir" ]]; then
        notification "Monitor baseline snapshot stored (${snap})" info
    elif [[ "$delta_total" -gt 0 ]]; then
        local alert_stats nuclei_u subs_u webs_u nuclei_s subs_s webs_s critical_u high_u
        alert_stats=$(_monitor_alert_summary "$snap")
        # IFS=' ' is REQUIRED, not stylistic. reconftw.sh:8 sets IFS=$'\n\t'
        # process-wide, so a bare `read` does not split on spaces: all eight
        # fields land in $nuclei_u and every numeric test below then dies with
        # "[[: 1 1 1 0 0 0 0 1: arithmetic syntax error". The alert summary was
        # therefore never parsed and monitor alerting could not work. Same
        # narrowing reconftw.sh:128 already applies for the axiom argv split.
        IFS=' ' read -r nuclei_u subs_u webs_u nuclei_s subs_s webs_s critical_u high_u <<<"$alert_stats"

        if [[ "${nuclei_u:-0}" -gt 0 ]]; then
            notification "[ALERT] New nuclei findings (>=${MONITOR_MIN_SEVERITY:-high}): ${nuclei_u:-0}" warn
        fi
        if [[ "${critical_u:-0}" -gt 0 || "${high_u:-0}" -gt 0 ]]; then
            notification "[ALERT] New high-impact findings: critical=${critical_u:-0}, high=${high_u:-0}" warn
        fi
        if [[ "${subs_u:-0}" -gt 0 || "${webs_u:-0}" -gt 0 ]]; then
            notification "New assets detected: subdomains=${subs_u:-0}, webs=${webs_u:-0}" good
        fi
        if [[ "${nuclei_u:-0}" -eq 0 && "${subs_u:-0}" -eq 0 && "${webs_u:-0}" -eq 0 ]]; then
            notification "Monitor detected deltas but all were suppressed by fingerprint history" info
        fi
    else
        notification "Monitor cycle: no changes detected" info
    fi

    # Update latest snapshot pointer.
    if [[ -L "$prev_link" || -d "$prev_link" ]]; then
        rm -rf -- "$prev_link"
    fi
    # Point at the directory actually written, not at $ts: the collision guard
    # above may have created ${ts}-2, and a pointer to $ts would then name a
    # DIFFERENT snapshot than the one just taken — the next cycle would diff
    # against the wrong baseline.
    ln -s "${snap##*/}" "$prev_link" 2>/dev/null || {
        cp -R "$snap" "$prev_link"
    }

    return 0
}

# Check if incremental mode should skip heavy operations
# Usage: incremental_should_skip
# Returns: 0 to skip, 1 to continue
function incremental_should_skip() {
    [[ "$INCREMENTAL_MODE" != "true" ]] && return 1 # Don't skip

    # Check if we have new findings
    local new_subs new_webs
    new_subs=$(cat .tmp/subs_new_count 2>/dev/null || echo 1)
    new_webs=$(cat .tmp/webs_new_count 2>/dev/null || echo 1)

    if [[ $new_subs -eq 0 && $new_webs -eq 0 ]]; then
        print_notice WARN "incremental" "No new assets found, skipping heavy scans"
        return 0 # Skip
    fi

    return 1 # Don't skip
}

###############################################################################################################

function zipSendOutputFolder {
    zip_name1=$(date +"%Y_%m_%d-%H.%M.%S")
    zip_name="${zip_name1}_${domain}.zip" 2>>"$LOGFILE" >/dev/null
    (cd "$dir" && zip -r "$zip_name" .) 2>>"$LOGFILE" >/dev/null
    _print_msg INFO "Sending zip file ${dir}/${zip_name}"
    if [[ -s "${dir}/$zip_name" ]]; then
        sendToNotify "$dir/$zip_name"
        rm -f "${dir}/$zip_name"
    else
        notification "No Zip file to send" warn
    fi
}

function isAsciiText {
    # shellcheck disable=SC2034  # IS_ASCII is consumed by callers after invoking this helper
    IS_ASCII=$([[ $(file "$1" | grep -o 'ASCII text$') == "ASCII text" ]] && echo "True" || echo "False")
}

function output() {
    mkdir -p "$dir_output"
    # Ensure both $dir and $dir_output are absolute paths
    dir="$(realpath "$dir")"
    dir_output="$(realpath "$dir_output")"

    # Prevent accidental deletion if $dir_output is a parent of $dir
    if [[ $dir == "$dir_output"* ]]; then
        _print_error "Output directory is a parent of the working directory. Aborting to prevent data loss."
        return 1
    fi

    cp -r "$dir" "$dir_output"

    # Only delete if source and destination are clearly different and safe
    # Safety: refuse to delete outside of Recon/ directory
    if [[ "$(dirname "$dir")" != "$dir_output" ]]; then
        if [[ "$dir" == "${SCRIPTPATH}/Recon/"* ]]; then
            rm -rf -- "$dir"
        else
            _print_error "Refusing to delete directory outside Recon/: $dir"
            return 1
        fi
    fi
}

function remove_big_files() {
    rm -rf .tmp/gotator*.txt 2>>"$LOGFILE"
    rm -rf .tmp/brute_recursive_wordlist.txt 2>>"$LOGFILE"
    rm -rf .tmp/subs_dns_tko.txt 2>>"$LOGFILE"
    rm -rf .tmp/subs_no_resolved.txt .tmp/subdomains_dns.txt .tmp/brute_dns_tko.txt .tmp/scrap_subs.txt .tmp/analytics_subs_clean.txt .tmp/gotator1.txt .tmp/gotator2.txt .tmp/passive_recursive.txt .tmp/brute_recursive_wordlist.txt .tmp/gotator1_recursive.txt .tmp/gotator2_recursive.txt 2>>"$LOGFILE"
    find .tmp -type f -size +200M -exec rm -f {} + 2>>"$LOGFILE"
}

function notification() {
    if [[ -n $1 ]] && [[ -n $2 ]]; then
        local -a notify_cmd=()
        if [[ $NOTIFICATION == true ]]; then
            notify_cmd=(notify -silent)
        fi
        local level="INFO"
        case $2 in
            info) level="INFO" ;;
            warn) level="WARN" ;;
            error) level="FAIL" ;;
            good) level="OK" ;;
        esac
 
        if declare -F ui_log_jsonl >/dev/null 2>&1; then
            local ui_level="INFO"
            case "$2" in
                info) ui_level="INFO" ;;
                warn) ui_level="WARN" ;;
                error) ui_level="ERROR" ;;
                good) ui_level="SUCCESS" ;;
            esac
            ui_log_jsonl "$ui_level" "${FUNCNAME[1]:-main}" "$1"
        fi

        # Quiet mode (OUTPUT_VERBOSITY==0): only print errors to terminal
        # Normal mode (OUTPUT_VERBOSITY==1): only print warn/error to terminal
        # Verbose mode (OUTPUT_VERBOSITY>=2): print everything
        local should_print=false
        case "$2" in
            error)      should_print=true ;;
            warn)       [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && should_print=true ;;
            info|good)  [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && should_print=true ;;
        esac

        if [[ "$should_print" != true ]]; then
            # Still send to notify if enabled, just skip terminal print
            if [[ ${#notify_cmd[@]} -gt 0 ]]; then
                printf "%s" "[${level}] ${1} - ${domain}" | "${notify_cmd[@]}" >/dev/null 2>&1
            fi
            return 0
        fi
 
        # Print to terminal in structured format (no counters)
        print_notice "$level" "${FUNCNAME[1]:-notice}" "$1"
        if [[ "$level" == "WARN" || "$level" == "FAIL" ]]; then
            record_incident "$level" "${FUNCNAME[1]:-notice}" "$1"
        fi

        # Send to notify if notifications are enabled
        if [[ ${#notify_cmd[@]} -gt 0 ]]; then
            # Remove color codes for the notification
            printf "%s" "[${level}] ${1} - ${domain}" | "${notify_cmd[@]}" >/dev/null 2>&1
        fi
    fi
}

function transfer {
    # WARNING: This uploads recon results to an external service (bashupload.com)
    # Requires explicit opt-in via ALLOW_TRANSFER=true in config
    if [[ "${ALLOW_TRANSFER:-false}" != "true" ]]; then
        notification "transfer() disabled by default (uploads to external service). Set ALLOW_TRANSFER=true to enable." warn
        return 1
    fi

    if [[ $# -eq 0 ]]; then
        echo "No arguments specified.\nUsage:\n transfer <file|directory>\n ... | transfer <file_name>" >&2
        return 1
    fi

    if tty -s; then
        local file="$1"
        local file_name
        file_name=$(basename "$file")
        if [[ ! -e $file ]]; then
            echo "$file: No such file or directory" >&2
            return 1
        fi
        run_command tar -czvf "/tmp/$file_name" "$file" >/dev/null 2>&1 && run_command curl -s "https://bashupload.com/${file_name}.tgz" --data-binary "@/tmp/$file_name" | grep wget
    else
        local file_name="$1"
        run_command tar -czvf "/tmp/$file_name" "$file" >/dev/null 2>&1 && run_command curl -s "https://bashupload.com/${file_name}.tgz" --data-binary "@/tmp/$file_name" | grep wget
    fi
}

function sendToNotify {
    if [[ -z "${1}" ]]; then
        _print_status WARN "No file provided to send"
    else
        if [[ -z "${NOTIFY_CONFIG}" ]]; then
            NOTIFY_CONFIG=~/.config/notify/provider-config.yaml
        fi
        if [[ -n "$(find "${1}" -prune -size +8000000c)" ]]; then
            _print_msg WARN "${1} is larger than 8MB, sending over external service"
            transfer "${1}" | notify -silent
            return 0
        fi
        if grep -q '^ telegram\|^telegram\|^    telegram' "${NOTIFY_CONFIG}"; then
            notification "Sending ${domain} data over Telegram" info
            telegram_chat_id=$(sed -n '/^telegram:/,/^[^ ]/p' "${NOTIFY_CONFIG}" | sed -n 's/^[ ]*telegram_chat_id:[ ]*"\([^"]*\)".*/\1/p')
            telegram_key=$(sed -n '/^telegram:/,/^[^ ]/p' "${NOTIFY_CONFIG}" | sed -n 's/^[ ]*telegram_api_key:[ ]*"\([^"]*\)".*/\1/p')
            register_secret "${telegram_key}"
            run_command curl -F "chat_id=${telegram_chat_id}" -F "document=@${1}" "https://api.telegram.org/bot${telegram_key}/sendDocument" 2>>"$LOGFILE" >/dev/null
        fi
        if grep -q '^ discord\|^discord\|^    discord' "${NOTIFY_CONFIG}"; then
            notification "Sending ${domain} data over Discord" info
            discord_url=$(sed -n '/^discord:/,/^[^ ]/p' "${NOTIFY_CONFIG}" | sed -n 's/^[ ]*discord_webhook_url:[ ]*"\([^"]*\)".*/\1/p')
            register_secret "${discord_url}"
            run_command curl -v -i -H "Accept: application/json" -H "Content-Type: multipart/form-data" -X POST -F 'payload_json={"username": "test", "content": "hello"}' -F "file1=@${1}" "${discord_url}" 2>>"$LOGFILE" >/dev/null
        fi
        if [[ -n "${slack_channel}" ]] && [[ -n "${slack_auth}" ]]; then
            register_secret "${slack_auth}"
            notification "Sending ${domain} data over Slack" info
            run_command curl -F "file=@${1}" -F "initial_comment=reconftw zip file" -F "channels=${slack_channel}" -H "Authorization: Bearer ${slack_auth}" https://slack.com/api/files.upload 2>>"$LOGFILE" >/dev/null
        fi
    fi
}

function start_func() {
    # Mid-run disk-full guard (D-07/D-09): abort BEFORE any state write so the
    # Plan 01-01 EXIT trap clears .inprogress_* with no orphan for this function.
    _check_disk_mid_run || _abort_disk_full
    local current_date
    current_date=$(date +'%Y-%m-%d %H:%M:%S')
    echo "[$current_date] Start function: ${1} " >>"${LOGFILE}"
    # Use per-function start time to avoid race conditions in parallel mode
    local _fn_name="${1}"
    printf -v "_start_time_${_fn_name//[^a-zA-Z0-9_]/_}" '%s' "$(date +%s)"
    # Keep global $start for backward compat (serial mode)
    start=$(date +%s)
    # Resume sentinel (RESIL-01 / D-01): touch .inprogress_<fn> as a crash-leftover
    # marker. end_func removes it before touching .<fn>. The EXIT trap clears it
    # ONLY when end() sets _RECON_CLEAN_EXIT=true (clean traversal). SIGINT/SIGTERM
    # via cleanup_on_exit does NOT set the flag, so the sentinel survives and the
    # next run's resume banner fires (CR-01 / SC1 fix; see modules/modes.sh start()
    # and end() for the flag set/init points).
    if [[ -n "${called_fn_dir:-}" ]]; then
        touch "$called_fn_dir/.inprogress_${1}" 2>/dev/null || true
    fi
    log_json "INFO" "${1}" "Function started" "description=${2}"
    if declare -F ui_log_jsonl >/dev/null 2>&1; then
        ui_log_jsonl "INFO" "${1}" "Function started" "description=${2}"
    fi
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        _print_msg "INFO" "Running ${1}..."
    fi
}

function end_func() {
    local message="${1:-}"
    local fn="${2:-${FUNCNAME[1]:-unknown}}"
    local status="${3:-}"

    # Backward-compatible status handling:
    # - end_func "msg" "func" warn
    # - end_func "msg" warn
    if [[ -n "$status" ]]; then
        case "$status" in
            info|warn|error|good|OK|WARN|FAIL|SKIP|SKIP_CONFIG|SKIP_NOINPUT|CACHE_HIT)
                ;; # keep as-is
            *)
                status="OK"
                ;;
        esac
    else
        case "$fn" in
            info|warn|error|good|OK|WARN|FAIL|SKIP|SKIP_CONFIG|SKIP_NOINPUT|CACHE_HIT)
                status="$fn"
                fn="${FUNCNAME[1]:-unknown}"
                ;;
            *)
                status="OK"
                ;;
        esac
    fi

    # Resume sentinel (RESIL-01 / D-01): remove .inprogress_<fn> BEFORE touching
    # the .<fn> success checkpoint. A crash between the rm and the touch leaves
    # .<fn> absent — the existing checkpoint guard re-enters the function on the
    # next run. .<fn> is the source of truth; .inprogress_<fn> is a surface
    # indicator only.
    if [[ -n "${called_fn_dir:-}" ]]; then
        rm -f "$called_fn_dir/.inprogress_${fn}" 2>/dev/null || true
        touch "$called_fn_dir/.${fn}" 2>/dev/null || true
    fi
    local end
    end=$(date +%s)
    # Try per-function start time first (parallel-safe), fall back to global $start
    local _start_var="_start_time_${fn//[^a-zA-Z0-9_]/_}"
    local _fn_start="${!_start_var:-$start}"
    local runtime=""
    getElapsedTime "$_fn_start" "$end"
    record_func_timing "${fn}" "$((end - _fn_start))"
    local duration=$((end - _fn_start))
    local end_date
    end_date=$(date +'%Y-%m-%d %H:%M:%S')
    echo "[$end_date] End function: ${fn} " >>"${LOGFILE}"
    local badge="OK"
    local reason_code=""
    case "$status" in
        info|INFO) badge="INFO" ;;
        warn|WARN) badge="WARN" ;;
        error|ERROR) badge="FAIL" ;;
        good|SUCCESS) badge="OK" ;;
        OK|FAIL|SKIP|CACHE) badge="$status" ;;
        SKIP_CONFIG) badge="SKIP"; reason_code="config" ;;
        SKIP_NOINPUT) badge="SKIP"; reason_code="noinput" ;;
        CACHE_HIT) badge="CACHE"; reason_code="cache" ;;
    esac

    # Auto-normalize common "no input" skip patterns when caller didn't set explicit status.
    if [[ "$badge" == "OK" ]] && [[ -n "$message" ]]; then
        local msg_lc
        msg_lc=$(printf "%s" "$message" | tr '[:upper:]' '[:lower:]')
        if [[ "$msg_lc" == no\ * ]] && [[ "$msg_lc" == *"skip"* ]]; then
            badge="SKIP"
            reason_code="noinput"
        elif [[ "$msg_lc" == *"missing url candidates"* ]]; then
            badge="SKIP"
            reason_code="noinput"
        fi
    fi

    # Persist per-function final status for parallel aggregator.
    if [[ -n "${called_fn_dir:-}" ]]; then
        printf "%s\n" "$badge" >"${called_fn_dir}/.status_${fn}" 2>/dev/null || true
        if [[ -n "$reason_code" ]]; then
            printf "%s\n" "$reason_code" >"${called_fn_dir}/.status_reason_${fn}" 2>/dev/null || true
        fi
    fi

    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then
        _print_status "$badge" "${fn}" "${duration}s"
        if [[ -n "$reason_code" ]] && { [[ "$badge" != "CACHE" ]] || [[ "${SHOW_CACHE:-false}" == "true" ]]; }; then
            printf "         reason: %s\n" "$reason_code"
        fi
        # Show detail line for non-OK statuses when message is present
        if [[ -n "$message" ]] && [[ "$badge" != "OK" && "$badge" != "INFO" ]]; then
            printf "         %s\n" "$message"
            if [[ "$badge" == "FAIL" || "$badge" == "WARN" ]]; then
                record_incident "$badge" "$fn" "$message"
            fi
        fi
    fi
    log_json "SUCCESS" "${fn}" "Function completed" "runtime=${runtime}" "duration_sec=${duration}"
    if declare -F ui_log_jsonl >/dev/null 2>&1; then
        ui_log_jsonl "SUCCESS" "${fn}" "Function completed" "runtime=${runtime}" "duration_sec=${duration}"
    fi

    # Post-checkpoint disk guard (D-07/D-09): runs AFTER .<fn> and .status_<fn>
    # are durably written so the completed function leaves a full success record;
    # abort here protects the NEXT function from running with no headroom.
    _check_disk_mid_run || _abort_disk_full

    :
}

function start_subfunc() {
    current_date=$(date +'%Y-%m-%d %H:%M:%S')
    echo "[$current_date] Start subfunction: ${1} " >>"${LOGFILE}"
    start_sub=$(date +%s)
    log_json "INFO" "${1}" "Subfunction started" "description=${2}"
    if declare -F ui_log_jsonl >/dev/null 2>&1; then
        ui_log_jsonl "INFO" "${1}" "Subfunction started" "description=${2}"
    fi
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        _print_msg "INFO" "Running ${1}..."
    fi
}

function end_subfunc() {
    if [[ -n "${called_fn_dir:-}" ]]; then
        touch "$called_fn_dir/.${2}" 2>/dev/null || true
    fi
    end_sub=$(date +%s)
    getElapsedTime "$start_sub" "$end_sub"
    local duration=$((end_sub - start_sub))
    local status="${3:-OK}"
    local badge="$status"
    local reason_code=""
    case "$status" in
        SKIP_CONFIG) badge="SKIP"; reason_code="config" ;;
        SKIP_NOINPUT) badge="SKIP"; reason_code="noinput" ;;
        CACHE_HIT) badge="CACHE"; reason_code="cache" ;;
    esac

    # Persist per-subfunction final status for parallel aggregator.
    if [[ -n "${called_fn_dir:-}" ]]; then
        printf "%s\n" "$badge" >"${called_fn_dir}/.status_${2}" 2>/dev/null || true
        if [[ -n "$reason_code" ]]; then
            printf "%s\n" "$reason_code" >"${called_fn_dir}/.status_reason_${2}" 2>/dev/null || true
        fi
    fi

    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then
        _print_status "$badge" "${2}" "${duration}s"
        if [[ -n "$reason_code" ]] && { [[ "$badge" != "CACHE" ]] || [[ "${SHOW_CACHE:-false}" == "true" ]]; }; then
            printf "         reason: %s\n" "$reason_code"
        fi
    fi
    local end_sub_date
    end_sub_date=$(date +'%Y-%m-%d %H:%M:%S')
    echo "[$end_sub_date] End subfunction: ${1} " >>"${LOGFILE}"
    log_json "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"
    if declare -F ui_log_jsonl >/dev/null 2>&1; then
        ui_log_jsonl "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"
    fi
    :
}

function check_inscope() {
    cat "$1" | inscope >"${1}_tmp" && cp "${1}_tmp" "$1" && rm -f "${1}_tmp"
}

function maybe_update_nuclei() {
    # Update nuclei templates once per run per target directory
    mkdir -p .tmp
    local stamp_file=".tmp/.nuclei_updated"
    if [[ ! -f $stamp_file ]]; then
        if [[ -n ${NUCLEI_TEMPLATES_PATH-} ]]; then
            run_command nuclei -update-templates -update-template-dir "${NUCLEI_TEMPLATES_PATH}" 2>>"$LOGFILE" >/dev/null || true # non-fatal: template update failure shouldn't block scan
        else
            run_command nuclei -update 2>>"$LOGFILE" >/dev/null || true # non-fatal: template update failure shouldn't block scan
        fi
        touch "$stamp_file"
    fi
}

# Plugin framework
function plugins_load() {
    local plugdir="${SCRIPTPATH}/plugins"
    if [[ -d $plugdir ]]; then
        for f in "$plugdir"/*.sh; do
            # shellcheck source=/dev/null
            [[ -f $f ]] && source "$f" || true
        done
    fi
}

function plugins_emit() {
    local event="$1"
    shift || true
    if declare -F reconftw_plugins >/dev/null 2>&1; then
        reconftw_plugins "$event" "$@" || true
    fi
}

# Asset store
function append_asset() {
    # Usage: append_asset type key value extras...
    [[ $ASSET_STORE != true ]] && return 0
    local type="$1"
    shift
    local key="$1"
    shift
    local val="$1"
    shift
    local now
    now=$(date +'%Y-%m-%d %H:%M:%S')
    mkdir -p .tmp
    printf '{"type":"%s","%s":"%s","ts":"%s","source":"%s"}\n' \
        "$type" "$key" "${val//\"/\\\"}" "$now" "${FUNCNAME[1]}" >>assets.jsonl 2>/dev/null
}

function append_assets_from_file() {
    # Usage: append_assets_from_file type key file
    [[ $ASSET_STORE != true ]] && return 0
    local type="$1"
    local key="$2"
    local file="$3"
    [[ ! -s $file ]] && return 0
    while IFS= read -r line; do
        [[ -z $line ]] && continue
        append_asset "$type" "$key" "$line"
    done <"$file"
}

# Chunk helper
function process_in_chunks() {
    # process_in_chunks <file> <chunk_size> <command with _chunk_ placeholder>
    local infile="$1"
    local chunksize="$2"
    shift 2
    [[ ! -s $infile ]] && return 0
    local lines
    lines=$(wc -l <"$infile" 2>/dev/null || echo 0)
    if [[ $lines -le $chunksize ]]; then
        # Use the original file path, properly quoted via array
        local cmd_str="${*//_chunk_/$infile}"
        bash -lc "$cmd_str"
        return $?
    fi
    mkdir -p .tmp/chunks
    # Clean up old chunks to avoid stale data
    rm -f .tmp/chunks/part_*
    split -l "$chunksize" -d "$infile" .tmp/chunks/part_
    for part in .tmp/chunks/part_*; do
        # Replace placeholder with the chunk path (already safe — generated by split)
        local cmd_str="${*//_chunk_/$part}"
        bash -lc "$cmd_str"
    done
    rm -f .tmp/chunks/part_*
}

###############################################################################################################
########################################## HEALTH CHECK #######################################################
###############################################################################################################

# Performance timing storage
declare -A FUNC_TIMINGS 2>/dev/null || true

function record_func_timing() {
    # Usage: record_func_timing function_name duration_seconds
    local fn="$1"
    local dur="$2"
    FUNC_TIMINGS["$fn"]=$dur 2>/dev/null || true
}

function print_timing_summary() {
    # Print sorted timing summary of all recorded functions (verbose only)
    [[ "${OUTPUT_VERBOSITY:-1}" -lt 2 ]] && return 0
    if [[ ${#FUNC_TIMINGS[@]} -eq 0 ]] 2>/dev/null; then
        return 0
    fi

    printf "\n"
    _print_rule
    printf "%b[%s] Performance Timing Summary%b\n\n" "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$reset"

    local total=0
    local fn dur
    local -a sorted_entries=()

    # First pass: calculate total and collect entries
    for fn in "${!FUNC_TIMINGS[@]}"; do
        dur=${FUNC_TIMINGS[$fn]}
        # Ensure dur is numeric
        if [[ "$dur" =~ ^[0-9]+$ ]]; then
            total=$((total + dur))
            sorted_entries+=("$dur|$fn")
        fi
    done

    # Sort entries by duration (descending) and print
    printf '%s\n' "${sorted_entries[@]}" | sort -t'|' -k1 -rn | while IFS='|' read -r dur fn; do
        local pct=0
        if [[ $total -gt 0 ]] && [[ "$dur" =~ ^[0-9]+$ ]]; then
            pct=$((dur * 100 / total))
        fi
        local mins=$((dur / 60))
        local secs=$((dur % 60))
        printf "  %-35s %3dm %02ds  (%2d%%)\n" "$fn" "$mins" "$secs" "$pct"
    done

    local total_mins=$((total / 60))
    local total_secs=$((total % 60))
    printf "\n  %-35s %3dm %02ds\n" "TOTAL" "$total_mins" "$total_secs"
    _print_rule
    printf "\n"
}

# Write machine-readable performance summary for dashboards/automation.
# Output: .log/perf_summary.json
function write_perf_summary() {
    local summary_file="${dir}/.log/perf_summary.json"
    local total=0
    local fn dur

    for fn in "${!FUNC_TIMINGS[@]}"; do
        dur=${FUNC_TIMINGS[$fn]}
        [[ "$dur" =~ ^[0-9]+$ ]] || continue
        total=$((total + dur))
    done

    local subs_count webs_count hosts_count findings_count
    subs_count=$(count_lines "${dir}/subdomains/subdomains.txt")
    webs_count=$(count_lines "${dir}/webs/webs_all.txt")
    hosts_count=$(count_lines "${dir}/hosts/ips.txt")
    if [[ -d "${dir}/nuclei_output" ]]; then
        findings_count=$(find "${dir}/nuclei_output" -type f -name '*_json.txt' -exec cat {} + 2>/dev/null | wc -l | tr -d ' ')
    else
        findings_count=0
    fi
    subs_count=${subs_count:-0}
    webs_count=${webs_count:-0}
    hosts_count=${hosts_count:-0}
    findings_count=${findings_count:-0}

    # Build top functions by duration.
    local top_json="[]"
    local -a sorted_entries=()
    for fn in "${!FUNC_TIMINGS[@]}"; do
        dur=${FUNC_TIMINGS[$fn]}
        [[ "$dur" =~ ^[0-9]+$ ]] || continue
        sorted_entries+=("$dur|$fn")
    done
    if [[ ${#sorted_entries[@]} -gt 0 ]]; then
        top_json=$(printf '%s\n' "${sorted_entries[@]}" \
            | sort -t'|' -k1 -rn \
            | head -n 15 \
            | awk -F'|' '{printf "{\"function\":\"%s\",\"duration_sec\":%s}\n",$2,$1}' \
            | jq -s '.')
    fi

    if command -v jq >/dev/null 2>&1; then
        jq -n \
            --arg ts "$(date -Iseconds)" \
            --arg domain "${domain:-unknown}" \
            --arg mode "${opt_mode:-unknown}" \
            --arg profile "${PERF_PROFILE:-balanced}" \
            --argjson total_sec "${total:-0}" \
            --argjson subdomains "${subs_count:-0}" \
            --argjson webs "${webs_count:-0}" \
            --argjson hosts "${hosts_count:-0}" \
            --argjson findings "${findings_count:-0}" \
            --argjson top "${top_json}" \
            '{
                timestamp:$ts,
                domain:$domain,
                mode:$mode,
                perf_profile:$profile,
                total_duration_sec:$total_sec,
                counts:{
                  subdomains:$subdomains,
                  webs:$webs,
                  hosts:$hosts,
                  findings:$findings
                },
                top_functions:$top
            }' >"$summary_file"
    else
        printf '{"timestamp":"%s","domain":"%s","mode":"%s","perf_profile":"%s","total_duration_sec":%s}\n' \
            "$(date -Iseconds)" "${domain:-unknown}" "${opt_mode:-unknown}" "${PERF_PROFILE:-balanced}" "${total:-0}" >"$summary_file"
    fi
}

# Strip control characters except tabs/newlines (pre-jq sanitation)
sanitize_control_chars() {
    LC_ALL=C tr -d '\000-\010\013\014\016-\037'
}

# Generate consolidated JSON + HTML report for the current scan.
# Outputs:
#   - report/report.json
#   - report/index.html
function generate_consolidated_report() {
    ensure_dirs report || return 1

    local report_json="report/report.json"
    local report_html="report/index.html"
    local subs_count webs_count hosts_count screenshots_count findings_total
    local sev_info sev_low sev_medium sev_high sev_critical
    local top_assets_json timeline_json links_json monitor_delta_json monitor_alerts_json

    subs_count=$(count_lines "subdomains/subdomains.txt")
    webs_count=$(count_lines "webs/webs_all.txt")
    hosts_count=$(count_lines "hosts/ips.txt")
    screenshots_count=$(find screenshots -type f -name '*.png' 2>/dev/null | wc -l | tr -d ' ')

    sev_info=$(count_lines "nuclei_output/info_json.txt")
    sev_low=$(count_lines "nuclei_output/low_json.txt")
    sev_medium=$(count_lines "nuclei_output/medium_json.txt")
    sev_high=$(count_lines "nuclei_output/high_json.txt")
    sev_critical=$(count_lines "nuclei_output/critical_json.txt")

    subs_count=${subs_count:-0}
    webs_count=${webs_count:-0}
    hosts_count=${hosts_count:-0}
    screenshots_count=${screenshots_count:-0}
    sev_info=${sev_info:-0}
    sev_low=${sev_low:-0}
    sev_medium=${sev_medium:-0}
    sev_high=${sev_high:-0}
    sev_critical=${sev_critical:-0}
    findings_total=$((sev_info + sev_low + sev_medium + sev_high + sev_critical))

    # Top assets from hotlist (if present)
    if [[ -s hotlist.txt ]] && command -v jq >/dev/null 2>&1; then
        top_assets_json=$(head -n "${HOTLIST_TOP:-50}" hotlist.txt \
            | awk '{score=$1; $1=""; sub(/^ /,"",$0); print score "\t" $0}' \
            | jq -Rn '[inputs | split("\t") | {asset: .[1], score: (.[0] | tonumber? // 0)}]')
    else
        top_assets_json="[]"
    fi

    # Timeline from structured log (preferred) or fallback from text log.
    if [[ -n "${STRUCTURED_LOG_FILE:-}" ]] && [[ -s "${STRUCTURED_LOG_FILE}" ]] && command -v jq >/dev/null 2>&1; then
        timeline_json=$(tail -n 80 "${STRUCTURED_LOG_FILE}" \
            | sanitize_control_chars \
            | jq -s 'map({timestamp:(.timestamp // ""), level:(.level // "INFO"), function:(.function // "unknown"), message:(.message // "")})')
    elif [[ -s "${LOGFILE:-}" ]] && command -v jq >/dev/null 2>&1; then
        timeline_json=$(tail -n 80 "${LOGFILE}" \
            | sanitize_control_chars \
            | awk -F'] ' '{
                ts=$1; gsub(/^\[/,"",ts);
                msg=$2;
                if (msg ~ /Start function:/) { print ts "\tINFO\t" msg "\tstarted" }
                else if (msg ~ /End function:/) { print ts "\tSUCCESS\t" msg "\tcompleted" }
            }' \
            | jq -Rn '[inputs | split("\t") | {timestamp: .[0], level: .[1], function: .[2], message: .[3]}]')
    else
        timeline_json="[]"
    fi

    # Quick links to important output files.
    if command -v jq >/dev/null 2>&1; then
        local link_path exists
        links_json=$(
            {
                for entry in \
                    "Subdomains|subdomains/subdomains.txt" \
                    "Web Targets|webs/webs_all.txt" \
                    "Takeovers|webs/takeover.txt" \
                    "Nuclei Critical|nuclei_output/critical.txt" \
                    "Nuclei High|nuclei_output/high.txt" \
                    "Hotlist|hotlist.txt" \
                    "Performance Summary|.log/perf_summary.json" \
                    "Assets JSONL|assets.jsonl" \
                    "Screenshots Changed|screenshots/diff_changed.txt" \
                    "Latest Report JSON|report/latest/report.json" \
                    "Latest Report HTML|report/latest/index.html"; do
                    link_path="${entry#*|}"
                    exists=false
                    [[ -s "$link_path" ]] && exists=true
                    printf '{"name":"%s","path":"%s","exists":%s}\n' "${entry%%|*}" "$link_path" "$exists"
                done
            } | jq -s '.'
        )
    else
        links_json="[]"
    fi

    if command -v jq >/dev/null 2>&1; then
        if [[ -s ".incremental/history/latest/delta.json" ]]; then
            monitor_delta_json=$(cat ".incremental/history/latest/delta.json")
        else
            monitor_delta_json='{}'
        fi
        if [[ -s ".incremental/history/latest/alerts.json" ]]; then
            monitor_alerts_json=$(cat ".incremental/history/latest/alerts.json")
        else
            monitor_alerts_json='{}'
        fi

        jq -n \
            --arg generated_at "$(date -Iseconds)" \
            --arg domain "${domain:-unknown}" \
            --arg mode "${opt_mode:-unknown}" \
            --arg runtime "${runtime:-unknown}" \
            --arg profile "${PERF_PROFILE:-balanced}" \
            --arg quick_rescan "${QUICK_RESCAN:-false}" \
            --arg incremental "${INCREMENTAL_MODE:-false}" \
            --arg parallel "${PARALLEL_MODE:-true}" \
            --arg axiom "${AXIOM:-false}" \
            --argjson subs "$subs_count" \
            --argjson webs "$webs_count" \
            --argjson hosts "$hosts_count" \
            --argjson screenshots "$screenshots_count" \
            --argjson findings_total "$findings_total" \
            --argjson sev_info "$sev_info" \
            --argjson sev_low "$sev_low" \
            --argjson sev_medium "$sev_medium" \
            --argjson sev_high "$sev_high" \
            --argjson sev_critical "$sev_critical" \
            --argjson top_assets "$top_assets_json" \
            --argjson timeline "$timeline_json" \
            --argjson links "$links_json" \
            --argjson delta "$monitor_delta_json" \
            --argjson alerts "$monitor_alerts_json" \
            '{
                generated_at:$generated_at,
                domain:$domain,
                mode:$mode,
                runtime:$runtime,
                metadata:{
                    perf_profile:$profile,
                    quick_rescan:($quick_rescan == "true"),
                    incremental:($incremental == "true"),
                    parallel:($parallel != "false"),
                    axiom:($axiom == "true")
                },
                summary:{
                    subdomains:$subs,
                    webs:$webs,
                    hosts:$hosts,
                    screenshots:$screenshots,
                    findings_total:$findings_total
                },
                severities:{
                    info:$sev_info,
                    low:$sev_low,
                    medium:$sev_medium,
                    high:$sev_high,
                    critical:$sev_critical
                },
                top_assets:$top_assets,
                timeline:$timeline,
                links:$links,
                delta_since_last:$delta,
                alerts_last:$alerts
            }' >"$report_json"
    else
        printf '{"generated_at":"%s","domain":"%s","mode":"%s","runtime":"%s"}\n' \
            "$(date -Iseconds)" "${domain:-unknown}" "${opt_mode:-unknown}" "${runtime:-unknown}" >"$report_json"
    fi

    # HTML renderer with inlined JSON payload.
    # Sanitize JSON for safe embedding in <script> tags:
    # - Escape </script> to prevent tag breakout (XSS)
    # - Escape <!-- to prevent HTML comment injection
    local report_payload
    report_payload=$(jq -c '.' "$report_json" 2>/dev/null || echo '{}')
    report_payload="${report_payload//</\u003c}"
    report_payload="${report_payload//>/\u003e}"
    cat >"$report_html" <<EOF
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8" />
  <meta name="viewport" content="width=device-width,initial-scale=1" />
  <title>reconFTW Report - ${domain}</title>
  <style>
    :root{--bg:#0b1320;--card:#111c2d;--text:#e8f0ff;--muted:#9ab0cc;--acc:#40c4ff;--ok:#32d583;--warn:#fdb022;--bad:#f97066}
    body{margin:0;font-family:ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Ubuntu,sans-serif;background:linear-gradient(160deg,#0b1320,#0f1e35);color:var(--text)}
    .wrap{max-width:1100px;margin:0 auto;padding:20px}
    .grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px}
    .card{background:var(--card);border:1px solid #1b2a44;border-radius:14px;padding:14px}
    h1,h2{margin:.2rem 0 .6rem}
    .muted{color:var(--muted);font-size:.92rem}
    .kpi{font-size:1.6rem;font-weight:700}
    table{width:100%;border-collapse:collapse}
    th,td{padding:8px;border-bottom:1px solid #213451;text-align:left;font-size:.93rem}
    a{color:var(--acc);text-decoration:none}
    .sev{display:flex;gap:10px;flex-wrap:wrap}
    .pill{padding:4px 10px;border-radius:999px;background:#15243b}
  </style>
</head>
<body>
  <div class="wrap">
    <h1>reconFTW Consolidated Report</h1>
    <div class="muted" id="meta"></div>
    <div class="grid" id="kpis"></div>
    <h2>Severities</h2>
    <div class="sev" id="severities"></div>
    <h2>Top Assets</h2>
    <div class="card"><table><thead><tr><th>Asset</th><th>Score</th></tr></thead><tbody id="topAssets"></tbody></table></div>
    <h2>Delta Since Last Run</h2>
    <div class="sev" id="delta"></div>
    <h2>Timeline</h2>
    <div class="card"><table><thead><tr><th>Timestamp</th><th>Level</th><th>Function</th><th>Message</th></tr></thead><tbody id="timeline"></tbody></table></div>
    <h2>Quick Links</h2>
    <div class="card"><table><thead><tr><th>Name</th><th>Path</th></tr></thead><tbody id="links"></tbody></table></div>
  </div>
  <script>
    const REPORT = ${report_payload};
    const s = REPORT.summary || {};
    const sev = REPORT.severities || {};
    document.getElementById('meta').textContent =
      "Domain: " + (REPORT.domain||"n/a") + " | Mode: " + (REPORT.mode||"n/a") + " | Runtime: " + (REPORT.runtime||"n/a") + " | Generated: " + (REPORT.generated_at||"");
    const kpis = [
      ["Subdomains", s.subdomains||0],["Webs", s.webs||0],["Hosts", s.hosts||0],["Screenshots", s.screenshots||0],["Findings", s.findings_total||0]
    ];
    document.getElementById('kpis').innerHTML = kpis.map(([k,v]) => '<div class="card"><div class="muted">'+k+'</div><div class="kpi">'+v+'</div></div>').join('');
    document.getElementById('severities').innerHTML =
      [["Critical",sev.critical||0],["High",sev.high||0],["Medium",sev.medium||0],["Low",sev.low||0],["Info",sev.info||0]]
      .map(([k,v]) => '<span class="pill">'+k+': '+v+'</span>').join('');
    document.getElementById('topAssets').innerHTML = (REPORT.top_assets||[]).map(x => '<tr><td>'+x.asset+'</td><td>'+x.score+'</td></tr>').join('') || '<tr><td colspan="2">No data</td></tr>';
    const d = (REPORT.delta_since_last && REPORT.delta_since_last.deltas) ? REPORT.delta_since_last.deltas : {};
    document.getElementById('delta').innerHTML =
      [["New Subdomains",d.subdomains_new||0],["New Webs",d.webs_new||0],["New High",d.high_findings_new||0],["New Critical",d.critical_findings_new||0]]
      .map(([k,v]) => '<span class="pill">'+k+': '+v+'</span>').join('');
    document.getElementById('timeline').innerHTML = (REPORT.timeline||[]).slice(-40).reverse().map(x =>
      '<tr><td>'+ (x.timestamp||'') +'</td><td>'+ (x.level||'') +'</td><td>'+ (x.function||'') +'</td><td>'+ (x.message||'') +'</td></tr>'
    ).join('') || '<tr><td colspan="4">No data</td></tr>';
    document.getElementById('links').innerHTML = (REPORT.links||[]).map(x => {
      const mark = x.exists ? 'ok' : 'missing';
      return '<tr><td>'+x.name+' ('+mark+')</td><td><a href="../'+x.path+'">'+x.path+'</a></td></tr>';
    }).join('');
  </script>
</body>
</html>
EOF

    # Stable latest report paths for automation/UI.
    mkdir -p report/latest
    cp "$report_json" report/latest/report.json 2>/dev/null || true
    cp "$report_html" report/latest/index.html 2>/dev/null || true

    notification "Consolidated report generated at report/index.html" info
}

# Escape a value for CSV output.
_csv_escape() {
    local v="${1:-}"
    v=${v//\"/\"\"}
    printf '"%s"' "$v"
}

# Normalize findings into line-delimited JSON for downstream automation.
# Outputs:
#   - report/findings_normalized.jsonl
#   - report/assets.jsonl (copy of assets.jsonl if present)
#   - report/export_all.jsonl (assets + normalized findings)
function export_findings_jsonl() {
    ensure_dirs report || return 1
    local out_norm="report/findings_normalized.jsonl"
    local out_assets="report/assets.jsonl"
    local out_merged="report/export_all.jsonl"
    : >"$out_norm"

    local sev f
    for sev in info low medium high critical; do
        f="nuclei_output/${sev}_json.txt"
        [[ -s "$f" ]] || continue
        if command -v jq >/dev/null 2>&1; then
            jq -c --arg sev "$sev" '
                {
                  type:"finding",
                  tool:"nuclei",
                  severity:(.info.severity // $sev),
                  template_id:(.["template-id"] // ""),
                  matcher_name:(.["matcher-name"] // ""),
                  finding_type:(.type // ""),
                  target:(.["matched-at"] // .host // ""),
                  host:(.host // ""),
                  name:(.info.name // ""),
                  tags:(.info.tags // []),
                  extracted_results:(.["extracted-results"] // []),
                  source_file:input_filename
                }' "$f" >>"$out_norm" 2>/dev/null || true
        else
            while IFS= read -r line; do
                [[ -z "$line" ]] && continue
                printf '{"type":"finding","tool":"nuclei","severity":"%s","raw":"%s"}\n' \
                    "$sev" "$(printf '%s' "$line" | sed 's/"/\\"/g')" >>"$out_norm"
            done <"$f"
        fi
    done

    if [[ -s assets.jsonl ]]; then
        cp assets.jsonl "$out_assets"
    else
        : >"$out_assets"
    fi

    cat "$out_assets" "$out_norm" >"$out_merged" 2>/dev/null || true
}

# Export CSV artifacts for quick triage.
# Outputs:
#   - report/subdomains.csv
#   - report/webs.csv
#   - report/hosts.csv
#   - report/findings.csv
function export_csv_artifacts() {
    ensure_dirs report || return 1

    # Subdomains CSV
    {
        printf "subdomain\n"
        if [[ -s subdomains/subdomains.txt ]]; then
            while IFS= read -r sub; do
                [[ -z "$sub" ]] && continue
                _csv_escape "$sub"; printf "\n"
            done <subdomains/subdomains.txt
        fi
    } >report/subdomains.csv

    # Webs CSV
    {
        printf "url,scheme,host\n"
        if [[ -s webs/webs_all.txt ]]; then
            while IFS= read -r url; do
                [[ -z "$url" ]] && continue
                local scheme host
                scheme=$(printf '%s' "$url" | awk -F:// '{print $1}')
                host=$(printf '%s' "$url" | awk -F/ '{print $3}' | sed 's/:.*$//')
                _csv_escape "$url"; printf ","
                _csv_escape "$scheme"; printf ","
                _csv_escape "$host"; printf "\n"
            done <webs/webs_all.txt
        fi
    } >report/webs.csv

    # Hosts CSV
    {
        printf "ip\n"
        if [[ -s hosts/ips.txt ]]; then
            while IFS= read -r ip; do
                [[ -z "$ip" ]] && continue
                _csv_escape "$ip"; printf "\n"
            done <hosts/ips.txt
        fi
    } >report/hosts.csv

    # Findings CSV (prefer normalized JSONL)
    if [[ ! -s report/findings_normalized.jsonl ]]; then
        export_findings_jsonl
    fi
    if command -v jq >/dev/null 2>&1 && [[ -s report/findings_normalized.jsonl ]]; then
        {
            printf "tool,severity,template_id,name,target,host,finding_type,matcher_name\n"
            jq -r '[.tool,.severity,.template_id,.name,.target,.host,.finding_type,.matcher_name] | @csv' report/findings_normalized.jsonl
        } >report/findings.csv
    else
        {
            printf "raw\n"
            [[ -s report/findings_normalized.jsonl ]] && while IFS= read -r line; do _csv_escape "$line"; printf "\n"; done <report/findings_normalized.jsonl
        } >report/findings.csv
    fi
}

# Export clean text artifacts for copy/paste workflows.
# Outputs:
#   - report/subdomains_clean.txt
#   - report/webs_clean.txt
#   - report/ips_clean.txt
function export_clean_text_artifacts() {
    ensure_dirs report || return 1

    if [[ -s subdomains/subdomains.txt ]]; then
        sed '/^$/d' subdomains/subdomains.txt | sort -u >report/subdomains_clean.txt
    else
        : >report/subdomains_clean.txt
    fi

    if [[ -s webs/webs_all.txt ]]; then
        sed '/^$/d' webs/webs_all.txt | sort -u >report/webs_clean.txt
    else
        : >report/webs_clean.txt
    fi

    cat hosts/ips.txt hosts/ips_v6.txt 2>/dev/null | sed '/^$/d' | sort -u >report/ips_clean.txt
}

# Export orchestrator.
# EXPORT_FORMAT values: json | html | csv | all
function export_reports() {
    local fmt="${EXPORT_FORMAT:-}"
    [[ -z "$fmt" ]] && return 0

    # Ensure base consolidated report exists for all export types.
    [[ -s report/report.json && -s report/index.html ]] || generate_consolidated_report || true

    case "$fmt" in
        json)
            export_findings_jsonl
            export_clean_text_artifacts
            notification "Exported JSON artifacts under report/" info
            ;;
        html)
            export_clean_text_artifacts
            notification "Exported HTML report at report/index.html" info
            ;;
        csv)
            export_findings_jsonl
            export_csv_artifacts
            export_clean_text_artifacts
            notification "Exported CSV artifacts under report/" info
            ;;
        all)
            export_findings_jsonl
            export_csv_artifacts
            export_clean_text_artifacts
            notification "Exported JSONL + CSV + HTML artifacts under report/" info
            ;;
        *)
            notification "Unknown EXPORT_FORMAT '$fmt' (expected json|html|csv|all)" warn
            ;;
    esac
}

function health_check() {
    local failures=0
    local warnings=0

    _print_section "Health Check"

    # 1. Check critical tools
    _print_status INFO "Checking critical tools..."
    local critical_tools=("bash" "python3" "curl" "git" "jq")
    for tool in "${critical_tools[@]}"; do
        if command -v "$tool" >/dev/null 2>&1; then
            _print_status OK "$tool"
        else
            _print_status FAIL "$tool" "not found"
            ((failures++))
        fi
    done

    # 2. Check key recon tools
    _print_status INFO "Checking recon tools..."
    local recon_tools=("subfinder" "httpx" "nuclei" "ffuf" "dnsx" "anew" "notify" "nmap" "nmapurls")
    for tool in "${recon_tools[@]}"; do
        if command -v "$tool" >/dev/null 2>&1; then
            _print_status OK "$tool"
        else
            _print_status WARN "$tool" "not found"
            ((warnings++))
        fi
    done

    # 3. Check network connectivity
    _print_status INFO "Checking network connectivity..."
    if run_command curl -s --connect-timeout 5 -o /dev/null -w "%{http_code}" "https://www.google.com" | grep -q "200\|301\|302"; then
        _print_status OK "Internet connectivity"
    else
        _print_status WARN "No internet connectivity"
        ((warnings++))
    fi

    # 4. Check disk space
    _print_status INFO "Checking disk space..."
    local avail_gb
    avail_gb=$(df -Pk . 2>/dev/null | awk 'NR==2 {print int($4 / 1024 / 1024)}')
    avail_gb=${avail_gb:-0}

    if [[ ${avail_gb} -ge 5 ]]; then
        _print_status OK "Disk space" "${avail_gb} GB available"
    elif [[ ${avail_gb} -ge 1 ]]; then
        _print_status WARN "Disk space" "Only ${avail_gb} GB available (recommend 5+ GB)"
        ((warnings++))
    else
        _print_status FAIL "Disk space" "Less than 1 GB available"
        ((failures++))
    fi

    # 5. Check resolver files
    _print_status INFO "Checking resolver files..."
    local resolvers_path="${resolvers:-${SCRIPTPATH}/resolvers.txt}"
    if [[ -s "$resolvers_path" ]]; then
        local count
        count=$(wc -l <"$resolvers_path")
        _print_status OK "Resolvers file" "${resolvers_path} (${count} entries)"
    else
        _print_status WARN "Resolvers file" "not found or empty: ${resolvers_path}"
        ((warnings++))
    fi

    # 6. Check API key status
    _print_status INFO "Checking API keys..."
    local -A api_keys=(
        ["SHODAN_API_KEY"]="${SHODAN_API_KEY:-}"
        ["WHOISXML_API"]="${WHOISXML_API:-}"
        ["GITHUB_TOKEN"]="${GITHUB_TOKEN:-}"
        ["VIRUSTOTAL_API_KEY"]="${VIRUSTOTAL_API_KEY:-}"
    )
    for key_name in "${!api_keys[@]}"; do
        if [[ -n "${api_keys[$key_name]}" ]]; then
            _print_status OK "$key_name" "configured"
        else
            _print_status INFO "$key_name" "not set"
        fi
    done

    # Summary
    _print_rule
    if [[ $failures -gt 0 ]]; then
        _print_status FAIL "Health check" "${failures} FAILURES, ${warnings} warnings"
    elif [[ $warnings -gt 0 ]]; then
        _print_status WARN "Health check" "PASSED with ${warnings} warnings"
    else
        _print_status OK "Health check" "ALL PASSED"
    fi

    return $failures
}