Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

37.8 KB
Raw Blame
#!/usr/bin/env bash
# lib/parallel.sh - Parallel execution utilities for reconFTW
# Part of reconFTW refactoring - Phase 3

# Prevent multiple sourcing
[[ -n "$_PARALLEL_SH_LOADED" ]] && return 0
declare -r _PARALLEL_SH_LOADED=1

###############################################################################
# Configuration
###############################################################################

# Default maximum concurrent jobs (can be overridden)
PARALLEL_MAX_JOBS="${PARALLEL_MAX_JOBS:-4}"
PARALLEL_HEARTBEAT_SECONDS="${PARALLEL_HEARTBEAT_SECONDS:-20}"
PARALLEL_UI_MODE="${PARALLEL_UI_MODE:-clean}"
PARALLEL_PROGRESS_SHOW_ETA="${PARALLEL_PROGRESS_SHOW_ETA:-true}"
PARALLEL_PROGRESS_SHOW_ACTIVE="${PARALLEL_PROGRESS_SHOW_ACTIVE:-true}"
PARALLEL_PROGRESS_COMPACT_ACTIVE_MAX="${PARALLEL_PROGRESS_COMPACT_ACTIVE_MAX:-4}"
PARALLEL_TRACE_SLOW_SECONDS="${PARALLEL_TRACE_SLOW_SECONDS:-30}"

# Track running PIDs for cleanup
declare -a _PARALLEL_PIDS=()
_PARALLEL_LAST_BADGE=""

###############################################################################
# Background-job throttle used by modules/{web,vulns,osint}.sh to parallelise
# per-target loops that replaced `interlace -c` (removed in the Naxus audit
# fix for command injection). Each iteration launches a subshell in the
# background; this helper blocks until the running-job count falls below the
# requested cap. Requires bash 4.3+ for `wait -n`.
###############################################################################
_throttle_jobs() {
    local max="${1:-4}"
    # Clamp to sane minimum so a misconfigured var doesn't busy-spawn.
    [[ "$max" =~ ^[0-9]+$ ]] || max=4
    (( max < 1 )) && max=1

    while (( $(jobs -rp | wc -l) >= max )); do
        wait -n 2>/dev/null || break
    done
}

# Recursively signal a process and all its descendants. Used by
# _timeout_kill_job to kill the actual external tool (puredns, dnsx, ffuf,
# axiom-scan, etc.) inside the parallel_funcs wrapper subshell, not just
# the wrapper itself (CR-03 fix). Walks the process tree via `pgrep -P`
# which is available on both Linux (procps) and macOS (BSD pgrep). If
# pgrep is missing on the host, the walk degrades gracefully to a
# wrapper-only kill (matches pre-patch behavior) so the run does not fail.
# Process-group kill via `kill -- -<pgid>` was rejected because start() at
# modules/modes.sh:16 explicitly disables job control (`set +m`), so the
# wrapper subshell does not get its own pgid; flipping `set -m` for one
# corner case is intrusive across the codebase.
function _kill_tree() {
    local parent="$1" sig="${2:-TERM}"
    local child
    # Recurse into children first so deeper leaves get signaled before the
    # parent — protects against a parent re-spawning children on signal.
    if command -v pgrep >/dev/null 2>&1; then
        for child in $(pgrep -P "$parent" 2>/dev/null); do
            _kill_tree "$child" "$sig"
        done
    fi
    kill "-$sig" "$parent" 2>/dev/null || true
}

# Kill a parallel job that has exceeded PARALLEL_JOB_TIMEOUT_SECONDS.
# Usage: _timeout_kill_job <pid> <func_name> <duration_sec>
# Sends SIGTERM, polls every second up to PARALLEL_KILL_GRACE_SECONDS, then SIGKILL
# if still alive. Persists FAIL + reason=timeout to the same .status_<fn> /
# .status_reason_<fn> files end_func writes, so _parallel_emit_job_output renders
# the timeout reason on the FAIL badge without any schema extension.
# Uses 'function' keyword per CONVENTIONS.md §Function Naming; sibling helpers in
# this file that omit it predate the rule and are a Phase 5 DOCS-01 followup, not
# a precedent for new code.
function _timeout_kill_job() {
    local pid="$1" func_name="$2" duration_sec="$3"
    local grace="${PARALLEL_KILL_GRACE_SECONDS:-10}"
    [[ "$grace" =~ ^[0-9]+$ ]] || grace=10

    # D-13: TERM first, then KILL after grace seconds for tools that ignore TERM.
    # CR-03 fix: kill the entire process tree under the wrapper PID so the actual
    # external tool (puredns, dnsx, ffuf, axiom-scan, etc.) terminates — not just
    # the wrapper subshell. Pre-patch behavior signaled only $pid (the wrapper),
    # leaving the tool orphaned to PID 1 and continuing past the timeout.
    _kill_tree "$pid" TERM
    local i
    for ((i=0; i<grace; i++)); do
        kill -0 "$pid" 2>/dev/null || break
        sleep 1
    done
    _kill_tree "$pid" KILL

    # D-14: persist FAIL + reason=timeout via the same pattern as modules/core.sh:1505-1509.
    if [[ -n "${called_fn_dir:-}" ]]; then
        printf "FAIL\n" >"${called_fn_dir}/.status_${func_name}" 2>/dev/null || true
        printf "timeout\n" >"${called_fn_dir}/.status_reason_${func_name}" 2>/dev/null || true
    fi

    # Structured log; no-op when STRUCTURED_LOGGING != true (modules/core.sh:680).
    if declare -F log_json >/dev/null 2>&1; then
        log_json "ERROR" "${func_name}" "Job timed out" "reason=timeout" "duration_sec=${duration_sec}"
    fi
}

_parallel_live_break() {
    if declare -F ui_live_progress_break >/dev/null 2>&1; then
        ui_live_progress_break
    fi
}

_parallel_get_ui_mode() {
    case "${PARALLEL_UI_MODE:-clean}" in
        clean|balanced|trace) printf "%s" "${PARALLEL_UI_MODE:-clean}" ;;
        *) printf "clean" ;;
    esac
}

_parallel_should_show_started() {
    local mode
    mode=$(_parallel_get_ui_mode)
    [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && [[ "$mode" == "trace" ]]
}

_parallel_should_show_finished() {
    local rc="${1:-0}" duration="${2:-0}" badge="${3:-OK}"
    local mode
    mode=$(_parallel_get_ui_mode)
    [[ "${OUTPUT_VERBOSITY:-1}" -lt 2 ]] && return 1
    case "$mode" in
        trace) return 0 ;;
        balanced)
            if [[ "$rc" -ne 0 ]] || [[ "$badge" == "WARN" ]] || [[ "$badge" == "FAIL" ]] || ((duration >= PARALLEL_TRACE_SLOW_SECONDS)); then
                return 0
            fi
            return 1
            ;;
        *) return 1 ;;
    esac
}

_parallel_effective_max_jobs() {
    local requested="${1:-$PARALLEL_MAX_JOBS}"
    local pressure="${PARALLEL_PRESSURE_LEVEL:-normal}"
    local effective="$requested"

    # Backpressure: shrink concurrency when rate-limit pressure is high.
    if [[ "$pressure" == "high" ]]; then
        effective=$((requested / 2))
        ((effective < 1)) && effective=1
    fi
    echo "$effective"
}

_parallel_compact_list() {
    local max="${2:-6}"
    local list="$1"
    if [[ -z "$list" ]]; then
        printf "none"
        return 0
    fi
    local count
    count=$(awk -F',' 'NF{print NF}' <<<"$list")
    if ((count <= max)); then
        printf "%s" "$list"
    else
        local trimmed
        trimmed=$(echo "$list" | awk -F',' -v m="$max" '{out=""; for(i=1;i<=m;i++){out=out $i (i<m? ",":"")} print out}')
        printf "%s, +%d more" "$trimmed" "$((count - max))"
    fi
}

_parallel_snapshot() {
    local active_list="$1"
    local done_list="$2"
    local queue_count="$3"
    local done_count="${4:-0}"
    local total_count="${5:-0}"
    local elapsed_batch="${6:-0}"
    [[ "${OUTPUT_VERBOSITY:-1}" -lt 1 ]] && return 0

    local mode active_max active_fmt done_fmt progress_pct eta="--"
    mode=$(_parallel_get_ui_mode)
    active_max="${PARALLEL_PROGRESS_COMPACT_ACTIVE_MAX:-4}"
    active_fmt=$(_parallel_compact_list "$active_list" "$active_max")
    done_fmt=$(_parallel_compact_list "$done_list" "$active_max")

    if ((total_count > 0)); then
        progress_pct=$((done_count * 100 / total_count))
    else
        progress_pct=0
    fi

    if [[ "${PARALLEL_PROGRESS_SHOW_ETA:-true}" == "true" ]] && ((done_count >= 2)) && ((total_count > done_count)) && ((elapsed_batch >= 60)); then
        local avg remaining eta_seconds
        avg=$((elapsed_batch / done_count))
        remaining=$((total_count - done_count))
        eta_seconds=$((avg * remaining))
        eta=$(format_duration "$eta_seconds")
    fi

    if declare -F ui_is_tty >/dev/null 2>&1 && ui_is_tty && declare -F ui_live_progress_update >/dev/null 2>&1; then
        local line
        line=$(printf "Progress: %d/%d (%d%%) | elapsed %s" \
            "$done_count" "$total_count" "$progress_pct" "$(format_duration "$elapsed_batch")")
        if [[ "${PARALLEL_PROGRESS_SHOW_ETA:-true}" == "true" ]]; then
            line="${line} | ETA: ${eta}"
        fi
        if [[ "${PARALLEL_PROGRESS_SHOW_ACTIVE:-true}" == "true" ]] && [[ "$active_fmt" != "none" || "$mode" == "trace" ]]; then
            line="${line} | Active: ${active_fmt}"
        fi
        if [[ "$mode" == "trace" ]]; then
            line="${line} | Done: ${done_fmt}"
        fi
        if [[ "$mode" != "clean" ]] || ((queue_count > 0)); then
            line="${line} | Queue: ${queue_count}"
        fi
        if declare -F ui_live_progress_begin >/dev/null 2>&1; then
            ui_live_progress_begin
        fi
        ui_live_progress_update "$line"
        if ((total_count > 0)) && ((done_count >= total_count)) && declare -F ui_live_progress_end >/dev/null 2>&1; then
            ui_live_progress_end
        fi
    fi
}

###############################################################################
# Output Helpers
###############################################################################

# Emit output for a single parallel job based on PARALLEL_LOG_MODE
# Usage: _parallel_emit_job_output func_name log_file rc start_ts end_ts
_parallel_emit_job_output() {
    local func_name="$1"
    local log_file="$2"
    local rc="$3"
    local start_ts="$4"
    local end_ts="$5"
    local mode="${PARALLEL_LOG_MODE:-summary}"
    local tail_lines="${PARALLEL_TAIL_LINES:-20}"
    local duration=$((end_ts - start_ts))

    local skip_marker=""
    local cache_marker=""
    local status_marker=""
    local reason_marker=""
    if [[ -n "${called_fn_dir:-}" ]]; then
        skip_marker="${called_fn_dir}/.skip_${func_name}"
        cache_marker="${called_fn_dir}/.cache_${func_name}"
        status_marker="${called_fn_dir}/.status_${func_name}"
        reason_marker="${called_fn_dir}/.status_reason_${func_name}"
    fi

    local badge=""
    local reason_code=""
    if [[ -n "$cache_marker" && -f "$cache_marker" ]]; then
        badge="CACHE"
        reason_code="cache"
        rm -f "$cache_marker" 2>/dev/null || true
    elif [[ -n "$skip_marker" && -f "$skip_marker" ]]; then
        badge="SKIP"
        reason_code="config"
        rm -f "$skip_marker" 2>/dev/null || true
    elif [[ -n "$status_marker" && -f "$status_marker" ]]; then
        badge=$(head -n 1 "$status_marker" 2>/dev/null | tr -d '\r\n' | tr '[:lower:]' '[:upper:]')
        case "$badge" in
            OK|WARN|FAIL|SKIP|CACHE|INFO) ;;
            *) badge="" ;;
        esac
        rm -f "$status_marker" 2>/dev/null || true
    elif [[ "$rc" -eq 0 ]]; then
        badge="OK"
    else
        badge="FAIL"
    fi

    if [[ -z "$badge" ]]; then
        if [[ "$rc" -eq 0 ]]; then
            badge="OK"
        else
            badge="FAIL"
        fi
    fi
    if [[ -n "$reason_marker" && -f "$reason_marker" ]]; then
        reason_code=$(head -n 1 "$reason_marker" 2>/dev/null | tr -d '\r\n' | tr '[:upper:]' '[:lower:]')
        rm -f "$reason_marker" 2>/dev/null || true
    fi
    _PARALLEL_LAST_BADGE="$badge"

    if declare -F ui_human_output_enabled >/dev/null 2>&1 && ! ui_human_output_enabled; then
        _print_status "$badge" "$func_name" "${duration}s"
        if [[ -n "$reason_code" ]] && declare -F ui_log_jsonl >/dev/null 2>&1; then
            ui_log_jsonl "INFO" "$func_name" "Status reason" "status=${badge}" "reason=${reason_code}"
        fi
        return 0
    fi

    # In quiet mode (OUTPUT_VERBOSITY==0), only show failures
    if [[ "${OUTPUT_VERBOSITY:-1}" -eq 0 ]] && [[ "$badge" != "FAIL" ]]; then
        return 0
    fi

    # For failures in quiet mode, print directly since _print_status is gated on verbosity
    local _force_print=false
    [[ "${OUTPUT_VERBOSITY:-1}" -eq 0 ]] && [[ "$badge" == "FAIL" ]] && _force_print=true

    case "$mode" in
        summary)
            if [[ "$_force_print" == true ]]; then
                if declare -F ui_count_inc >/dev/null 2>&1; then
                    ui_count_inc "$badge"
                fi
                (OUTPUT_VERBOSITY=1; print_task "$badge" "$func_name" "$duration" "")
            else
                _print_status "$badge" "$func_name" "${duration}s"
            fi
            if [[ "$badge" == "FAIL" ]] && [[ -s "$log_file" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
                tail -n 5 "$log_file" | while IFS= read -r line; do
                    printf "         %s\n" "$line"
                done
            fi
            if [[ -n "$reason_code" ]] && { [[ "$badge" == "SKIP" ]] || [[ "$badge" == "FAIL" ]] || { [[ "$badge" == "CACHE" ]] && [[ "${SHOW_CACHE:-false}" == "true" ]]; }; }; then
                printf "         reason: %s\n" "$reason_code"
            fi
            ;;
        tail)
            local show_lines="$tail_lines"
            [[ "$rc" -ne 0 ]] && show_lines=$((tail_lines * 2))
            if [[ "$_force_print" == true ]]; then
                if declare -F ui_count_inc >/dev/null 2>&1; then
                    ui_count_inc "$badge"
                fi
                (OUTPUT_VERBOSITY=1; print_task "$badge" "$func_name" "$duration" "")
            else
                _print_status "$badge" "$func_name" "${duration}s"
            fi
            if [[ -s "$log_file" ]]; then
                tail -n "$show_lines" "$log_file" | strip_ansi_stream
            fi
            if [[ -n "$reason_code" ]] && { [[ "$badge" == "SKIP" ]] || [[ "$badge" == "FAIL" ]] || { [[ "$badge" == "CACHE" ]] && [[ "${SHOW_CACHE:-false}" == "true" ]]; }; }; then
                printf "         reason: %s\n" "$reason_code"
            fi
            ;;
        full)
            if [[ "$_force_print" == true ]]; then
                if declare -F ui_count_inc >/dev/null 2>&1; then
                    ui_count_inc "$badge"
                fi
                (OUTPUT_VERBOSITY=1; print_task "$badge" "$func_name" "$duration" "")
            else
                _print_status "$badge" "$func_name" "${duration}s"
            fi
            if [[ -s "$log_file" ]]; then
                cat "$log_file" | strip_ansi_stream
            fi
            if [[ -n "$reason_code" ]] && { [[ "$badge" == "SKIP" ]] || [[ "$badge" == "FAIL" ]] || { [[ "$badge" == "CACHE" ]] && [[ "${SHOW_CACHE:-false}" == "true" ]]; }; }; then
                printf "         reason: %s\n" "$reason_code"
            fi
            ;;
    esac
}

# Print a summary line for a completed batch
# Usage: _parallel_batch_summary total_jobs failed_count batch_start_ts
_parallel_batch_summary() {
    local ok="$1"
    local warn="$2"
    local fail="$3"
    local skip="$4"
    local cache="$5"
    local batch_start="$6"
    local completed="${7:-0}"
    local total="${8:-0}"
    local batch_end
    batch_end=$(date +%s)
    local elapsed=$((batch_end - batch_start))
    local jsonl_enabled=false
    if declare -F ui_is_jsonl >/dev/null 2>&1 && ui_is_jsonl; then
        jsonl_enabled=true
    fi

    # Skip in quiet mode unless JSONL is enabled (events still useful to machines).
    if [[ "${OUTPUT_VERBOSITY:-1}" -eq 0 ]] && [[ "$jsonl_enabled" != "true" ]]; then
        return 0
    fi
    # Skip human batch envelope in summary mode at normal verbosity unless JSONL is enabled.
    if [[ "${PARALLEL_LOG_MODE:-summary}" == "summary" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -lt 2 ]] && [[ "$jsonl_enabled" != "true" ]]; then
        return 0
    fi

    if declare -F ui_batch_end >/dev/null 2>&1; then
        ui_batch_end "$ok" "$warn" "$fail" "$elapsed" "$skip" "$cache" "$completed" "$total"
    else
        printf "  --- batch: ok:%d warn:%d fail:%d skip:%d cache:%d, %d/%d completed, %ds elapsed ---\n" \
            "$ok" "$warn" "$fail" "$skip" "$cache" "$completed" "$total" "$elapsed"
    fi
}

###############################################################################
# Core Parallel Execution
###############################################################################

# Run multiple commands in parallel with a job limit
# Usage: parallel_run max_jobs "cmd1" "cmd2" "cmd3" ...
# Example: parallel_run 4 "subfinder -d $domain" "amass enum -d $domain"
# parallel_run() removed — was unused and had injection vector (bash -c "$cmd")

# Run multiple functions in parallel with a job limit
# Usage: parallel_funcs max_jobs func1 func2 func3 ...
# Example: parallel_funcs 4 sub_passive sub_crt sub_active
parallel_funcs() {
    local max_jobs
    max_jobs=$(_parallel_effective_max_jobs "${1:-$PARALLEL_MAX_JOBS}")
    shift

    local parallel_log_root
    if [[ -n "${dir:-}" ]]; then
        parallel_log_root="${dir}/.tmp/parallel"
    else
        parallel_log_root="$(mktemp -d /tmp/reconftw_parallel.XXXXXX)"
    fi
    mkdir -p "$parallel_log_root" 2>/dev/null || true

    local -a batch_pids=()
    local -a batch_funcs=()
    local -a batch_logs=()
    local -a batch_starts=()
    local func
    local failed=0
    local batch_start_ts
    batch_start_ts=$(date +%s)
    local batch_label="${PARALLEL_LABEL:-parallel}"
    local total_funcs=$#
    local queued_count=0
    local done_list=""

    for func in "$@"; do
        # Check if function exists
        if ! declare -f "$func" >/dev/null 2>&1; then
            print_warnf "Function %s not found, skipping" "$func"
            continue
        fi

        if ((${#batch_pids[@]} == 0)); then
            if declare -F ui_batch_start >/dev/null 2>&1; then
                # Skip batch envelope in summary mode at normal verbosity - individual status lines suffice
                if { declare -F ui_is_jsonl >/dev/null 2>&1 && ui_is_jsonl; } \
                    || { [[ "$(_parallel_get_ui_mode)" != "clean" ]] && { [[ "${PARALLEL_LOG_MODE:-summary}" != "summary" ]] || [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; }; }; then
                    ui_batch_start "$batch_label" "$max_jobs"
                fi
            fi
        fi

        local log_file="${parallel_log_root}/${func}.$$.$RANDOM.log"
        local job_start_ts
        job_start_ts=$(date +%s)

        # Run function in background subshell; write end timestamp after completion
        (
            local _rc=0
            "$func" || _rc=$?
            date +%s >"${log_file%.log}.endts"
            exit "$_rc"
        ) >"$log_file" 2>&1 &
        batch_pids+=("$!")
        batch_funcs+=("$func")
        batch_logs+=("$log_file")
        batch_starts+=("$job_start_ts")
        queued_count=$((queued_count + 1))

        # Log if verbose (OUTPUT_VERBOSITY >= 2)
        if _parallel_should_show_started; then
            _parallel_live_break
            printf "%b[*] Started %s (PID: %d)%b\n" "${cyan:-}" "$func" "${batch_pids[${#batch_pids[@]}-1]}" "${reset:-}"
        fi

        # If we've reached max jobs, flush the current batch.
        if ((${#batch_pids[@]} >= max_jobs)); then
            if [[ "${PARALLEL_MODE:-true}" == "true" ]]; then
                local active_list=""
                local i
                for i in "${batch_funcs[@]}"; do
                    if [[ -z "$active_list" ]]; then
                        active_list="${i} 00:00"
                    else
                        active_list="${active_list}, ${i} 00:00"
                    fi
                done
                local queue_count=$((total_funcs - queued_count))
                _parallel_snapshot "$active_list" "$done_list" "$queue_count" "0" "${#batch_pids[@]}" "0"
            fi

            # Heartbeat while long-running jobs are executing, to avoid "stuck" perception.
            # Loop runs when EITHER timeout enforcement is active OR verbose progress is on;
            # snapshots fire only when verbose progress is on. Decoupled per CR-02 fix so
            # PARALLEL_JOB_TIMEOUT_SECONDS works under --quiet (the documented CI scenario at
            # reconftw.cfg:317).
            local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"
            local _to="${PARALLEL_JOB_TIMEOUT_SECONDS:-0}"
            [[ "$_to" =~ ^[0-9]+$ ]] || _to=0
            local _verbose_progress=false
            [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && _verbose_progress=true
            # NB: PARALLEL_HEARTBEAT_SECONDS=0 disables the loop entirely, which also disables timeout enforcement. WR-05 documented this; resolve in Phase 5 DOCS-01 if it surfaces in practice.
            local _loop_active=false
            [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)) && _loop_active=true
            if [[ "$_loop_active" == "true" ]] && { [[ "$_verbose_progress" == "true" ]] || (( _to > 0 )); }; then
                local last_hb now alive hb_active_list job_dur dur_fmt queue_count batch_elapsed hb_done_count
                last_hb=$(date +%s)
                while :; do
                    alive=0
                    hb_done_count=0
                    hb_active_list=""
                    now=$(date +%s)
                    for idx in "${!batch_pids[@]}"; do
                        if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then
                            alive=1
                            job_dur=$((now - batch_starts[$idx]))
                            # Timeout enforcement (RESIL-03 / D-11..D-14): fires regardless of
                            # verbosity (CR-02 fix). Uses hoisted _to from outer scope.
                            if (( _to > 0 )) && (( job_dur > _to )); then
                                _timeout_kill_job "${batch_pids[$idx]}" "${batch_funcs[$idx]}" "$job_dur"
                            fi
                            dur_fmt=$(format_duration "$job_dur")
                            if [[ -z "$hb_active_list" ]]; then
                                hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"
                            else
                                hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"
                            fi
                        else
                            hb_done_count=$((hb_done_count + 1))
                        fi
                    done
                    ((alive == 0)) && break
                    if [[ "$_verbose_progress" == "true" ]] && ((now - last_hb >= hb)); then
                        queue_count=$((total_funcs - queued_count))
                        batch_elapsed=$((now - batch_start_ts))
                        _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count" "$hb_done_count" "${#batch_pids[@]}" "$batch_elapsed"
                        last_hb=$now
                    fi
                    sleep 1
                done
            fi

            local idx rc
            local batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0
            for idx in "${!batch_pids[@]}"; do
                if wait "${batch_pids[$idx]}" 2>/dev/null; then
                    rc=0
                else
                    rc=$?
                fi
                local job_end_ts endts_file
                endts_file="${batch_logs[$idx]%.log}.endts"
                if [[ -f "$endts_file" ]]; then
                    job_end_ts=$(< "$endts_file")
                    rm -f "$endts_file" 2>/dev/null || true
                else
                    job_end_ts=$(date +%s)
                fi
                _parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"
                case "${_PARALLEL_LAST_BADGE:-OK}" in
                    OK) ((batch_ok++)) || true ;;
                    WARN) ((batch_warn++)) || true ;;
                    FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;
                    SKIP) ((batch_skip++)) || true ;;
                    CACHE) ((batch_cache++)) || true ;;
                    *) ((batch_ok++)) || true ;;
                esac
                local job_dur=$((job_end_ts - batch_starts[$idx]))
                local dur_fmt
                dur_fmt=$(format_duration "$job_dur")
                if [[ -z "$done_list" ]]; then
                    done_list="${batch_funcs[$idx]} ${dur_fmt}"
                else
                    done_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"
                fi
                if _parallel_should_show_finished "$rc" "$job_dur" "${_PARALLEL_LAST_BADGE:-OK}"; then
                    _parallel_live_break
                    printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
                fi
                rm -f "${batch_logs[$idx]}" 2>/dev/null || true
            done
            _parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts" "${#batch_pids[@]}" "${#batch_pids[@]}"
            if [[ "${PARALLEL_MODE:-true}" == "true" ]]; then
                local queue_count=$((total_funcs - queued_count))
                local now_ts
                now_ts=$(date +%s)
                _parallel_snapshot "none" "$done_list" "$queue_count" "${#batch_pids[@]}" "${#batch_pids[@]}" "$((now_ts - batch_start_ts))"
            fi
            batch_pids=()
            batch_funcs=()
            batch_logs=()
            batch_starts=()
            batch_start_ts=$(date +%s)
        fi
    done

    # Wait for all remaining jobs and collect exit codes
    if ((${#batch_pids[@]} > 0)); then
        if [[ "${PARALLEL_MODE:-true}" == "true" ]]; then
            local active_list=""
            local i
            for i in "${batch_funcs[@]}"; do
                if [[ -z "$active_list" ]]; then
                    active_list="${i} 00:00"
                else
                    active_list="${active_list}, ${i} 00:00"
                fi
            done
            local queue_count=$((total_funcs - queued_count))
            _parallel_snapshot "$active_list" "$done_list" "$queue_count" "0" "${#batch_pids[@]}" "0"
        fi

        # Heartbeat while long-running jobs are executing, to avoid "stuck" perception.
        # Loop runs when EITHER timeout enforcement is active OR verbose progress is on;
        # snapshots fire only when verbose progress is on. Decoupled per CR-02 fix so
        # PARALLEL_JOB_TIMEOUT_SECONDS works under --quiet (the documented CI scenario at
        # reconftw.cfg:317).
        local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"
        local _to="${PARALLEL_JOB_TIMEOUT_SECONDS:-0}"
        [[ "$_to" =~ ^[0-9]+$ ]] || _to=0
        local _verbose_progress=false
        [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && _verbose_progress=true
        # NB: PARALLEL_HEARTBEAT_SECONDS=0 disables the loop entirely, which also disables timeout enforcement. WR-05 documented this; resolve in Phase 5 DOCS-01 if it surfaces in practice.
        local _loop_active=false
        [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)) && _loop_active=true
        if [[ "$_loop_active" == "true" ]] && { [[ "$_verbose_progress" == "true" ]] || (( _to > 0 )); }; then
            local last_hb now alive hb_active_list job_dur dur_fmt queue_count batch_elapsed hb_done_count
            last_hb=$(date +%s)
            while :; do
                alive=0
                hb_done_count=0
                hb_active_list=""
                now=$(date +%s)
                for idx in "${!batch_pids[@]}"; do
                    if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then
                        alive=1
                        job_dur=$((now - batch_starts[$idx]))
                        # Timeout enforcement (RESIL-03 / D-11..D-14): fires regardless of
                        # verbosity (CR-02 fix). Uses hoisted _to from outer scope.
                        if (( _to > 0 )) && (( job_dur > _to )); then
                            _timeout_kill_job "${batch_pids[$idx]}" "${batch_funcs[$idx]}" "$job_dur"
                        fi
                        dur_fmt=$(format_duration "$job_dur")
                        if [[ -z "$hb_active_list" ]]; then
                            hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"
                        else
                            hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"
                        fi
                    else
                        hb_done_count=$((hb_done_count + 1))
                    fi
                done
                ((alive == 0)) && break
                if [[ "$_verbose_progress" == "true" ]] && ((now - last_hb >= hb)); then
                    queue_count=$((total_funcs - queued_count))
                    batch_elapsed=$((now - batch_start_ts))
                    _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count" "$hb_done_count" "${#batch_pids[@]}" "$batch_elapsed"
                    last_hb=$now
                fi
                sleep 1
            done
        fi

        local idx rc
        local batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0
        for idx in "${!batch_pids[@]}"; do
            if wait "${batch_pids[$idx]}" 2>/dev/null; then
                rc=0
            else
                rc=$?
            fi
            local job_end_ts endts_file
            endts_file="${batch_logs[$idx]%.log}.endts"
            if [[ -f "$endts_file" ]]; then
                job_end_ts=$(< "$endts_file")
                rm -f "$endts_file" 2>/dev/null || true
            else
                job_end_ts=$(date +%s)
            fi
            _parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"
            case "${_PARALLEL_LAST_BADGE:-OK}" in
                OK) ((batch_ok++)) || true ;;
                WARN) ((batch_warn++)) || true ;;
                FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;
                SKIP) ((batch_skip++)) || true ;;
                CACHE) ((batch_cache++)) || true ;;
                *) ((batch_ok++)) || true ;;
            esac
            local job_dur=$((job_end_ts - batch_starts[$idx]))
            local dur_fmt
            dur_fmt=$(format_duration "$job_dur")
            if [[ -z "$done_list" ]]; then
                done_list="${batch_funcs[$idx]} ${dur_fmt}"
            else
                done_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"
            fi
            if _parallel_should_show_finished "$rc" "$job_dur" "${_PARALLEL_LAST_BADGE:-OK}"; then
                _parallel_live_break
                printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
            fi
            rm -f "${batch_logs[$idx]}" 2>/dev/null || true
        done
        _parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts" "${#batch_pids[@]}" "${#batch_pids[@]}"
        if [[ "${PARALLEL_MODE:-true}" == "true" ]]; then
            local now_ts
            now_ts=$(date +%s)
            _parallel_snapshot "none" "$done_list" "0" "${#batch_pids[@]}" "${#batch_pids[@]}" "$((now_ts - batch_start_ts))"
        fi
    fi
    rmdir "$parallel_log_root" 2>/dev/null || true

    return $failed
}

# Run functions in groups (batch parallelization)
# Usage: parallel_batch group_size func1 func2 func3 ...
# Example: parallel_batch 3 func1 func2 func3 func4 func5
# Runs func1,func2,func3 in parallel, waits, then func4,func5
parallel_batch() {
    local batch_size="${1:-3}"
    shift
    
    local -a batch=()
    local func
    
    for func in "$@"; do
        batch+=("$func")
        
        if ((${#batch[@]} >= batch_size)); then
            parallel_funcs "$batch_size" "${batch[@]}"
            batch=()
        fi
    done
    
    # Run remaining functions
    if ((${#batch[@]} > 0)); then
        parallel_funcs "${#batch[@]}" "${batch[@]}"
    fi
}

###############################################################################
# Job Control Helpers
###############################################################################

# Wait for all background jobs with timeout
# Usage: wait_for_jobs [timeout_seconds]
wait_for_jobs() {
    local timeout="${1:-0}"
    local start_time
    start_time=$(date +%s)
    
    while jobs -p | grep -q .; do
        if ((timeout > 0)); then
            local elapsed
            elapsed=$(($(date +%s) - start_time))
            if ((elapsed >= timeout)); then
                print_warnf "Timeout reached, killing remaining jobs"
                jobs -p | xargs -r kill 2>/dev/null || true
                return 1
            fi
        fi
        sleep 1
    done
    
    return 0
}

# Get count of running background jobs
# Usage: running_jobs=$(get_running_jobs)
get_running_jobs() {
    jobs -p | wc -l | tr -d ' '
}

# Kill all tracked parallel jobs (for cleanup)
# Usage: cleanup_parallel_jobs
cleanup_parallel_jobs() {
    local pid
    for pid in "${_PARALLEL_PIDS[@]}"; do
        if kill -0 "$pid" 2>/dev/null; then
            kill "$pid" 2>/dev/null || true
        fi
    done
    _PARALLEL_PIDS=()
}

###############################################################################
# Subdomain Enumeration Parallelization
###############################################################################

# Run passive subdomain enumeration in parallel
# Usage: parallel_passive_enum
# Runs: sub_passive, sub_crt in parallel (sources that don't need resolved subs)
parallel_passive_enum() {
    local funcs=(
        "sub_passive"
        "sub_crt"
    )

    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running passive enumeration in parallel (%d functions)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_SUB_PASSIVE_GROUP_SIZE:-2}" "${funcs[@]}"
}

# Run active subdomain enumeration in parallel
# Usage: parallel_active_enum
# Runs: sub_active, sub_noerror, sub_dns in parallel
parallel_active_enum() {
    local funcs=(
        "sub_active"
        "sub_noerror"
        "sub_dns"
    )

    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running active enumeration in parallel (%d functions)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_SUB_DEP_ACTIVE_GROUP_SIZE:-3}" "${funcs[@]}"
}

# Run post-active subdomain enumeration in parallel
# Usage: parallel_postactive_enum
# Runs: sub_tls, sub_analytics (require resolved subdomains from sub_active)
parallel_postactive_enum() {
    local funcs=(
        "sub_tls"
        "sub_analytics"
    )

    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running post-active enumeration in parallel (%d functions)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_SUB_POST_ACTIVE_GROUP_SIZE:-2}" "${funcs[@]}"
}

# Run brute force enumeration sequentially (resource usage and shared artifacts)
# Usage: parallel_brute_enum
parallel_brute_enum() {
    local funcs=(
        "sub_brute"
        "sub_permut"
        "sub_regex_permut"
        "sub_ia_permut"
    )

    # Kept for compatibility with existing helper-based orchestrators.
    # Brute force/permutation stages should not run concurrently.
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running brute force enumeration sequentially%b\n" \
            "${bblue:-}" "${reset:-}"
    fi

    local func rc failed=0
    for func in "${funcs[@]}"; do
        if ! declare -f "$func" >/dev/null 2>&1; then
            print_warnf "Function %s not found, skipping" "$func"
            continue
        fi

        "$func"
        rc=$?
        if ((rc > 0)); then
            if [[ "${CONTINUE_ON_TOOL_ERROR:-true}" == "true" ]]; then
                print_warnf "Brute phase function %s failed (rc=%d); continuing" "$func" "$rc"
                ((failed++)) || true
            else
                print_errorf "Brute phase function %s failed (rc=%d)" "$func" "$rc"
                return 1
            fi
        fi
    done

    return "$failed"
}

###############################################################################
# Vulnerability Scanning Parallelization
###############################################################################

# Run web vulnerability checks in parallel
# Usage: parallel_web_vulns
parallel_web_vulns() {
    local funcs=(
        "crlf_checks"
        "xss"
    )
    
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running web vulnerability checks in parallel (%d checks)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_VULNS_GROUP1_SIZE:-4}" "${funcs[@]}"
}

# Run injection vulnerability checks in parallel
# Usage: parallel_injection_vulns
parallel_injection_vulns() {
    local funcs=(
        "sqli"
        "ssti"
        "lfi"
        "command_injection"
    )
    
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running injection vulnerability checks in parallel (%d checks)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_VULNS_GROUP2_SIZE:-4}" "${funcs[@]}"
}

# Run server-side vulnerability checks in parallel
# Usage: parallel_server_vulns
parallel_server_vulns() {
    local funcs=(
        "ssrf_checks"
        "lfi"
    )
    
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running server-side vulnerability checks in parallel (%d checks)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_VULNS_GROUP1_SIZE:-4}" "${funcs[@]}"
}

###############################################################################
# OSINT Parallelization
###############################################################################

# Run OSINT gathering in parallel
# Usage: parallel_osint
parallel_osint() {
    local funcs=(
        "google_dorks"
        "github_dorks"
        "metadata"
        "emails"
        "domain_info"
    )
    
    if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
        printf "%b[*] Running OSINT gathering in parallel (%d sources)%b\n" \
            "${bblue:-}" "${#funcs[@]}" "${reset:-}"
    fi

    parallel_funcs "${PAR_OSINT_GROUP1_SIZE:-4}" "${funcs[@]}"
}

###############################################################################
# Full Pipeline Parallelization
###############################################################################

# Orchestrate full subdomain enumeration with parallelization
# parallel_subdomains_full() and parallel_vulns_full() removed — defined but never called