reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities
Shell bug-bountybugbountybugbounty-tooldnshackingnucleiosintpenetration-testingpentestpentest-toolpentestingreconreconnaissancescannersecuritysecurity-toolssubdomainsubdomain-enumerationvulnerabilities
more fixes on its way
six2dezfc5356be38f9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 69175e9..7e91ee6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -153,6 +153,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- `test_ssl` now respects `run_command`, preventing real execution during dry-run.- `sub_asn` now applies timeout fencing with kill semantics (`-k`) when timeout tooling is available and reports non-OK outcomes correctly.- `subdomains` finalization now safely creates missing incremental files before counting to avoid stderr noise.+- Fixed benign non-zero paths (`anew`, optional pipelines) that were surfacing as false `ERR(1)` noise in logs.+- Hardened `sub_crt` JSON parsing to avoid noisy `jq` parse errors on malformed/empty upstream responses.+- Parallel job output now strips ANSI/control sequences before replaying tail/full logs, reducing report/export parser issues.+- Function/subfunction end log timestamps now use real end times (not reused start timestamps).+- Added heartbeat progress lines for long-running phases (`nuclei`, `ffuf/interlace`, `cmseek`, and selected passive collectors) to avoid "stuck" perception.**Parallel Execution and Status Reporting**- Parallel aggregation now preserves per-function final badges (`OK/WARN/FAIL/SKIP/CACHE`) from module functions/subfunctions instead of inferring only from process exit code.diff --git a/lib/common.sh b/lib/common.shindex 6780dbc..73f231b 100644--- a/lib/common.sh+++ b/lib/common.sh@@ -450,6 +450,60 @@ run_tool() {"$@"}+# Remove ANSI/control sequences from a text stream.+# Usage: some_command | strip_ansi_stream+strip_ansi_stream() {+ # CSI + OSC sequence stripping, keeping regular text intact.+ sed -E $'s/\x1B\\[[0-9;?]*[ -/]*[@-~]//g; s/\x1B\\][^\a]*(\a|\x1B\\\\)//g'+}++# Run a command with periodic heartbeat status lines for long-running tasks.+# Usage: run_with_heartbeat "label" [interval_seconds] command [args...]+run_with_heartbeat() {+ local label="${1:-task}"+ shift+ local interval="${HEARTBEAT_INTERVAL_SECONDS:-20}"+ if [[ "${1:-}" =~ ^[0-9]+$ ]]; then+ interval="$1"+ shift+ fi++ if [[ $# -eq 0 ]]; then+ return 1+ fi++ if [[ "${DRY_RUN:-false}" == "true" ]]; then+ run_command "$@"+ return $?+ fi++ local start_ts now_ts elapsed last_hb+ start_ts=$(date +%s)+ last_hb="$start_ts"++ local hb_log="/dev/null"+ if [[ -n "${LOGFILE:-}" ]]; then+ hb_log="$LOGFILE"+ fi++ run_command "$@" >>"$hb_log" 2>&1 &+ local cmd_pid=$!++ while kill -0 "$cmd_pid" 2>/dev/null; do+ sleep 1+ now_ts=$(date +%s)+ if ((now_ts - last_hb >= interval)); then+ elapsed=$((now_ts - start_ts))+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then+ _print_msg INFO "${label} still running (${elapsed}s elapsed)"+ fi+ last_hb="$now_ts"+ fi+ done++ wait "$cmd_pid"+}+# Execute command and capture line count result with validation# Usage: NUMOFLINES=$(safe_count "command | pipeline")# Always returns a valid number (0 on failure)diff --git a/lib/parallel.sh b/lib/parallel.shindex b57284e..23946b3 100644--- a/lib/parallel.sh+++ b/lib/parallel.sh@@ -12,6 +12,7 @@ declare -r _PARALLEL_SH_LOADED=1# Default maximum concurrent jobs (can be overridden)PARALLEL_MAX_JOBS="${PARALLEL_MAX_JOBS:-4}"+PARALLEL_HEARTBEAT_SECONDS="${PARALLEL_HEARTBEAT_SECONDS:-20}"# Track running PIDs for cleanupdeclare -a _PARALLEL_PIDS=()@@ -105,7 +106,13 @@ _parallel_emit_job_output() {badge="FAIL"fi- [[ -z "$badge" ]] && badge=$([[ "$rc" -eq 0 ]] && echo "OK" || echo "FAIL")+ if [[ -z "$badge" ]]; then+ if [[ "$rc" -eq 0 ]]; then+ badge="OK"+ else+ badge="FAIL"+ fi+ fi_PARALLEL_LAST_BADGE="$badge"# In quiet mode (OUTPUT_VERBOSITY==0), only show failures@@ -145,7 +152,7 @@ _parallel_emit_job_output() {_print_status "$badge" "$func_name" "${duration}s"fiif [[ -s "$log_file" ]]; then- tail -n "$show_lines" "$log_file"+ tail -n "$show_lines" "$log_file" | strip_ansi_streamfi;;full)@@ -158,7 +165,7 @@ _parallel_emit_job_output() {_print_status "$badge" "$func_name" "${duration}s"fiif [[ -s "$log_file" ]]; then- cat "$log_file"+ cat "$log_file" | strip_ansi_streamfi;;esac@@ -283,8 +290,9 @@ parallel_funcs() {queued_count=$((queued_count + 1))# Log if verbose (OUTPUT_VERBOSITY >= 2)- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Started %s (PID: %d)%b\n" "${cyan:-}" "$func" "${batch_pids[${#batch_pids[@]}-1]}" "${reset:-}"+ fi# If we've reached max jobs, flush the current batch.if ((${#batch_pids[@]} >= max_jobs)); then@@ -292,11 +300,47 @@ parallel_funcs() {local active_list=""local ifor i in "${batch_funcs[@]}"; do- [[ -z "$active_list" ]] && active_list="${i} 00:00" || active_list="${active_list}, ${i} 00:00"+ if [[ -z "$active_list" ]]; then+ active_list="${i} 00:00"+ else+ active_list="${active_list}, ${i} 00:00"+ fidonelocal queue_count=$((total_funcs - queued_count))_parallel_snapshot "$active_list" "$done_list" "$queue_count"fi++ # Heartbeat while long-running jobs are executing, to avoid "stuck" perception.+ local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"+ if [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)); then+ local last_hb now alive hb_active_list job_dur dur_fmt queue_count+ last_hb=$(date +%s)+ while :; do+ alive=0+ hb_active_list=""+ now=$(date +%s)+ for idx in "${!batch_pids[@]}"; do+ if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then+ alive=1+ job_dur=$((now - batch_starts[$idx]))+ dur_fmt=$(format_duration "$job_dur")+ if [[ -z "$hb_active_list" ]]; then+ hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"+ else+ hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"+ fi+ fi+ done+ ((alive == 0)) && break+ if ((now - last_hb >= hb)); then+ queue_count=$((total_funcs - queued_count))+ _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count"+ last_hb=$now+ fi+ sleep 1+ done+ fi+local idx rclocal batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0for idx in "${!batch_pids[@]}"; do@@ -309,12 +353,12 @@ parallel_funcs() {job_end_ts=$(date +%s)_parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"case "${_PARALLEL_LAST_BADGE:-OK}" in- OK) ((batch_ok++)) ;;- WARN) ((batch_warn++)) ;;- FAIL) ((batch_fail++)); ((failed++)) ;;- SKIP) ((batch_skip++)) ;;- CACHE) ((batch_cache++)) ;;- *) ((batch_ok++)) ;;+ OK) ((batch_ok++)) || true ;;+ WARN) ((batch_warn++)) || true ;;+ FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;+ SKIP) ((batch_skip++)) || true ;;+ CACHE) ((batch_cache++)) || true ;;+ *) ((batch_ok++)) || true ;;esaclocal job_dur=$((job_end_ts - batch_starts[$idx]))local dur_fmt@@ -324,7 +368,9 @@ parallel_funcs() {elsedone_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"fi- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then+ printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"+ firm -f "${batch_logs[$idx]}" 2>/dev/null || truedone_parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts"@@ -346,11 +392,46 @@ parallel_funcs() {local active_list=""local ifor i in "${batch_funcs[@]}"; do- [[ -z "$active_list" ]] && active_list="${i} 00:00" || active_list="${active_list}, ${i} 00:00"+ if [[ -z "$active_list" ]]; then+ active_list="${i} 00:00"+ else+ active_list="${active_list}, ${i} 00:00"+ fidonelocal queue_count=$((total_funcs - queued_count))_parallel_snapshot "$active_list" "$done_list" "$queue_count"fi++ local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"+ if [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)); then+ local last_hb now alive hb_active_list job_dur dur_fmt queue_count+ last_hb=$(date +%s)+ while :; do+ alive=0+ hb_active_list=""+ now=$(date +%s)+ for idx in "${!batch_pids[@]}"; do+ if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then+ alive=1+ job_dur=$((now - batch_starts[$idx]))+ dur_fmt=$(format_duration "$job_dur")+ if [[ -z "$hb_active_list" ]]; then+ hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"+ else+ hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"+ fi+ fi+ done+ ((alive == 0)) && break+ if ((now - last_hb >= hb)); then+ queue_count=$((total_funcs - queued_count))+ _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count"+ last_hb=$now+ fi+ sleep 1+ done+ fi+local idx rclocal batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0for idx in "${!batch_pids[@]}"; do@@ -363,12 +444,12 @@ parallel_funcs() {job_end_ts=$(date +%s)_parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"case "${_PARALLEL_LAST_BADGE:-OK}" in- OK) ((batch_ok++)) ;;- WARN) ((batch_warn++)) ;;- FAIL) ((batch_fail++)); ((failed++)) ;;- SKIP) ((batch_skip++)) ;;- CACHE) ((batch_cache++)) ;;- *) ((batch_ok++)) ;;+ OK) ((batch_ok++)) || true ;;+ WARN) ((batch_warn++)) || true ;;+ FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;+ SKIP) ((batch_skip++)) || true ;;+ CACHE) ((batch_cache++)) || true ;;+ *) ((batch_ok++)) || true ;;esaclocal job_dur=$((job_end_ts - batch_starts[$idx]))local dur_fmt@@ -378,7 +459,9 @@ parallel_funcs() {elsedone_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"fi- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then+ printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"+ firm -f "${batch_logs[$idx]}" 2>/dev/null || truedone_parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts"@@ -475,9 +558,10 @@ parallel_passive_enum() {"sub_crt")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running passive enumeration in parallel (%d functions)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 2 "${funcs[@]}"}@@ -492,9 +576,10 @@ parallel_active_enum() {"sub_dns")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running active enumeration in parallel (%d functions)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 3 "${funcs[@]}"}@@ -508,9 +593,10 @@ parallel_postactive_enum() {"sub_analytics")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running post-active enumeration in parallel (%d functions)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 2 "${funcs[@]}"}@@ -525,9 +611,10 @@ parallel_brute_enum() {)# Brute force is resource intensive, run with limit of 2- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running brute force enumeration (limited parallelism)%b\n" \"${bblue:-}" "${reset:-}"+ fiparallel_funcs 2 "${funcs[@]}"}@@ -546,9 +633,10 @@ parallel_web_vulns() {"xss")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running web vulnerability checks in parallel (%d checks)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 4 "${funcs[@]}"}@@ -563,9 +651,10 @@ parallel_injection_vulns() {"command_injection")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running injection vulnerability checks in parallel (%d checks)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 4 "${funcs[@]}"}@@ -579,9 +668,10 @@ parallel_server_vulns() {"smuggling")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running server-side vulnerability checks in parallel (%d checks)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 3 "${funcs[@]}"}@@ -601,9 +691,10 @@ parallel_osint() {"domain_info")- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Running OSINT gathering in parallel (%d sources)%b\n" \"${bblue:-}" "${#funcs[@]}" "${reset:-}"+ fiparallel_funcs 4 "${funcs[@]}"}@@ -616,16 +707,21 @@ parallel_osint() {# Usage: parallel_subdomains_full# This replaces the sequential execution in subdomains_full()parallel_subdomains_full() {- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Starting parallelized subdomain enumeration%b\n" \"${bblue:-}" "${reset:-}"+ fi# Phase 1: Passive enumeration (parallel - no dependencies)parallel_passive_enum# Merge passive results before active phase- [[ -s ".tmp/passive_subs.txt" ]] && cat .tmp/passive_subs.txt | anew -q subdomains/subdomains.txt- [[ -s ".tmp/crtsh_subs.txt" ]] && cat .tmp/crtsh_subs.txt | anew -q subdomains/subdomains.txt+ if [[ -s ".tmp/passive_subs.txt" ]]; then+ cat .tmp/passive_subs.txt | anew -q subdomains/subdomains.txt+ fi+ if [[ -s ".tmp/crtsh_subs.txt" ]]; then+ cat .tmp/crtsh_subs.txt | anew -q subdomains/subdomains.txt+ fi# Phase 2: Active enumeration (parallel - resolves subdomains)parallel_active_enum@@ -639,20 +735,26 @@ parallel_subdomains_full() {fi# Phase 5: Scraping and recursive (sequential - depends on previous results)- [[ ${SUBSCRAPING:-false} == true ]] && sub_scraping- [[ ${SUB_RECURSIVE_BRUTE:-false} == true ]] && sub_recursive_brute+ if [[ ${SUBSCRAPING:-false} == true ]]; then+ sub_scraping+ fi+ if [[ ${SUB_RECURSIVE_BRUTE:-false} == true ]]; then+ sub_recursive_brute+ fi- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Parallelized subdomain enumeration complete%b\n" \"${bgreen:-}" "${reset:-}"+ fi}# Orchestrate full vulnerability scanning with parallelization# Usage: parallel_vulns_fullparallel_vulns_full() {- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Starting parallelized vulnerability scanning%b\n" \"${bblue:-}" "${reset:-}"+ fi# Run nuclei first (it's comprehensive)nuclei_check@@ -662,7 +764,8 @@ parallel_vulns_full() {parallel_injection_vulnsparallel_server_vulns- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; thenprintf "%b[*] Parallelized vulnerability scanning complete%b\n" \"${bgreen:-}" "${reset:-}"+ fi}diff --git a/modules/core.sh b/modules/core.shindex adaf3e2..8cf9e51 100644--- a/modules/core.sh+++ b/modules/core.sh@@ -1129,7 +1129,9 @@ function end_func() {getElapsedTime "$start" "$end"record_func_timing "${fn}" "$((end - start))"local duration=$((end - start))- echo "[$current_date] End function: ${fn} " >>"${LOGFILE}"+ local end_date+ end_date=$(date +'%Y-%m-%d %H:%M:%S')+ echo "[$end_date] End function: ${fn} " >>"${LOGFILE}"local badge="OK"case "$status" ininfo|INFO) badge="INFO" ;;@@ -1190,7 +1192,9 @@ function end_subfunc() {if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then_print_status "$status" " ${2}" "${duration}s"fi- echo "[$current_date] End subfunction: ${1} " >>"${LOGFILE}"+ local end_sub_date+ end_sub_date=$(date +'%Y-%m-%d %H:%M:%S')+ echo "[$end_sub_date] End subfunction: ${1} " >>"${LOGFILE}"log_json "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"if declare -F ui_log_jsonl >/dev/null 2>&1; thenui_log_jsonl "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"diff --git a/modules/osint.sh b/modules/osint.shindex e0e66a7..e966b70 100644--- a/modules/osint.sh+++ b/modules/osint.sh@@ -371,7 +371,7 @@ function third_party_misconfigs() {firun_command misconfig-mapper -update-templates 1>>"$LOGFILE"- run_command misconfig-mapper -target visma.com -as-domain true -permutations false -skip-ssl \+ run_command misconfig-mapper -target "$domain" -as-domain true -permutations false -skip-ssl \-service "*" -verbose 0 | anew -q "${dir}/osint/3rdparts_misconfigurations.txt"run_command misconfig-mapper -target "$company_name" -skip-ssl -verbose 0 -service "*" \| anew -q "${dir}/osint/3rdparts_misconfigurations.txt"diff --git a/modules/subdomains.sh b/modules/subdomains.shindex 5d31ec1..3fa9999 100644--- a/modules/subdomains.sh+++ b/modules/subdomains.sh@@ -478,7 +478,7 @@ function sub_asn() {| grep -E '^([a-zA-Z0-9\.\-]+\.)+[a-zA-Z]{1,}$' \| grep "\.${DOMAIN_ESCAPED}$\|^${DOMAIN_ESCAPED}$" \| sort -u \- | anew -q .tmp/subs_no_resolved.txt+ | anew -q .tmp/subs_no_resolved.txt || trueif [[ -s .tmp/subs_no_resolved.txt ]]; then_print_msg INFO "ASN sources added $(wc -l <.tmp/subs_no_resolved.txt | tr -d ' ') in-scope domains"fi@@ -604,7 +604,7 @@ function sub_crt() {firun_command curl -s "https://bgp.he.net/certs/api/list?domain=$domain" \- | jq -r 'try .domains[].domain' \+ | jq -r 'try .domains[]?.domain // empty' 2>>"$LOGFILE" \| sed -e 's/^\*\.//' \| anew -q .tmp/crtsh_subdomains.txt || true@@ -861,13 +861,13 @@ function sub_dns() {# Extract various DNS records and process themjq -r --arg domain "$domain" '.. | strings | select(test("\\." + $domain + "$"))' <"subdomains/subdomains_dnsregs.json" \| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \- | sort -u | anew -q .tmp/subdomains_dns.txt+ | sort -u | anew -q .tmp/subdomains_dns.txt || truejq -r '.. | strings | select(test("^(\\d{1,3}\\.){3}\\d{1,3}$|^[0-9a-fA-F:]+$"))' <"subdomains/subdomains_dnsregs.json" \| sort -u | run_command hakip2host | awk '{print $3}' | unfurl -u domains \| sed -e 's/^\*\.//' -e 's/\.$//' -e '/\./!d' | grep "\.${DOMAIN_ESCAPED}$" \| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' | sort -u \- | anew -q .tmp/subdomains_dns.txt+ | anew -q .tmp/subdomains_dns.txt || truejq -r '.. | strings | select(test("^(\\d{1,3}\\.){3}\\d{1,3}$|^[0-9a-fA-F:]+$"))' <"subdomains/subdomains_dnsregs.json" \| sort -u \@@ -881,7 +881,7 @@ function sub_dns() {donejq -r 'select(.host) |"\(.host) - \((.a // [])[])", "\(.host) - \((.aaaa // [])[])"' <"subdomains/subdomains_dnsregs.json" \- | grep -E ' - [0-9a-fA-F:.]+$' | sort -u | anew -q "subdomains/subdomains_ips.txt"+ | grep -E ' - [0-9a-fA-F:.]+$' | sort -u | anew -q "subdomains/subdomains_ips.txt" || truefiif [[ $AXIOM != true ]]; then@@ -1214,7 +1214,7 @@ function sub_analytics() {if [[ -s ".tmp/analytics_subs_tmp.txt" ]]; thengrep "\.${DOMAIN_ESCAPED}$\|^${DOMAIN_ESCAPED}$" .tmp/analytics_subs_tmp.txt \| grep -E '^([a-zA-Z0-9\.\-]+\.)+[a-zA-Z]{1,}$' \- | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt+ | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt || trueif [[ $AXIOM != true ]]; thendiff --git a/modules/web.sh b/modules/web.shindex cceb9f7..3629c4d 100644--- a/modules/web.sh+++ b/modules/web.sh@@ -745,13 +745,17 @@ _nuclei_scan_local() {_print_msg INFO "Running: Nuclei Severity: ${crit}"# Non-WAF at default rateif [[ -s "$NOWAF_LIST" ]]; then- run_command nuclei -l "$NOWAF_LIST" -severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" -silent -retries 2 "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_json.txt" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "nuclei ${crit} (normal targets)" nuclei \+ -l "$NOWAF_LIST" -severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" -silent -retries 2 \+ "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_json.txt"fi# WAF hosts at slower rateif [[ -s "$WAF_LIST" ]]; thenlocal slow_rlslow_rl=$((NUCLEI_RATELIMIT / 3 + 1))- run_command nuclei -l "$WAF_LIST" -severity "$crit" -nh -rl "$slow_rl" -silent -retries 2 "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_waf_json.txt" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "nuclei ${crit} (waf targets)" nuclei \+ -l "$WAF_LIST" -severity "$crit" -nh -rl "$slow_rl" -silent -retries 2 \+ "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_waf_json.txt"[[ -s "nuclei_output/${crit}_waf_json.txt" ]] && cat "nuclei_output/${crit}_waf_json.txt" >>"nuclei_output/${crit}_json.txt"fi_nuclei_parse_results "$crit"@@ -766,10 +770,10 @@ _nuclei_scan_axiom() {for crit in "${severity_array[@]}"; do_print_msg INFO "Running: Axiom Nuclei Severity: ${crit}. Check results in nuclei_output folder."- run_command axiom-scan .tmp/webs_subs.txt -m nuclei \+ run_with_heartbeat "axiom nuclei ${crit}" axiom-scan .tmp/webs_subs.txt -m nuclei \--nuclei-templates "$NUCLEI_TEMPLATES_PATH" \-severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" \- -silent -retries 2 "$NUCLEI_EXTRA_ARGS" -j -o "nuclei_output/${crit}_json.txt" "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null+ -silent -retries 2 "$NUCLEI_EXTRA_ARGS" -j -o "nuclei_output/${crit}_json.txt" "$AXIOM_EXTRA_ARGS"_nuclei_parse_results "$crit"doneprintf "\n\n"@@ -963,7 +967,10 @@ _fuzz_run_local() {_fuzz_classify_targetsif [[ -s .tmp/webs_normal.txt ]]; then- run_command interlace -tL .tmp/webs_normal.txt -threads "${INTERLACE_THREADS}" -c "ffuf ${FFUF_FLAGS} -t ${FFUF_THREADS} -rate ${FFUF_RATELIMIT} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" -o "$dir/.tmp/fuzzing" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "ffuf normal targets" interlace \+ -tL .tmp/webs_normal.txt -threads "${INTERLACE_THREADS}" \+ -c "ffuf ${FFUF_FLAGS} -t ${FFUF_THREADS} -rate ${FFUF_RATELIMIT} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \+ -o "$dir/.tmp/fuzzing"fiif [[ -s .tmp/webs_slow.txt ]]; then@@ -971,14 +978,17 @@ _fuzz_run_local() {[[ $slow_threads -lt 5 ]] && slow_threads=5local slow_rate=$FFUF_RATELIMIT[[ ${FFUF_RATELIMIT:-0} -eq 0 ]] && slow_rate=50 || slow_rate=$((FFUF_RATELIMIT / 3 + 1))- run_command interlace -tL .tmp/webs_slow.txt -threads "${INTERLACE_THREADS}" -c "ffuf ${FFUF_FLAGS} -t ${slow_threads} -rate ${slow_rate} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" -o "$dir/.tmp/fuzzing" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "ffuf slow targets" interlace \+ -tL .tmp/webs_slow.txt -threads "${INTERLACE_THREADS}" \+ -c "ffuf ${FFUF_FLAGS} -t ${slow_threads} -rate ${slow_rate} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \+ -o "$dir/.tmp/fuzzing"fi}# Run ffuf via axiom-scan and parse per-subdomain results_fuzz_run_axiom() {cached_download_typed "${fuzzing_remote_list}" ".tmp/fuzzing_remote_list.txt" "onelistforallmicro.txt" "wordlists"- run_command axiom-scan webs/webs_all.txt -m ffuf -wL .tmp/fuzzing_remote_list.txt -H "${HEADER}" "$FFUF_FLAGS" -s -maxtime "$FFUF_MAXTIME" -oJ "$dir/.tmp/ffuf-content.json" "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "axiom ffuf" axiom-scan webs/webs_all.txt -m ffuf -wL .tmp/fuzzing_remote_list.txt -H "${HEADER}" "$FFUF_FLAGS" -s -maxtime "$FFUF_MAXTIME" -oJ "$dir/.tmp/ffuf-content.json" "$AXIOM_EXTRA_ARGS"while read -r sub; dolocal sub_out@@ -1124,9 +1134,9 @@ function cms_scanner() {"${tools}/CMSeeK/venv/bin/python3" "${tools}/CMSeeK/cmseek.py"-l webs/webs_all.txt --batch -r)- local exit_status=0- if ! run_command "${cmseek_cmd[@]}" &>>"$LOGFILE"; then- exit_status=$?+ run_with_heartbeat "cmseek batch scan" "${cmseek_cmd[@]}"+ local exit_status=$?+ if [[ ${exit_status} -ne 0 ]]; then# Attempt one-time repair on known CMSeeK index corruption.if [[ ${exit_status} -eq 124 || ${exit_status} -eq 137 ]]; thenecho "TIMEOUT cmseek.py - investigate manually for $dir" >>"$LOGFILE"@@ -1200,7 +1210,7 @@ function urlchecks() {log_note "urlchecks: reusing waymore output from sub_scraping" "${FUNCNAME[0]}" "${LINENO}"cat .tmp/waymore_urls_subs.txt | anew -q .tmp/url_extract_tmp.txt || trueelse- if ! run_command "$waymore_timeout_cmd" "${WAYMORE_TIMEOUT:-30m}" waymore -i "$domain" -mode U -oU .tmp/waymore_urls.txt 2>>"$LOGFILE" >/dev/null; then+ if ! run_with_heartbeat "waymore passive urls" "$waymore_timeout_cmd" "${WAYMORE_TIMEOUT:-30m}" waymore -i "$domain" -mode U -oU .tmp/waymore_urls.txt; thenlog_note "urlchecks: waymore failed or timed out; continuing with other passive sources" "${FUNCNAME[0]}" "${LINENO}"fiif [[ -s ".tmp/waymore_urls.txt" ]]; then@@ -1289,9 +1299,9 @@ function urlchecks() {if [[ $diff_webs != "0" ]] || [[ ! -s ".tmp/katana.txt" ]]; thenif [[ $URL_CHECK_ACTIVE == true ]]; thenif [[ $DEEP == true ]]; then- run_command axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 3 -fs rdn --max-runtime 4h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "axiom katana (deep)" axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 3 -fs rdn --max-runtime 4h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS"else- run_command axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 2 -fs rdn --max-runtime 3h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null+ run_with_heartbeat "axiom katana" axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 2 -fs rdn --max-runtime 3h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS"fififi@@ -1700,20 +1710,20 @@ function wordlist_gen() {start_func "${FUNCNAME[0]}" "Wordlist Generation"- [[ -s ".tmp/url_extract_tmp.txt" ]] && cat webs/url_extract.txt | anew -q .tmp/url_extract_tmp.txt+ [[ -s ".tmp/url_extract_tmp.txt" ]] && cat webs/url_extract.txt | anew -q .tmp/url_extract_tmp.txt || true# Ensure url_extract_tmp.txt exists and is not emptyif [[ -s ".tmp/url_extract_tmp.txt" ]]; then# Define patterns for keys and valuescat ".tmp/url_extract_tmp.txt" | unfurl -u keys 2>>"$LOGFILE" \| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \- | anew -q webs/dict_keys.txt+ | anew -q webs/dict_keys.txt || truecat ".tmp/url_extract_tmp.txt" | unfurl -u values 2>>"$LOGFILE" \| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \- | anew -q webs/dict_values.txt+ | anew -q webs/dict_values.txt || true_print_msg INFO "Extracting words..."- tr "[:punct:]" "\n" <".tmp/url_extract_tmp.txt" | anew -q "webs/dict_words.txt"+ tr "[:punct:]" "\n" <".tmp/url_extract_tmp.txt" | anew -q "webs/dict_words.txt" || truefiend_func "Results are saved in $domain/webs/dict_[words|paths].txt" "${FUNCNAME[0]}"