Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

more fixes on its way

six2dezfc5356be38f9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 69175e9..7e91ee6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -153,6 +153,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `test_ssl` now respects `run_command`, preventing real execution during dry-run.
- `sub_asn` now applies timeout fencing with kill semantics (`-k`) when timeout tooling is available and reports non-OK outcomes correctly.
- `subdomains` finalization now safely creates missing incremental files before counting to avoid stderr noise.
+- Fixed benign non-zero paths (`anew`, optional pipelines) that were surfacing as false `ERR(1)` noise in logs.
+- Hardened `sub_crt` JSON parsing to avoid noisy `jq` parse errors on malformed/empty upstream responses.
+- Parallel job output now strips ANSI/control sequences before replaying tail/full logs, reducing report/export parser issues.
+- Function/subfunction end log timestamps now use real end times (not reused start timestamps).
+- Added heartbeat progress lines for long-running phases (`nuclei`, `ffuf/interlace`, `cmseek`, and selected passive collectors) to avoid "stuck" perception.
**Parallel Execution and Status Reporting**
- Parallel aggregation now preserves per-function final badges (`OK/WARN/FAIL/SKIP/CACHE`) from module functions/subfunctions instead of inferring only from process exit code.
diff --git a/lib/common.sh b/lib/common.sh
index 6780dbc..73f231b 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -450,6 +450,60 @@ run_tool() {
"$@"
}
+# Remove ANSI/control sequences from a text stream.
+# Usage: some_command | strip_ansi_stream
+strip_ansi_stream() {
+ # CSI + OSC sequence stripping, keeping regular text intact.
+ sed -E $'s/\x1B\\[[0-9;?]*[ -/]*[@-~]//g; s/\x1B\\][^\a]*(\a|\x1B\\\\)//g'
+}
+
+# Run a command with periodic heartbeat status lines for long-running tasks.
+# Usage: run_with_heartbeat "label" [interval_seconds] command [args...]
+run_with_heartbeat() {
+ local label="${1:-task}"
+ shift
+ local interval="${HEARTBEAT_INTERVAL_SECONDS:-20}"
+ if [[ "${1:-}" =~ ^[0-9]+$ ]]; then
+ interval="$1"
+ shift
+ fi
+
+ if [[ $# -eq 0 ]]; then
+ return 1
+ fi
+
+ if [[ "${DRY_RUN:-false}" == "true" ]]; then
+ run_command "$@"
+ return $?
+ fi
+
+ local start_ts now_ts elapsed last_hb
+ start_ts=$(date +%s)
+ last_hb="$start_ts"
+
+ local hb_log="/dev/null"
+ if [[ -n "${LOGFILE:-}" ]]; then
+ hb_log="$LOGFILE"
+ fi
+
+ run_command "$@" >>"$hb_log" 2>&1 &
+ local cmd_pid=$!
+
+ while kill -0 "$cmd_pid" 2>/dev/null; do
+ sleep 1
+ now_ts=$(date +%s)
+ if ((now_ts - last_hb >= interval)); then
+ elapsed=$((now_ts - start_ts))
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then
+ _print_msg INFO "${label} still running (${elapsed}s elapsed)"
+ fi
+ last_hb="$now_ts"
+ fi
+ done
+
+ wait "$cmd_pid"
+}
+
# Execute command and capture line count result with validation
# Usage: NUMOFLINES=$(safe_count "command | pipeline")
# Always returns a valid number (0 on failure)
diff --git a/lib/parallel.sh b/lib/parallel.sh
index b57284e..23946b3 100644
--- a/lib/parallel.sh
+++ b/lib/parallel.sh
@@ -12,6 +12,7 @@ declare -r _PARALLEL_SH_LOADED=1
# Default maximum concurrent jobs (can be overridden)
PARALLEL_MAX_JOBS="${PARALLEL_MAX_JOBS:-4}"
+PARALLEL_HEARTBEAT_SECONDS="${PARALLEL_HEARTBEAT_SECONDS:-20}"
# Track running PIDs for cleanup
declare -a _PARALLEL_PIDS=()
@@ -105,7 +106,13 @@ _parallel_emit_job_output() {
badge="FAIL"
fi
- [[ -z "$badge" ]] && badge=$([[ "$rc" -eq 0 ]] && echo "OK" || echo "FAIL")
+ if [[ -z "$badge" ]]; then
+ if [[ "$rc" -eq 0 ]]; then
+ badge="OK"
+ else
+ badge="FAIL"
+ fi
+ fi
_PARALLEL_LAST_BADGE="$badge"
# In quiet mode (OUTPUT_VERBOSITY==0), only show failures
@@ -145,7 +152,7 @@ _parallel_emit_job_output() {
_print_status "$badge" "$func_name" "${duration}s"
fi
if [[ -s "$log_file" ]]; then
- tail -n "$show_lines" "$log_file"
+ tail -n "$show_lines" "$log_file" | strip_ansi_stream
fi
;;
full)
@@ -158,7 +165,7 @@ _parallel_emit_job_output() {
_print_status "$badge" "$func_name" "${duration}s"
fi
if [[ -s "$log_file" ]]; then
- cat "$log_file"
+ cat "$log_file" | strip_ansi_stream
fi
;;
esac
@@ -283,8 +290,9 @@ parallel_funcs() {
queued_count=$((queued_count + 1))
# Log if verbose (OUTPUT_VERBOSITY >= 2)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Started %s (PID: %d)%b\n" "${cyan:-}" "$func" "${batch_pids[${#batch_pids[@]}-1]}" "${reset:-}"
+ fi
# If we've reached max jobs, flush the current batch.
if ((${#batch_pids[@]} >= max_jobs)); then
@@ -292,11 +300,47 @@ parallel_funcs() {
local active_list=""
local i
for i in "${batch_funcs[@]}"; do
- [[ -z "$active_list" ]] && active_list="${i} 00:00" || active_list="${active_list}, ${i} 00:00"
+ if [[ -z "$active_list" ]]; then
+ active_list="${i} 00:00"
+ else
+ active_list="${active_list}, ${i} 00:00"
+ fi
done
local queue_count=$((total_funcs - queued_count))
_parallel_snapshot "$active_list" "$done_list" "$queue_count"
fi
+
+ # Heartbeat while long-running jobs are executing, to avoid "stuck" perception.
+ local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"
+ if [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)); then
+ local last_hb now alive hb_active_list job_dur dur_fmt queue_count
+ last_hb=$(date +%s)
+ while :; do
+ alive=0
+ hb_active_list=""
+ now=$(date +%s)
+ for idx in "${!batch_pids[@]}"; do
+ if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then
+ alive=1
+ job_dur=$((now - batch_starts[$idx]))
+ dur_fmt=$(format_duration "$job_dur")
+ if [[ -z "$hb_active_list" ]]; then
+ hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"
+ else
+ hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"
+ fi
+ fi
+ done
+ ((alive == 0)) && break
+ if ((now - last_hb >= hb)); then
+ queue_count=$((total_funcs - queued_count))
+ _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count"
+ last_hb=$now
+ fi
+ sleep 1
+ done
+ fi
+
local idx rc
local batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0
for idx in "${!batch_pids[@]}"; do
@@ -309,12 +353,12 @@ parallel_funcs() {
job_end_ts=$(date +%s)
_parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"
case "${_PARALLEL_LAST_BADGE:-OK}" in
- OK) ((batch_ok++)) ;;
- WARN) ((batch_warn++)) ;;
- FAIL) ((batch_fail++)); ((failed++)) ;;
- SKIP) ((batch_skip++)) ;;
- CACHE) ((batch_cache++)) ;;
- *) ((batch_ok++)) ;;
+ OK) ((batch_ok++)) || true ;;
+ WARN) ((batch_warn++)) || true ;;
+ FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;
+ SKIP) ((batch_skip++)) || true ;;
+ CACHE) ((batch_cache++)) || true ;;
+ *) ((batch_ok++)) || true ;;
esac
local job_dur=$((job_end_ts - batch_starts[$idx]))
local dur_fmt
@@ -324,7 +368,9 @@ parallel_funcs() {
else
done_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"
fi
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
+ printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
+ fi
rm -f "${batch_logs[$idx]}" 2>/dev/null || true
done
_parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts"
@@ -346,11 +392,46 @@ parallel_funcs() {
local active_list=""
local i
for i in "${batch_funcs[@]}"; do
- [[ -z "$active_list" ]] && active_list="${i} 00:00" || active_list="${active_list}, ${i} 00:00"
+ if [[ -z "$active_list" ]]; then
+ active_list="${i} 00:00"
+ else
+ active_list="${active_list}, ${i} 00:00"
+ fi
done
local queue_count=$((total_funcs - queued_count))
_parallel_snapshot "$active_list" "$done_list" "$queue_count"
fi
+
+ local hb="${PARALLEL_HEARTBEAT_SECONDS:-20}"
+ if [[ "${PARALLEL_MODE:-true}" == "true" ]] && [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]] && [[ "$hb" =~ ^[0-9]+$ ]] && ((hb > 0)); then
+ local last_hb now alive hb_active_list job_dur dur_fmt queue_count
+ last_hb=$(date +%s)
+ while :; do
+ alive=0
+ hb_active_list=""
+ now=$(date +%s)
+ for idx in "${!batch_pids[@]}"; do
+ if kill -0 "${batch_pids[$idx]}" 2>/dev/null; then
+ alive=1
+ job_dur=$((now - batch_starts[$idx]))
+ dur_fmt=$(format_duration "$job_dur")
+ if [[ -z "$hb_active_list" ]]; then
+ hb_active_list="${batch_funcs[$idx]} ${dur_fmt}"
+ else
+ hb_active_list="${hb_active_list}, ${batch_funcs[$idx]} ${dur_fmt}"
+ fi
+ fi
+ done
+ ((alive == 0)) && break
+ if ((now - last_hb >= hb)); then
+ queue_count=$((total_funcs - queued_count))
+ _parallel_snapshot "${hb_active_list:-none}" "$done_list" "$queue_count"
+ last_hb=$now
+ fi
+ sleep 1
+ done
+ fi
+
local idx rc
local batch_ok=0 batch_warn=0 batch_fail=0 batch_skip=0 batch_cache=0
for idx in "${!batch_pids[@]}"; do
@@ -363,12 +444,12 @@ parallel_funcs() {
job_end_ts=$(date +%s)
_parallel_emit_job_output "${batch_funcs[$idx]}" "${batch_logs[$idx]}" "$rc" "${batch_starts[$idx]}" "$job_end_ts"
case "${_PARALLEL_LAST_BADGE:-OK}" in
- OK) ((batch_ok++)) ;;
- WARN) ((batch_warn++)) ;;
- FAIL) ((batch_fail++)); ((failed++)) ;;
- SKIP) ((batch_skip++)) ;;
- CACHE) ((batch_cache++)) ;;
- *) ((batch_ok++)) ;;
+ OK) ((batch_ok++)) || true ;;
+ WARN) ((batch_warn++)) || true ;;
+ FAIL) ((batch_fail++)) || true; ((failed++)) || true ;;
+ SKIP) ((batch_skip++)) || true ;;
+ CACHE) ((batch_cache++)) || true ;;
+ *) ((batch_ok++)) || true ;;
esac
local job_dur=$((job_end_ts - batch_starts[$idx]))
local dur_fmt
@@ -378,7 +459,9 @@ parallel_funcs() {
else
done_list="${done_list}, ${batch_funcs[$idx]} ${dur_fmt}"
fi
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
+ printf "%b[*] Finished %s (PID: %s, rc=%s)%b\n" "${cyan:-}" "${batch_funcs[$idx]}" "${batch_pids[$idx]}" "$rc" "${reset:-}"
+ fi
rm -f "${batch_logs[$idx]}" 2>/dev/null || true
done
_parallel_batch_summary "$batch_ok" "$batch_warn" "$batch_fail" "$batch_skip" "$batch_cache" "$batch_start_ts"
@@ -475,9 +558,10 @@ parallel_passive_enum() {
"sub_crt"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running passive enumeration in parallel (%d functions)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 2 "${funcs[@]}"
}
@@ -492,9 +576,10 @@ parallel_active_enum() {
"sub_dns"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running active enumeration in parallel (%d functions)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 3 "${funcs[@]}"
}
@@ -508,9 +593,10 @@ parallel_postactive_enum() {
"sub_analytics"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running post-active enumeration in parallel (%d functions)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 2 "${funcs[@]}"
}
@@ -525,9 +611,10 @@ parallel_brute_enum() {
)
# Brute force is resource intensive, run with limit of 2
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running brute force enumeration (limited parallelism)%b\n" \
"${bblue:-}" "${reset:-}"
+ fi
parallel_funcs 2 "${funcs[@]}"
}
@@ -546,9 +633,10 @@ parallel_web_vulns() {
"xss"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running web vulnerability checks in parallel (%d checks)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 4 "${funcs[@]}"
}
@@ -563,9 +651,10 @@ parallel_injection_vulns() {
"command_injection"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running injection vulnerability checks in parallel (%d checks)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 4 "${funcs[@]}"
}
@@ -579,9 +668,10 @@ parallel_server_vulns() {
"smuggling"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running server-side vulnerability checks in parallel (%d checks)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 3 "${funcs[@]}"
}
@@ -601,9 +691,10 @@ parallel_osint() {
"domain_info"
)
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Running OSINT gathering in parallel (%d sources)%b\n" \
"${bblue:-}" "${#funcs[@]}" "${reset:-}"
+ fi
parallel_funcs 4 "${funcs[@]}"
}
@@ -616,16 +707,21 @@ parallel_osint() {
# Usage: parallel_subdomains_full
# This replaces the sequential execution in subdomains_full()
parallel_subdomains_full() {
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Starting parallelized subdomain enumeration%b\n" \
"${bblue:-}" "${reset:-}"
+ fi
# Phase 1: Passive enumeration (parallel - no dependencies)
parallel_passive_enum
# Merge passive results before active phase
- [[ -s ".tmp/passive_subs.txt" ]] && cat .tmp/passive_subs.txt | anew -q subdomains/subdomains.txt
- [[ -s ".tmp/crtsh_subs.txt" ]] && cat .tmp/crtsh_subs.txt | anew -q subdomains/subdomains.txt
+ if [[ -s ".tmp/passive_subs.txt" ]]; then
+ cat .tmp/passive_subs.txt | anew -q subdomains/subdomains.txt
+ fi
+ if [[ -s ".tmp/crtsh_subs.txt" ]]; then
+ cat .tmp/crtsh_subs.txt | anew -q subdomains/subdomains.txt
+ fi
# Phase 2: Active enumeration (parallel - resolves subdomains)
parallel_active_enum
@@ -639,20 +735,26 @@ parallel_subdomains_full() {
fi
# Phase 5: Scraping and recursive (sequential - depends on previous results)
- [[ ${SUBSCRAPING:-false} == true ]] && sub_scraping
- [[ ${SUB_RECURSIVE_BRUTE:-false} == true ]] && sub_recursive_brute
+ if [[ ${SUBSCRAPING:-false} == true ]]; then
+ sub_scraping
+ fi
+ if [[ ${SUB_RECURSIVE_BRUTE:-false} == true ]]; then
+ sub_recursive_brute
+ fi
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Parallelized subdomain enumeration complete%b\n" \
"${bgreen:-}" "${reset:-}"
+ fi
}
# Orchestrate full vulnerability scanning with parallelization
# Usage: parallel_vulns_full
parallel_vulns_full() {
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Starting parallelized vulnerability scanning%b\n" \
"${bblue:-}" "${reset:-}"
+ fi
# Run nuclei first (it's comprehensive)
nuclei_check
@@ -662,7 +764,8 @@ parallel_vulns_full() {
parallel_injection_vulns
parallel_server_vulns
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
printf "%b[*] Parallelized vulnerability scanning complete%b\n" \
"${bgreen:-}" "${reset:-}"
+ fi
}
diff --git a/modules/core.sh b/modules/core.sh
index adaf3e2..8cf9e51 100644
--- a/modules/core.sh
+++ b/modules/core.sh
@@ -1129,7 +1129,9 @@ function end_func() {
getElapsedTime "$start" "$end"
record_func_timing "${fn}" "$((end - start))"
local duration=$((end - start))
- echo "[$current_date] End function: ${fn} " >>"${LOGFILE}"
+ local end_date
+ end_date=$(date +'%Y-%m-%d %H:%M:%S')
+ echo "[$end_date] End function: ${fn} " >>"${LOGFILE}"
local badge="OK"
case "$status" in
info|INFO) badge="INFO" ;;
@@ -1190,7 +1192,9 @@ function end_subfunc() {
if [[ "${OUTPUT_VERBOSITY:-1}" -ge 1 ]]; then
_print_status "$status" " ${2}" "${duration}s"
fi
- echo "[$current_date] End subfunction: ${1} " >>"${LOGFILE}"
+ local end_sub_date
+ end_sub_date=$(date +'%Y-%m-%d %H:%M:%S')
+ echo "[$end_sub_date] End subfunction: ${1} " >>"${LOGFILE}"
log_json "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"
if declare -F ui_log_jsonl >/dev/null 2>&1; then
ui_log_jsonl "SUCCESS" "${2}" "Subfunction completed" "runtime=${runtime}" "duration_sec=${duration}"
diff --git a/modules/osint.sh b/modules/osint.sh
index e0e66a7..e966b70 100644
--- a/modules/osint.sh
+++ b/modules/osint.sh
@@ -371,7 +371,7 @@ function third_party_misconfigs() {
fi
run_command misconfig-mapper -update-templates 1>>"$LOGFILE"
- run_command misconfig-mapper -target visma.com -as-domain true -permutations false -skip-ssl \
+ run_command misconfig-mapper -target "$domain" -as-domain true -permutations false -skip-ssl \
-service "*" -verbose 0 | anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
run_command misconfig-mapper -target "$company_name" -skip-ssl -verbose 0 -service "*" \
| anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
diff --git a/modules/subdomains.sh b/modules/subdomains.sh
index 5d31ec1..3fa9999 100644
--- a/modules/subdomains.sh
+++ b/modules/subdomains.sh
@@ -478,7 +478,7 @@ function sub_asn() {
| grep -E '^([a-zA-Z0-9\.\-]+\.)+[a-zA-Z]{1,}$' \
| grep "\.${DOMAIN_ESCAPED}$\|^${DOMAIN_ESCAPED}$" \
| sort -u \
- | anew -q .tmp/subs_no_resolved.txt
+ | anew -q .tmp/subs_no_resolved.txt || true
if [[ -s .tmp/subs_no_resolved.txt ]]; then
_print_msg INFO "ASN sources added $(wc -l <.tmp/subs_no_resolved.txt | tr -d ' ') in-scope domains"
fi
@@ -604,7 +604,7 @@ function sub_crt() {
fi
run_command curl -s "https://bgp.he.net/certs/api/list?domain=$domain" \
- | jq -r 'try .domains[].domain' \
+ | jq -r 'try .domains[]?.domain // empty' 2>>"$LOGFILE" \
| sed -e 's/^\*\.//' \
| anew -q .tmp/crtsh_subdomains.txt || true
@@ -861,13 +861,13 @@ function sub_dns() {
# Extract various DNS records and process them
jq -r --arg domain "$domain" '.. | strings | select(test("\\." + $domain + "$"))' <"subdomains/subdomains_dnsregs.json" \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \
- | sort -u | anew -q .tmp/subdomains_dns.txt
+ | sort -u | anew -q .tmp/subdomains_dns.txt || true
jq -r '.. | strings | select(test("^(\\d{1,3}\\.){3}\\d{1,3}$|^[0-9a-fA-F:]+$"))' <"subdomains/subdomains_dnsregs.json" \
| sort -u | run_command hakip2host | awk '{print $3}' | unfurl -u domains \
| sed -e 's/^\*\.//' -e 's/\.$//' -e '/\./!d' | grep "\.${DOMAIN_ESCAPED}$" \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' | sort -u \
- | anew -q .tmp/subdomains_dns.txt
+ | anew -q .tmp/subdomains_dns.txt || true
jq -r '.. | strings | select(test("^(\\d{1,3}\\.){3}\\d{1,3}$|^[0-9a-fA-F:]+$"))' <"subdomains/subdomains_dnsregs.json" \
| sort -u \
@@ -881,7 +881,7 @@ function sub_dns() {
done
jq -r 'select(.host) |"\(.host) - \((.a // [])[])", "\(.host) - \((.aaaa // [])[])"' <"subdomains/subdomains_dnsregs.json" \
- | grep -E ' - [0-9a-fA-F:.]+$' | sort -u | anew -q "subdomains/subdomains_ips.txt"
+ | grep -E ' - [0-9a-fA-F:.]+$' | sort -u | anew -q "subdomains/subdomains_ips.txt" || true
fi
if [[ $AXIOM != true ]]; then
@@ -1214,7 +1214,7 @@ function sub_analytics() {
if [[ -s ".tmp/analytics_subs_tmp.txt" ]]; then
grep "\.${DOMAIN_ESCAPED}$\|^${DOMAIN_ESCAPED}$" .tmp/analytics_subs_tmp.txt \
| grep -E '^([a-zA-Z0-9\.\-]+\.)+[a-zA-Z]{1,}$' \
- | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt
+ | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt || true
if [[ $AXIOM != true ]]; then
diff --git a/modules/web.sh b/modules/web.sh
index cceb9f7..3629c4d 100644
--- a/modules/web.sh
+++ b/modules/web.sh
@@ -745,13 +745,17 @@ _nuclei_scan_local() {
_print_msg INFO "Running: Nuclei Severity: ${crit}"
# Non-WAF at default rate
if [[ -s "$NOWAF_LIST" ]]; then
- run_command nuclei -l "$NOWAF_LIST" -severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" -silent -retries 2 "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_json.txt" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "nuclei ${crit} (normal targets)" nuclei \
+ -l "$NOWAF_LIST" -severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" -silent -retries 2 \
+ "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_json.txt"
fi
# WAF hosts at slower rate
if [[ -s "$WAF_LIST" ]]; then
local slow_rl
slow_rl=$((NUCLEI_RATELIMIT / 3 + 1))
- run_command nuclei -l "$WAF_LIST" -severity "$crit" -nh -rl "$slow_rl" -silent -retries 2 "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_waf_json.txt" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "nuclei ${crit} (waf targets)" nuclei \
+ -l "$WAF_LIST" -severity "$crit" -nh -rl "$slow_rl" -silent -retries 2 \
+ "${NUCLEI_EXTRA_ARGS}" -t "${NUCLEI_TEMPLATES_PATH}" -j -o "nuclei_output/${crit}_waf_json.txt"
[[ -s "nuclei_output/${crit}_waf_json.txt" ]] && cat "nuclei_output/${crit}_waf_json.txt" >>"nuclei_output/${crit}_json.txt"
fi
_nuclei_parse_results "$crit"
@@ -766,10 +770,10 @@ _nuclei_scan_axiom() {
for crit in "${severity_array[@]}"; do
_print_msg INFO "Running: Axiom Nuclei Severity: ${crit}. Check results in nuclei_output folder."
- run_command axiom-scan .tmp/webs_subs.txt -m nuclei \
+ run_with_heartbeat "axiom nuclei ${crit}" axiom-scan .tmp/webs_subs.txt -m nuclei \
--nuclei-templates "$NUCLEI_TEMPLATES_PATH" \
-severity "$crit" -nh -rl "$NUCLEI_RATELIMIT" \
- -silent -retries 2 "$NUCLEI_EXTRA_ARGS" -j -o "nuclei_output/${crit}_json.txt" "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null
+ -silent -retries 2 "$NUCLEI_EXTRA_ARGS" -j -o "nuclei_output/${crit}_json.txt" "$AXIOM_EXTRA_ARGS"
_nuclei_parse_results "$crit"
done
printf "\n\n"
@@ -963,7 +967,10 @@ _fuzz_run_local() {
_fuzz_classify_targets
if [[ -s .tmp/webs_normal.txt ]]; then
- run_command interlace -tL .tmp/webs_normal.txt -threads "${INTERLACE_THREADS}" -c "ffuf ${FFUF_FLAGS} -t ${FFUF_THREADS} -rate ${FFUF_RATELIMIT} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" -o "$dir/.tmp/fuzzing" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "ffuf normal targets" interlace \
+ -tL .tmp/webs_normal.txt -threads "${INTERLACE_THREADS}" \
+ -c "ffuf ${FFUF_FLAGS} -t ${FFUF_THREADS} -rate ${FFUF_RATELIMIT} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \
+ -o "$dir/.tmp/fuzzing"
fi
if [[ -s .tmp/webs_slow.txt ]]; then
@@ -971,14 +978,17 @@ _fuzz_run_local() {
[[ $slow_threads -lt 5 ]] && slow_threads=5
local slow_rate=$FFUF_RATELIMIT
[[ ${FFUF_RATELIMIT:-0} -eq 0 ]] && slow_rate=50 || slow_rate=$((FFUF_RATELIMIT / 3 + 1))
- run_command interlace -tL .tmp/webs_slow.txt -threads "${INTERLACE_THREADS}" -c "ffuf ${FFUF_FLAGS} -t ${slow_threads} -rate ${slow_rate} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" -o "$dir/.tmp/fuzzing" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "ffuf slow targets" interlace \
+ -tL .tmp/webs_slow.txt -threads "${INTERLACE_THREADS}" \
+ -c "ffuf ${FFUF_FLAGS} -t ${slow_threads} -rate ${slow_rate} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \
+ -o "$dir/.tmp/fuzzing"
fi
}
# Run ffuf via axiom-scan and parse per-subdomain results
_fuzz_run_axiom() {
cached_download_typed "${fuzzing_remote_list}" ".tmp/fuzzing_remote_list.txt" "onelistforallmicro.txt" "wordlists"
- run_command axiom-scan webs/webs_all.txt -m ffuf -wL .tmp/fuzzing_remote_list.txt -H "${HEADER}" "$FFUF_FLAGS" -s -maxtime "$FFUF_MAXTIME" -oJ "$dir/.tmp/ffuf-content.json" "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "axiom ffuf" axiom-scan webs/webs_all.txt -m ffuf -wL .tmp/fuzzing_remote_list.txt -H "${HEADER}" "$FFUF_FLAGS" -s -maxtime "$FFUF_MAXTIME" -oJ "$dir/.tmp/ffuf-content.json" "$AXIOM_EXTRA_ARGS"
while read -r sub; do
local sub_out
@@ -1124,9 +1134,9 @@ function cms_scanner() {
"${tools}/CMSeeK/venv/bin/python3" "${tools}/CMSeeK/cmseek.py"
-l webs/webs_all.txt --batch -r
)
- local exit_status=0
- if ! run_command "${cmseek_cmd[@]}" &>>"$LOGFILE"; then
- exit_status=$?
+ run_with_heartbeat "cmseek batch scan" "${cmseek_cmd[@]}"
+ local exit_status=$?
+ if [[ ${exit_status} -ne 0 ]]; then
# Attempt one-time repair on known CMSeeK index corruption.
if [[ ${exit_status} -eq 124 || ${exit_status} -eq 137 ]]; then
echo "TIMEOUT cmseek.py - investigate manually for $dir" >>"$LOGFILE"
@@ -1200,7 +1210,7 @@ function urlchecks() {
log_note "urlchecks: reusing waymore output from sub_scraping" "${FUNCNAME[0]}" "${LINENO}"
cat .tmp/waymore_urls_subs.txt | anew -q .tmp/url_extract_tmp.txt || true
else
- if ! run_command "$waymore_timeout_cmd" "${WAYMORE_TIMEOUT:-30m}" waymore -i "$domain" -mode U -oU .tmp/waymore_urls.txt 2>>"$LOGFILE" >/dev/null; then
+ if ! run_with_heartbeat "waymore passive urls" "$waymore_timeout_cmd" "${WAYMORE_TIMEOUT:-30m}" waymore -i "$domain" -mode U -oU .tmp/waymore_urls.txt; then
log_note "urlchecks: waymore failed or timed out; continuing with other passive sources" "${FUNCNAME[0]}" "${LINENO}"
fi
if [[ -s ".tmp/waymore_urls.txt" ]]; then
@@ -1289,9 +1299,9 @@ function urlchecks() {
if [[ $diff_webs != "0" ]] || [[ ! -s ".tmp/katana.txt" ]]; then
if [[ $URL_CHECK_ACTIVE == true ]]; then
if [[ $DEEP == true ]]; then
- run_command axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 3 -fs rdn --max-runtime 4h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "axiom katana (deep)" axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 3 -fs rdn --max-runtime 4h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS"
else
- run_command axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 2 -fs rdn --max-runtime 3h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null
+ run_with_heartbeat "axiom katana" axiom-scan webs/webs_all.txt -m katana -jc -kf all -d 2 -fs rdn --max-runtime 3h -o .tmp/katana.txt "$AXIOM_EXTRA_ARGS"
fi
fi
fi
@@ -1700,20 +1710,20 @@ function wordlist_gen() {
start_func "${FUNCNAME[0]}" "Wordlist Generation"
- [[ -s ".tmp/url_extract_tmp.txt" ]] && cat webs/url_extract.txt | anew -q .tmp/url_extract_tmp.txt
+ [[ -s ".tmp/url_extract_tmp.txt" ]] && cat webs/url_extract.txt | anew -q .tmp/url_extract_tmp.txt || true
# Ensure url_extract_tmp.txt exists and is not empty
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
# Define patterns for keys and values
cat ".tmp/url_extract_tmp.txt" | unfurl -u keys 2>>"$LOGFILE" \
| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \
- | anew -q webs/dict_keys.txt
+ | anew -q webs/dict_keys.txt || true
cat ".tmp/url_extract_tmp.txt" | unfurl -u values 2>>"$LOGFILE" \
| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \
- | anew -q webs/dict_values.txt
+ | anew -q webs/dict_values.txt || true
_print_msg INFO "Extracting words..."
- tr "[:punct:]" "\n" <".tmp/url_extract_tmp.txt" | anew -q "webs/dict_words.txt"
+ tr "[:punct:]" "\n" <".tmp/url_extract_tmp.txt" | anew -q "webs/dict_words.txt" || true
fi
end_func "Results are saved in $domain/webs/dict_[words|paths].txt" "${FUNCNAME[0]}"