reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities
Shell bug-bountybugbountybugbounty-tooldnshackingnucleiosintpenetration-testingpentestpentest-toolpentestingreconreconnaissancescannersecuritysecurity-toolssubdomainsubdomain-enumerationvulnerabilities
Fix webprobe simple
six2dez79318154b0d0
diff --git a/modules/modes.sh b/modules/modes.shindex 46f1de9..9533a85 100644--- a/modules/modes.sh+++ b/modules/modes.sh@@ -679,7 +679,6 @@ function recon() {subtakeoverremove_big_filess3buckets- cloud_extra_providersui_module_end "Subdomains" "subdomains/subdomains.txt" "webs/webs.txt"progress_module "Subdomains"diff --git a/modules/subdomains.sh b/modules/subdomains.shindex daf87b2..a84b08b 100644--- a/modules/subdomains.sh+++ b/modules/subdomains.sh@@ -5,7 +5,7 @@# sub_noerror, sub_dns, sub_brute, sub_scraping, sub_analytics,# sub_permut, sub_regex_permut, sub_ia_permut, sub_recursive_passive,# sub_recursive_brute, subtakeover, zonetransfer, s3buckets,-# cloud_extra_providers, geo_info+# geo_info# Helpers: deep_wildcard_filter, _is_sensitive_domain# This file is sourced by reconftw.sh - do not execute directly[[ -z "${SCRIPTPATH:-}" ]] && {@@ -2089,61 +2089,6 @@ function s3buckets() {fi}-function cloud_extra_providers() {- ensure_dirs subdomains .tmp-- if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then- start_func "${FUNCNAME[0]}" "Extra cloud providers checks"- # Candidate names from domain and subdomains- company=$(unfurl format %r <<<"$domain")- printf "%s\n%s\n${domain%%.*}" "$company" "$domain" | sed 's/[^a-zA-Z0-9-]//g' | awk 'length>2' | sort -u >.tmp/cloudnames.txt- # shellcheck disable=SC2046 # Word splitting intended for awk- [[ -s subdomains/subdomains.txt ]] && awk -F. -v r="$(echo "$domain" | awk -F. '{print $(NF-1)"."$NF}')" '{print $(NF-2)}' subdomains/subdomains.txt 2>/dev/null | sed 's/[^a-zA-Z0-9-]//g' | awk 'length>2' | sort -u >>.tmp/cloudnames.txt- sed_i 's/^\///; s/\.$//' .tmp/cloudnames.txt- sort -u .tmp/cloudnames.txt -o .tmp/cloudnames.txt-- # Common container names for Azure (best-effort)- printf "public\nstatic\nmedia\nimages\nassets\nbackup\nbackups\nfiles\ncdn" >.tmp/azcontainers.txt-- : >subdomains/cloud_extra.txt-- while IFS= read -r name; do- # GCS- for u in \- "https://storage.googleapis.com/$name/" \- "https://$name.storage.googleapis.com/"; do- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)- if [[ $code =~ ^20|403$ ]]; then printf "GCS %s %s\n" "$name" "$u" | anew -q subdomains/cloud_extra.txt; fi- done-- # Azure Blob (requires container; try a few)- while IFS= read -r c; do- u="https://$name.blob.core.windows.net/$c?restype=container&comp=list"- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)- if [[ $code =~ ^20|403$ ]]; then printf "AZURE %s %s\n" "$name/$c" "$u" | anew -q subdomains/cloud_extra.txt; fi- done <.tmp/azcontainers.txt-- # DigitalOcean Spaces (regional)- for r in ams3 nyc3 sfo3 sgp1 fra1 blr1; do- u="https://$name.$r.digitaloceanspaces.com/"- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)- if [[ $code =~ ^20|403$ ]]; then printf "DOSPACE %s %s\n" "$name.$r" "$u" | anew -q subdomains/cloud_extra.txt; fi- done-- # Backblaze B2 (best-effort listing)- u="https://f000.backblazeb2.com/file/$name/"- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)- if [[ $code =~ ^20|403$ ]]; then printf "B2 %s %s\n" "$name" "$u" | anew -q subdomains/cloud_extra.txt; fi-- done <.tmp/cloudnames.txt-- append_assets_from_file cloud asset subdomains/cloud_extra.txt- end_func "Results are saved in subdomains/cloud_extra.txt" "${FUNCNAME[0]}"- else- _print_msg WARN "${FUNCNAME[0]} skipped or already processed."- fi-}-############################################################################################################################################################ GEOLOCALIZATION INFO ###########################################################################################################################################################diff --git a/modules/web.sh b/modules/web.shindex 25f1385..d1e3907 100644--- a/modules/web.sh+++ b/modules/web.sh@@ -103,7 +103,7 @@ function webprobe_simple() {start_subfunc "${FUNCNAME[0]}" "Running: HTTP probing $domain"# Baseline cache files (some modules merge into these).- touch .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt 2>/dev/null || true+ touch .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt webs/webs.txt 2>/dev/null || true# If in multi mode and subdomains.txt doesn't exist, create itif [[ -n $multi ]] && [[ ! -f "$dir/subdomains/subdomains.txt" ]]; then@@ -124,33 +124,51 @@ function webprobe_simple() {-o .tmp/web_full_info_probe.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/nullfi- # Merge current probe output with prior cache/state.- touch .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>/dev/null || true- if ! cat .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>>"$LOGFILE" \- | jq -cs 'unique_by(.input)[]' 2>>"$LOGFILE" >webs/web_full_info.txt; then- log_note "webprobe_simple: failed to merge httpx JSON; falling back to probe-only" "${FUNCNAME[0]}" "${LINENO}"- awk 'match($0, /^[[:space:]]*\\{/) {print}' .tmp/web_full_info_probe.txt >.tmp/web_full_info_merge_input.jsonl 2>/dev/null || true- if [[ -s ".tmp/web_full_info_merge_input.jsonl" ]]; then- jq -cs 'unique_by(.input)[]' .tmp/web_full_info_merge_input.jsonl 2>>"$LOGFILE" >webs/web_full_info.txt || : >webs/web_full_info.txt- else- : >webs/web_full_info.txt- fi- fi- # Keep cache as JSONL for later merges.- cp webs/web_full_info.txt .tmp/web_full_info.txt 2>/dev/null || true-- # Extract URLs- if [[ -s "webs/web_full_info.txt" ]]; then- jq -r 'try .url' webs/web_full_info.txt 2>/dev/null \- | grep "$domain" \- | grep -aEo 'https?://[^ ]+' \- | sed 's/*.//' | anew -q .tmp/probed_tmp.txt- fi-- # Adaptive throttling heuristics: mark slow hosts (429/403) from httpx- if [[ -s "webs/web_full_info.txt" ]]; then- jq -r 'try select(.status_code==403 or .status_code==429) | .url' webs/web_full_info.txt 2>/dev/null \- | awk -F/ '{print $3}' | sed 's/\:$//' | sort -u >.tmp/slow_hosts.txt+ # webprobe_simple is expected to write JSONL when using httpx -json.+ # Some runners (or wrappers) may produce a plain URL list instead.+ # Detect the format early to avoid jq parse errors and missing webs/webs.txt.+ local probe_first_line probe_is_json+ probe_first_line="$(awk 'NF {print; exit}' .tmp/web_full_info_probe.txt 2>/dev/null || true)"+ probe_is_json=false+ [[ "$probe_first_line" =~ ^[[:space:]]*\\{ ]] && probe_is_json=true++ # Always start fresh for this run (used by urlchecks diff too).+ : >.tmp/probed_tmp.txt 2>/dev/null || true++ if [[ "$probe_is_json" == true ]]; then+ # Merge current probe output with prior cache/state.+ touch .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>/dev/null || true+ if ! cat .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>>"$LOGFILE" \+ | jq -cs 'unique_by(.input)[]' 2>>"$LOGFILE" >webs/web_full_info.txt; then+ log_note "webprobe_simple: failed to merge httpx JSON; falling back to probe-only" "${FUNCNAME[0]}" "${LINENO}"+ awk 'match($0, /^[[:space:]]*\\{/) {print}' .tmp/web_full_info_probe.txt >.tmp/web_full_info_merge_input.jsonl 2>/dev/null || true+ if [[ -s ".tmp/web_full_info_merge_input.jsonl" ]]; then+ jq -cs 'unique_by(.input)[]' .tmp/web_full_info_merge_input.jsonl 2>>"$LOGFILE" >webs/web_full_info.txt || : >webs/web_full_info.txt+ else+ : >webs/web_full_info.txt+ fi+ fi+ # Keep cache as JSONL for later merges.+ cp webs/web_full_info.txt .tmp/web_full_info.txt 2>/dev/null || true++ # Extract URLs from JSONL+ if [[ -s "webs/web_full_info.txt" ]]; then+ jq -r 'try (.url // empty)' webs/web_full_info.txt 2>/dev/null \+ | awk -v dom="$domain" 'index($0, dom) && $0 ~ /^https?:\\/\\// {print}' \+ | sed 's/*.//' | anew -q .tmp/probed_tmp.txt+ fi+ else+ log_note "webprobe_simple: probe output not JSON; treating as URL list" "${FUNCNAME[0]}" "${LINENO}"+ if [[ -s ".tmp/web_full_info_probe.txt" ]]; then+ awk -v dom="$domain" 'index($0, dom) && $0 ~ /^https?:\\/\\// {print}' .tmp/web_full_info_probe.txt 2>/dev/null \+ | sed 's/*.//' | anew -q .tmp/probed_tmp.txt+ fi+ fi++ # Adaptive throttling heuristics: mark slow hosts (429/403) from httpx+ if [[ -s "webs/web_full_info.txt" ]]; then+ jq -r 'try select(.status_code==403 or .status_code==429) | .url' webs/web_full_info.txt 2>/dev/null \+ | awk -F/ '{print $3}' | sed 's/\:$//' | sort -u >.tmp/slow_hosts.txtfi# Extract web info to plain text@@ -164,14 +182,14 @@ function webprobe_simple() {if ! deleteOutScoped "$outOfScope_file" .tmp/probed_tmp.txt; thenprint_warnf "Failed to delete out-of-scope entries."fi- fi+ fi- touch .tmp/probed_tmp.txt+ touch .tmp/probed_tmp.txt- # Count new websites- if ! NUMOFLINES=$(anew webs/webs.txt <.tmp/probed_tmp.txt 2>/dev/null | sed '/^$/d' | wc -l); then- print_warnf "Failed to count new websites."- NUMOFLINES=0+ # Count new websites+ if ! NUMOFLINES=$(anew webs/webs.txt <.tmp/probed_tmp.txt 2>/dev/null | sed '/^$/d' | wc -l); then+ print_warnf "Failed to count new websites."+ NUMOFLINES=0fi# Update webs_all.txtdiff --git a/tests/unit/test_webprobe_simple_merge.bats b/tests/unit/test_webprobe_simple_merge.batsindex c0e8c63..f8f569c 100644--- a/tests/unit/test_webprobe_simple_merge.bats+++ b/tests/unit/test_webprobe_simple_merge.bats@@ -121,3 +121,35 @@ SHgrep -q "new.example.com" "webs/web_full_info.txt"! grep -q "NOT_JSON" ".tmp/web_full_info.txt"}++@test "webprobe_simple promotes URL list output into webs/webs.txt and webs_all" {+ mkdir -p .tmp webs subdomains+ printf "a.example.com\n" > subdomains/subdomains.txt++ cat > "$MOCK_BIN/httpx" <<'SH'+#!/usr/bin/env bash+out=""+while [[ $# -gt 0 ]]; do+ case "$1" in+ -o)+ out="$2"+ shift 2+ ;;+ *)+ shift+ ;;+ esac+done+printf '%s\n' "https://a.example.com" "https://b.example.com" > "$out"+SH+ chmod +x "$MOCK_BIN/httpx"++ run webprobe_simple+ [ "$status" -eq 0 ]+ [ -s "webs/webs.txt" ]+ grep -q "https://a.example.com" "webs/webs.txt"+ grep -q "https://b.example.com" "webs/webs.txt"+ [ -s "webs/webs_all.txt" ]+ grep -q "https://a.example.com" "webs/webs_all.txt"+ grep -q "https://b.example.com" "webs/webs_all.txt"+}