Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

Fix webprobe simple

six2dez79318154b0d0
diff --git a/modules/modes.sh b/modules/modes.sh
index 46f1de9..9533a85 100644
--- a/modules/modes.sh
+++ b/modules/modes.sh
@@ -679,7 +679,6 @@ function recon() {
subtakeover
remove_big_files
s3buckets
- cloud_extra_providers
ui_module_end "Subdomains" "subdomains/subdomains.txt" "webs/webs.txt"
progress_module "Subdomains"
diff --git a/modules/subdomains.sh b/modules/subdomains.sh
index daf87b2..a84b08b 100644
--- a/modules/subdomains.sh
+++ b/modules/subdomains.sh
@@ -5,7 +5,7 @@
# sub_noerror, sub_dns, sub_brute, sub_scraping, sub_analytics,
# sub_permut, sub_regex_permut, sub_ia_permut, sub_recursive_passive,
# sub_recursive_brute, subtakeover, zonetransfer, s3buckets,
-# cloud_extra_providers, geo_info
+# geo_info
# Helpers: deep_wildcard_filter, _is_sensitive_domain
# This file is sourced by reconftw.sh - do not execute directly
[[ -z "${SCRIPTPATH:-}" ]] && {
@@ -2089,61 +2089,6 @@ function s3buckets() {
fi
}
-function cloud_extra_providers() {
- ensure_dirs subdomains .tmp
-
- if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
- start_func "${FUNCNAME[0]}" "Extra cloud providers checks"
- # Candidate names from domain and subdomains
- company=$(unfurl format %r <<<"$domain")
- printf "%s\n%s\n${domain%%.*}" "$company" "$domain" | sed 's/[^a-zA-Z0-9-]//g' | awk 'length>2' | sort -u >.tmp/cloudnames.txt
- # shellcheck disable=SC2046 # Word splitting intended for awk
- [[ -s subdomains/subdomains.txt ]] && awk -F. -v r="$(echo "$domain" | awk -F. '{print $(NF-1)"."$NF}')" '{print $(NF-2)}' subdomains/subdomains.txt 2>/dev/null | sed 's/[^a-zA-Z0-9-]//g' | awk 'length>2' | sort -u >>.tmp/cloudnames.txt
- sed_i 's/^\///; s/\.$//' .tmp/cloudnames.txt
- sort -u .tmp/cloudnames.txt -o .tmp/cloudnames.txt
-
- # Common container names for Azure (best-effort)
- printf "public\nstatic\nmedia\nimages\nassets\nbackup\nbackups\nfiles\ncdn" >.tmp/azcontainers.txt
-
- : >subdomains/cloud_extra.txt
-
- while IFS= read -r name; do
- # GCS
- for u in \
- "https://storage.googleapis.com/$name/" \
- "https://$name.storage.googleapis.com/"; do
- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)
- if [[ $code =~ ^20|403$ ]]; then printf "GCS %s %s\n" "$name" "$u" | anew -q subdomains/cloud_extra.txt; fi
- done
-
- # Azure Blob (requires container; try a few)
- while IFS= read -r c; do
- u="https://$name.blob.core.windows.net/$c?restype=container&comp=list"
- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)
- if [[ $code =~ ^20|403$ ]]; then printf "AZURE %s %s\n" "$name/$c" "$u" | anew -q subdomains/cloud_extra.txt; fi
- done <.tmp/azcontainers.txt
-
- # DigitalOcean Spaces (regional)
- for r in ams3 nyc3 sfo3 sgp1 fra1 blr1; do
- u="https://$name.$r.digitaloceanspaces.com/"
- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)
- if [[ $code =~ ^20|403$ ]]; then printf "DOSPACE %s %s\n" "$name.$r" "$u" | anew -q subdomains/cloud_extra.txt; fi
- done
-
- # Backblaze B2 (best-effort listing)
- u="https://f000.backblazeb2.com/file/$name/"
- code=$(run_command curl -sk -o /dev/null -w '%{http_code}' "$u" || true)
- if [[ $code =~ ^20|403$ ]]; then printf "B2 %s %s\n" "$name" "$u" | anew -q subdomains/cloud_extra.txt; fi
-
- done <.tmp/cloudnames.txt
-
- append_assets_from_file cloud asset subdomains/cloud_extra.txt
- end_func "Results are saved in subdomains/cloud_extra.txt" "${FUNCNAME[0]}"
- else
- _print_msg WARN "${FUNCNAME[0]} skipped or already processed."
- fi
-}
-
###############################################################################################################
############################################# GEOLOCALIZATION INFO ############################################
###############################################################################################################
diff --git a/modules/web.sh b/modules/web.sh
index 25f1385..d1e3907 100644
--- a/modules/web.sh
+++ b/modules/web.sh
@@ -103,7 +103,7 @@ function webprobe_simple() {
start_subfunc "${FUNCNAME[0]}" "Running: HTTP probing $domain"
# Baseline cache files (some modules merge into these).
- touch .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt 2>/dev/null || true
+ touch .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt webs/webs.txt 2>/dev/null || true
# If in multi mode and subdomains.txt doesn't exist, create it
if [[ -n $multi ]] && [[ ! -f "$dir/subdomains/subdomains.txt" ]]; then
@@ -124,33 +124,51 @@ function webprobe_simple() {
-o .tmp/web_full_info_probe.txt "$AXIOM_EXTRA_ARGS" 2>>"$LOGFILE" >/dev/null
fi
- # Merge current probe output with prior cache/state.
- touch .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>/dev/null || true
- if ! cat .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>>"$LOGFILE" \
- | jq -cs 'unique_by(.input)[]' 2>>"$LOGFILE" >webs/web_full_info.txt; then
- log_note "webprobe_simple: failed to merge httpx JSON; falling back to probe-only" "${FUNCNAME[0]}" "${LINENO}"
- awk 'match($0, /^[[:space:]]*\\{/) {print}' .tmp/web_full_info_probe.txt >.tmp/web_full_info_merge_input.jsonl 2>/dev/null || true
- if [[ -s ".tmp/web_full_info_merge_input.jsonl" ]]; then
- jq -cs 'unique_by(.input)[]' .tmp/web_full_info_merge_input.jsonl 2>>"$LOGFILE" >webs/web_full_info.txt || : >webs/web_full_info.txt
- else
- : >webs/web_full_info.txt
- fi
- fi
- # Keep cache as JSONL for later merges.
- cp webs/web_full_info.txt .tmp/web_full_info.txt 2>/dev/null || true
-
- # Extract URLs
- if [[ -s "webs/web_full_info.txt" ]]; then
- jq -r 'try .url' webs/web_full_info.txt 2>/dev/null \
- | grep "$domain" \
- | grep -aEo 'https?://[^ ]+' \
- | sed 's/*.//' | anew -q .tmp/probed_tmp.txt
- fi
-
- # Adaptive throttling heuristics: mark slow hosts (429/403) from httpx
- if [[ -s "webs/web_full_info.txt" ]]; then
- jq -r 'try select(.status_code==403 or .status_code==429) | .url' webs/web_full_info.txt 2>/dev/null \
- | awk -F/ '{print $3}' | sed 's/\:$//' | sort -u >.tmp/slow_hosts.txt
+ # webprobe_simple is expected to write JSONL when using httpx -json.
+ # Some runners (or wrappers) may produce a plain URL list instead.
+ # Detect the format early to avoid jq parse errors and missing webs/webs.txt.
+ local probe_first_line probe_is_json
+ probe_first_line="$(awk 'NF {print; exit}' .tmp/web_full_info_probe.txt 2>/dev/null || true)"
+ probe_is_json=false
+ [[ "$probe_first_line" =~ ^[[:space:]]*\\{ ]] && probe_is_json=true
+
+ # Always start fresh for this run (used by urlchecks diff too).
+ : >.tmp/probed_tmp.txt 2>/dev/null || true
+
+ if [[ "$probe_is_json" == true ]]; then
+ # Merge current probe output with prior cache/state.
+ touch .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>/dev/null || true
+ if ! cat .tmp/web_full_info_probe.txt .tmp/web_full_info.txt 2>>"$LOGFILE" \
+ | jq -cs 'unique_by(.input)[]' 2>>"$LOGFILE" >webs/web_full_info.txt; then
+ log_note "webprobe_simple: failed to merge httpx JSON; falling back to probe-only" "${FUNCNAME[0]}" "${LINENO}"
+ awk 'match($0, /^[[:space:]]*\\{/) {print}' .tmp/web_full_info_probe.txt >.tmp/web_full_info_merge_input.jsonl 2>/dev/null || true
+ if [[ -s ".tmp/web_full_info_merge_input.jsonl" ]]; then
+ jq -cs 'unique_by(.input)[]' .tmp/web_full_info_merge_input.jsonl 2>>"$LOGFILE" >webs/web_full_info.txt || : >webs/web_full_info.txt
+ else
+ : >webs/web_full_info.txt
+ fi
+ fi
+ # Keep cache as JSONL for later merges.
+ cp webs/web_full_info.txt .tmp/web_full_info.txt 2>/dev/null || true
+
+ # Extract URLs from JSONL
+ if [[ -s "webs/web_full_info.txt" ]]; then
+ jq -r 'try (.url // empty)' webs/web_full_info.txt 2>/dev/null \
+ | awk -v dom="$domain" 'index($0, dom) && $0 ~ /^https?:\\/\\// {print}' \
+ | sed 's/*.//' | anew -q .tmp/probed_tmp.txt
+ fi
+ else
+ log_note "webprobe_simple: probe output not JSON; treating as URL list" "${FUNCNAME[0]}" "${LINENO}"
+ if [[ -s ".tmp/web_full_info_probe.txt" ]]; then
+ awk -v dom="$domain" 'index($0, dom) && $0 ~ /^https?:\\/\\// {print}' .tmp/web_full_info_probe.txt 2>/dev/null \
+ | sed 's/*.//' | anew -q .tmp/probed_tmp.txt
+ fi
+ fi
+
+ # Adaptive throttling heuristics: mark slow hosts (429/403) from httpx
+ if [[ -s "webs/web_full_info.txt" ]]; then
+ jq -r 'try select(.status_code==403 or .status_code==429) | .url' webs/web_full_info.txt 2>/dev/null \
+ | awk -F/ '{print $3}' | sed 's/\:$//' | sort -u >.tmp/slow_hosts.txt
fi
# Extract web info to plain text
@@ -164,14 +182,14 @@ function webprobe_simple() {
if ! deleteOutScoped "$outOfScope_file" .tmp/probed_tmp.txt; then
print_warnf "Failed to delete out-of-scope entries."
fi
- fi
+ fi
- touch .tmp/probed_tmp.txt
+ touch .tmp/probed_tmp.txt
- # Count new websites
- if ! NUMOFLINES=$(anew webs/webs.txt <.tmp/probed_tmp.txt 2>/dev/null | sed '/^$/d' | wc -l); then
- print_warnf "Failed to count new websites."
- NUMOFLINES=0
+ # Count new websites
+ if ! NUMOFLINES=$(anew webs/webs.txt <.tmp/probed_tmp.txt 2>/dev/null | sed '/^$/d' | wc -l); then
+ print_warnf "Failed to count new websites."
+ NUMOFLINES=0
fi
# Update webs_all.txt
diff --git a/tests/unit/test_webprobe_simple_merge.bats b/tests/unit/test_webprobe_simple_merge.bats
index c0e8c63..f8f569c 100644
--- a/tests/unit/test_webprobe_simple_merge.bats
+++ b/tests/unit/test_webprobe_simple_merge.bats
@@ -121,3 +121,35 @@ SH
grep -q "new.example.com" "webs/web_full_info.txt"
! grep -q "NOT_JSON" ".tmp/web_full_info.txt"
}
+
+@test "webprobe_simple promotes URL list output into webs/webs.txt and webs_all" {
+ mkdir -p .tmp webs subdomains
+ printf "a.example.com\n" > subdomains/subdomains.txt
+
+ cat > "$MOCK_BIN/httpx" <<'SH'
+#!/usr/bin/env bash
+out=""
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -o)
+ out="$2"
+ shift 2
+ ;;
+ *)
+ shift
+ ;;
+ esac
+done
+printf '%s\n' "https://a.example.com" "https://b.example.com" > "$out"
+SH
+ chmod +x "$MOCK_BIN/httpx"
+
+ run webprobe_simple
+ [ "$status" -eq 0 ]
+ [ -s "webs/webs.txt" ]
+ grep -q "https://a.example.com" "webs/webs.txt"
+ grep -q "https://b.example.com" "webs/webs.txt"
+ [ -s "webs/webs_all.txt" ]
+ grep -q "https://a.example.com" "webs/webs_all.txt"
+ grep -q "https://b.example.com" "webs/webs_all.txt"
+}