reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities
Shell bug-bountybugbountybugbounty-tooldnshackingnucleiosintpenetration-testingpentestpentest-toolpentestingreconreconnaissancescannersecuritysecurity-toolssubdomainsubdomain-enumerationvulnerabilities
#!/bin/bash
# shellcheck disable=SC2154 # Variables defined in reconftw.cfg
# reconFTW - Web analysis module
# Contains: webprobe_full, screenshot, virtualhosts,
# favirecon_tech,
# portscan, cdnprovider, waf_checks, nuclei_check, graphql_scan,
# param_discovery, grpc_reflection, fuzz, iishortname, cms_scanner,
# urlchecks, url_gf, url_ext, jschecks, websocket_checks,
# wordlist_gen, wordlist_gen_roboxtractor, password_dict, brokenLinks
# This file is sourced by reconftw.sh - do not execute directly
[[ -z "${SCRIPTPATH:-}" ]] && {
echo "Error: This module must be sourced by reconftw.sh" >&2
exit 1
}
###############################################################################
# Helper Functions for Web Module
###############################################################################
# Normalize wildcard-prefixed hosts in URL-like lines.
# Examples:
# *.api.example.com -> api.example.com
# https://*.api.example.com -> https://api.example.com
_normalize_probe_urls() {
sed -E 's#^(https?://)\*\.#\1#; s#^\*\.##'
}
# Validate probe output as JSONL. Empty files are valid.
# Usage: _validate_probe_jsonl <input_file>
_validate_probe_jsonl() {
local input_file="$1"
[[ ! -s "$input_file" ]] && return 0
jq -e . "$input_file" >/dev/null 2>>"$LOGFILE"
}
# Extract in-scope URLs from a JSONL probe file.
# Uses filter_in_scope_urls (lib/validation.sh) to anchor the host match and
# reject URLs with userinfo (`user@host`), which previously allowed off-scope
# hosts like `http://[email protected]:8080/` to slip into webs_all.txt.
# Usage: _extract_probe_urls_from_json <input_file> <domain_filter> <output_file>
_extract_probe_urls_from_json() {
local input_file="$1"
local dom_filter="$2"
local output_file="$3"
[[ ! -s "$input_file" ]] && return 0
jq -r 'try (.url // empty)' "$input_file" 2>/dev/null \
| filter_in_scope_urls "$dom_filter" \
| _normalize_probe_urls \
| sed '/^$/d' \
| anew_q_safe "$output_file"
}
# Render full-info JSONL into plain text entries.
# Usage: _write_plain_webinfo <input_jsonl> <domain_filter> <output_file>
_write_plain_webinfo() {
local json_file="$1"
local dom_filter="$2"
local output_file="$3"
[[ ! -s "$json_file" ]] && return 0
# "Multi" is the sentinel for list-mode runs (no single in-scope root);
# accept every line in that case. Otherwise require the domain substring.
if [[ "$dom_filter" == "Multi" ]]; then
jq -r 'try . | "\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' "$json_file" \
| anew_q_safe "$output_file"
else
jq -r 'try . | "\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' "$json_file" \
| awk -v dom="$dom_filter" 'index($0, dom)' \
| anew_q_safe "$output_file"
fi
}
# Send URLs to proxy if enabled
# Usage: _send_to_proxy urls_file
_send_to_proxy() {
local urls_file="$1"
local max_urls="${2:-$DEEP_LIMIT2}"
if [[ $PROXY == true ]] && [[ -n $proxy_url ]] && [[ -s "$urls_file" ]]; then
local count
count=$(wc -l < "$urls_file" | tr -d ' ')
if [[ $count -le $max_urls ]]; then
notification "Sending websites to proxy" "info"
run_command ffuf -mc all -w "$urls_file" -u FUZZ -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
fi
}
# Display summary of discovered webs
print_webs_summary() {
[[ "${OUTPUT_VERBOSITY:-1}" -lt 1 ]] && return 0
local webs_file="webs/webs_all.txt"
[[ -s "$webs_file" ]] || webs_file="webs/webs.txt"
[[ -s "$webs_file" ]] || return 0
sort -u "$webs_file" -o "$webs_file" 2>/dev/null || true
local web_count
web_count=$(wc -l <"$webs_file" | tr -d ' ')
print_artifacts "${web_count} hosts -> ${webs_file}"
}
###############################################################################
# Main Web Functions
###############################################################################
function webprobe_full() {
# Create necessary directories
if ! ensure_dirs .tmp webs subdomains; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $WEBPROBEFULL == true ]]; then
start_func "${FUNCNAME[0]}" "HTTP Probing Full Port Set"
# Prepare target/output files.
touch webs/webs.txt webs/webs_uncommon_ports.txt \
webs/web_full_info.txt webs/web_full_info_plain.txt \
webs/web_full_info_uncommon.txt webs/web_full_info_uncommon_plain.txt 2>/dev/null || true
: >.tmp/probed_common_ports_tmp.txt 2>/dev/null || true
: >.tmp/probed_uncommon_ports_tmp.txt 2>/dev/null || true
: >.tmp/probed_tmp.txt 2>/dev/null || true
: >.tmp/slow_hosts.txt 2>/dev/null || true
# If in multi mode and subdomains.txt doesn't exist, create it
if [[ -n $multi ]] && [[ ! -f "$dir/subdomains/subdomains.txt" ]]; then
printf "%b\n" "$domain" >"$dir/subdomains/subdomains.txt"
fi
if [[ ! -s "subdomains/subdomains.txt" ]]; then
end_func "No subdomains/subdomains.txt file found, web probing skipped." "${FUNCNAME[0]}" "SKIP_NOINPUT"
plugins_emit after_webprobe "$domain" "$dir"
return 0
fi
local probe_ports="${WEBPROBE_PORTS:-80,443,${UNCOMMON_PORTS_WEB:-}}"
probe_ports=$(printf '%s' "$probe_ports" | sed -E 's/,+/,/g; s/^,//; s/,$//')
local probe_out=".tmp/web_full_info_probe.txt"
local common_json_tmp=".tmp/web_full_info_common_current.txt"
local uncommon_json_tmp=".tmp/web_full_info_uncommon_current.txt"
: >"$probe_out" 2>/dev/null || true
: >"$common_json_tmp" 2>/dev/null || true
: >"$uncommon_json_tmp" 2>/dev/null || true
if [[ $AXIOM != true ]]; then
local -a httpx_cmd=(
httpx
-follow-host-redirects
-random-agent
-status-code
-p "$probe_ports"
-threads "$HTTPX_UNCOMMONPORTS_THREADS"
-rl "$HTTPX_RATELIMIT"
-timeout "$HTTPX_UNCOMMONPORTS_TIMEOUT"
-silent
-retries 2
-title
-web-server
-tech-detect
-location
-no-color
-json
-o "$probe_out"
)
run_command "${httpx_cmd[@]}" <subdomains/subdomains.txt 2>>"$LOGFILE" >/dev/null
else
local -a axiom_cmd=(
axiom-scan
subdomains/subdomains.txt
-m httpx
-follow-host-redirects
-H "${HEADER}"
-status-code
-p "$probe_ports"
-threads "$HTTPX_UNCOMMONPORTS_THREADS"
-timeout "$HTTPX_UNCOMMONPORTS_TIMEOUT"
-silent
-retries 2
-title
-web-server
-tech-detect
-location
-no-color
-json
-o "$probe_out"
)
if [[ ${#AXIOM_EXTRA_ARGS_ARR[@]} -gt 0 ]]; then
axiom_cmd+=("${AXIOM_EXTRA_ARGS_ARR[@]}")
fi
run_command "${axiom_cmd[@]}" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s "$probe_out" ]] && ! _validate_probe_jsonl "$probe_out"; then
end_func "Invalid non-JSON output detected in ${probe_out}" "${FUNCNAME[0]}" "FAIL"
plugins_emit after_webprobe "$domain" "$dir"
return 1
fi
if [[ -s "$probe_out" ]]; then
jq -cr --arg dom "$domain" '
def effective_port:
if (.port? != null and ((.port | tostring | length) > 0)) then (.port | tostring)
elif ((.url // "") | test(":[0-9]+(/|$)")) then ((.url // "") | capture(":(?<p>[0-9]+)(/|$)").p)
elif ((.url // "") | startswith("https://")) then "443"
elif ((.url // "") | startswith("http://")) then "80"
else ""
end;
select($dom == "Multi" or ((.url // "") | contains($dom)))
| select(effective_port == "80" or effective_port == "443")
' "$probe_out" 2>>"$LOGFILE" >"$common_json_tmp"
jq -cr --arg dom "$domain" '
def effective_port:
if (.port? != null and ((.port | tostring | length) > 0)) then (.port | tostring)
elif ((.url // "") | test(":[0-9]+(/|$)")) then ((.url // "") | capture(":(?<p>[0-9]+)(/|$)").p)
elif ((.url // "") | startswith("https://")) then "443"
elif ((.url // "") | startswith("http://")) then "80"
else ""
end;
select($dom == "Multi" or ((.url // "") | contains($dom)))
| select(effective_port != "80" and effective_port != "443")
' "$probe_out" 2>>"$LOGFILE" >"$uncommon_json_tmp"
fi
if [[ -s "$common_json_tmp" ]]; then
cat "$common_json_tmp" | anew_q_safe "webs/web_full_info.txt"
fi
if [[ -s "$uncommon_json_tmp" ]]; then
cat "$uncommon_json_tmp" | anew_q_safe "webs/web_full_info_uncommon.txt"
fi
_write_plain_webinfo "$common_json_tmp" "$domain" "webs/web_full_info_plain.txt"
_write_plain_webinfo "$uncommon_json_tmp" "$domain" "webs/web_full_info_uncommon_plain.txt"
_extract_probe_urls_from_json "$common_json_tmp" "$domain" ".tmp/probed_common_ports_tmp.txt"
_extract_probe_urls_from_json "$uncommon_json_tmp" "$domain" ".tmp/probed_uncommon_ports_tmp.txt"
if [[ -s "$outOfScope_file" ]]; then
deleteOutScoped "$outOfScope_file" ".tmp/probed_common_ports_tmp.txt" || true
deleteOutScoped "$outOfScope_file" ".tmp/probed_uncommon_ports_tmp.txt" || true
fi
local common_new=0
local uncommon_new=0
if ! common_new=$(anew_safe "webs/webs.txt" <".tmp/probed_common_ports_tmp.txt" 2>/dev/null | sed '/^$/d' | wc -l); then
print_warnf "Failed to count common web targets."
common_new=0
fi
if ! uncommon_new=$(anew_safe "webs/webs_uncommon_ports.txt" <".tmp/probed_uncommon_ports_tmp.txt" 2>/dev/null | sed '/^$/d' | wc -l); then
print_warnf "Failed to count uncommon web targets."
uncommon_new=0
fi
cat ".tmp/probed_common_ports_tmp.txt" ".tmp/probed_uncommon_ports_tmp.txt" 2>/dev/null \
| sed '/^$/d' \
| sort -u >".tmp/probed_tmp.txt"
if [[ -s "$probe_out" ]]; then
jq -r 'try select(.status_code==403 or .status_code==429) | .url' "$probe_out" 2>/dev/null \
| awk -F/ '{print $3}' \
| sed 's/:.*$//' \
| sed '/^$/d' \
| sort -u >".tmp/slow_hosts.txt"
fi
notification "Common web ports: ${common_new} new websites" "good"
notification "Uncommon web ports: ${uncommon_new} new websites" "good"
ensure_webs_all || true
append_assets_from_file web url webs/webs.txt
append_assets_from_file web url webs/webs_uncommon_ports.txt
if [[ $PROXY == true ]] && [[ -n $proxy_url ]]; then
if [[ -s "webs/webs.txt" ]] && [[ $(wc -l <webs/webs.txt) -le $DEEP_LIMIT2 ]]; then
notification "Sending websites to proxy" "info"
run_command ffuf -mc all -w webs/webs.txt -u FUZZ -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s "webs/webs_uncommon_ports.txt" ]] && [[ $(wc -l <webs/webs_uncommon_ports.txt) -le $DEEP_LIMIT2 ]]; then
notification "Sending websites with uncommon ports to proxy" "info"
run_command ffuf -mc all -w webs/webs_uncommon_ports.txt -u FUZZ -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
fi
end_func "Results are saved in $domain/webs/[webs.txt|webs_uncommon_ports.txt|webs_all.txt]" "${FUNCNAME[0]}"
print_webs_summary
else
if [[ $WEBPROBEFULL == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
# Emit plugin event
plugins_emit after_webprobe "$domain" "$dir"
}
function screenshot() {
# Create necessary directories
if ! ensure_dirs webs screenshots; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $WEBSCREENSHOT == true ]]; then
if [[ $AXIOM != true ]]; then
if ! command -v nuclei >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: nuclei binary not found in PATH - install nuclei first"
return 0
fi
if [[ ! -d "${NUCLEI_TEMPLATES_PATH:-}" ]]; then
_print_msg WARN "${FUNCNAME[0]}: nuclei templates not found at '${NUCLEI_TEMPLATES_PATH}'"
return 0
fi
fi
start_func "${FUNCNAME[0]}" "Web Screenshots"
# Build/refresh webs_all.txt from current web targets.
ensure_webs_all || true
# Run nuclei or axiom-scan based on AXIOM flag
if [[ $AXIOM != true ]]; then
if [[ -s "webs/webs_all.txt" ]]; then
run_command nuclei -headless -id screenshot -V dir='screenshots' <webs/webs_all.txt 2>>"$LOGFILE" >/dev/null
fi
else
if [[ -s "webs/webs_all.txt" ]]; then
run_command axiom-scan webs/webs_all.txt -m nuclei-screenshots -o screenshots "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
fi
end_func "Results are saved in $domain/screenshots" "${FUNCNAME[0]}"
else
if [[ $WEBSCREENSHOT == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function virtualhosts() {
# Create necessary directories
if ! ensure_dirs .tmp/virtualhosts virtualhosts webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $VIRTUALHOSTS == true ]]; then
if ! command -v VhostFinder >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: VhostFinder not found in PATH"
return 0
fi
start_func "${FUNCNAME[0]}" "Virtual Hosts Discovery"
# Build/refresh webs_all.txt from current web targets.
ensure_webs_all || true
# Proceed only if input files exist
if [[ -s "subdomains/subdomains.txt" ]] && [[ -s "hosts/ips.txt" ]]; then
run_command VhostFinder -ips hosts/ips.txt -wordlist subdomains/subdomains.txt -verify | grep "+" | anew -q "$dir/webs/virtualhosts.txt"
fi
# Optionally send to proxy if conditions are met
if [[ $PROXY == true ]] && [[ -n $proxy_url ]] && [[ $(wc -l <webs/webs_uncommon_ports.txt) -le $DEEP_LIMIT2 ]]; then
notification "Sending websites with uncommon ports to proxy" "info"
run_command ffuf -mc all -w webs/webs_uncommon_ports.txt -u FUZZ -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
end_func "Results are saved in $domain/webs/virtualhosts.txt" "${FUNCNAME[0]}"
else
if [[ $VIRTUALHOSTS == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function tls_ip_pivots() {
ensure_dirs .tmp subdomains hosts webs
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ ${TLS_IP_PIVOTS:-false} == true ]]; then
start_func "${FUNCNAME[0]}" "TLS Certificate Harvesting from IPs"
# Collect IP sources
: >.tmp/tls_ip_targets.txt
[[ -s "hosts/ips.txt" ]] && cat hosts/ips.txt >> .tmp/tls_ip_targets.txt
[[ -s "hosts/origin_ips.txt" ]] && cat hosts/origin_ips.txt >> .tmp/tls_ip_targets.txt
sort -u .tmp/tls_ip_targets.txt -o .tmp/tls_ip_targets.txt
if [[ ! -s ".tmp/tls_ip_targets.txt" ]]; then
end_func "No IPs available for TLS IP pivoting" "${FUNCNAME[0]}" "SKIP_NOINPUT"
return
fi
local ip_count tls_ports batch_size
ip_count=$(wc -l <.tmp/tls_ip_targets.txt | tr -d ' ')
tls_ports="${TLS_PORTS:-443}"
batch_size="${TLS_IP_SNI_BATCH_SIZE:-1000}"
_print_msg INFO "TLS IP pivots: scanning ${ip_count} IPs for certificates"
# Phase A: Passive cert harvest — extract SAN/CN from TLS certs on IPs
: >hosts/tls_ip_certs.jsonl
if [[ $AXIOM != true ]]; then
run_command tlsx -san -cn -silent -ro -resp-only \
-p "$tls_ports" \
-c "$TLSX_THREADS" \
-json \
<.tmp/tls_ip_targets.txt \
>hosts/tls_ip_certs.jsonl 2>>"$LOGFILE" || true
else
run_command axiom-scan .tmp/tls_ip_targets.txt -m tlsx \
-san -cn -silent -ro -resp-only \
-p "$tls_ports" \
-c "$TLSX_THREADS" \
-json \
-o hosts/tls_ip_certs.jsonl "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
# Extract hostnames from SAN/CN in JSONL
: >.tmp/tls_ip_sans_all.txt
if [[ -s "hosts/tls_ip_certs.jsonl" ]]; then
jq -r '
(.subject_cn // empty),
(.subject_an[]? // empty)
' hosts/tls_ip_certs.jsonl 2>/dev/null \
| sed -e 's/^\*\.//' -e 's/\.$//' -e '/^$/d' \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \
| sort -u >.tmp/tls_ip_sans_all.txt || true
fi
# Phase B: SNI brute — use discovered labels + domain as SNI candidates (only in DEEP mode)
: >.tmp/tls_ip_sni_results.txt
if [[ -s ".tmp/tls_ip_sans_all.txt" ]] && [[ ${DEEP:-false} == true ]]; then
# Build SNI candidate list from discovered names + existing subdomains
: >.tmp/tls_ip_sni_candidates.txt
cat .tmp/tls_ip_sans_all.txt >> .tmp/tls_ip_sni_candidates.txt
[[ -s "subdomains/subdomains.txt" ]] && cat subdomains/subdomains.txt >> .tmp/tls_ip_sni_candidates.txt
sort -u .tmp/tls_ip_sni_candidates.txt -o .tmp/tls_ip_sni_candidates.txt
head -n "$batch_size" .tmp/tls_ip_targets.txt \
| run_command tlsx -san -cn -silent -ro -resp-only \
-p "$tls_ports" \
-c "$TLSX_THREADS" \
-sni-file .tmp/tls_ip_sni_candidates.txt \
>.tmp/tls_ip_sni_results.txt 2>>"$LOGFILE" || true
fi
# Phase C: Reverse PTR + SNI probing
if [[ -s ".tmp/tls_ip_targets.txt" ]]; then
head -n "$batch_size" .tmp/tls_ip_targets.txt \
| run_command tlsx -san -cn -silent -ro -resp-only \
-rev-ptr-sni \
-p "$tls_ports" \
-c "$TLSX_THREADS" \
>>.tmp/tls_ip_sni_results.txt 2>>"$LOGFILE" || true
fi
# Merge all discovered hostnames
: >.tmp/tls_ip_all_hosts.txt
[[ -s ".tmp/tls_ip_sans_all.txt" ]] && cat .tmp/tls_ip_sans_all.txt >> .tmp/tls_ip_all_hosts.txt
if [[ -s ".tmp/tls_ip_sni_results.txt" ]]; then
sed -e 's/^\*\.//' -e 's/\.$//' -e '/^$/d' .tmp/tls_ip_sni_results.txt \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \
>> .tmp/tls_ip_all_hosts.txt 2>/dev/null || true
fi
sort -u .tmp/tls_ip_all_hosts.txt -o .tmp/tls_ip_all_hosts.txt
# Filter in-scope
: >subdomains/tls_ip_pivots.txt
if [[ -s ".tmp/tls_ip_all_hosts.txt" ]]; then
grep -E "$DOMAIN_MATCH_REGEX" .tmp/tls_ip_all_hosts.txt \
| sort -u >subdomains/tls_ip_pivots.txt 2>/dev/null || true
fi
if [[ $INSCOPE == true ]] && [[ -s "subdomains/tls_ip_pivots.txt" ]]; then
if ! check_inscope subdomains/tls_ip_pivots.txt 2>>"$LOGFILE" >/dev/null; then
print_warnf "check_inscope command failed."
fi
fi
# Resolve new candidates and merge
NUMOFLINES=0
if [[ -s "subdomains/tls_ip_pivots.txt" ]]; then
: >.tmp/tls_ip_pivots_resolved.txt
if [[ $AXIOM != true ]]; then
_resolve_domains subdomains/tls_ip_pivots.txt .tmp/tls_ip_pivots_resolved.txt
else
run_command axiom-scan subdomains/tls_ip_pivots.txt -m puredns-resolve \
-r "${AXIOM_RESOLVERS_PATH}" --resolvers-trusted "${AXIOM_RESOLVERS_TRUSTED_PATH}" \
--wildcard-tests "$PUREDNS_WILDCARDTEST_LIMIT" --wildcard-batch "$PUREDNS_WILDCARDBATCH_LIMIT" \
-o .tmp/tls_ip_pivots_resolved.txt "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s ".tmp/tls_ip_pivots_resolved.txt" ]]; then
NUMOFLINES=$(anew subdomains/subdomains.txt <.tmp/tls_ip_pivots_resolved.txt | sed '/^$/d' | wc -l | tr -d ' ' || true)
[[ "$NUMOFLINES" =~ ^[0-9]+$ ]] || NUMOFLINES=0
fi
fi
# Delta probe: run targeted httpx on new subs only
if [[ ${TLS_IP_DELTA_PROBE:-true} == true ]] && [[ ${NUMOFLINES:-0} -gt 0 ]] && [[ -s ".tmp/tls_ip_pivots_resolved.txt" ]]; then
_print_msg INFO "TLS IP pivots: delta-probing ${NUMOFLINES} new subdomains"
# Identify genuinely new subs by comparing against pre-existing list
if [[ -s ".tmp/subdomains_old.txt" ]]; then
comm -23 <(sort -u .tmp/tls_ip_pivots_resolved.txt) \
<(sort -u .tmp/subdomains_old.txt) \
>.tmp/tls_ip_delta_subs.txt 2>/dev/null || true
else
cp .tmp/tls_ip_pivots_resolved.txt .tmp/tls_ip_delta_subs.txt
fi
if [[ -s ".tmp/tls_ip_delta_subs.txt" ]]; then
run_command httpx -follow-host-redirects -random-agent -silent \
-status-code -p "${WEBPROBE_PORTS:-80,443}" \
-threads "${HTTPX_THREADS:-50}" -rl "${HTTPX_RATELIMIT:-0}" \
-timeout "${HTTPX_TIMEOUT:-10}" -retries 2 -no-color \
<.tmp/tls_ip_delta_subs.txt \
| awk '{print $1}' | anew -q webs/webs.txt 2>/dev/null || true
fi
fi
end_func "${NUMOFLINES} new subs (tls ip pivots)" "${FUNCNAME[0]}"
else
if [[ ${TLS_IP_PIVOTS:-false} == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
###############################################################################################################
############################################# HOST SCAN #######################################################
###############################################################################################################
function favirecon_tech() {
# Create necessary directories
if ! ensure_dirs .tmp webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ ${FAVIRECON:-true} == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Favicon Technology Recon"
ensure_webs_all || true
if [[ -s "webs/webs_all.txt" ]]; then
local fav_cmd=(
favirecon
-l "webs/webs_all.txt"
-c "${FAVIRECON_CONCURRENCY:-50}"
-t "${FAVIRECON_TIMEOUT:-10}"
-s
-j
-o "webs/favirecon.json"
)
if [[ -n "${FAVIRECON_PROXY:-}" ]]; then
fav_cmd+=(-px "${FAVIRECON_PROXY}")
fi
if [[ "${FAVIRECON_RATE_LIMIT:-0}" -gt 0 ]]; then
fav_cmd+=(-rl "${FAVIRECON_RATE_LIMIT}")
fi
if ! run_command "${fav_cmd[@]}" 2>>"$LOGFILE" >/dev/null; then
log_note "favirecon_tech: favirecon command failed" "${FUNCNAME[0]}" "${LINENO}"
fi
if [[ -s "webs/favirecon.json" ]]; then
jq -r '[(.URL // .url // ""), (.Name // .name // "unknown"), (.Hash // .hash // "unknown")] | @tsv' "webs/favirecon.json" 2>/dev/null \
| awk -F'\t' 'NF>=1 && $1 ~ /^https?:\/\// {printf "%s [%s] [%s]\n", $1, $2, $3}' \
| anew -q "webs/favirecon.txt"
fi
else
log_note "favirecon_tech: no webs/webs_all.txt found" "${FUNCNAME[0]}" "${LINENO}"
fi
end_func "Results are saved in webs/favirecon.[json|txt]" "${FUNCNAME[0]}"
else
if [[ ${FAVIRECON:-true} == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function portscan() {
# Create necessary directories
if ! ensure_dirs .tmp subdomains hosts webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $PORTSCANNER == true ]]; then
start_func "${FUNCNAME[0]}" "Port scan"
# Determine if domain is IP address or domain name
if ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
# Not an IP address
if [[ -s "subdomains/subdomains_ips.txt" ]]; then
cut -d ' ' -f3 subdomains/subdomains_ips.txt \
| grep -aEiv "^(127|10|169\.254|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)\." \
| grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" \
| anew -q hosts/ips.txt
fi
else
# Domain is an IP address
printf "%b\n" "$domain" \
| grep -aEiv "^(127|10|169\.254|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)\." \
| grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' \
| anew -q hosts/ips.txt || true
fi
# Check for CDN providers
if [[ ! -s "hosts/cdn_providers.txt" ]]; then
if [[ -s "hosts/ips.txt" ]]; then
cat hosts/ips.txt | run_command cdncheck -silent -resp -cdn -waf -nc 2>/dev/null | anew -q hosts/cdn_providers.txt || true
fi
fi
if [[ -s "hosts/ips.txt" ]]; then
# Remove CDN IPs.
comm -23 <(sort -u hosts/ips.txt) <(cut -d'[' -f1 hosts/cdn_providers.txt | sed 's/[[:space:]]*$//' | sort -u) \
| grep -aEiv "^(127|10|169\.254|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)\." | grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' \
| sort -u | anew -q .tmp/ips_nocdn.txt || true
fi
# Optional CDN bypass to recover origin IPs.
if [[ ${CDN_BYPASS:-true} == true ]] && command -v hakoriginfinder >/dev/null 2>&1; then
: >".tmp/origin_input_hosts.txt"
if [[ -s "subdomains/subdomains.txt" ]]; then
cat subdomains/subdomains.txt | anew -q ".tmp/origin_input_hosts.txt"
elif [[ -s "webs/webs_all.txt" ]]; then
awk -F/ '{print $3}' webs/webs_all.txt | sed 's/:.*$//' | sed '/^$/d' | sort -u | anew -q ".tmp/origin_input_hosts.txt"
fi
if [[ -s ".tmp/origin_input_hosts.txt" ]]; then
if run_command hakoriginfinder <".tmp/origin_input_hosts.txt" >".tmp/hakoriginfinder_raw.txt" 2>>"$LOGFILE"; then
grep -aoE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' ".tmp/hakoriginfinder_raw.txt" \
| grep -aEiv "^(127|10|169\.254|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)\." \
| sort -u >"hosts/origin_ips.txt"
if [[ -s "hosts/origin_ips.txt" ]]; then
cat hosts/origin_ips.txt | anew -q .tmp/ips_nocdn.txt
cat hosts/origin_ips.txt | anew -q hosts/ips.txt
fi
else
log_note "portscan: hakoriginfinder failed, skipping CDN bypass origins" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
fi
# Display resolved IPs without CDN.
ips_nocdn_count=0
if [[ -s ".tmp/ips_nocdn.txt" ]]; then
ips_nocdn_count=$(sort -u ".tmp/ips_nocdn.txt" | wc -l | tr -d ' ')
fi
if ((ips_nocdn_count > 0)); then
_print_msg INFO "Resolved IP addresses (No CDN): ${ips_nocdn_count} (see .tmp/ips_nocdn.txt)"
else
_print_msg INFO "Resolved IP addresses (No CDN): none"
fi
_print_msg INFO "Scanning ports..."
ips_file="${dir}/hosts/ips.txt"
# Discover IPv6 addresses from DNS JSON if enabled.
if [[ $IPV6_SCAN == true ]]; then
if [[ -s "subdomains/subdomains_dnsregs.json" ]]; then
jq -r '.. | strings | select(test("^[0-9a-fA-F:]+$"))' <"subdomains/subdomains_dnsregs.json" \
| grep -v '\\.' | sort -u | anew -q hosts/ips_v6.txt || true
fi
# Add target if it's an IPv6 literal.
if [[ $domain =~ : ]]; then echo "$domain" | anew -q hosts/ips_v6.txt || true; fi
fi
if [[ $PORTSCAN_PASSIVE == true ]]; then
if [[ ! -f $ips_file ]]; then
print_warnf "File %s does not exist." "$ips_file"
else
json_array=()
while IFS= read -r cip; do
if ! json_result=$(run_command curl -s "https://internetdb.shodan.io/${cip}"); then
print_warnf "Failed to retrieve data for IP %s." "$cip"
else
json_array+=("$json_result")
fi
done <"$ips_file"
formatted_json="["
for ((i = 0; i < ${#json_array[@]}; i++)); do
formatted_json+="$(echo "${json_array[i]}" | tr -d '\n')"
if [ $i -lt $((${#json_array[@]} - 1)) ]; then
formatted_json+=", "
fi
done
formatted_json+="]"
if ! echo "$formatted_json" >"${dir}/hosts/portscan_shodan.txt"; then
print_warnf "Failed to write portscan_shodan.txt."
fi
fi
fi
if [[ $PORTSCAN_PASSIVE == true ]] && [[ ! -f "hosts/portscan_passive.txt" ]] && [[ -s ".tmp/ips_nocdn.txt" ]]; then
run_command smap -iL .tmp/ips_nocdn.txt >hosts/portscan_passive.txt
fi
if [[ $PORTSCAN_ACTIVE == true ]]; then
# Resolve active nmap options (deep profile optional).
local active_opts_raw
active_opts_raw="${PORTSCAN_ACTIVE_OPTIONS:-}"
if [[ $DEEP == true ]] && [[ -n "${PORTSCAN_DEEP_OPTIONS:-}" ]]; then
active_opts_raw="${PORTSCAN_DEEP_OPTIONS}"
fi
local portscan_opts=()
if [[ -n "$active_opts_raw" ]]; then
local _ifs="$IFS"
IFS=' '
read -r -a portscan_opts <<<"$active_opts_raw"
IFS="$_ifs"
fi
local strategy="${PORTSCAN_STRATEGY:-legacy}"
local used_targeted_output=false
if [[ "$strategy" == "naabu_nmap" ]] && [[ ${NAABU_ENABLE:-true} == true ]] && [[ $AXIOM != true ]]; then
if command -v naabu >/dev/null 2>&1; then
local naabu_ports_raw="${NAABU_PORTS:---top-ports 1000}"
local naabu_opts=()
if [[ -n "$naabu_ports_raw" ]]; then
local _ifs="$IFS"
IFS=' '
read -r -a naabu_opts <<<"$naabu_ports_raw"
IFS="$_ifs"
fi
if [[ -s ".tmp/ips_nocdn.txt" ]]; then
run_command naabu -list ".tmp/ips_nocdn.txt" -silent -rate "${NAABU_RATE:-1000}" "${naabu_opts[@]}" -o "hosts/naabu_open.txt" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s "hosts/naabu_open.txt" ]]; then
local naabu_ports_csv=""
naabu_ports_csv=$(cut -d':' -f2 "hosts/naabu_open.txt" | sed '/^$/d' | sort -un | paste -sd, -)
if [[ -n "$naabu_ports_csv" ]]; then
run_command "$SUDO" nmap "${portscan_opts[@]}" -p "$naabu_ports_csv" -iL ".tmp/ips_nocdn.txt" -oA "hosts/portscan_active_targeted" 2>>"$LOGFILE" >/dev/null
used_targeted_output=true
fi
fi
else
_print_msg WARN "PORTSCAN_STRATEGY=naabu_nmap requested but naabu is missing. Falling back to legacy nmap."
fi
elif [[ "$strategy" == "naabu_nmap" ]] && [[ $AXIOM == true ]]; then
_print_msg WARN "PORTSCAN_STRATEGY=naabu_nmap is not enabled for AXIOM yet. Falling back to legacy nmapx."
fi
# Legacy fallback.
if [[ "$used_targeted_output" != true ]]; then
if [[ $AXIOM != true ]]; then
if [[ -s ".tmp/ips_nocdn.txt" ]]; then
run_command "$SUDO" nmap "${portscan_opts[@]}" -iL .tmp/ips_nocdn.txt -oA hosts/portscan_active 2>>"$LOGFILE" >/dev/null
fi
else
if [[ -s ".tmp/ips_nocdn.txt" ]]; then
run_command axiom-scan .tmp/ips_nocdn.txt -m nmapx "${portscan_opts[@]}" \
-oA hosts/portscan_active "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
fi
fi
# Keep compatibility output names if targeted flow ran.
if [[ "$used_targeted_output" == true ]] && [[ ! -s "hosts/portscan_active.xml" ]] && [[ -s "hosts/portscan_active_targeted.xml" ]]; then
cp "hosts/portscan_active_targeted.xml" "hosts/portscan_active.xml"
[[ -s "hosts/portscan_active_targeted.gnmap" ]] && cp "hosts/portscan_active_targeted.gnmap" "hosts/portscan_active.gnmap"
[[ -s "hosts/portscan_active_targeted.nmap" ]] && cp "hosts/portscan_active_targeted.nmap" "hosts/portscan_active.nmap"
fi
if [[ $IPV6_SCAN == true && -s "hosts/ips_v6.txt" ]] && [[ $AXIOM != true ]]; then
run_command "$SUDO" nmap -6 "${portscan_opts[@]}" -iL hosts/ips_v6.txt -oA hosts/portscan_active_v6 2>>"$LOGFILE" >/dev/null
fi
fi
# Optional UDP top-ports scan (local-only, requires privileges).
if [[ ${PORTSCAN_UDP:-false} == true ]] && [[ $AXIOM != true ]] && [[ -s ".tmp/ips_nocdn.txt" ]]; then
local udp_opts_raw="${PORTSCAN_UDP_OPTIONS:---top-ports 20 -sU -sV -n -Pn --open}"
local udp_opts=()
if [[ -n "$udp_opts_raw" ]]; then
local _ifs="$IFS"
IFS=' '
read -r -a udp_opts <<<"$udp_opts_raw"
IFS="$_ifs"
fi
run_command "$SUDO" nmap "${udp_opts[@]}" -iL ".tmp/ips_nocdn.txt" -oA "hosts/portscan_active_udp" 2>>"$LOGFILE" >/dev/null
elif [[ ${PORTSCAN_UDP:-false} == true ]] && [[ $AXIOM == true ]]; then
_print_msg WARN "PORTSCAN_UDP is local-only for now; skipped in AXIOM mode."
fi
if [[ -s "hosts/portscan_active.xml" ]]; then
nmapurls <hosts/portscan_active.xml 2>>"$LOGFILE" | anew -q hosts/webs.txt
# Feed IPv4 nmap URL findings back into the main web target set.
[[ -s hosts/webs.txt ]] && cat hosts/webs.txt | anew -q webs/webs.txt
fi
if [[ -s "hosts/portscan_active_v6.xml" ]]; then
nmapurls <hosts/portscan_active_v6.xml 2>>"$LOGFILE" | anew -q hosts/webs_v6.txt
[[ -s hosts/webs_v6.txt ]] && cat hosts/webs_v6.txt | anew -q webs/webs.txt
fi
if [[ $FARADAY == true ]]; then
# Check if the Faraday server is running.
if ! faraday-cli status 2>>"$LOGFILE" >/dev/null; then
print_warnf "Faraday server is not running. Skipping Faraday integration."
else
if [[ -s "hosts/portscan_active.xml" ]]; then
faraday-cli tool report -w "$FARADAY_WORKSPACE" --plugin-id nmap hosts/portscan_active.xml 2>>"$LOGFILE" >/dev/null
fi
if [[ -s "hosts/portscan_active_udp.xml" ]]; then
faraday-cli tool report -w "$FARADAY_WORKSPACE" --plugin-id nmap hosts/portscan_active_udp.xml 2>>"$LOGFILE" >/dev/null
fi
fi
fi
if [[ -s "hosts/webs.txt" ]]; then
if ! NUMOFLINES=$(wc -l <hosts/webs.txt); then
print_warnf "Failed to count lines in hosts/webs.txt."
NUMOFLINES=0
fi
notification "Webs detected from port scan: ${NUMOFLINES} new websites" "good"
append_assets_from_file web url hosts/webs.txt
fi
service_fingerprint
end_func "Results are saved in hosts/portscan_[passive|active|active_targeted|active_udp|shodan].*" "${FUNCNAME[0]}"
else
if [[ $PORTSCANNER == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
_build_service_fp_targets_from_nmap() {
local nmap_gnmap="$1"
local out_file="$2"
[[ ! -s "$nmap_gnmap" ]] && return 0
awk '
/^Host: / {
host = $2
if (host == "") next
ports = $0
sub(/^.*Ports: /, "", ports)
n = split(ports, arr, ",")
for (i = 1; i <= n; i++) {
gsub(/^ +| +$/, "", arr[i])
split(arr[i], f, "/")
if (f[2] == "open" && f[1] ~ /^[0-9]+$/) {
print host ":" f[1]
}
}
}' "$nmap_gnmap" | sort -u | anew -q "$out_file"
}
function service_fingerprint() {
ensure_dirs hosts .tmp
if [[ "${SERVICE_FINGERPRINT:-true}" != "true" ]]; then
return 0
fi
if [[ "${SERVICE_FINGERPRINT_ENGINE:-nerva}" != "nerva" ]]; then
_print_msg WARN "${FUNCNAME[0]}: unsupported SERVICE_FINGERPRINT_ENGINE=${SERVICE_FINGERPRINT_ENGINE}"
return 0
fi
if [[ $AXIOM == true ]]; then
log_note "service_fingerprint: local-only for now, skipped in AXIOM mode" "${FUNCNAME[0]}" "${LINENO}"
return 0
fi
if ! command -v nerva >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: nerva not found in PATH"
return 0
fi
start_subfunc "${FUNCNAME[0]}" "Service fingerprinting (nerva)"
local targets_file=".tmp/service_fp_targets.txt"
: >"$targets_file"
if [[ -s "hosts/naabu_open.txt" ]]; then
awk -F: 'NF==2 && $2 ~ /^[0-9]+$/ {print $1 ":" $2}' "hosts/naabu_open.txt" | sort -u | anew -q "$targets_file"
fi
if [[ ! -s "$targets_file" ]]; then
_build_service_fp_targets_from_nmap "hosts/portscan_active.gnmap" "$targets_file"
fi
if [[ ! -s "$targets_file" ]]; then
_build_service_fp_targets_from_nmap "hosts/portscan_active_targeted.gnmap" "$targets_file"
fi
if [[ -s "$targets_file" ]]; then
local timeout_ms="${SERVICE_FINGERPRINT_TIMEOUT_MS:-2000}"
run_command nerva --json -l "$targets_file" -w "$timeout_ms" -o "hosts/service_fingerprints.jsonl" 2>>"$LOGFILE" >/dev/null || true
if [[ -s "hosts/service_fingerprints.jsonl" ]]; then
jq -r '[(.host // .ip // .target // "unknown"), (.port // "unknown"), (.protocol // .service // "unknown")] | @tsv' "hosts/service_fingerprints.jsonl" 2>/dev/null \
| awk -F'\t' '{printf "%s:%s [%s]\n", $1, $2, $3}' \
| anew -q "hosts/service_fingerprints.txt"
fi
fi
end_subfunc "Results are saved in hosts/service_fingerprints.[jsonl|txt]" "${FUNCNAME[0]}"
}
function cdnprovider() {
# Create necessary directories
if ! ensure_dirs .tmp subdomains hosts; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $CDN_IP == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "CDN Provider Check"
# Check if subdomains_dnsregs.json exists and is not empty
if [[ -s "subdomains/subdomains_ips.txt" ]]; then
cut -d ' ' -f3 subdomains/subdomains_ips.txt \
| grep -aEiv "^(127|10|169\.254|172\.1[6-9]|172\.2[0-9]|172\.3[0-1]|192\.168)\." \
| grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" \
| anew -q hosts/ips.txt
fi
# Check if ips_cdn.txt exists and is not empty
if [[ -s "hosts/ips.txt" ]]; then
# Run cdncheck on the IPs and save to cdn_providers.txt
run_command cdncheck -silent -resp -nc <hosts/ips.txt | anew -q "$dir/hosts/cdn_providers.txt"
fi
end_func "Results are saved in hosts/cdn_providers.txt" "${FUNCNAME[0]}"
else
if [[ $CDN_IP == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
###############################################################################################################
############################################# WEB SCAN ########################################################
###############################################################################################################
function waf_checks() {
# Create necessary directories
if ! ensure_dirs .tmp webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $WAF_DETECTION == true ]]; then
if [[ $AXIOM != true ]] && ! command -v wafw00f >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: wafw00f not found in PATH"
return 0
fi
start_func "${FUNCNAME[0]}" "Website's WAF Detection"
ensure_webs_all || true
# Proceed only if webs_all.txt exists and is non-empty
if [[ -s "webs/webs_all.txt" ]]; then
if [[ $AXIOM != true ]]; then
# Parallelize wafw00f across hosts using interlace for speed
if command -v interlace >/dev/null 2>&1; then
mkdir -p .tmp/wafs_parts
run_command interlace -tL "webs/webs_all.txt" -threads "${INTERLACE_THREADS:-10}" \
-c "${TIMEOUT_CMD:+$TIMEOUT_CMD ${WAF_PER_HOST_TIMEOUT:-120}} wafw00f _target_ -o .tmp/wafs_parts/_cleantarget_.txt" 2>>"$LOGFILE" >/dev/null
cat .tmp/wafs_parts/*.txt >.tmp/wafs.txt 2>/dev/null || true
rm -rf .tmp/wafs_parts
else
run_command wafw00f -i "webs/webs_all.txt" -o ".tmp/wafs.txt" 2>>"$LOGFILE" >/dev/null
fi
else
# Run axiom-scan with wafw00f module on webs_all.txt
run_command axiom-scan "webs/webs_all.txt" -m wafw00f -o ".tmp/wafs.txt" "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
# Process wafs.txt if it exists and is not empty
if [[ -s ".tmp/wafs.txt" ]]; then
# Format the wafs.txt file
sed -e 's/^[ \t]*//' -e 's/ \+ /\t/g' -e '/(None)/d' ".tmp/wafs.txt" | tr -s "\t" ";" >"webs/webs_wafs.txt"
# Count the number of websites protected by WAF
if ! NUMOFLINES=$(sed '/^$/d' "webs/webs_wafs.txt" 2>>"$LOGFILE" | wc -l); then
print_warnf "Failed to count lines in webs_wafs.txt."
NUMOFLINES=0
fi
# Send a notification about the number of WAF-protected websites
notification "${NUMOFLINES} websites protected by WAF" "info"
# End the function with a success message
end_func "Results are saved in webs/webs_wafs.txt" "${FUNCNAME[0]}"
else
# End the function indicating no results were found
end_func "No results found" "${FUNCNAME[0]}"
fi
else
# End the function indicating there are no websites to scan
end_func "No websites to scan" "${FUNCNAME[0]}"
fi
else
if [[ $WAF_DETECTION == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
# Prepare WAF-aware target lists for nuclei scanning
# Splits webs into WAF-protected and non-WAF lists for rate-limiting
# Sets: WAF_LIST, NOWAF_LIST variables
_nuclei_prepare_waf_lists() {
WAF_LIST=.tmp/webs_waf.txt
NOWAF_LIST=.tmp/webs_nowaf.txt
sort -u .tmp/webs_subs.txt -o .tmp/webs_subs.txt 2>/dev/null || true
: >"$WAF_LIST"
: >"$NOWAF_LIST"
if [[ -s webs/webs_wafs.txt ]]; then
cut -d';' -f1 webs/webs_wafs.txt | sed 's/https\?:\/\///' | sed 's/\/$//' | sort -u >.tmp/waf_hosts.txt
# Single-pass classification: NR==FNR loads WAF hosts, then classifies URLs
awk -F/ '
NR==FNR { waf[$1]=1; next }
{
host=$3; sub(/:.*/, "", host)
if (host in waf) print $0 > waf_out
else print $0 > nowaf_out
}
' waf_out="$WAF_LIST" nowaf_out="$NOWAF_LIST" .tmp/waf_hosts.txt .tmp/webs_subs.txt
else
cp .tmp/webs_subs.txt "$NOWAF_LIST"
fi
# Include slow hosts (429/403) into WAF list as well
if [[ -s .tmp/slow_hosts.txt ]]; then
while read -r host; do
awk -v h="$host" -F/ '{u=$3; sub(/:.*/,"",u); if (u==h) print $0}' .tmp/webs_subs.txt | anew -q "$WAF_LIST"
done <.tmp/slow_hosts.txt
fi
}
# Parse nuclei JSON output to human-readable text and display results
# Usage: _nuclei_parse_results severity_level
_nuclei_parse_results() {
local crit="$1"
if [[ -s "nuclei_output/${crit}_json.txt" ]]; then
jq -r '["[" + .["template-id"] + (if .["matcher-name"] != null then ":" + .["matcher-name"] else "" end) + "] [" + .["type"] + "] [" + .info.severity + "] " + (.["matched-at"] // .host) + (if .["extracted-results"] != null then " " + (.["extracted-results"] | @json) else "" end)] | .[]' "nuclei_output/${crit}_json.txt" >"nuclei_output/${crit}.txt"
append_assets_from_file finding value "nuclei_output/${crit}.txt"
if [[ -s "nuclei_output/${crit}.txt" ]]; then
local _count
_count=$(wc -l <"nuclei_output/${crit}.txt" | tr -d ' ')
if [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]]; then
_print_status OK "nuclei_${crit}" "${_count} findings -> nuclei_output/${crit}.txt"
fi
fi
fi
}
# Split combined nuclei JSON output into per-severity files for backward compatibility
# Usage: _nuclei_split_by_severity
_nuclei_split_by_severity() {
if [[ ! -s ".tmp/nuclei_combined_json.txt" ]]; then
return 0
fi
IFS=',' read -ra severity_array <<<"$NUCLEI_SEVERITY"
for crit in "${severity_array[@]}"; do
jq -c "select(.info.severity == \"${crit}\")" ".tmp/nuclei_combined_json.txt" >"nuclei_output/${crit}_json.txt" 2>/dev/null || true
_nuclei_parse_results "$crit"
done
}
# Run nuclei scan locally with WAF-aware rate limiting (single pass, all severities)
# Usage: _nuclei_scan_local
_nuclei_scan_local() {
local all_severities="$NUCLEI_SEVERITY"
_print_msg INFO "Running: Nuclei Severities: ${all_severities}"
: >".tmp/nuclei_combined_json.txt"
# Non-WAF at default rate
if [[ -s "$NOWAF_LIST" ]]; then
# shellcheck disable=SC2086 # Intentionally allow user-provided nuclei args
run_with_heartbeat "nuclei (normal targets)" nuclei \
-l "$NOWAF_LIST" -severity "$all_severities" -nh -rl "$NUCLEI_RATELIMIT" -silent -retries 2 \
$NUCLEI_EXTRA_ARGS -t "${NUCLEI_TEMPLATES_PATH}" -j -o ".tmp/nuclei_combined_json.txt"
fi
# WAF hosts at slower rate
if [[ -s "$WAF_LIST" ]]; then
local slow_rl
slow_rl=$((NUCLEI_RATELIMIT / 3 + 1))
# shellcheck disable=SC2086 # Intentionally allow user-provided nuclei args
run_with_heartbeat "nuclei (waf targets)" nuclei \
-l "$WAF_LIST" -severity "$all_severities" -nh -rl "$slow_rl" -silent -retries 2 \
$NUCLEI_EXTRA_ARGS -t "${NUCLEI_TEMPLATES_PATH}" -j -o ".tmp/nuclei_waf_combined_json.txt"
[[ -s ".tmp/nuclei_waf_combined_json.txt" ]] && cat ".tmp/nuclei_waf_combined_json.txt" >>".tmp/nuclei_combined_json.txt"
fi
# Split combined results into per-severity files for backward compatibility
_nuclei_split_by_severity
}
# Run nuclei scan via Axiom distributed fleet (single pass, all severities)
# Usage: _nuclei_scan_axiom
_nuclei_scan_axiom() {
local all_severities="$NUCLEI_SEVERITY"
_print_msg INFO "Running: Axiom Nuclei Severities: ${all_severities}. Check results in nuclei_output folder."
# shellcheck disable=SC2086 # Intentionally allow user-provided nuclei args
run_with_heartbeat "axiom nuclei" axiom-scan .tmp/webs_subs.txt -m nuclei \
--nuclei-templates "$NUCLEI_TEMPLATES_PATH" \
-severity "$all_severities" -nh -rl "$NUCLEI_RATELIMIT" \
-silent -retries 2 $NUCLEI_EXTRA_ARGS -j -o ".tmp/nuclei_combined_json.txt" "${AXIOM_EXTRA_ARGS_ARR[@]}"
# Split combined results into per-severity files for backward compatibility
_nuclei_split_by_severity
}
function nuclei_check() {
# Create necessary directories
if ! ensure_dirs .tmp webs subdomains nuclei_output; then return 1; fi
# Check if the function should run
if should_run "NUCLEICHECK"; then
# Verify nuclei binary is available
if ! command -v nuclei >/dev/null 2>&1; then
_print_msg WARN "nuclei_check: nuclei binary not found in PATH - install nuclei first"
return 0
fi
# Verify templates directory exists (needed for -t flag)
if [[ ! -d "${NUCLEI_TEMPLATES_PATH:-}" ]]; then
_print_msg WARN "nuclei_check: templates directory '${NUCLEI_TEMPLATES_PATH}' not found - run 'nuclei -update-templates' first"
return 0
fi
start_func "${FUNCNAME[0]}" "Templates-based Web Scanner"
maybe_update_nuclei
# Handle multi mode and initialize subdomains.txt if necessary
if [[ -n $multi ]] && [[ ! -f "$dir/subdomains/subdomains.txt" ]]; then
printf "%b\n" "$domain" >"$dir/subdomains/subdomains.txt"
touch webs/webs.txt webs/webs_uncommon_ports.txt
fi
ensure_webs_all || true
# Combine webs_all.txt targets (with protocol) - avoid duplicate scans
if [[ ! -s ".tmp/webs_subs.txt" ]]; then
cat webs/webs_all.txt 2>>"$LOGFILE" | sort -u >.tmp/webs_subs.txt
fi
# Prepare WAF-aware lists and run scans
_nuclei_prepare_waf_lists
if [[ $AXIOM != true ]]; then
_nuclei_scan_local
else
_nuclei_scan_axiom
fi
# Faraday integration
if [[ $FARADAY == true ]]; then
if ! faraday-cli status 2>>"$LOGFILE" >/dev/null; then
print_warnf "Faraday server is not running. Skipping Faraday integration."
else
# Report all severity levels to Faraday
IFS=',' read -ra severity_array <<<"$NUCLEI_SEVERITY"
for crit in "${severity_array[@]}"; do
if [[ -s "nuclei_output/${crit}_json.txt" ]]; then
faraday-cli tool report -w "$FARADAY_WORKSPACE" --plugin-id nuclei "nuclei_output/${crit}_json.txt" 2>>"$LOGFILE" >/dev/null
fi
done
fi
fi
end_func "Results are saved in $domain/nuclei_output folder" "${FUNCNAME[0]}"
plugins_emit after_nuclei "$domain" "$dir"
else
if [[ $NUCLEICHECK == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function graphql_scan() {
# Reuse nuclei_check findings for GraphQL detection and optional introspection
ensure_dirs .tmp webs nuclei_output vulns/graphql
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GRAPHQL_CHECK == true ]]; then
start_func "${FUNCNAME[0]}" "GraphQL detection"
: >nuclei_output/graphql_json.txt
: >.tmp/graphql_endpoints.txt
IFS=',' read -ra severity_array <<<"$NUCLEI_SEVERITY"
for crit in "${severity_array[@]}"; do
local src_json="nuclei_output/${crit}_json.txt"
if [[ -s "$src_json" ]]; then
jq -c 'select(."template-id" == "graphql-detect")' "$src_json" \
| tee -a nuclei_output/graphql_json.txt \
| jq -r '.["matched-at"] // .host // empty' \
| sed '/^$/d' \
| sort -u \
| anew -q .tmp/graphql_endpoints.txt
fi
done
if [[ -s nuclei_output/graphql_json.txt ]]; then
jq -r '["[" + .["template-id"] + "] [" + .info.severity + "] " + (.["matched-at"] // .host)] | .[]' nuclei_output/graphql_json.txt >nuclei_output/graphql.txt
else
_print_msg WARN "No graphql-detect findings in nuclei_check outputs; skipping GraphQL deep checks."
log_note "No graphql-detect findings in nuclei_check outputs; skipping GraphQL deep checks." "${FUNCNAME[0]}" "${LINENO}"
fi
# Optionally run GQLSpection on endpoints discovered by nuclei_check (graphql-detect)
if [[ $GQLSPECTION == true ]] && [[ -s .tmp/graphql_endpoints.txt ]]; then
while read -r ep; do
[[ -z $ep ]] && continue
hostfile="vulns/graphql/$(echo "$ep" | sed 's|^[^/]*//||; s|/.*$||')"
mkdir -p "$(dirname "$hostfile")" 2>>"$LOGFILE" || true
gqlspection -t "$ep" -o "${hostfile}.json" 2>>"$LOGFILE" >/dev/null || true
done <.tmp/graphql_endpoints.txt
fi
end_func "Results are saved in nuclei_output/graphql* and vulns/graphql" "${FUNCNAME[0]}"
else
if [[ $GRAPHQL_CHECK == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function param_discovery() {
ensure_dirs webs .tmp
_parse_arjun_text_output() {
local arjun_out_file="$1"
[[ ! -s "$arjun_out_file" ]] && return 0
grep -aEo 'https?://[^ ]+' "$arjun_out_file" | sed 's/^/URL: /' | anew -q webs/params_discovered.txt || true
grep -aE 'Scanning [0-9]+/[0-9]+: https?://' "$arjun_out_file" | sed -E 's/^.*Scanning [0-9]+\/[0-9]+:[[:space:]]*//' \
| sed 's/^/URL: /' | anew -q webs/params_discovered.txt || true
grep -aE 'Parameters found:[[:space:]]*' "$arjun_out_file" | sed -E 's/^.*Parameters found:[[:space:]]*//' \
| tr ',' '\n' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' | sed '/^$/d' \
| sed 's/^/PARAM: /' | sort -u | anew -q webs/params_discovered.txt || true
grep -aEo '([?&][A-Za-z0-9_.-]+)=' "$arjun_out_file" \
| sed -E 's/^[?&]//; s/=.*$//' | sed '/^$/d' \
| sed 's/^/PARAM: /' | sort -u | anew -q webs/params_discovered.txt || true
}
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $PARAM_DISCOVERY == true ]]; then
# Only run in deep mode - arjun is too slow for normal scans
if [[ $DEEP != true ]]; then
skip_notification "mode"
return 0
fi
if ! command -v arjun >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: arjun not found in PATH"
return 0
fi
start_func "${FUNCNAME[0]}" "Parameter discovery (arjun)"
local input_file="webs/url_extract_nodupes.txt"
local arjun_urls_raw=".tmp/arjun_urls_raw.txt"
local arjun_urls_scoped=".tmp/arjun_urls_scoped.txt"
local arjun_urls_params=".tmp/arjun_urls_params.txt"
: >"$arjun_urls_raw"
: >"$arjun_urls_scoped"
: >"$arjun_urls_params"
[[ ! -s $input_file ]] && input_file="webs/webs_all.txt"
# Limit input URLs to avoid multi-hour runs
local arjun_input="$input_file"
if [[ -s "$input_file" ]] && [[ "${PARAM_MAX_URLS:-0}" -gt 0 ]]; then
local url_count
url_count=$(wc -l <"$input_file" | tr -d ' ')
if [[ "$url_count" -gt "${PARAM_MAX_URLS}" ]]; then
_print_msg INFO "${FUNCNAME[0]}: limiting ${url_count} URLs to ${PARAM_MAX_URLS} (PARAM_MAX_URLS)"
head -n "${PARAM_MAX_URLS}" "$input_file" >".tmp/arjun_input_limited.txt"
arjun_input=".tmp/arjun_input_limited.txt"
fi
fi
# Build timeout prefix for arjun commands
local arjun_timeout_cmd=""
if [[ -n "${TIMEOUT_CMD:-}" ]] && [[ "${PARAM_DISCOVERY_TIMEOUT:-0}" -gt 0 ]]; then
arjun_timeout_cmd="$TIMEOUT_CMD ${PARAM_DISCOVERY_TIMEOUT}m"
fi
if [[ -s "$arjun_input" ]]; then
# shellcheck disable=SC2086 # Intentional word splitting for timeout command
run_command $arjun_timeout_cmd arjun -i "$arjun_input" -t "$ARJUN_THREADS" -oT .tmp/arjun.txt 2>>"$LOGFILE" >/dev/null || true
_parse_arjun_text_output .tmp/arjun.txt
# Feed useful parameterized URLs discovered by arjun back into the main
# URL pipelines so gf/vuln modules can consume them.
if [[ -s ".tmp/arjun.txt" ]]; then
grep -aEo 'https?://[^ ]+' ".tmp/arjun.txt" \
| sed -E 's/[[:space:]]+$//' \
| sed -E 's/[),;]$//' \
| sed 's/"$//' \
| sed "s/'$//" \
| grep -a '=' \
| grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg)$" \
| anew -q "$arjun_urls_raw" || true
fi
if [[ -s "$arjun_urls_raw" ]]; then
local domain_regex
domain_regex=$(domain_match_regex "$domain")
awk -v re="$domain_regex" -F/ '
/^https?:\/\// {
h=$3
sub(/:.*/, "", h)
if (h ~ re) print
}
' "$arjun_urls_raw" | anew -q "$arjun_urls_scoped" || true
if [[ -s "$outOfScope_file" ]] && [[ -s "$arjun_urls_scoped" ]]; then
deleteOutScoped "$outOfScope_file" "$arjun_urls_scoped" || true
fi
if [[ $INSCOPE == true ]] && [[ -s "$arjun_urls_scoped" ]]; then
if ! check_inscope "$arjun_urls_scoped" 2>>"$LOGFILE" >/dev/null; then
print_warnf "check_inscope command failed."
fi
fi
fi
if [[ -s "$arjun_urls_scoped" ]]; then
if command -v urless >/dev/null 2>&1; then
urless <"$arjun_urls_scoped" | anew -q "$arjun_urls_params" 2>>"$LOGFILE" >/dev/null || true
else
cat "$arjun_urls_scoped" | anew -q "$arjun_urls_params" || true
fi
fi
if [[ -s "$arjun_urls_params" ]]; then
local arjun_added=0
if ! arjun_added=$(anew webs/url_extract.txt <"$arjun_urls_params" | sed '/^$/d' | wc -l); then
arjun_added=0
fi
append_assets_from_file url value webs/url_extract.txt
if command -v p1radup >/dev/null 2>&1; then
p1radup -i webs/url_extract.txt -o webs/url_extract_nodupes.txt -s 2>>"$LOGFILE" >/dev/null || true
else
sort -u webs/url_extract.txt >webs/url_extract_nodupes.txt 2>>"$LOGFILE" || true
fi
notification "Parameter discovery merged: ${arjun_added} new parameterized URLs" "info"
fi
fi
end_func "Results are saved in webs/params_discovered.txt" "${FUNCNAME[0]}"
else
if [[ $PARAM_DISCOVERY == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function grpc_reflection() {
ensure_dirs hosts .tmp
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GRPC_SCAN == true ]]; then
if ! command -v grpcurl >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: grpcurl not found in PATH"
return 0
fi
start_func "${FUNCNAME[0]}" "gRPC reflection probing"
# Build target IP list
local ips_file="hosts/ips.txt"
local targets=()
# shellcheck disable=SC2207 # Word splitting intended for array population
[[ -s $ips_file ]] && targets+=($(cat "$ips_file"))
# shellcheck disable=SC2207 # Word splitting intended for array population
[[ -s "hosts/ips_v6.txt" ]] && targets+=($(cat "hosts/ips_v6.txt"))
printf "%s\n" "${targets[@]}" | sort -u >.tmp/grpc_ips.txt
if [[ -s .tmp/grpc_ips.txt ]]; then
# Probe common plaintext gRPC ports
while read -r ip; do
for p in 50051 50052; do
run_command grpcurl -plaintext -max-msg-sz 10485760 -d '{}' "$ip:$p" list 2>>"$LOGFILE" | sed "s/^/[$ip:$p] /" | anew -q hosts/grpc_reflection.txt || true
done
done <.tmp/grpc_ips.txt
fi
end_func "Results are saved in hosts/grpc_reflection.txt" "${FUNCNAME[0]}"
else
if [[ $GRPC_SCAN == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function llm_probe() {
ensure_dirs webs .tmp
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ "${LLM_PROBE:-false}" == "true" ]]; then
if ! command -v julius >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: julius not found in PATH"
return 0
fi
if [[ ! -s "webs/webs_all.txt" ]]; then
skip_notification "noinput"
return 0
fi
start_func "${FUNCNAME[0]}" "LLM service probing (julius)"
local -a julius_cmd=(julius -o jsonl -q probe -f "webs/webs_all.txt")
if [[ "${LLM_PROBE_AUGUSTUS:-false}" == "true" ]]; then
julius_cmd=(julius -o jsonl -q probe --augustus -f "webs/webs_all.txt")
fi
run_command "${julius_cmd[@]}" >"webs/llm_probe.jsonl" 2>>"$LOGFILE" || true
if [[ -s "webs/llm_probe.jsonl" ]]; then
jq -r '[(.target // .url // "unknown"), (.provider // .service // "unknown"), (.probe // "n/a")] | @tsv' "webs/llm_probe.jsonl" 2>/dev/null \
| awk -F'\t' '{printf "%s [%s] [%s]\n", $1, $2, $3}' \
| anew -q "webs/llm_probe.txt"
fi
end_func "Results are saved in webs/llm_probe.[jsonl|txt]" "${FUNCNAME[0]}"
else
if [[ "${LLM_PROBE:-false}" == "false" ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
# Classify targets into normal and slow lists for adaptive fuzzing
_fuzz_classify_targets() {
sort -u webs/webs_all.txt -o webs/webs_all.txt 2>/dev/null || true
: >.tmp/webs_slow.txt
: >.tmp/webs_normal.txt
if [[ -s .tmp/slow_hosts.txt ]]; then
while read -r host; do
awk -v h="$host" -F/ '{u=$3; sub(/:.*/,"",u); if (u==h) print $0}' webs/webs_all.txt | anew -q .tmp/webs_slow.txt
done <.tmp/slow_hosts.txt
comm -23 <(sort -u webs/webs_all.txt) <(sort -u .tmp/webs_slow.txt) >.tmp/webs_normal.txt
else
cp webs/webs_all.txt .tmp/webs_normal.txt
fi
}
# Run ffuf locally with interlace, handling normal and slow targets separately
_fuzz_run_local() {
_fuzz_classify_targets
local ffuf_recursion_flags=""
if [[ ${DEEP:-false} == true ]]; then
ffuf_recursion_flags="-recursion -recursion-depth ${FUZZ_RECURSION_DEPTH:-2}"
fi
if [[ -s .tmp/webs_normal.txt ]]; then
run_with_heartbeat "ffuf normal targets" interlace \
-tL .tmp/webs_normal.txt -threads "${INTERLACE_THREADS}" \
-c "ffuf ${FFUF_FLAGS} ${ffuf_recursion_flags} -t ${FFUF_THREADS} -rate ${FFUF_RATELIMIT} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \
-o "$dir/.tmp/fuzzing"
fi
if [[ -s .tmp/webs_slow.txt ]]; then
local slow_threads=$((FFUF_THREADS / 3))
[[ $slow_threads -lt 5 ]] && slow_threads=5
local slow_rate=$FFUF_RATELIMIT
[[ ${FFUF_RATELIMIT:-0} -eq 0 ]] && slow_rate=50 || slow_rate=$((FFUF_RATELIMIT / 3 + 1))
run_with_heartbeat "ffuf slow targets" interlace \
-tL .tmp/webs_slow.txt -threads "${INTERLACE_THREADS}" \
-c "ffuf ${FFUF_FLAGS} ${ffuf_recursion_flags} -t ${slow_threads} -rate ${slow_rate} -H \"${HEADER}\" -w ${fuzz_wordlist} -maxtime ${FFUF_MAXTIME} -u _target_/FUZZ -o _output_/_cleantarget_.json" \
-o "$dir/.tmp/fuzzing"
fi
# Convert JSON results to TXT in fuzzing/ directory (like axiom path does)
find "$dir/.tmp/fuzzing/" -type f -name "*.json" -print0 2>/dev/null | while IFS= read -r -d '' json_file; do
local base
base=$(basename "${json_file%.json}")
# Preserve raw JSON in final output directory
cp "$json_file" "$dir/fuzzing/${base}.json" 2>>"$LOGFILE" || true
# Extract human-readable TXT summary
jq -r 'try .results[] | "\(.status) \(.length) \(.url)"' "$json_file" 2>>"$LOGFILE" \
| sort -k1 | anew -q "$dir/fuzzing/${base}.txt" 2>>"$LOGFILE" || true
done
}
# Run ffuf via axiom-scan and parse per-subdomain results
_fuzz_run_axiom() {
cached_download_typed "${fuzzing_remote_list}" ".tmp/fuzzing_remote_list.txt" "onelistforallmicro.txt" "wordlists"
run_with_heartbeat "axiom ffuf" axiom-scan webs/webs_all.txt -m ffuf -wL .tmp/fuzzing_remote_list.txt -H "${HEADER}" "$FFUF_FLAGS" -s -maxtime "$FFUF_MAXTIME" -oJ "$dir/.tmp/ffuf-content.json" "${AXIOM_EXTRA_ARGS_ARR[@]}"
while read -r sub; do
local sub_out
sub_out=$(echo "$sub" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
if [[ -s "$dir/.tmp/ffuf-content.json" ]]; then
local tmp_out=".tmp/ffuf_${sub_out}.txt"
jq -r --arg sub "$sub" 'try .results[] | select(.url | contains($sub)) | "\(.status) \(.length) \(.url)"' "$dir/.tmp/ffuf-content.json" 2>>"$LOGFILE" \
| sort -k1 >"$tmp_out" || true
if [[ -s "$tmp_out" ]]; then
anew -q "$dir/fuzzing/${sub_out}.txt" <"$tmp_out" 2>>"$LOGFILE" || true
fi
rm -f "$tmp_out" 2>/dev/null || true
fi
done <webs/webs_all.txt
}
# Merge all per-subdomain fuzzing results into fuzzing_full.txt
_fuzz_merge_results() {
find "$dir/fuzzing/" -type f -iname "*.txt" ! -name "fuzzing_full.txt" -exec cat {} + 2>>"$LOGFILE" | sort -k1 | anew -q "$dir/fuzzing/fuzzing_full.txt"
}
function fuzz() {
ensure_dirs .tmp/fuzzing webs fuzzing
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $FUZZ == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Web Directory Fuzzing"
# Handle multi mode initialization
if [[ -n $multi ]] && [[ ! -f "$dir/webs/webs.txt" ]]; then
printf "%b\n" "$domain" >"$dir/webs/webs.txt" || print_warnf "Failed to create webs.txt."
touch webs/webs_uncommon_ports.txt 2>>"$LOGFILE" || true
fi
ensure_webs_all || true
if [[ -s "webs/webs_all.txt" ]]; then
if [[ $AXIOM != true ]]; then
_fuzz_run_local
else
_fuzz_run_axiom
fi
_fuzz_merge_results
end_func "Results are saved in $domain/fuzzing/*subdomain*.txt" "${FUNCNAME[0]}"
else
end_func "No $domain/webs/webs_all.txt file found, fuzzing skipped " "${FUNCNAME[0]}" "SKIP_NOINPUT"
fi
else
if [[ $FUZZ == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function iishortname() {
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $IIS_SHORTNAME == true ]]; then
start_func "${FUNCNAME[0]}" "IIS Shortname Scanner"
ensure_dirs .tmp || true
: >.tmp/iis_sites.txt
# Ensure nuclei_output/info.txt exists and is not empty
if [[ -s "nuclei_output/info.txt" ]]; then
# Extract IIS targets (if any) without triggering pipefail on "no matches".
awk '/iis-version/ {print $4}' "nuclei_output/info.txt" >.tmp/iis_sites.txt
fi
# Proceed only if iis_sites.txt exists and is non-empty
if [[ -s ".tmp/iis_sites.txt" ]]; then
# Create necessary directory
mkdir -p "$dir/vulns/iis-shortname-shortscan/"
# Run shortscan using interlace
run_command interlace -tL .tmp/iis_sites.txt -threads "$INTERLACE_THREADS" \
-c "shortscan _target_ -F -s -p 1 > _output_/_cleantarget_.txt" \
-o "$dir/vulns/iis-shortname-shortscan/" 2>>"$LOGFILE" >/dev/null
# Remove non-vulnerable shortscan results
while IFS= read -r -d '' file; do
if ! grep -q 'Vulnerable: Yes' "$file" 2>/dev/null; then
rm -f "$file" 2>>"$LOGFILE"
fi
done < <(find "$dir/vulns/iis-shortname-shortscan/" -type f -iname "*.txt" -print0 2>/dev/null)
fi
end_func "Results are saved in vulns/iis-shortname/" "${FUNCNAME[0]}"
else
if [[ $IIS_SHORTNAME == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function swagger_check() {
ensure_dirs .tmp webs vulns/swagger
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $SJ_CHECK == true ]]; then
if ! command -v sj >/dev/null 2>&1; then
end_func "sj not found in PATH, skipping" "${FUNCNAME[0]}" "SKIP_NOINPUT"
return 0
fi
start_func "${FUNCNAME[0]}" "Swagger/OpenAPI analysis"
: >.tmp/swagger_urls_all.txt
# ── Phase 1: Brute-force discovery with sj brute ─────────────
if [[ $SJ_BRUTE == true ]] && [[ -s "webs/webs_all.txt" ]]; then
mkdir -p .tmp/sj_brute
local brute_targets=".tmp/sj_brute_targets.txt"
if [[ ${SJ_MAX_TARGETS:-0} -gt 0 ]]; then
head -n "${SJ_MAX_TARGETS}" webs/webs_all.txt >"$brute_targets"
else
cp webs/webs_all.txt "$brute_targets"
fi
# Wrap the whole interlace+sj pool in run_with_heartbeat_shell so the
# UI shows "Running: sj brute (swagger) | elapsed Xm Ys | ETA: --"
# every 20s — otherwise swagger_check appears to hang in -l/-z modes
# where sj brute can run for tens of minutes against 200 targets.
local _brute_targets_count
_brute_targets_count=$(wc -l <"$brute_targets" 2>/dev/null | tr -d ' ' || echo "?")
run_with_heartbeat_shell "sj brute (swagger, ${_brute_targets_count} targets)" \
"interlace -tL \"$brute_targets\" -threads \"$INTERLACE_THREADS\" -c \"sj brute -u _target_ -qi -t ${SJ_TIMEOUT:-30} 2>/dev/null | grep -o 'Definition file found: .*' > _output_/_cleantarget_.txt\" -o .tmp/sj_brute 2>>\"$LOGFILE\" >/dev/null" \
|| true
find .tmp/sj_brute/ -type f -name "*.txt" -print0 2>/dev/null \
| xargs -0 grep -hoP 'Definition file found: \K\S+' 2>/dev/null \
| sort -u \
| anew -q .tmp/swagger_urls_all.txt
fi
# ── Phase 2: Aggregate swagger URLs from other sources ────────
# Nuclei: template-ids "swagger-api" and "openapi" (already in scope)
if [[ -d "nuclei_output" ]]; then
IFS=',' read -ra severity_array <<<"$NUCLEI_SEVERITY"
for crit in "${severity_array[@]}"; do
local src_json="nuclei_output/${crit}_json.txt"
if [[ -s "$src_json" ]]; then
jq -r 'select(."template-id" == "swagger-api" or ."template-id" == "openapi") | .["matched-at"] // .host // empty' "$src_json" 2>/dev/null \
| sed '/^$/d' \
| anew -q .tmp/swagger_urls_all.txt
fi
done
fi
# SwaggerSpy OSINT output: filter against scope before adding
if [[ -s "${dir}/osint/swagger_leaks.txt" ]]; then
grep -hioP 'https?://\S+' "${dir}/osint/swagger_leaks.txt" 2>/dev/null \
| grep -iE '\.(json|yaml|yml)$|swagger|openapi|api-docs' \
>.tmp/swagger_spy_candidates.txt || true
if [[ -s .tmp/swagger_spy_candidates.txt ]]; then
check_inscope .tmp/swagger_spy_candidates.txt 2>/dev/null || true
if [[ -s .tmp/swagger_spy_candidates.txt ]]; then
cat .tmp/swagger_spy_candidates.txt | anew -q .tmp/swagger_urls_all.txt
fi
fi
fi
# Deduplicate
if [[ -s .tmp/swagger_urls_all.txt ]]; then
sort -u .tmp/swagger_urls_all.txt -o .tmp/swagger_urls_all.txt
cp .tmp/swagger_urls_all.txt "webs/swagger_urls.txt"
fi
local swagger_count
swagger_count=$(wc -l <.tmp/swagger_urls_all.txt 2>/dev/null || echo 0)
swagger_count=$((swagger_count + 0))
if [[ $swagger_count -eq 0 ]]; then
end_func "No swagger/openapi definitions discovered" "${FUNCNAME[0]}"
return
fi
# ── Phase 3: Endpoint extraction + auth testing ───────────────
: >vulns/swagger/endpoints_all.txt
mkdir -p vulns/swagger/automate
while IFS= read -r swagger_url; do
[[ -z "$swagger_url" ]] && continue
# Extract endpoints
sj endpoints -u "$swagger_url" -qi -t "${SJ_TIMEOUT:-30}" 2>>"$LOGFILE" \
| anew -q vulns/swagger/endpoints_all.txt || true
# Auth testing with sj automate
if [[ $SJ_AUTOMATE == true ]]; then
local clean_name
clean_name=$(echo "$swagger_url" | sed -e 's|^[^/]*//||' -e 's|[^a-zA-Z0-9._-]|_|g')
local automate_out="vulns/swagger/automate/${clean_name}.json"
sj automate -u "$swagger_url" -qi -t "${SJ_TIMEOUT:-30}" \
-F json -o "$automate_out" 2>>"$LOGFILE" || true
if [[ -s "$automate_out" ]]; then
jq -r 'select(.status == 200) | "\(.method) \(.target)"' "$automate_out" 2>/dev/null \
| anew -q vulns/swagger/accessible_endpoints.txt || true
fi
fi
done <.tmp/swagger_urls_all.txt
end_func "Results are saved in webs/swagger_urls.txt and vulns/swagger/" "${FUNCNAME[0]}"
else
if [[ $SJ_CHECK == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function cms_scanner() {
# Create necessary directories
if ! mkdir -p .tmp/fuzzing webs cms; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $CMS_SCANNER == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "CMS Scanner"
rm -rf "$dir/cms/"*
# Handle multi mode and initialize webs.txt if necessary
if [[ -n $multi ]] && [[ ! -f "$dir/webs/webs.txt" ]]; then
printf "%b\n" "$domain" >"$dir/webs/webs.txt"
touch webs/webs_uncommon_ports.txt
fi
# Ensure webs_all.txt exists
if [[ ! -s "webs/webs_all.txt" ]]; then
end_func "No webs/webs_all.txt file found, cms scanner skipped." "${FUNCNAME[0]}" "SKIP_NOINPUT"
return
fi
# Run CMSeeK with timeout
local cmseek_cmd=(
timeout -k 1m "${CMSSCAN_TIMEOUT}s"
"${tools}/CMSeeK/venv/bin/python3" "${tools}/CMSeeK/cmseek.py"
-l webs/webs_all.txt --batch -r
)
run_with_heartbeat "cmseek batch scan" "${cmseek_cmd[@]}"
local exit_status=$?
if [[ ${exit_status} -ne 0 ]]; then
# Attempt one-time repair on known CMSeeK index corruption.
if [[ ${exit_status} -eq 124 || ${exit_status} -eq 137 ]]; then
echo "TIMEOUT cmseek.py - investigate manually for $dir" >>"$LOGFILE"
end_func "TIMEOUT cmseek.py - investigate manually for $dir" "${FUNCNAME[0]}"
return
elif [[ ${exit_status} -ne 0 ]]; then
echo "ERROR cmseek.py - investigate manually for $dir" >>"$LOGFILE"
end_func "ERROR cmseek.py - investigate manually for $dir" "${FUNCNAME[0]}"
return
fi
fi
# Process CMSeeK results
while IFS= read -r sub; do
sub_out=$(echo "$sub" | sed -e 's|^[^/]*//||' -e 's|/.*$||')
sub_out="${sub_out//$'\r'/}"
sub_out="${sub_out//:/_}"
cms_json_path="${tools}/CMSeeK/Result/${sub_out}/cms.json"
if [[ -s $cms_json_path ]]; then
cms_id=$(jq -r 'try .cms_id // empty' "$cms_json_path")
mv -f "${tools}/CMSeeK/Result/${sub_out}" "$dir/cms/" 2>>"$LOGFILE"
if [[ -z $cms_id ]]; then
log_note "cms_scanner: empty cms_id for ${sub_out}, results moved for inspection" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
done <"webs/webs_all.txt"
end_func "Results are saved in $domain/cms/*subdomain* folder" "${FUNCNAME[0]}"
else
if [[ $CMS_SCANNER == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed-visible"
fi
fi
}
_katana_headless_flags() {
local target_count="${1:-0}"
local profile="${KATANA_HEADLESS_PROFILE:-off}"
case "$profile" in
off) echo "" ;;
full) echo "-headless" ;;
smart)
if [[ "$target_count" -le "${KATANA_HEADLESS_SMART_LIMIT:-15}" ]]; then
echo "-headless"
else
echo ""
fi
;;
*)
log_note "urlchecks: invalid KATANA_HEADLESS_PROFILE='${profile}', using off" "${FUNCNAME[0]}" "${LINENO}"
echo ""
;;
esac
}
function urlchecks() {
# Create necessary directories
if ! ensure_dirs .tmp webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $URL_CHECK == true ]]; then
start_func "${FUNCNAME[0]}" "URL Extraction"
local end_message="Results are saved in $domain/webs/url_extract.txt"
local katana_timeout_cmd="${TIMEOUT_CMD:-timeout}"
local waymore_timeout_cmd="${TIMEOUT_CMD:-timeout}"
local katana_headless_flags=""
ensure_webs_all || true
if [[ -s "webs/webs_all.txt" ]]; then
local katana_target_count=0
katana_target_count=$(wc -l <"webs/webs_all.txt" 2>/dev/null || echo 0)
katana_headless_flags=$(_katana_headless_flags "$katana_target_count")
if [[ $URL_CHECK_PASSIVE == true ]]; then
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
log_note "urlchecks: reusing urlfinder output from sub_scraping" "${FUNCNAME[0]}" "${LINENO}"
else
: >.tmp/url_extract_tmp.txt
run_command urlfinder -d "$domain" -all -o .tmp/url_extract_tmp.txt 2>>"$LOGFILE" >/dev/null
fi
if command -v waymore &>/dev/null; then
if [[ -s ".tmp/waymore_urls_subs.txt" ]]; then
log_note "urlchecks: reusing waymore output from sub_scraping" "${FUNCNAME[0]}" "${LINENO}"
cat .tmp/waymore_urls_subs.txt | anew -q .tmp/url_extract_tmp.txt || true
else
if ! run_with_heartbeat "waymore passive urls" "$waymore_timeout_cmd" "${WAYMORE_TIMEOUT:-30m}" waymore -i "$domain" -mode U -oU .tmp/waymore_urls.txt; then
log_note "urlchecks: waymore failed or timed out; continuing with other passive sources" "${FUNCNAME[0]}" "${LINENO}"
fi
if [[ -s ".tmp/waymore_urls.txt" ]]; then
cat .tmp/waymore_urls.txt | anew -q .tmp/url_extract_tmp.txt || true
fi
fi
else
log_note "urlchecks: waymore not found; skipping passive waymore collection" "${FUNCNAME[0]}" "${LINENO}"
fi
if [[ -s $GITHUB_TOKENS ]]; then
run_command github-endpoints -q -k -d "$domain" -t "$GITHUB_TOKENS" -o .tmp/github-endpoints.txt 2>>"$LOGFILE" >/dev/null
if [[ -s ".tmp/github-endpoints.txt" ]]; then
cat .tmp/github-endpoints.txt | anew -q .tmp/url_extract_tmp.txt || true
fi
fi
fi
if [[ $AXIOM != true ]]; then
if [[ -s ".tmp/probed_tmp.txt" && -s "webs/webs_all.txt" ]]; then
diff_webs=$(comm -3 <(sort -u .tmp/probed_tmp.txt 2>>"$LOGFILE") <(sort -u webs/webs_all.txt 2>>"$LOGFILE") | wc -l)
else
log_note "urlchecks: missing .tmp/probed_tmp.txt or webs/webs_all.txt; skipping diff" "${FUNCNAME[0]}" "${LINENO}"
diff_webs=1
fi
if [[ $diff_webs != "0" ]] || [[ ! -s ".tmp/katana.txt" ]]; then
if [[ $URL_CHECK_ACTIVE == true ]]; then
# Only crawl NEW URLs when previous results exist (incremental)
local katana_input_file="webs/webs_all.txt"
if [[ -s ".tmp/katana.txt" ]] && [[ -s ".tmp/probed_tmp.txt" ]] && [[ -s "webs/webs_all.txt" ]]; then
comm -23 <(sort -u webs/webs_all.txt) <(sort -u .tmp/probed_tmp.txt) > .tmp/katana_new_webs.txt 2>/dev/null || true
if [[ -s ".tmp/katana_new_webs.txt" ]]; then
katana_input_file=".tmp/katana_new_webs.txt"
fi
fi
# Split slow vs normal targets based on httpx status (403/429)
: >.tmp/katana_targets_slow.txt
: >.tmp/katana_targets_normal.txt
if [[ -s .tmp/slow_hosts.txt ]]; then
while read -r host; do
grep "://${host}[:/\n]" "$katana_input_file" | anew -q .tmp/katana_targets_slow.txt
done <.tmp/slow_hosts.txt
comm -23 <(sort -u "$katana_input_file") <(sort -u .tmp/katana_targets_slow.txt) >.tmp/katana_targets_normal.txt
else
cp "$katana_input_file" .tmp/katana_targets_normal.txt
fi
: >.tmp/katana.txt
# Normal targets
if [[ -s .tmp/katana_targets_normal.txt ]]; then
LINES=$(wc -l <.tmp/katana_targets_normal.txt)
if [[ $LINES -gt ${CHUNK_LIMIT:-2000} ]]; then
if [[ $DEEP == true ]]; then
process_in_chunks .tmp/katana_targets_normal.txt "${CHUNK_LIMIT:-2000}" "katana -silent -list _chunk_ ${katana_headless_flags} -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn >> .tmp/katana.txt 2>>\"$LOGFILE\" >/dev/null"
else
process_in_chunks .tmp/katana_targets_normal.txt "${CHUNK_LIMIT:-2000}" "katana -silent -list _chunk_ ${katana_headless_flags} -jc -kf all -c $KATANA_THREADS -d 2 -fs rdn >> .tmp/katana.txt 2>>\"$LOGFILE\""
fi
else
if [[ $DEEP == true ]]; then
run_with_heartbeat_shell "katana normal targets (deep)" "$katana_timeout_cmd 4h katana -silent -list .tmp/katana_targets_normal.txt ${katana_headless_flags} -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn >> .tmp/katana.txt 2>> \"$LOGFILE\""
else
run_with_heartbeat_shell "katana normal targets" "$katana_timeout_cmd 3h katana -silent -list .tmp/katana_targets_normal.txt ${katana_headless_flags} -jc -kf all -c $KATANA_THREADS -d 2 -fs rdn >> .tmp/katana.txt 2>> \"$LOGFILE\""
fi
fi
fi
# Slow targets with reduced concurrency
if [[ -s .tmp/katana_targets_slow.txt ]]; then
slow_c=$((KATANA_THREADS / 3))
[[ $slow_c -lt 2 ]] && slow_c=2
LINES=$(wc -l <.tmp/katana_targets_slow.txt)
if [[ $LINES -gt ${CHUNK_LIMIT:-2000} ]]; then
if [[ $DEEP == true ]]; then
process_in_chunks .tmp/katana_targets_slow.txt "${CHUNK_LIMIT:-2000}" "katana -silent -list _chunk_ ${katana_headless_flags} -jc -kf all -c $slow_c -d 3 -fs rdn >> .tmp/katana.txt 2>>\"$LOGFILE\""
else
process_in_chunks .tmp/katana_targets_slow.txt "${CHUNK_LIMIT:-2000}" "katana -silent -list _chunk_ ${katana_headless_flags} -jc -kf all -c $slow_c -d 2 -fs rdn >> .tmp/katana.txt 2>>\"$LOGFILE\""
fi
else
if [[ $DEEP == true ]]; then
run_with_heartbeat_shell "katana slow targets (deep)" "$katana_timeout_cmd 4h katana -silent -list .tmp/katana_targets_slow.txt ${katana_headless_flags} -jc -kf all -c $slow_c -d 3 -fs rdn >> .tmp/katana.txt 2>> \"$LOGFILE\""
else
run_with_heartbeat_shell "katana slow targets" "$katana_timeout_cmd 3h katana -silent -list .tmp/katana_targets_slow.txt ${katana_headless_flags} -jc -kf all -c $slow_c -d 2 -fs rdn >> .tmp/katana.txt 2>> \"$LOGFILE\""
fi
fi
fi
fi
fi
else
if [[ -s ".tmp/probed_tmp.txt" && -s "webs/webs_all.txt" ]]; then
diff_webs=$(comm -3 <(sort -u .tmp/probed_tmp.txt) <(sort -u webs/webs_all.txt) | wc -l)
else
log_note "urlchecks: missing .tmp/probed_tmp.txt or webs/webs_all.txt; skipping diff" "${FUNCNAME[0]}" "${LINENO}"
diff_webs=1
fi
if [[ $diff_webs != "0" ]] || [[ ! -s ".tmp/katana.txt" ]]; then
if [[ $URL_CHECK_ACTIVE == true ]]; then
if [[ $DEEP == true ]]; then
run_with_heartbeat "axiom katana (deep)" axiom-scan webs/webs_all.txt -m katana $katana_headless_flags -jc -kf all -d 3 -fs rdn --max-runtime 4h -o .tmp/katana.txt "${AXIOM_EXTRA_ARGS_ARR[@]}"
else
run_with_heartbeat "axiom katana" axiom-scan webs/webs_all.txt -m katana $katana_headless_flags -jc -kf all -d 2 -fs rdn --max-runtime 3h -o .tmp/katana.txt "${AXIOM_EXTRA_ARGS_ARR[@]}"
fi
fi
fi
fi
if [[ -s ".tmp/katana.txt" ]]; then
sed_i '/^.\{2048\}./d' .tmp/katana.txt
cat .tmp/katana.txt | anew -q .tmp/url_extract_tmp.txt || true
fi
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
# Use filter_in_scope_urls (validation.sh) to anchor host match and
# reject userinfo/off-scope look-alikes. Replaces fragile grep -a "$domain".
grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep -iE '\.js([?#].*)?$|\.js([/?&].*)' | anew -q .tmp/url_extract_js.txt || true
grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep -iE '\.map([/?#].*)?$' | anew -q .tmp/url_extract_jsmap.txt || true
if [[ $DEEP == true ]] && [[ -s ".tmp/url_extract_js.txt" ]]; then
# argv-safe parallel replacement for interlace -c. Each target runs
# in a background subshell; outputs are written to per-target temp
# files (anew is not safe for concurrent writes) then merged once
# all workers finish.
local _jsa_parts
_jsa_parts=$(mktemp -d)
while IFS= read -r _jsa_target; do
[[ -z "$_jsa_target" ]] && continue
(
_jsa_hash=$(printf '%s' "$_jsa_target" | sha256sum | cut -d' ' -f1)
"${tools}/JSA/venv/bin/python3" "${tools}/JSA/jsa.py" -f "$_jsa_target" \
>"$_jsa_parts/$_jsa_hash" 2>/dev/null || true
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/url_extract_js.txt
wait
cat "$_jsa_parts"/* 2>/dev/null | anew -q .tmp/url_extract_tmp.txt
rm -rf "$_jsa_parts"
fi
grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg)$" | anew -q .tmp/url_extract_tmp2.txt || true
if [[ -s ".tmp/url_extract_tmp2.txt" ]]; then
urless <.tmp/url_extract_tmp2.txt | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/null || true
fi
if [[ -s ".tmp/url_extract_uddup.txt" ]]; then
if ! NUMOFLINES=$(anew webs/url_extract.txt <.tmp/url_extract_uddup.txt | sed '/^$/d' | wc -l); then
print_warnf "Failed to update url_extract.txt."
NUMOFLINES=0
fi
notification "${NUMOFLINES} new URLs with parameters" "info"
# Asset store: append new URLs
append_assets_from_file url value webs/url_extract.txt
else
NUMOFLINES=0
fi
if [[ -s "webs/url_extract.txt" ]]; then
p1radup -i webs/url_extract.txt -o webs/url_extract_nodupes.txt -s 2>>"$LOGFILE" >/dev/null || true
if [[ $PROXY == true ]] && [[ -n $proxy_url ]] && [[ $(wc -l <webs/url_extract.txt) -le $DEEP_LIMIT2 ]]; then
notification "Sending URLs to proxy" "info"
run_command ffuf -mc all -w webs/url_extract.txt -u FUZZ -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
else
_print_msg WARN "No URL extraction output generated; skipping p1radup/proxy replay."
log_note "No URL extraction output generated; skipping p1radup/proxy replay." "${FUNCNAME[0]}" "${LINENO}"
end_message="No URL extraction output generated."
fi
else
end_message="No URL extraction candidates generated."
_print_msg WARN "No URL extraction candidates generated."
fi
else
end_message="No web targets available for URL extraction."
_print_msg WARN "No web targets available for URL extraction."
fi
if [[ "$end_message" == No\ * ]]; then
end_func "${end_message}" "${FUNCNAME[0]}" "SKIP_NOINPUT"
else
end_func "${end_message}" "${FUNCNAME[0]}"
fi
else
if [[ $URL_CHECK == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function url_gf() {
# Create necessary directories
if ! mkdir -p .tmp webs gf; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $URL_GF == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
if ! command -v gf >/dev/null 2>&1; then
_print_msg WARN "${FUNCNAME[0]}: gf not found in PATH"
return 0
fi
start_func "${FUNCNAME[0]}" "Vulnerable Pattern Search"
# Ensure webs/url_extract.txt exists and is not empty
if [[ -s "webs/url_extract.txt" ]]; then
# Define an array of GF patterns
declare -A gf_patterns=(
["xss"]="gf/xss.txt"
["ssti"]="gf/ssti.txt"
["ssrf"]="gf/ssrf.txt"
["sqli"]="gf/sqli.txt"
["redirect"]="gf/redirect.txt"
["rce"]="gf/rce.txt"
["potential"]="gf/potential.txt"
["lfi"]="gf/lfi.txt"
)
# Iterate over GF patterns and process each
for pattern in "${!gf_patterns[@]}"; do
output_file="${gf_patterns[$pattern]}"
_print_msg INFO "Running: GF Pattern '${pattern}'"
if [[ $pattern == "potential" ]]; then
# Special handling for 'potential' pattern
run_command gf "$pattern" "webs/url_extract.txt" | cut -d ':' -f3-5 | anew -q "$output_file"
elif [[ $pattern == "redirect" && -s "gf/ssrf.txt" ]]; then
# Append SSFR results to redirect if ssrf.txt exists
run_command gf "$pattern" "webs/url_extract.txt" | anew -q "$output_file"
else
# General handling for other patterns
run_command gf "$pattern" "webs/url_extract.txt" | anew -q "$output_file"
fi
done
# Process endpoints extraction
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
_print_msg INFO "Extracting endpoints..."
grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" ".tmp/url_extract_tmp.txt" \
| unfurl -u format '%s://%d%p' 2>>"$LOGFILE" | anew -q "gf/endpoints.txt"
fi
else
end_func "No webs/url_extract.txt file found, URL_GF check skipped." "${FUNCNAME[0]}"
return
fi
end_func "Results are saved in $domain/gf folder" "${FUNCNAME[0]}"
else
if [[ $URL_GF == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function url_ext() {
# Create necessary directories
if ! mkdir -p .tmp webs gf; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $URL_EXT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
start_func "${FUNCNAME[0]}" "Vulnerable Pattern Search"
# Define an array of file extensions
ext=("7z" "achee" "action" "adr" "apk" "arj" "ascx" "asmx" "asp" "aspx" "axd" "backup" "bak" "bat" "bin" "bkf" "bkp" "bok" "cab" "cer" "cfg" "cfm" "cnf" "conf" "config" "cpl" "crt" "csr" "csv" "dat" "db" "dbf" "deb" "dmg" "dmp" "doc" "docx" "drv" "email" "eml" "emlx" "env" "exe" "gadget" "gz" "html" "ica" "inf" "ini" "iso" "jar" "java" "jhtml" "json" "jsp" "key" "log" "lst" "mai" "mbox" "mbx" "md" "mdb" "msg" "msi" "nsf" "ods" "oft" "old" "ora" "ost" "pac" "passwd" "pcf" "pdf" "pem" "pgp" "php" "php3" "php4" "php5" "phtm" "phtml" "pkg" "pl" "plist" "pst" "pwd" "py" "rar" "rb" "rdp" "reg" "rpm" "rtf" "sav" "sh" "shtm" "shtml" "skr" "sql" "swf" "sys" "tar" "tar.gz" "tmp" "toast" "tpl" "txt" "url" "vcd" "vcf" "wml" "wpd" "wsdl" "wsf" "xls" "xlsm" "xlsx" "xml" "xsd" "yaml" "yml" "z" "zip")
# Initialize the output file
if ! : >webs/urls_by_ext.txt; then
print_warnf "Failed to initialize webs/urls_by_ext.txt."
fi
# Build a single regex pattern for all extensions (single-pass extraction)
local ext_pattern
ext_pattern=$(printf '%s|' "${ext[@]}")
ext_pattern="${ext_pattern%|}" # remove trailing pipe
# Single grep pass to classify URLs by extension using awk
awk -v exts="${ext_pattern}" '
BEGIN { split(exts, ea, "|"); for (i in ea) ext_map[ea[i]] = 1 }
{
url = $0
# Extract extension from URL
gsub(/[?#].*/, "", url)
gsub(/\/$/, "", url)
n = split(url, parts, ".")
if (n > 0) {
ext = tolower(parts[n])
if (ext in ext_map) {
print ext "\t" $0
}
}
}' ".tmp/url_extract_tmp.txt" 2>>"$LOGFILE" | sort -t$'\t' -k1,1 -k2,2 -u > ".tmp/urls_by_ext_all.tmp" || true
# Group by extension and write output
local current_ext=""
while IFS=$'\t' read -r t url; do
if [[ "$t" != "$current_ext" ]]; then
current_ext="$t"
printf "\n############################\n + %s + \n############################\n" "$t" >>webs/urls_by_ext.txt
fi
printf "%s\n" "$url" >>webs/urls_by_ext.txt
done < ".tmp/urls_by_ext_all.tmp"
rm -f ".tmp/urls_by_ext_all.tmp"
# Append ssrf.txt to redirect.txt if ssrf.txt exists and is not empty
if [[ -s "gf/ssrf.txt" ]]; then
cat "gf/ssrf.txt" | anew -q "gf/redirect.txt" || true
fi
end_func "Results are saved in $domain/webs/urls_by_ext.txt" "${FUNCNAME[0]}"
else
end_func "No .tmp/url_extract_tmp.txt file found, URL_EXT check skipped." "${FUNCNAME[0]}"
fi
else
if [[ $URL_EXT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function jschecks() {
# Create necessary directories
if ! mkdir -p .tmp webs subdomains js; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $JSCHECKS == true ]]; then
start_func "${FUNCNAME[0]}" "JavaScript Scan"
# If .tmp/url_extract_js.txt doesn't exist, try to recover from previous run output
if [[ ! -s ".tmp/url_extract_js.txt" ]] && [[ -s "js/url_extract_js.txt" ]]; then
cat js/url_extract_js.txt | anew -q .tmp/url_extract_js.txt
fi
if [[ -s ".tmp/url_extract_js.txt" ]]; then
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Fetching URLs 1/6%b\n" "$yellow" "$reset"
if [[ $AXIOM != true ]]; then
run_command subjs -ua "Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" -c 40 <.tmp/url_extract_js.txt \
| grep -F "$domain" \
| grep -aEo 'https?://[^ ]+' | anew -q .tmp/subjslinks.txt || true
else
run_command axiom-scan .tmp/url_extract_js.txt -m subjs -o .tmp/subjslinks.txt "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s ".tmp/subjslinks.txt" ]]; then
grep -Eiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" .tmp/subjslinks.txt \
| anew -q js/nojs_links.txt || true
grep -iE '\.js([?#].*)?$|\.js([/?&].*)' .tmp/subjslinks.txt | anew -q .tmp/url_extract_js.txt || true
fi
urless <.tmp/url_extract_js.txt \
| anew -q js/url_extract_js.txt 2>>"$LOGFILE" >/dev/null
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Resolving JS URLs 2/6%b\n" "$yellow" "$reset"
if [[ $AXIOM != true ]]; then
if [[ -s "js/url_extract_js.txt" ]]; then
run_command httpx -follow-redirects -random-agent -silent -timeout "$HTTPX_TIMEOUT" -threads "$HTTPX_THREADS" \
-rl "$HTTPX_RATELIMIT" -status-code -content-type -retries 2 -no-color <js/url_extract_js.txt \
| awk '/\\[200\\]/ && /javascript/ {print $1}' | anew -q js/js_livelinks.txt || true
fi
else
if [[ -s "js/url_extract_js.txt" ]]; then
run_command axiom-scan js/url_extract_js.txt -m httpx -follow-host-redirects -H "$HEADER" -status-code \
-threads "$HTTPX_THREADS" -rl "$HTTPX_RATELIMIT" -timeout "$HTTPX_TIMEOUT" -silent \
-content-type -retries 2 -no-color -o .tmp/js_livelinks.txt "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
if [[ -s ".tmp/js_livelinks.txt" ]]; then
awk '/\\[200\\]/ && /javascript/ {print $1}' .tmp/js_livelinks.txt | anew -q js/js_livelinks.txt || true
fi
fi
fi
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Extracting JS from sourcemaps 3/6%b\n" "$yellow" "$reset"
if ! mkdir -p .tmp/sourcemapper; then
print_warnf "Failed to create sourcemapper directory."
fi
if [[ -s "js/js_livelinks.txt" ]]; then
# argv-safe parallel replacement for interlace -c. sourcemapper writes
# its own per-target output dir so no post-merge is needed.
while IFS= read -r _sm_target; do
[[ -z "$_sm_target" ]] && continue
(
_sm_safe=$(printf '%s' "$_sm_target" | sha256sum | cut -d' ' -f1)
sourcemapper -jsurl "$_sm_target" -output ".tmp/sourcemapper/${_sm_safe}" 2>>"$LOGFILE" >/dev/null || true
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <js/js_livelinks.txt
wait
fi
if [[ -s ".tmp/url_extract_jsmap.txt" ]]; then
# argv-safe parallel replacement for interlace -c.
while IFS= read -r _smu_target; do
[[ -z "$_smu_target" ]] && continue
(
_smu_safe=$(printf '%s' "$_smu_target" | sha256sum | cut -d' ' -f1)
sourcemapper -url "$_smu_target" -output ".tmp/sourcemapper/${_smu_safe}" 2>>"$LOGFILE" >/dev/null || true
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/url_extract_jsmap.txt
wait
fi
find .tmp/sourcemapper/ \( -name "*.js" -o -name "*.ts" \) -type f \
| run_command jsluice urls | jq -r .url | anew -q .tmp/js_endpoints.txt || true
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Gathering endpoints 4/6%b\n" "$yellow" "$reset"
if [[ -s "js/js_livelinks.txt" ]]; then
xnLinkFinder -i js/js_livelinks.txt -sf subdomains/subdomains.txt -d "$XNLINKFINDER_DEPTH" \
-o .tmp/js_endpoints.txt 2>>"$LOGFILE" >/dev/null
fi
if [[ -s ".tmp/js_endpoints.txt" ]]; then
sed_i '/^\//!d' .tmp/js_endpoints.txt
cat .tmp/js_endpoints.txt | anew -q js/js_endpoints.txt || true
fi
merge_scoped_urls_into_url_extract "js/nojs_links.txt" "js_nojs"
merge_scoped_urls_into_url_extract "js/js_endpoints.txt" "js_endpoints"
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Gathering secrets 5/6%b\n" "$yellow" "$reset"
if [[ -s "js/js_livelinks.txt" ]]; then
if [[ $AXIOM != true ]]; then
cat js/js_livelinks.txt | mantra -ua "$HEADER" -s | anew -q js/js_secrets.txt 2>>"$LOGFILE" >/dev/null || true
else
axiom-exec "go install github.com/brosck/mantra@latest" 2>>"$LOGFILE" >/dev/null
run_command axiom-scan js/js_livelinks.txt -m mantra -ua "$HEADER" -s -o js/js_secrets.txt "${AXIOM_EXTRA_ARGS_ARR[@]}" &>/dev/null
fi
mkdir -p .tmp/sourcemapper/secrets
if [[ -s "js/js_secrets.txt" ]]; then
# Parallel downloads instead of serial wget loop
cut -d' ' -f2 js/js_secrets.txt | xargs -P 10 -I {} wget -q -P .tmp/sourcemapper/secrets -- {} 2>/dev/null || true
fi
run_command trufflehog filesystem .tmp/sourcemapper/ -j 2>/dev/null | jq -c | anew -q js/js_secrets_jsmap.txt
find .tmp/sourcemapper/ -type f -name "*.js" | run_command jsluice secrets -j --patterns="${PATTERNS_DIR}/jsluice_patterns.json" | anew -q js/js_secrets_jsmap_jsluice.txt
fi
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && printf "%bRunning: Building wordlist 6/6%b\n" "$yellow" "$reset"
if [[ -s "js/js_livelinks.txt" ]]; then
if [[ -n "${GETJSWORDS_VENV:-}" ]]; then
if [[ -f "${GETJSWORDS_VENV}/bin/activate" ]]; then
(
# shellcheck source=/dev/null
source "${GETJSWORDS_VENV}/bin/activate"
if python3 -c "import jsbeautifier, requests" 2>/dev/null; then
# argv-safe parallel replacement for interlace -c. Per-target
# output is written to a temp dir, then merged into dict_words.txt
# (anew is not concurrent-write safe).
_gjs_parts=$(mktemp -d)
while IFS= read -r _gjs_target; do
[[ -z "$_gjs_target" ]] && continue
(
_gjs_hash=$(printf '%s' "$_gjs_target" | sha256sum | cut -d' ' -f1)
python3 "${tools}/getjswords.py" "$_gjs_target" \
>"$_gjs_parts/$_gjs_hash" 2>>"$LOGFILE" || true
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <js/js_livelinks.txt
wait
cat "$_gjs_parts"/* 2>/dev/null | anew -q webs/dict_words.txt
rm -rf "$_gjs_parts"
else
log_note "jschecks: jsbeautifier/requests missing in venv ${GETJSWORDS_VENV}; skipping getjswords wordlist step" "${FUNCNAME[0]}" "${LINENO}"
fi
deactivate || true
)
else
log_note "GETJSWORDS_VENV invalid: ${GETJSWORDS_VENV}; skipping getjswords wordlist step" "${FUNCNAME[0]}" "${LINENO}"
fi
else
local getjs_py="${GETJSWORDS_PYTHON:-python3}"
if ! command -v "$getjs_py" >/dev/null 2>&1; then
log_note "getjswords python not found: ${getjs_py}; skipping wordlist step" "${FUNCNAME[0]}" "${LINENO}"
elif "$getjs_py" -c "import jsbeautifier, requests" 2>/dev/null; then
# argv-safe parallel replacement for interlace -c.
_gjs_parts=$(mktemp -d)
while IFS= read -r _gjs_target; do
[[ -z "$_gjs_target" ]] && continue
(
_gjs_hash=$(printf '%s' "$_gjs_target" | sha256sum | cut -d' ' -f1)
"$getjs_py" "${tools}/getjswords.py" "$_gjs_target" \
>"$_gjs_parts/$_gjs_hash" 2>>"$LOGFILE" || true
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <js/js_livelinks.txt
wait
cat "$_gjs_parts"/* 2>/dev/null | anew -q webs/dict_words.txt
rm -rf "$_gjs_parts"
else
log_note "jschecks: jsbeautifier/requests missing in ${getjs_py}; skipping getjswords wordlist step" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
fi
end_func "Results are saved in $domain/js folder" "${FUNCNAME[0]}"
else
end_func "No JS files to process" "${FUNCNAME[0]}"
fi
else
if [[ $JSCHECKS == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function sub_js_extract() {
ensure_dirs .tmp subdomains webs
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ ${JS_SUB_EXTRACT:-true} == true ]]; then
start_func "${FUNCNAME[0]}" "Hostname extraction from JS/crawl outputs"
: >.tmp/js_extracted_hosts.txt
# Extract hostnames from URLs in JS/crawl output files
for f in js/nojs_links.txt js/js_livelinks.txt .tmp/subjslinks.txt .tmp/katana.txt webs/url_extract.txt; do
if [[ -s "$f" ]]; then
grep -aoE 'https?://[a-zA-Z0-9][-a-zA-Z0-9]*(\.[a-zA-Z0-9][-a-zA-Z0-9]*)+' "$f" \
| sed -E 's|^https?://||' \
| sed -E 's|[:/].*||' \
| grep -E "$DOMAIN_MATCH_REGEX" \
>> .tmp/js_extracted_hosts.txt 2>/dev/null || true
fi
done
# Extract hostnames from JS secrets files (may reference internal hosts)
for f in js/js_secrets.txt js/js_secrets_jsmap.txt; do
if [[ -s "$f" ]]; then
grep -aoE '[a-zA-Z0-9][-a-zA-Z0-9]*(\.[a-zA-Z0-9][-a-zA-Z0-9]*)+' "$f" \
| grep -E "$DOMAIN_MATCH_REGEX" \
>> .tmp/js_extracted_hosts.txt 2>/dev/null || true
fi
done
if [[ ! -s ".tmp/js_extracted_hosts.txt" ]]; then
end_func "No hostnames found in JS/crawl outputs" "${FUNCNAME[0]}" "SKIP_NOINPUT"
return
fi
# Deduplicate and filter valid domains
sort -u .tmp/js_extracted_hosts.txt \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \
>.tmp/js_extracted_hosts_clean.txt 2>/dev/null || true
# Remove already-known subdomains
if [[ -s "subdomains/subdomains.txt" ]] && [[ -s ".tmp/js_extracted_hosts_clean.txt" ]]; then
comm -23 <(sort -u .tmp/js_extracted_hosts_clean.txt) \
<(sort -u subdomains/subdomains.txt) \
>.tmp/js_extracted_hosts_new.txt 2>/dev/null || true
else
cp .tmp/js_extracted_hosts_clean.txt .tmp/js_extracted_hosts_new.txt 2>/dev/null || true
fi
NUMOFLINES=0
if [[ -s ".tmp/js_extracted_hosts_new.txt" ]]; then
# Resolve to validate
: >.tmp/js_extracted_hosts_resolved.txt
if [[ $AXIOM != true ]]; then
_resolve_domains .tmp/js_extracted_hosts_new.txt .tmp/js_extracted_hosts_resolved.txt
else
run_command axiom-scan .tmp/js_extracted_hosts_new.txt -m puredns-resolve \
-r "${AXIOM_RESOLVERS_PATH}" --resolvers-trusted "${AXIOM_RESOLVERS_TRUSTED_PATH}" \
--wildcard-tests "$PUREDNS_WILDCARDTEST_LIMIT" --wildcard-batch "$PUREDNS_WILDCARDBATCH_LIMIT" \
-o .tmp/js_extracted_hosts_resolved.txt "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
if [[ -s ".tmp/js_extracted_hosts_resolved.txt" ]]; then
NUMOFLINES=$(anew subdomains/subdomains.txt <.tmp/js_extracted_hosts_resolved.txt | sed '/^$/d' | wc -l | tr -d ' ' || true)
[[ "$NUMOFLINES" =~ ^[0-9]+$ ]] || NUMOFLINES=0
# Delta probe new subs
if [[ $NUMOFLINES -gt 0 ]]; then
_print_msg INFO "JS extract: delta-probing ${NUMOFLINES} new subdomains"
run_command httpx -follow-host-redirects -random-agent -silent \
-status-code -p "${WEBPROBE_PORTS:-80,443}" \
-threads "${HTTPX_THREADS:-50}" -rl "${HTTPX_RATELIMIT:-0}" \
-timeout "${HTTPX_TIMEOUT:-10}" -retries 2 -no-color \
<.tmp/js_extracted_hosts_resolved.txt \
| awk '{print $1}' | anew -q webs/webs.txt 2>/dev/null || true
fi
fi
fi
end_func "${NUMOFLINES} new subs from JS/crawl extraction" "${FUNCNAME[0]}"
else
if [[ ${JS_SUB_EXTRACT:-true} == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function well_known_pivots() {
ensure_dirs .tmp subdomains webs
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ ${WELLKNOWN_PIVOTS:-false} == true ]]; then
start_func "${FUNCNAME[0]}" "Well-known metadata endpoint pivots"
ensure_webs_all || true
if [[ ! -s "webs/webs_all.txt" ]]; then
end_func "No web targets for well-known pivots" "${FUNCNAME[0]}" "SKIP_NOINPUT"
return
fi
: >.tmp/wellknown_hosts.txt
local max_targets="${WELLKNOWN_MAX_TARGETS:-200}"
head -n "$max_targets" webs/webs_all.txt | while IFS= read -r url; do
[[ -z "$url" ]] && continue
# Defense-in-depth: even if something bypasses _extract_probe_urls_from_json,
# re-validate each URL here and skip if it contains userinfo or is off-scope.
url=$(is_in_scope_url "$url" "$domain") || continue
# security.txt (RFC 9116)
local resp
for path in "/.well-known/security.txt" "/security.txt"; do
resp=$(curl -s -m 5 -L "${url}${path}" 2>/dev/null || true)
if [[ -n "$resp" ]]; then
echo "$resp" \
| grep -aoE '[a-zA-Z0-9][-a-zA-Z0-9]*(\.[a-zA-Z0-9][-a-zA-Z0-9]*)+' \
| grep -E "$DOMAIN_MATCH_REGEX" \
>> .tmp/wellknown_hosts.txt 2>/dev/null || true
fi
done
# OpenID Configuration
local oidc_resp
oidc_resp=$(curl -s -m 5 -L "${url}/.well-known/openid-configuration" 2>/dev/null || true)
if [[ -n "$oidc_resp" ]] && echo "$oidc_resp" | jq -e . >/dev/null 2>&1; then
echo "$oidc_resp" | jq -r '.. | strings' 2>/dev/null \
| grep -aoE '[a-zA-Z0-9][-a-zA-Z0-9]*(\.[a-zA-Z0-9][-a-zA-Z0-9]*)+' \
| grep -E "$DOMAIN_MATCH_REGEX" \
>> .tmp/wellknown_hosts.txt 2>/dev/null || true
fi
# OAuth authorization server metadata (RFC 8414)
local oauth_resp
oauth_resp=$(curl -s -m 5 -L "${url}/.well-known/oauth-authorization-server" 2>/dev/null || true)
if [[ -n "$oauth_resp" ]] && echo "$oauth_resp" | jq -e . >/dev/null 2>&1; then
echo "$oauth_resp" | jq -r '.. | strings' 2>/dev/null \
| grep -aoE '[a-zA-Z0-9][-a-zA-Z0-9]*(\.[a-zA-Z0-9][-a-zA-Z0-9]*)+' \
| grep -E "$DOMAIN_MATCH_REGEX" \
>> .tmp/wellknown_hosts.txt 2>/dev/null || true
fi
done
NUMOFLINES=0
if [[ -s ".tmp/wellknown_hosts.txt" ]]; then
sort -u .tmp/wellknown_hosts.txt \
| grep -E '^([a-zA-Z0-9][-a-zA-Z0-9]*\.)+[a-zA-Z]{2,}$' \
>.tmp/wellknown_hosts_clean.txt 2>/dev/null || true
# Remove already known
if [[ -s "subdomains/subdomains.txt" ]]; then
comm -23 <(sort -u .tmp/wellknown_hosts_clean.txt) \
<(sort -u subdomains/subdomains.txt) \
>.tmp/wellknown_hosts_new.txt 2>/dev/null || true
else
cp .tmp/wellknown_hosts_clean.txt .tmp/wellknown_hosts_new.txt 2>/dev/null || true
fi
if [[ -s ".tmp/wellknown_hosts_new.txt" ]]; then
# Resolve
: >.tmp/wellknown_hosts_resolved.txt
_resolve_domains .tmp/wellknown_hosts_new.txt .tmp/wellknown_hosts_resolved.txt
if [[ -s ".tmp/wellknown_hosts_resolved.txt" ]]; then
NUMOFLINES=$(anew subdomains/subdomains.txt <.tmp/wellknown_hosts_resolved.txt | sed '/^$/d' | wc -l | tr -d ' ' || true)
[[ "$NUMOFLINES" =~ ^[0-9]+$ ]] || NUMOFLINES=0
if [[ $NUMOFLINES -gt 0 ]]; then
_print_msg INFO "Well-known pivots: delta-probing ${NUMOFLINES} new subdomains"
run_command httpx -follow-host-redirects -random-agent -silent \
-status-code -p "${WEBPROBE_PORTS:-80,443}" \
-threads "${HTTPX_THREADS:-50}" -rl "${HTTPX_RATELIMIT:-0}" \
-timeout "${HTTPX_TIMEOUT:-10}" -retries 2 -no-color \
<.tmp/wellknown_hosts_resolved.txt \
| awk '$2 ~ /^\[?[123]/ {print $1}' | anew -q webs/webs.txt 2>/dev/null || true
fi
fi
fi
fi
end_func "${NUMOFLINES} new subs from well-known endpoints" "${FUNCNAME[0]}"
else
if [[ ${WELLKNOWN_PIVOTS:-false} == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function websocket_checks() {
mkdir -p webs vulns .tmp
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $URL_CHECK == true ]]; then
start_func "${FUNCNAME[0]}" "WebSocket discovery and handshake"
# Collect ws/wss endpoints from JS endpoints and URLs
touch .tmp/ws_endpoints_raw.txt
[[ -s js/js_endpoints.txt ]] && grep -aEo 'wss?://[^ ]+' js/js_endpoints.txt | anew -q .tmp/ws_endpoints_raw.txt || true
[[ -s webs/url_extract.txt ]] && grep -aEo 'wss?://[^ ]+' webs/url_extract.txt | anew -q .tmp/ws_endpoints_raw.txt || true
if [[ -s .tmp/ws_endpoints_raw.txt ]]; then
# Normalize and de-dup
cat .tmp/ws_endpoints_raw.txt | sed 's/\"//g' | sed 's/[\]\[ ]//g' | sort -u >.tmp/ws_endpoints.txt
# Handshake test and origin check
while IFS= read -r ws; do
[[ -z $ws ]] && continue
url_no_proto=$(echo "$ws" | sed 's|^wss\?://||')
host=$(echo "$url_no_proto" | cut -d'/' -f1)
# Prepare Sec-WebSocket headers for accurate handshake
wskey=$(head -c 16 /dev/urandom | base64 2>/dev/null || echo dGVzdGtleQ==)
code=$(run_command curl -sk --http1.1 -o /dev/null -w '%{http_code}' \
-H 'Connection: Upgrade' -H 'Upgrade: websocket' \
-H "Host: $host" \
-H "Sec-WebSocket-Key: $wskey" \
-H 'Sec-WebSocket-Version: 13' \
"$ws" || true)
if [[ $code == "101" ]]; then
printf "HANDSHAKE %s\n" "$ws" | anew -q vulns/websockets.txt || true
# Origin test: send a cross-origin header, expect failure ideally
wskey2=$(head -c 16 /dev/urandom | base64 2>/dev/null || echo dGVzdGtleQ==)
code2=$(run_command curl -sk --http1.1 -o /dev/null -w '%{http_code}' \
-H 'Connection: Upgrade' -H 'Upgrade: websocket' \
-H "Origin: https://evil.example" -H "Host: $host" \
-H "Sec-WebSocket-Key: $wskey2" \
-H 'Sec-WebSocket-Version: 13' \
"$ws" || true)
if [[ $code2 == "101" ]]; then
printf "ORIGIN-ALLOWED %s\n" "$ws" | anew -q vulns/websocket_misconfig.txt || true
fi
fi
done <.tmp/ws_endpoints.txt
fi
end_func "Results are saved in vulns/websocket_misconfig.txt" "${FUNCNAME[0]}"
else
skip_notification "processed"
fi
}
function wordlist_gen() {
# Create necessary directories
if ! ensure_dirs .tmp webs; then return 1; fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $WORDLIST == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Wordlist Generation"
[[ -s ".tmp/url_extract_tmp.txt" ]] && [[ -s "webs/url_extract.txt" ]] && cat webs/url_extract.txt | anew -q .tmp/url_extract_tmp.txt || true
# Ensure url_extract_tmp.txt exists and is not empty
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
# Define patterns for keys and values
cat ".tmp/url_extract_tmp.txt" | unfurl -u keys 2>>"$LOGFILE" \
| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \
| anew -q webs/dict_keys.txt || true
cat ".tmp/url_extract_tmp.txt" | unfurl -u values 2>>"$LOGFILE" \
| sed 's/[][]//g' | sed 's/[#]//g' | sed 's/[}{]//g' \
| anew -q webs/dict_values.txt || true
_print_msg INFO "Extracting words..."
tr "[:punct:]" "\n" <".tmp/url_extract_tmp.txt" | anew -q "webs/dict_words.txt" || true
fi
end_func "Results are saved in $domain/webs/dict_[words|paths].txt" "${FUNCNAME[0]}"
else
if [[ $WORDLIST == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function wordlist_gen_roboxtractor() {
# Create necessary directories
if ! mkdir -p .tmp webs gf; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $ROBOTSWORDLIST == true ]] \
&& [[ ${DEEP:-false} == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Robots Wordlist Generation"
ensure_webs_all || true
# Proceed only if webs_all.txt exists and is non-empty
if [[ -s "webs/webs_all.txt" ]]; then
# Extract URLs using roboxtractor and append unique entries to robots_wordlist.txt
_print_msg INFO "Running: Roboxtractor for Robots Wordlist"
roboxtractor -m 1 -wb <"webs/webs_all.txt" 2>>"$LOGFILE" | anew -q "webs/robots_wordlist.txt"
else
end_func "No webs/webs_all.txt file found, Robots Wordlist generation skipped." "${FUNCNAME[0]}"
return
fi
end_func "Results are saved in $domain/webs/robots_wordlist.txt" "${FUNCNAME[0]}"
# Handle Proxy if conditions are met
if [[ $PROXY == true ]] && [[ -n $proxy_url ]] && [[ "$(wc -l <"webs/robots_wordlist.txt")" -le $DEEP_LIMIT2 ]]; then
notification "Sending URLs to proxy" info
run_command ffuf -mc all -w "webs/robots_wordlist.txt" -u "FUZZ" -replay-proxy "$proxy_url" 2>>"$LOGFILE" >/dev/null
fi
else
if [[ $ROBOTSWORDLIST == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
elif [[ ${DEEP:-false} != true ]]; then
skip_notification "mode"
else
skip_notification "processed"
fi
fi
}
function password_dict() {
# Create necessary directories
if ! mkdir -p "$dir/webs" "$dir/.tmp"; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $PASSWORD_DICT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Password Dictionary Generation"
local dict_engine="${PASSWORD_DICT_ENGINE:-cewler}"
local dict_output="$dir/webs/password_dict.txt"
: >"$dict_output"
# Ensure web targets exist for cewler.
ensure_webs_all || true
if [[ "$dict_engine" == "cewler" ]]; then
local cewler_cmd=()
if command -v cewler >/dev/null 2>&1; then
cewler_cmd=(cewler)
fi
if [[ ${#cewler_cmd[@]} -gt 0 ]] && [[ -s "webs/webs_all.txt" ]]; then
local max_targets="${PASSWORD_DICT_MAX_TARGETS:-50}"
if [[ ${DEEP:-false} == true ]]; then
max_targets=0
fi
if [[ "$max_targets" -gt 0 ]]; then
head -n "$max_targets" "webs/webs_all.txt" >".tmp/password_dict_targets.txt"
else
cp "webs/webs_all.txt" ".tmp/password_dict_targets.txt"
fi
: >".tmp/password_dict_raw.txt"
while IFS= read -r target; do
[[ -z "$target" ]] && continue
if [[ -n "${TIMEOUT_CMD:-}" ]]; then
run_command "$TIMEOUT_CMD" -k 5s "${PASSWORD_DICT_CEWLER_TIMEOUT:-45}" "${cewler_cmd[@]}" \
-d "${PASSWORD_DICT_CEWLER_DEPTH:-1}" -m "${PASSWORD_MIN_LENGTH:-5}" \
-l -o ".tmp/password_dict_part.txt" "$target" 2>>"$LOGFILE" >/dev/null || true
else
run_command "${cewler_cmd[@]}" \
-d "${PASSWORD_DICT_CEWLER_DEPTH:-1}" -m "${PASSWORD_MIN_LENGTH:-5}" \
-l -o ".tmp/password_dict_part.txt" "$target" 2>>"$LOGFILE" >/dev/null || true
fi
if [[ -s ".tmp/password_dict_part.txt" ]]; then
cat ".tmp/password_dict_part.txt" >>".tmp/password_dict_raw.txt"
: >".tmp/password_dict_part.txt"
fi
done <".tmp/password_dict_targets.txt"
if [[ -s ".tmp/password_dict_raw.txt" ]]; then
awk -v min="${PASSWORD_MIN_LENGTH:-5}" -v max="${PASSWORD_MAX_LENGTH:-14}" '
{
w=$0;
gsub(/[^a-z0-9._-]/, "", w);
if (length(w) >= min && length(w) <= max) print w;
}' ".tmp/password_dict_raw.txt" | sort -u >"$dict_output"
fi
fi
fi
# Fallback to pydictor when cewler is unavailable/empty or explicitly requested.
if [[ ! -s "$dict_output" ]] || [[ "$dict_engine" == "pydictor" ]]; then
local word="${domain%%.*}"
if [[ -s "${tools}/pydictor/pydictor.py" ]]; then
run_command python3 "${tools}/pydictor/pydictor.py" -extend "$word" --leet 0 1 2 11 21 --len "$PASSWORD_MIN_LENGTH" "$PASSWORD_MAX_LENGTH" -o "$dict_output" 2>>"$LOGFILE" >/dev/null
[[ "$dict_engine" == "cewler" ]] && log_note "password_dict: cewler unavailable/empty, pydictor fallback used" "${FUNCNAME[0]}" "${LINENO}"
elif [[ "$dict_engine" == "cewler" ]]; then
log_note "password_dict: cewler and pydictor unavailable, skipping output generation" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
end_func "Results are saved in $domain/webs/password_dict.txt" "${FUNCNAME[0]}"
# Optionally, create a marker file to indicate the function has been processed
touch "$called_fn_dir/.${FUNCNAME[0]}"
else
if [[ $PASSWORD_DICT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
###############################################################################################################
######################################### VULNERABILITIES #####################################################
###############################################################################################################
function brokenLinks() {
# Create necessary directories
if ! mkdir -p .tmp webs vulns; then
print_warnf "Failed to create directories."
return 1
fi
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $BROKENLINKS == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Broken Links Checks"
ensure_webs_all || true
# Check if webs_all.txt exists and is not empty
if [[ -s "webs/webs_all.txt" ]]; then
local bl_engine="${BROKENLINKS_ENGINE:-second-order}"
if [[ "$bl_engine" == "second-order" ]]; then
local so_depth="${SECOND_ORDER_DEPTH:-1}"
[[ $DEEP == true ]] && so_depth=$((so_depth + 1))
rm -rf ".tmp/second_order" >/dev/null 2>&1 || true
mkdir -p ".tmp/second_order"
: >".tmp/brokenLinks_total.txt"
while IFS= read -r target; do
[[ -z "$target" ]] && continue
local safe_target
safe_target=$(echo "$target" | sed 's|https\?://||; s|[^A-Za-z0-9._-]|_|g')
local out_dir=".tmp/second_order/${safe_target}"
mkdir -p "$out_dir"
local so_cmd=(
second-order
-target "$target"
-config "${SECOND_ORDER_CONFIG:-${tools}/second-order/config/takeover.json}"
-depth "$so_depth"
-threads "${SECOND_ORDER_THREADS:-10}"
-output "$out_dir"
)
if [[ ${SECOND_ORDER_INSECURE:-false} == true ]]; then
so_cmd+=(-insecure)
fi
if [[ -n "${HEADER:-}" ]]; then
so_cmd+=(-header "${HEADER}")
fi
run_command "${so_cmd[@]}" 2>>"$LOGFILE" >/dev/null || true
if [[ -s "${out_dir}/non-200-url-attributes.json" ]]; then
jq -r 'to_entries[]?.value | to_entries[]?.value[]? // empty' "${out_dir}/non-200-url-attributes.json" 2>/dev/null \
| grep -aE '^https?://' \
| anew -q ".tmp/brokenLinks_total.txt"
fi
done <"webs/webs_all.txt"
else
local katana_legacy_headless=""
katana_legacy_headless=$(_katana_headless_flags "$(wc -l <webs/webs_all.txt 2>/dev/null || echo 0)")
if [[ $AXIOM != true ]]; then
# Use katana for scanning
if [[ ! -s ".tmp/katana.txt" ]]; then
if [[ $DEEP == true ]]; then
timeout 4h katana -silent -list "webs/webs_all.txt" $katana_legacy_headless -jc -kf all -c "$KATANA_THREADS" -d 3 -o ".tmp/katana.txt" 2>>"$LOGFILE" >/dev/null
else
timeout 3h katana -silent -list "webs/webs_all.txt" $katana_legacy_headless -jc -kf all -c "$KATANA_THREADS" -d 2 -o ".tmp/katana.txt" 2>>"$LOGFILE" >/dev/null
fi
fi
# Remove lines longer than 2048 characters
if [[ -s ".tmp/katana.txt" ]]; then
sed_i '/^.\{2048\}./d' ".tmp/katana.txt"
fi
else
# Use axiom-scan for scanning
if [[ ! -s ".tmp/katana.txt" ]]; then
if [[ $DEEP == true ]]; then
run_command axiom-scan "webs/webs_all.txt" -m katana $katana_legacy_headless -jc -kf all -d 3 --max-runtime 4h -o ".tmp/katana.txt" "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
else
run_command axiom-scan "webs/webs_all.txt" -m katana $katana_legacy_headless -jc -kf all -d 2 --max-runtime 3h -o ".tmp/katana.txt" "${AXIOM_EXTRA_ARGS_ARR[@]}" 2>>"$LOGFILE" >/dev/null
fi
# Remove lines longer than 2048 characters
if [[ -s ".tmp/katana.txt" ]]; then
sed_i '/^.\{2048\}./d' ".tmp/katana.txt"
fi
fi
fi
# Process katana.txt to find broken links
if [[ -s ".tmp/katana.txt" ]]; then
run_command httpx -follow-redirects -random-agent -status-code -threads "$HTTPX_THREADS" -rl "$HTTPX_RATELIMIT" -timeout "$HTTPX_TIMEOUT" -silent -retries 2 -no-color <".tmp/katana.txt" 2>>"$LOGFILE" \
| grep "\[4" | cut -d ' ' -f1 | anew -q ".tmp/brokenLinks_total.txt"
fi
fi
# Update brokenLinks.txt with unique entries
if [[ -s ".tmp/brokenLinks_total.txt" ]]; then
cat ".tmp/brokenLinks_total.txt" | anew -q "vulns/brokenLinks.txt"
NUMOFLINES=$(sed '/^$/d' "vulns/brokenLinks.txt" | wc -l)
notification "${NUMOFLINES} broken links found" info
fi
end_func "Results are saved in vulns/brokenLinks.txt" "${FUNCNAME[0]}"
else
end_func "No webs/webs_all.txt file found, Broken Links check skipped." "${FUNCNAME[0]}"
return
fi
else
if [[ $BROKENLINKS == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}