reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities
Shell bug-bountybugbountybugbounty-tooldnshackingnucleiosintpenetration-testingpentestpentest-toolpentestingreconreconnaissancescannersecuritysecurity-toolssubdomainsubdomain-enumerationvulnerabilities
Pre v3 release
six2dez174d475f9d53
diff --git a/README.md b/README.mdindex 5484390..5f65f3e 100644--- a/README.md+++ b/README.md@@ -7,8 +7,8 @@</h1><p align="center">- <a href="https://github.com/six2dez/reconftw/releases/tag/v2.7">- <img src="https://img.shields.io/badge/release-v2.7-green">+ <a href="https://github.com/six2dez/reconftw/releases/tag/v2.7.1">+ <img src="https://img.shields.io/badge/release-v2.7.1-green"></a></a><a href="https://opensource.org/licenses/MIT">@@ -173,7 +173,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATHAMASS_CONFIG=~/.config/amass/config.iniGITHUB_TOKENS=${tools}/.github_tokensGITLAB_TOKENS=${tools}/.gitlab_tokens-SUBGPT_COOKIE=${tools}/subgpt_cookies.json#CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path# APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line@@ -212,7 +211,6 @@ SUBBRUTE=true # DNS bruteforcingSUBSCRAPING=true # Subdomains extraction from web crawlingSUBPERMUTE=true # DNS permutationsSUBREGEXPERMUTE=true # Permutations by regex analysis-SUBGPT=true # Permutations by BingGPT predictionPERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper)GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotatorSUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check thisdiff --git a/Terraform/files/reconftw.cfg b/Terraform/files/reconftw.cfgindex 6b5ad22..31152a3 100644--- a/Terraform/files/reconftw.cfg+++ b/Terraform/files/reconftw.cfg@@ -15,6 +15,7 @@ fuzzing_remote_list="https://raw.githubusercontent.com/six2dez/OneListForAll/maiproxy_url="http://127.0.0.1:8080/" # Proxy urlinstall_golang=true # Set it to false if you already have Golang configured and readyupgrade_tools=true+upgrade_before_running=false # Upgrade tools before running#dir_output=/custom/output/path# Golang Vars (Comment or change on your own)@@ -27,7 +28,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATHAMASS_CONFIG=~/.config/amass/config.iniGITHUB_TOKENS=${tools}/.github_tokensGITLAB_TOKENS=${tools}/.gitlab_tokens-SUBGPT_COOKIE=${tools}/subgpt_cookies.json#CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path# APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line@@ -52,6 +52,7 @@ EMAILS=true # Fetch emails from differents sitesDOMAIN_INFO=true # whois infoREVERSE_WHOIS=true # amass intel reverse whois info, takes some timeIP_INFO=true # Reverse IP search, geolocation and whois+POSTMAN_LEAKS=true # Check for postman leaksMETAFINDER_LIMIT=20 # Max 250# Subdomains@@ -60,16 +61,16 @@ RUNSUBFINDER=trueSUBDOMAINS_GENERAL=true # Enable or disable the whole Subdomains moduleSUBPASSIVE=true # Passive subdomains searchSUBCRT=true # crtsh search-SUBNOERROR=true # Check DNS NOERROR response and BF on them+CTR_LIMIT=999999 # Limit the number of results+SUBNOERROR=false # Check DNS NOERROR response and BF on themSUBANALYTICS=true # Google Analytics searchSUBBRUTE=true # DNS bruteforcingSUBSCRAPING=true # Subdomains extraction from web crawlingSUBPERMUTE=true # DNS permutationsSUBREGEXPERMUTE=true # Permutations by regex analysis-SUBGPT=true # Permutations by BingGPT predictionPERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper)GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator-SUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check this+SUBTAKEOVER=true # Check subdomain takeovers, false by default cuz nuclei already check thisSUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queriesDEEP_RECURSIVE_PASSIVE=10 # Number of top subdomains for recursionSUB_RECURSIVE_BRUTE=false # Needs big disk space and time to resolve@@ -96,8 +97,9 @@ CDN_IP=true # Check which IPs belongs to CDN# Web analysisWAF_DETECTION=true # Detect WAFsNUCLEICHECK=true # Enable or disable nuclei+NUCLEI_TEMPLATES_PATH="$HOME/nuclei-templates" # Set nuclei templates pathNUCLEI_SEVERITY="info,low,medium,high,critical" # Set templates criticity-NUCLEI_FLAGS=" -silent -t $HOME/nuclei-templates/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like " -etags openssh"+NUCLEI_FLAGS=" -silent -t ${NUCLEI_TEMPLATES_PATH}/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like " -etags openssh"NUCLEI_FLAGS_JS=" -silent -tags exposure,token -severity info,low,medium,high,critical" # Additional nuclei extra flags for js secretsURL_CHECK=true # Enable or disable URL collectionURL_CHECK_PASSIVE=true # Search for urls, passive methods from Archive, OTX, CommonCrawl, etc@@ -110,8 +112,8 @@ CMS_SCANNER=true # CMS scannerWORDLIST=true # Wordlist generationROBOTSWORDLIST=true # Check historic disallow entries on waybackMachinePASSWORD_DICT=true # Generate password dictionary-PASSWORD_MIN_LENGTH=5 # Min password lenght-PASSWORD_MAX_LENGTH=14 # Max password lenght+PASSWORD_MIN_LENGTH=5 # Min password length+PASSWORD_MAX_LENGTH=14 # Max password length# VulnsVULNS_GENERAL=false # Enable or disable the vulnerability module (very intrusive and slow)@@ -133,6 +135,7 @@ PROTO_POLLUTION=true # Check for prototype pollution flawsSMUGGLING=true # Check for HTTP request smuggling flawsWEBCACHE=true # Check for Web Cache issuesBYPASSER4XX=true # Check for 4XX bypasses+FUZZPARAMS=true # Fuzz parameters values# Extra featuresNOTIFICATION=false # Notification for every function@@ -146,7 +149,7 @@ REMOVELOG=false # Delete logs after executionPROXY=false # Send to proxy the websites foundSENDZIPNOTIFY=false # Send to zip the results (over notify)PRESERVE=true # set to true to avoid deleting the .called_fn files on really large scans-FFUF_FLAGS=" -mc all -fc 404 -ac -sf" # Ffuf flags+FFUF_FLAGS=" -mc all -fc 404 -ach -sf -of json" # Ffuf flagsHTTPX_FLAGS=" -follow-redirects -random-agent -status-code -silent -title -web-server -tech-detect -location -content-length" # Httpx flags for simple web probingGOWITNESS_FLAGS=" --disable-logging --timeout 5"@@ -189,6 +192,7 @@ FFUF_MAXTIME=900 # SecondsHTTPX_TIMEOUT=10 # SecondsHTTPX_UNCOMMONPORTS_TIMEOUT=10 # SecondsPERMUTATIONS_LIMIT=21474836480 # Bytes, default is 20 GB+GOWITNESS_TIMEOUT_PER_SITE=20 # Seconds# listsfuzz_wordlist=${tools}/fuzz_wordlist.txtdiff --git a/install.sh b/install.shindex 6a7a42f..9a7090f 100755--- a/install.sh+++ b/install.sh@@ -6,45 +6,37 @@ dir=${tools}double_check=false# ARM Detection-if [[ $(uname -m) == "amd64" ]] || [[ $(uname -m) == "x86_64" ]]; then- IS_ARM="False"-fi-if [[ $(uname -m) == "arm64" ]] || [[ $(uname -m) == "armv6l" ]]; then- IS_ARM="True"- if [[ $(uname -m) == "arm64" ]]; then- RPI_4="False"- else- RPI_3="True"- fi-fi+ARCH=$(uname -m)+case $ARCH in+ amd64|x86_64) IS_ARM="False" ;;+ arm64|armv6l)+ IS_ARM="True"+ RPI_4=$([[ $ARCH == "arm64" ]] && echo "True" || echo "False")+ RPI_3=$([[ $ARCH == "arm64" ]] && echo "False" || echo "True")+ ;;+esac#Mac Osx Detecting-if [[ "$OSTYPE" == "darwin"* ]]; then- IS_MAC="True"-else- IS_MAC="False"-fi--# Check Bash version-#(bash --version | awk 'NR==1{print $4}' | cut -d'.' -f1) 2&>/dev/null || echo "Unable to get bash version, for MacOS run 'brew install bash' and rerun installer in a new terminal" && exit 1+IS_MAC=$([[ "$OSTYPE" == "darwin"* ]] && echo "True" || echo "False")BASH_VERSION=$(bash --version | awk 'NR==1{print $4}' | cut -d'.' -f1)if [ "${BASH_VERSION}" -lt 4 ]; thenprintf "${bred} Your Bash version is lower than 4, please update${reset}\n"- printf "%s Your Bash version is lower than 4, please update%s\n" "${bred}" "${reset}"+ printf "%s Your Bash version is lower than 4, please update%s\n" "${bred}" "${reset}" >&2if [ "True" = "$IS_MAC" ]; thenprintf "${yellow} For MacOS run 'brew install bash' and rerun installer in a new terminal${reset}\n\n"exit 1;fifi+# Declaring Go tools and their installation commandsdeclare -A gotoolsgotools["gf"]="go install -v github.com/tomnomnom/gf@latest"gotools["qsreplace"]="go install -v github.com/tomnomnom/qsreplace@latest"gotools["amass"]="go install -v github.com/owasp-amass/amass/v3/...@master"gotools["ffuf"]="go install -v github.com/ffuf/ffuf/v2@latest"gotools["github-subdomains"]="go install -v github.com/gwen001/github-subdomains@latest"-gotools["gitlab-subdomains"]="go install github.com/gwen001/gitlab-subdomains@latest"+gotools["gitlab-subdomains"]="go install -v github.com/gwen001/gitlab-subdomains@latest"gotools["nuclei"]="go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest"gotools["anew"]="go install -v github.com/tomnomnom/anew@latest"gotools["notify"]="go install -v github.com/projectdiscovery/notify/cmd/notify@latest"@@ -54,7 +46,7 @@ gotools["github-endpoints"]="go install -v github.com/gwen001/github-endpoints@lgotools["dnsx"]="go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest"gotools["subjs"]="go install -v github.com/lc/subjs@latest"gotools["Gxss"]="go install -v github.com/KathanP19/Gxss@latest"-gotools["katana"]="go install github.com/projectdiscovery/katana/cmd/katana@latest"+gotools["katana"]="go install -v github.com/projectdiscovery/katana/cmd/katana@latest"gotools["crlfuzz"]="go install -v github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@latest"gotools["dalfox"]="go install -v github.com/hahwul/dalfox/v2@latest"gotools["puredns"]="go install -v github.com/d3mondev/puredns/v2@latest"@@ -66,21 +58,22 @@ gotools["mapcidr"]="go install -v github.com/projectdiscovery/mapcidr/cmd/mapcidgotools["cdncheck"]="go install -v github.com/projectdiscovery/cdncheck/cmd/cdncheck@latest"gotools["dnstake"]="go install -v github.com/pwnesia/dnstake/cmd/dnstake@latest"gotools["gowitness"]="go install -v github.com/sensepost/gowitness@latest"-gotools["tlsx"]="go install github.com/projectdiscovery/tlsx/cmd/tlsx@latest"+gotools["tlsx"]="go install -v github.com/projectdiscovery/tlsx/cmd/tlsx@latest"gotools["gitdorks_go"]="go install -v github.com/damit5/gitdorks_go@latest"gotools["smap"]="go install -v github.com/s0md3v/smap/cmd/smap@latest"gotools["dsieve"]="go install -v github.com/trickest/dsieve@master"-gotools["inscope"]="go install github.com/tomnomnom/hacks/inscope@latest"-gotools["enumerepo"]="go install github.com/trickest/enumerepo@latest"+gotools["inscope"]="go install -v github.com/tomnomnom/hacks/inscope@latest"+gotools["enumerepo"]="go install -v github.com/trickest/enumerepo@latest"gotools["Web-Cache-Vulnerability-Scanner"]="go install -v github.com/Hackmanit/Web-Cache-Vulnerability-Scanner@latest"gotools["subfinder"]="go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest"gotools["byp4xx"]="go install -v github.com/lobuhi/byp4xx@latest"-gotools["hakip2host"]="go install github.com/hakluke/hakip2host@latest"+gotools["hakip2host"]="go install -v github.com/hakluke/hakip2host@latest"gotools["gau"]="go install -v github.com/lc/gau/v2/cmd/gau@latest"-gotools["Mantra"]="go install github.com/MrEmpy/Mantra@latest"-gotools["crt"]="go install github.com/cemulus/crt@latest"+gotools["Mantra"]="go install -v github.com/MrEmpy/Mantra@latest"+gotools["crt"]="go install -v github.com/cemulus/crt@latest"gotools["s3scanner"]="go install -v github.com/sa7mon/s3scanner@latest"+# Declaring repositories and their pathsdeclare -A reposrepos["dorks_hunter"]="six2dez/dorks_hunter"repos["pwndb"]="davidtavarez/pwndb"@@ -115,7 +108,7 @@ repos["trufflehog"]="trufflesecurity/trufflehog"function banner_web(){- echo -en "\033c"+ tput clearprintf "\n${bgreen}"printf " ██▀███ ▓█████ ▄████▄ ▒█████ ███▄ █ █████▒▄▄▄█████▓ █ █░\n"printf " ▓██ ▒ ██▒▓█ ▀ ▒██▀ ▀█ ▒██▒ ██▒ ██ ▀█ █ ▓██ ▒ ▓ ██▒ ▓▒▓█░ █ ░█░\n"@@ -130,8 +123,18 @@ function banner_web(){printf " ${reconftw_version} by @six2dez\n"}+function install_ppfuzz() {+ local url=$1+ local tar_file=$2++ wget -N -c "$url" $DEBUG_STD+ eval $SUDO tar -C /usr/local/bin/ -xzf "$tar_file" $DEBUG_STD+ eval $SUDO rm -rf "$tar_file" $DEBUG_STD+}++# This function installs various tools and repositories as per the configuration.function install_tools(){- #eval ln -s /usr/local/bin/pip3 /usr/local/bin/pip3 $DEBUG_STD+eval pip3 install -I -r requirements.txt $DEBUG_STDprintf "${bblue} Running: Installing Golang tools (${#gotools[@]})${reset}\n\n"@@ -219,28 +222,18 @@ function install_tools(){if [ "True" = "$IS_ARM" ]; thenif [ "True" = "$RPI_3" ]; then- eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO rm -rf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD+ install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz"elif [ "True" = "$RPI_4" ]; then- eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO rm -rf ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD+ install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz"fielif [ "True" = "$IS_MAC" ]; thenif [ "True" = "$IS_ARM" ]; then- eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD- eval $SUDO rm -rf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD+ install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz"else- eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD- eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD- eval $SUDO rm -rf ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD+ install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz" "ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz"fielse- eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD- eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD- eval $SUDO rm -rf ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD+ install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz" "ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz"fieval $SUDO chmod 755 /usr/local/bin/ppfuzzeval $SUDO strip -s /usr/local/bin/ppfuzz $DEBUG_STD@@ -272,7 +265,14 @@ install_webserver(){$SUDO pip3 install -r $SCRIPTPATH/web/requirements.txt &>/dev/nullprintf "${yellow} Installing tools...${reset}\n\n"+ if command -v apt > /dev/null; then$SUDO apt install redis-server -y &>/dev/null+ elif command -v yum > /dev/null; then+ $SUDO yum install redis -y &>/dev/null+ else+ printf '[ERROR] Unable to find a supported package manager. Please install redis manually.\n'+ exit 1+ fiprintf "${yellow} Creating WEB User...${reset}\n\n"$SUDO rm $SCRIPTPATH/web/db.sqlite3 &>/dev/null@@ -305,9 +305,15 @@ display_menu(){printf "${bblue} 3. Setup Web Interface${reset} ${yellow}(User Interaction needed!)${reset}\n\n"printf "${bblue} 4. Exit${reset}\n\n"printf "${bgreen}#######################################################################${reset}\n\n"- read -p "$(echo -e ${bblue} "Insert option: "${reset})" option+ read -p "${bblue}Insert option: ${reset}" optionprintf "\n\n${bgreen}#######################################################################${reset}\n\n"+ option=$(echo "$option" | tr -d '[:space:]')+ if ! [[ "$option" =~ ^[1-4]$ ]]; then+ printf "${bred} Invalid option. Please try again.${reset}\n\n"+ continue+ fi+case $option in1)web=false@@ -354,7 +360,7 @@ display_menu(){exit 1;;*)- printf "${bblue} Invalid option. Exiting...${reset}\n\n"+ printf "${bred} Invalid option. Please try again.${reset}\n\n"exit 1;;esac@@ -362,14 +368,18 @@ display_menu(){done}-if [ "$1" = '--tools' ]; then- install_tools-fi--if [ "$1" != '--auto' ]; then- echo "$1"- display_menu-fi+case "$1" in+ --tools)+ install_tools+ ;;+ --auto)+ # possibly some other actions+ ;;+ *)+ echo "$1"+ display_menu+ ;;+esacprintf "${yellow} This may take time. So, go grab a coffee! ${reset}\n\n"@@ -432,7 +442,11 @@ eval git config --global --unset https.proxy $DEBUG_STDprintf "${bblue} Running: Looking for new reconFTW version${reset}\n\n"-eval git fetch $DEBUG_STD+if ! eval git fetch $DEBUG_STD; then+ echo "Failed to fetch updates."+ exit 1+fi+BRANCH=$(git rev-parse --abbrev-ref HEAD)HEADHASH=$(git rev-parse HEAD)UPSTREAMHASH=$(git rev-parse "${BRANCH}@{upstream}")@@ -608,6 +622,6 @@ if [ "$web" = true ]; thenprintf "\n${bgreen} Web server is installed, to set it up run ./install.sh and select option 3 ${reset}\n\n"fi-printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/provider-config.yaml)\n - GitLab (~/Tools/.gitlab_tokens)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n - subgpt_cookies.json (subgpt_cookies.json file, follow instructions at https://github.com/s0md3v/SubGPT#getting-bing-cookie)\n\n\n${reset}"+printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/provider-config.yaml)\n - GitLab (~/Tools/.gitlab_tokens)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n\n${reset}"printf "${bgreen} Finished!${reset}\n\n"printf "\n\n${bgreen}#######################################################################${reset}\n"diff --git a/reconftw.cfg b/reconftw.cfgindex 294b38a..40be4fd 100644--- a/reconftw.cfg+++ b/reconftw.cfg@@ -28,7 +28,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATHAMASS_CONFIG=~/.config/amass/config.iniGITHUB_TOKENS=${tools}/.github_tokensGITLAB_TOKENS=${tools}/.gitlab_tokens-SUBGPT_COOKIE=${tools}/subgpt_cookies.json#CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path# APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line@@ -53,6 +52,7 @@ EMAILS=true # Fetch emails from differents sitesDOMAIN_INFO=true # whois infoREVERSE_WHOIS=true # amass intel reverse whois info, takes some timeIP_INFO=true # Reverse IP search, geolocation and whois+POSTMAN_LEAKS=true # Check for postman leaksMETAFINDER_LIMIT=20 # Max 250# Subdomains@@ -68,7 +68,6 @@ SUBBRUTE=true # DNS bruteforcingSUBSCRAPING=true # Subdomains extraction from web crawlingSUBPERMUTE=true # DNS permutationsSUBREGEXPERMUTE=true # Permutations by regex analysis-SUBGPT=true # Permutations by BingGPT predictionPERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper)GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotatorSUBTAKEOVER=true # Check subdomain takeovers, false by default cuz nuclei already check thisdiff --git a/reconftw.sh b/reconftw.shindex 2d5792a..8740a4a 100755--- a/reconftw.sh+++ b/reconftw.sh@@ -114,7 +114,6 @@ function tools_installed(){which hakip2host &>/dev/null || { printf "${bred} [*] hakip2host [NO]${reset}\n${reset}"; allinstalled=false;}which gau &>/dev/null || { printf "${bred} [*] gau [NO]${reset}\n${reset}"; allinstalled=false;}which crt &>/dev/null || { printf "${bred} [*] crt [NO]${reset}\n${reset}"; allinstalled=false;}- which subgpt &>/dev/null || { printf "${bred} [*] subgpt [NO]${reset}\n${reset}"; allinstalled=false;}which gitleaks &>/dev/null || { printf "${bred} [*] gitleaks [NO]${reset}\n${reset}"; allinstalled=false;}which trufflehog &>/dev/null || { printf "${bred} [*] trufflehog [NO]${reset}\n${reset}"; allinstalled=false;}which s3scanner &>/dev/null || { printf "${bred} [*] s3scanner [NO]${reset}\n${reset}"; allinstalled=false;}@@ -178,7 +177,7 @@ function github_repos(){if [ -s "${GITHUB_TOKENS}" ]; thenGH_TOKEN=$(cat ${GITHUB_TOKENS} | head -1)echo $domain | unfurl format %r > .tmp/company_name.txt- enumerepo -token-string "${GH_TOKEN}" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null &+ enumerepo -token-string "${GH_TOKEN}" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null[ -s ".tmp/company_repos.txt" ] && jq -r '.[].repos[]|.url' < .tmp/company_repos.txt > .tmp/company_repos_url.txt 2>>"$LOGFILE"mkdir -p .tmp/github_repos 2>>"$LOGFILE" >>"$LOGFILE"mkdir -p .tmp/github 2>>"$LOGFILE" >>"$LOGFILE"@@ -224,6 +223,28 @@ function metadata(){fi}+function postleaks(){+ if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$POSTMAN_LEAKS" = true ] && [ "$OSINT" = true ] && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then+ start_func ${FUNCNAME[0]} "Scanning for leaks in postman public directory"++ postleaksNg -k "$domain" > .tmp/postleaks.txt || { echo "postleaksNg command failed"; exit 1; }++ end_func "Results are saved in $domain/osint/[software/authors/metadata_results].txt" ${FUNCNAME[0]}+ else+ if [ "$POSTMAN_LEAKS" = false ] || [ "$OSINT" = false ]; then+ printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n"+ elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then+ return+ else+ if [ "$POSTMAN_LEAKS" = false ] || [ "$OSINT" = false ]; then+ printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n"+ else+ printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n"+ fi+ fi+ fi+}+function emails(){if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$EMAILS" = true ] && [ "$OSINT" = true ] && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; thenstart_func ${FUNCNAME[0]} "Searching emails/users/passwords leaks"@@ -425,7 +446,7 @@ function sub_active(){cat .tmp/subdomains_tmp.txt | tlsx -san -cn -silent -ro -c $TLSX_THREADS | anew -q .tmp/subdomains_tmp.txtfi[[ "$INSCOPE" = true ]] && check_inscope .tmp/subdomains_tmp.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/subdomains_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/subdomains_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)end_subfunc "${NUMOFLINES} subs DNS resolved from passive" ${FUNCNAME[0]}elseprintf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n"@@ -443,7 +464,7 @@ function sub_noerror(){dnsx -d $domain -r $resolvers -silent -rcode noerror -w $subs_wordlist | cut -d' ' -f1 | anew -q .tmp/subs_noerror.txt 2>>"$LOGFILE" >/dev/nullfi[[ "$INSCOPE" = true ]] && check_inscope .tmp/subs_noerror.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/subs_noerror.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/subs_noerror.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)end_subfunc "${NUMOFLINES} new subs (DNS noerror)" ${FUNCNAME[0]}elseprintf "\n${yellow} Detected DNSSEC black lies, skipping this technique ${reset}\n"@@ -462,22 +483,22 @@ function sub_dns(){start_subfunc ${FUNCNAME[0]} "Running : DNS Subdomain Enumeration and PTR search"if [ ! "$AXIOM" = true ]; then[ -s "subdomains/subdomains.txt" ] && cat subdomains/subdomains.txt | dnsx -r $resolvers_trusted -a -aaaa -cname -ns -ptr -mx -soa -silent -retry 3 -json -o subdomains/subdomains_dnsregs.json 2>>"$LOGFILE" >/dev/null- [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt- [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt+ [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt+ [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt[ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try "\(.host) - \(.a[])"' 2>/dev/null | sort -u -k2 | anew -q subdomains/subdomains_ips.txtresolvers_update_quick_local[ -s ".tmp/subdomains_dns.txt" ] && puredns resolve .tmp/subdomains_dns.txt -w .tmp/subdomains_dns_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/nullelse[ -s "subdomains/subdomains.txt" ] && axiom-scan subdomains/subdomains.txt -m dnsx -retry 3 -a -aaaa -cname -ns -ptr -mx -soa -json -o subdomains/subdomains_dnsregs.json $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null[ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | anew -q .tmp/subdomains_dns_a_records.txt- [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt- [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt+ [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt+ [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt[ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try "\(.host) - \(.a[])"' 2>/dev/null | sort -u -k2 | anew -q subdomains/subdomains_ips.txtresolvers_update_quick_axiom[ -s ".tmp/subdomains_dns.txt" ] && axiom-scan .tmp/subdomains_dns.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/subdomains_dns_resolved.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullfi[[ "$INSCOPE" = true ]] && check_inscope .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)end_subfunc "${NUMOFLINES} new subs (dns resolution)" ${FUNCNAME[0]}elseprintf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n"@@ -505,7 +526,7 @@ function sub_brute(){[ -s ".tmp/subs_brute.txt" ] && axiom-scan .tmp/subs_brute.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/subs_brute_valid.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullfi[[ "$INSCOPE" = true ]] && check_inscope .tmp/subs_brute_valid.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/subs_brute_valid.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/subs_brute_valid.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)end_subfunc "${NUMOFLINES} new subs (bruteforce)" ${FUNCNAME[0]}elseif [ "$SUBBRUTE" = false ]; then@@ -525,9 +546,9 @@ function sub_scraping(){if [ ! "$AXIOM" = true ]; thenresolvers_update_quick_localcat subdomains/subdomains.txt | httpx -follow-host-redirects -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info1.txt 2>>"$LOGFILE" >/dev/null- [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt[ -s ".tmp/probed_tmp_scrap.txt" ] && cat .tmp/probed_tmp_scrap.txt | httpx -tls-grab -tls-probe -csp-probe -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info2.txt 2>>"$LOGFILE" >/dev/null- [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txtif [ "$DEEP" = true ]; then[ -s ".tmp/probed_tmp_scrap.txt" ] && katana -silent -list .tmp/probed_tmp_scrap.txt -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn -o .tmp/katana.txt 2>>"$LOGFILE" >/dev/null@@ -537,9 +558,9 @@ function sub_scraping(){elseresolvers_update_quick_axiomaxiom-scan subdomains/subdomains.txt -m httpx -follow-host-redirects -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info1.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null- [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt[ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m httpx -tls-grab -tls-probe -csp-probe -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info2.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null- [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txtif [ "$DEEP" = true ]; then[ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m katana -jc -kf all -d 3 -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullelse@@ -547,14 +568,14 @@ function sub_scraping(){fifised -i '/^.\{2048\}./d' .tmp/katana.txt- [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/scrap_subs.txt[ -s ".tmp/scrap_subs.txt" ] && puredns resolve .tmp/scrap_subs.txt -w .tmp/scrap_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/nullif [ "$INSCOPE" = true ]; thencheck_inscope .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" >/dev/nullfi- NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l)[ -s ".tmp/diff_scrap.txt" ] && cat .tmp/diff_scrap.txt | httpx -follow-host-redirects -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info3.txt 2>>"$LOGFILE" >/dev/null- [ -s ".tmp/web_full_info3.txt" ] && cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt+ [ -s ".tmp/web_full_info3.txt" ] && cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txtcat .tmp/web_full_info1.txt .tmp/web_full_info2.txt .tmp/web_full_info3.txt 2>>"$LOGFILE" | jq -s 'try .' | jq 'try unique_by(.input)' | jq 'try .[]' 2>>"$LOGFILE" > .tmp/web_full_info.txtend_subfunc "${NUMOFLINES} new subs (code scraping)" ${FUNCNAME[0]}else@@ -579,7 +600,7 @@ function sub_analytics(){mkdir -p .tmp/output_analytics/analyticsrelationships -ch < .tmp/probed_tmp_scrap.txt >> .tmp/analytics_subs_tmp.txt 2>>"$LOGFILE"- [ -s ".tmp/analytics_subs_tmp.txt" ] && cat .tmp/analytics_subs_tmp.txt | grep "\.$domain$\|^$domain$" | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt+ [ -s ".tmp/analytics_subs_tmp.txt" ] && cat .tmp/analytics_subs_tmp.txt | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txtif [ ! "$AXIOM" = true ]; thenresolvers_update_quick_local[ -s ".tmp/analytics_subs_clean.txt" ] && puredns resolve .tmp/analytics_subs_clean.txt -w .tmp/analytics_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null@@ -643,7 +664,7 @@ function sub_permut(){if [ -s ".tmp/permute_subs.txt" ]; then[ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/permute_subs.txt[[ "$INSCOPE" = true ]] && check_inscope .tmp/permute_subs.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/permute_subs.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/permute_subs.txt 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)elseNUMOFLINES=0fi@@ -675,11 +696,11 @@ function sub_regex_permut(){if [ -s ".tmp/regulator.txt" ]; then[ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/regulator.txt[[ "$INSCOPE" = true ]] && check_inscope .tmp/regulator.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/regulator.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)+ NUMOFLINES=$(cat .tmp/regulator.txt 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)elseNUMOFLINES=0fi- end_subfunc "${NUMOFLINES} new subs (permutations)" ${FUNCNAME[0]}+ end_subfunc "${NUMOFLINES} new subs (permutations by regex)" ${FUNCNAME[0]}elseif [ "$SUBREGEXPERMUTE" = false ]; thenprintf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n"@@ -689,37 +710,6 @@ function sub_regex_permut(){fi}-function sub_gpt(){- if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$SUBGPT" = true ] && [ -s "$SUBGPT_COOKIE" ]; then- start_subfunc ${FUNCNAME[0]} "Running : Permutations by BingGPT prediction"- subgpt -i ${dir}/subdomains/subdomains.txt -c $SUBGPT_COOKIE --dont-resolve -o ${dir}/.tmp/gpt_subs.txt 2>>"$LOGFILE"- if [ ! "$AXIOM" = true ]; then- resolvers_update_quick_local- [ -s "${dir}/.tmp/gpt_subs.txt" ] && puredns resolve ${dir}/.tmp/gpt_subs.txt -w .tmp/gpt_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null- else- resolvers_update_quick_axiom- [ -s "${dir}/.tmp/gpt_subs.txt" ] && axiom-scan ${dir}/.tmp/gpt_subs.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/gpt_resolved.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null- fi-- if [ -s ".tmp/gpt_resolved.txt" ]; then- [ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/gpt_resolved.txt- [[ "$INSCOPE" = true ]] && check_inscope .tmp/gpt_resolved.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/gpt_resolved.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l)- else- NUMOFLINES=0- fi- end_subfunc "${NUMOFLINES} new subs (permutations)" ${FUNCNAME[0]}- else- if [ "$SUBGPT" = false ]; then- printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n"- elif [ ! -s "$SUBGPT_COOKIE" ]; then- printf "\n${yellow} ${FUNCNAME[0]} SUBGPT_COOKIE not defined on config file (reconftw.cfg by default) ${reset}\n"- else- printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n"- fi- fi-}-function sub_recursive_passive(){if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$SUB_RECURSIVE_PASSIVE" = true ] && [ -s "subdomains/subdomains.txt" ]; thenstart_subfunc ${FUNCNAME[0]} "Running : Subdomains recursive search passive"@@ -736,7 +726,7 @@ function sub_recursive_passive(){[ -s ".tmp/passive_recursive.txt" ] && axiom-scan .tmp/passive_recursive.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/passive_recurs_tmp.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullfi[[ "$INSCOPE" = true ]] && check_inscope .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" >/dev/null- NUMOFLINES=$(cat .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l)+ NUMOFLINES=$(cat .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l)end_subfunc "${NUMOFLINES} new subs (recursive)" ${FUNCNAME[0]}elseif [ "$SUB_RECURSIVE_PASSIVE" = false ]; then@@ -802,8 +792,8 @@ function sub_recursive_brute(){[ -s ".tmp/brute_recursive.txt" ] && axiom-scan .tmp/brute_perm_recursive.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/brute_perm_recursive_final.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullfi- NUMOFLINES=$(cat .tmp/brute_perm_recursive_final.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l)- end_subfunc "${NUMOFLINES} new subs (recursive)" ${FUNCNAME[0]}+ NUMOFLINES=$(cat .tmp/brute_perm_recursive_final.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l)+ end_subfunc "${NUMOFLINES} new subs (recursive active)" ${FUNCNAME[0]}elseif [ "$SUB_RECURSIVE_BRUTE" = false ]; thenprintf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n"@@ -921,7 +911,7 @@ function webprobe_simple(){axiom-scan subdomains/subdomains.txt -m httpx ${HTTPX_FLAGS} -no-color -json -random-agent -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -retries 2 -timeout $HTTPX_TIMEOUT -o .tmp/web_full_info_probe.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullficat .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt 2>>"$LOGFILE" | jq -s 'try .' | jq 'try unique_by(.input)' | jq 'try .[]' 2>>"$LOGFILE" > webs/web_full_info.txt- [ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew -q .tmp/probed_tmp.txt+ [ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew -q .tmp/probed_tmp.txt[ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try . |"\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' | grep "$domain" | anew -q webs/web_full_info_plain.txt[ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/probed_tmp.txtNUMOFLINES=$(cat .tmp/probed_tmp.txt 2>>"$LOGFILE" | anew webs/webs.txt | sed '/^$/d' | wc -l)@@ -954,7 +944,7 @@ function webprobe_full(){fififi- [ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew -q .tmp/probed_uncommon_ports_tmp.txt+ [ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew -q .tmp/probed_uncommon_ports_tmp.txt[ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try . |"\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' | anew -q webs/web_full_info_uncommon_plain.txtif [ -s ".tmp/web_full_info_uncommon.txt" ]; thenif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then@@ -1314,11 +1304,11 @@ function urlchecks(){fi[ -s ".tmp/katana.txt" ] && sed -i '/^.\{2048\}./d' .tmp/katana.txt[ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | anew -q .tmp/url_extract_tmp.txt- [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -aEi "\.(js)" | anew -q .tmp/url_extract_js.txt+ [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | grep -aEi "\.(js)" | anew -q .tmp/url_extract_js.txtif [ "$DEEP" = true ]; then[ -s ".tmp/url_extract_js.txt" ] && interlace -tL .tmp/url_extract_js.txt -threads 10 -c "python3 $tools/JSA/jsa.py -f target | anew -q .tmp/url_extract_tmp.txt" &>/dev/nullfi- [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" | anew -q .tmp/url_extract_tmp2.txt+ [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" | anew -q .tmp/url_extract_tmp2.txt[ -s ".tmp/url_extract_tmp2.txt" ] && cat .tmp/url_extract_tmp2.txt | python3 $tools/urless/urless/urless.py | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/nullNUMOFLINES=$(cat .tmp/url_extract_uddup.txt 2>>"$LOGFILE" | anew webs/url_extract.txt | sed '/^$/d' | wc -l)notification "${NUMOFLINES} new urls with params" info@@ -1393,7 +1383,7 @@ function jschecks(){if [ -s ".tmp/url_extract_js.txt" ]; thenprintf "${yellow} Running : Fetching Urls 1/5${reset}\n"if [ ! "$AXIOM" = true ]; then- cat .tmp/url_extract_js.txt | subjs -ua "Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" -c 40 | grep "$domain" | anew -q .tmp/subjslinks.txt+ cat .tmp/url_extract_js.txt | subjs -ua "Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" -c 40 | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subjslinks.txtelseaxiom-scan .tmp/url_extract_js.txt -m subjs -o .tmp/subjslinks.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/nullfi@@ -1418,7 +1408,7 @@ function jschecks(){if [ ! "$AXIOM" = true ]; then[ -s "js/js_livelinks.txt" ] && cat js/js_livelinks.txt | Mantra -ua ${HEADER} -s | anew -q js/js_secrets.txtelse- [ -s "js/js_livelinks.txt" ] && axiom-scan js/js_livelinks.txt -m mantra -ua ${HEADER} -s -o js/js_secrets.txt $AXIOM_EXTRA_ARGS &>/dev/null+ [ -s "js/js_livelinks.txt" ] && axiom-scan js/js_livelinks.txt -m mantra -ua \"${HEADER}\" -s -o js/js_secrets.txt $AXIOM_EXTRA_ARGS &>/dev/nullfi[ -s "js/js_secrets.txt" ] && sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" -i js/js_secrets.txtprintf "${yellow} Running : Building wordlist 5/5${reset}\n"@@ -1953,7 +1943,7 @@ function getElapsedTime {function zipSnedOutputFolder {zip_name1=$(date +"%Y_%m_%d-%H.%M.%S")- zip_name="${zip_name1}_${domain}.zip"+ zip_name="${zip_name1}_${domain}.zip" 2>>"$LOGFILE" >/dev/null(cd "$dir" && zip -r "$zip_name" .)echo "Sending zip file "${dir}/${zip_name}""@@ -2353,7 +2343,6 @@ function passive(){SUBSCRAPING=falseSUBPERMUTE=falseSUBREGEXPERMUTE=false- SUBGPT=falseSUB_RECURSIVE_BRUTE=falseWEBPROBESIMPLE=falseif [ "$AXIOM" = true ]; then@@ -2411,6 +2400,7 @@ function vulns(){sprayingbrokenLinksfuzzparams+ 4xxbypasstest_sslfi}diff --git a/requirements.txt b/requirements.txtindex 30305df..e36da7f 100644--- a/requirements.txt+++ b/requirements.txt@@ -34,4 +34,4 @@ tldextract # dorks_huntertqdm # multipleujson # multipleurllib3 # multiple-subgpt # Tool+postleaksNeg # Tool