Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

katana replacing gospider

six2dez386f79ec0d03
diff --git a/README.md b/README.md
index 75ee79e..cdfc981 100644
--- a/README.md
+++ b/README.md
@@ -280,7 +280,7 @@ HEADER="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Fire
FFUF_THREADS=40
HTTPX_THREADS=50
HTTPX_UNCOMMONPORTS_THREADS=100
-GOSPIDER_THREADS=20
+KATANA_THREADS=20
BRUTESPRAY_THREADS=20
BRUTESPRAY_CONCURRENCE=10
GAU_THREADS=10
@@ -469,7 +469,7 @@ reset='\033[0m'
- NOERROR subdomain discovery ([dnsx](https://github.com/projectdiscovery/dnsx), more info [here](https://www.securesystems.de/blog/enhancing-subdomain-enumeration-ents-and-noerror/))
- Bruteforce ([puredns](https://github.com/d3mondev/puredns))
- Permutations ([Gotator](https://github.com/Josue87/gotator), [ripgen](https://github.com/resyncgg/ripgen) and [regulator](https://github.com/cramppet/regulator))
- - JS files & Source Code Scraping ([gospider](https://github.com/jaeles-project/gospider))
+ - JS files & Source Code Scraping ([katana](https://github.com/projectdiscovery/katana))
- DNS Records ([dnsx](https://github.com/projectdiscovery/dnsx))
- Google Analytics ID ([AnalyticsRelationships](https://github.com/Josue87/AnalyticsRelationships))
- TLS handshake ([tlsx](https://github.com/projectdiscovery/tlsx))
@@ -492,7 +492,7 @@ reset='\033[0m'
- Web screenshoting ([webscreenshot](https://github.com/maaaaz/webscreenshot) or [gowitness](https://github.com/sensepost/gowitness))
- Web templates scanner ([nuclei](https://github.com/projectdiscovery/nuclei) and [nuclei geeknik](https://github.com/geeknik/the-nuclei-templates.git))
- CMS Scanner ([CMSeeK](https://github.com/Tuhinshubhra/CMSeeK))
-- Url extraction ([waymore](https://github.com/xnl-h4ck3r/waymore), [gospider](https://github.com/jaeles-project/gospider), [github-endpoints](https://gist.github.com/six2dez/d1d516b606557526e9a78d7dd49cacd3) and [JSA](https://github.com/w9w/JSA))
+- Url extraction ([waymore](https://github.com/xnl-h4ck3r/waymore), [katana](https://github.com/projectdiscovery/katana), [github-endpoints](https://gist.github.com/six2dez/d1d516b606557526e9a78d7dd49cacd3) and [JSA](https://github.com/w9w/JSA))
- URL patterns Search and filtering ([urless](https://github.com/xnl-h4ck3r/urless), [gf](https://github.com/tomnomnom/gf) and [gf-patterns](https://github.com/1ndianl33t/Gf-Patterns))
- Favicon Real IP ([fav-up](https://github.com/pielco11/fav-up))
- Javascript analysis ([subjs](https://github.com/lc/subjs), [JSA](https://github.com/w9w/JSA), [xnLinkFinder](https://github.com/xnl-h4ck3r/xnLinkFinder), [getjswords](https://github.com/m4ll0k/BBTz))
@@ -511,7 +511,7 @@ reset='\033[0m'
- SQLi Check ([SQLMap](https://github.com/sqlmapproject/sqlmap))
- SSTI ([ffuf](https://github.com/ffuf/ffuf))
- SSL tests ([testssl](https://github.com/drwetter/testssl.sh))
-- Broken Links Checker ([gospider](https://github.com/jaeles-project/gospider))
+- Broken Links Checker ([katana](https://github.com/projectdiscovery/katana))
- Prototype Pollution ([ppfuzz](https://github.com/dwisiswant0/ppfuzz))
- Web Cache Vulnerabilities ([Web-Cache-Vulnerability-Scanner](https://github.com/Hackmanit/Web-Cache-Vulnerability-Scanner))
- 4XX Bypasser ([byp4xx](https://github.com/lobuhi/byp4xx))
diff --git a/Terraform/files/reconftw.cfg b/Terraform/files/reconftw.cfg
index 81f1567..e9c0220 100644
--- a/Terraform/files/reconftw.cfg
+++ b/Terraform/files/reconftw.cfg
@@ -1,194 +1,194 @@
-#################################################################
-# reconFTW config file #
-#################################################################
-
-# General values
-tools=~/Tools
-SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-profile_shell=".$(basename $(echo $SHELL))rc"
-reconftw_version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --tags)
-generate_resolvers=false
-proxy_url="http://127.0.0.1:8080/"
-#dir_output=/custom/output/path
-
-# Golang Vars (Comment or change on your own)
-export GOROOT=/usr/local/go
-export GOPATH=$HOME/go
-export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH
-
-# Tools config files
-#NOTIFY_CONFIG=~/.config/notify/provider-config.yaml # No need to define
-AMASS_CONFIG=~/.config/amass/config.ini
-GITHUB_TOKENS=${tools}/.github_tokens
-#CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path
-
-# APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line
-#SHODAN_API_KEY="XXXXXXXXXXXXX"
-#WHOISXML_API="XXXXXXXXXX"
-#XSS_SERVER="XXXXXXXXXXXXXXXXX"
-#COLLAB_SERVER="XXXXXXXXXXXXXXXXX"
-#slack_channel="XXXXXXXX"
-#slack_auth="xoXX-XXX-XXX-XXX"
-
-# File descriptors
-DEBUG_STD="&>/dev/null"
-DEBUG_ERROR="2>/dev/null"
-
-# Osint
-OSINT=true
-GOOGLE_DORKS=true
-GITHUB_DORKS=true
-METADATA=true
-EMAILS=true
-DOMAIN_INFO=true
-IP_INFO=true
-METAFINDER_LIMIT=20 # Max 250
-
-# Subdomains
-SUBDOMAINS_GENERAL=true
-SUBPASSIVE=true
-SUBCRT=true
-SUBANALYTICS=true
-SUBBRUTE=true
-SUBSCRAPING=true
-SUBPERMUTE=true
-SUBTAKEOVER=true
-SUBRECURSIVE=true
-SUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queries
-SUB_RECURSIVE_BRUTE=false # Needs big disk space and time to resolve
-ZONETRANSFER=true
-S3BUCKETS=true
-REVERSE_IP=false
-TLS_PORTS="21,22,25,80,110,135,143,261,271,324,443,448,465,563,614,631,636,664,684,695,832,853,854,990,993,989,990,992,993,994,995,1129,1131,1184,2083,2087,2089,2096,2221,2252,2376,2381,2478,2479,2482,2484,2679,2762,3077,3078,3183,3191,3220,3269,3306,3410,3424,3471,3496,3509,3529,3539,3535,3660,36611,3713,3747,3766,3864,3885,3995,3896,4031,4036,4062,4064,4081,4083,4116,4335,4336,4536,4590,4740,4843,4843,4849,5443,5007,5061,5321,5349,5671,5783,5868,5986,5989,5990,6209,6251,6443,6513,6514,6619,6697,6771,6697,7202,7443,7673,7674,7677,7775,8243,8443,8991,8989,9089,9295,9318,9443,9444,9614,9802,10161,10162,11751,12013,12109,14143,15002,16995,41230,16993,20003"
-
-# Web detection
-WEBPROBESIMPLE=true
-WEBPROBEFULL=true
-WEBSCREENSHOT=true
-VIRTUALHOSTS=true
-UNCOMMON_PORTS_WEB="81,300,591,593,832,981,1010,1311,1099,2082,2095,2096,2480,3000,3128,3333,4243,4567,4711,4712,4993,5000,5104,5108,5280,5281,5601,5800,6543,7000,7001,7396,7474,8000,8001,8008,8014,8042,8060,8069,8080,8081,8083,8088,8090,8091,8095,8118,8123,8172,8181,8222,8243,8280,8281,8333,8337,8443,8500,8834,8880,8888,8983,9000,9001,9043,9060,9080,9090,9091,9092,9200,9443,9502,9800,9981,10000,10250,11371,12443,15672,16080,17778,18091,18092,20720,32000,55440,55672"
-# You can change to aquatone if gowitness fails, comment the one you don't want
-AXIOM_SCREENSHOT_MODULE=webscreenshot # Choose between aquatone,gowitness,webscreenshot
-
-# Host
-FAVICON=true
-PORTSCANNER=true
-PORTSCAN_PASSIVE=true
-PORTSCAN_ACTIVE=true
-CDN_IP=true
-
-# Web analysis
-WAF_DETECTION=true
-NUCLEICHECK=true
-NUCLEI_SEVERITY="info,low,medium,high,critical"
-URL_CHECK=true
-URL_GF=true
-URL_EXT=true
-JSCHECKS=true
-FUZZ=true
-CMS_SCANNER=true
-WORDLIST=true
-ROBOTSWORDLIST=true
-PASSWORD_DICT=true
-PASSWORD_MIN_LENGTH=5
-PASSWORD_MAX_LENGTH=14
-
-# Vulns
-VULNS_GENERAL=false
-XSS=true
-CORS=true
-TEST_SSL=true
-OPEN_REDIRECT=true
-SSRF_CHECKS=true
-CRLF_CHECKS=true
-LFI=true
-SSTI=true
-SQLI=true
-BROKENLINKS=true
-SPRAY=true
-COMM_INJ=true
-PROTO_POLLUTION=true
-
-# Extra features
-NOTIFICATION=false # Notification for every function
-SOFT_NOTIFICATION=false # Only for start/end
-DEEP=false
-DEEP_LIMIT=500
-DEEP_LIMIT2=1500
-DIFF=false
-REMOVETMP=false
-REMOVELOG=false
-PROXY=false
-SENDZIPNOTIFY=false
-PRESERVE=true # set to true to avoid deleting the .called_fn files on really large scans
-FFUF_FLAGS="-mc all -fc 404 -ac -sf -s"
-
-# HTTP options
-HEADER="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0"
-
-# Threads
-FFUF_THREADS=40
-HTTPX_THREADS=50
-HTTPX_UNCOMMONPORTS_THREADS=100
-GOSPIDER_THREADS=20
-BRUTESPRAY_THREADS=20
-BRUTESPRAY_CONCURRENCE=10
-GAUPLUS_THREADS=10
-DNSTAKE_THREADS=100
-DALFOX_THREADS=200
-PUREDNS_PUBLIC_LIMIT=0 # Set between 2000 - 10000 if your router blows up, 0 is unlimited
-PUREDNS_TRUSTED_LIMIT=400
-PUREDNS_WILDCARDTEST_LIMIT=30
-PUREDNS_WILDCARDBATCH_LIMIT=1500000
-WEBSCREENSHOT_THREADS=200
-GOWITNESS_THREADS=8
-RESOLVE_DOMAINS_THREADS=150
-PPFUZZ_THREADS=30
-DNSVALIDATOR_THREADS=200
-INTERLACE_THREADS=10
-TLSX_THREADS=1000
-
-# Timeouts
-CMSSCAN_TIMEOUT=3600
-FFUF_MAXTIME=900 # Seconds
-HTTPX_TIMEOUT=10 # Seconds
-HTTPX_UNCOMMONPORTS_TIMEOUT=10 # Seconds
-
-# lists
-fuzz_wordlist=${tools}/fuzz_wordlist.txt
-lfi_wordlist=${tools}/lfi_wordlist.txt
-ssti_wordlist=${tools}/ssti_wordlist.txt
-subs_wordlist=${tools}/subdomains.txt
-subs_wordlist_big=${tools}/subdomains_big.txt
-resolvers=${tools}/resolvers.txt
-resolvers_trusted=${tools}/resolvers_trusted.txt
-
-# Axiom Fleet
-# Will not start a new fleet if one exist w/ same name and size (or larger)
-# AXIOM=false Uncomment only to overwrite command line flags
-AXIOM_FLEET_LAUNCH=false
-AXIOM_FLEET_NAME="reconFTW"
-AXIOM_FLEET_COUNT=5
-AXIOM_FLEET_REGIONS="eu-central"
-AXIOM_FLEET_SHUTDOWN=true
-# This is a script on your reconftw host that might prep things your way...
-#AXIOM_POST_START="~/Tools/axiom_config.sh"
-AXIOM_EXTRA_ARGS="" # Leave empty if you don't want to add extra arguments
-#AXIOM_EXTRA_ARGS="--rm-logs" # Example
-
-# BBRF
-BBRF_CONNECTION=false
-BBRF_SERVER=https://demo.bbrf.me/bbrf
-BBRF_USERNAME="user"
-BBRF_PASSWORD="password"
-
-# TERM COLORS
-bred='\033[1;31m'
-bblue='\033[1;34m'
-bgreen='\033[1;32m'
-byellow='\033[1;33m'
-red='\033[0;31m'
-blue='\033[0;34m'
-green='\033[0;32m'
-yellow='\033[0;33m'
-reset='\033[0m'
+#################################################################
+# reconFTW config file #
+#################################################################
+
+# General values
+tools=~/Tools
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+profile_shell=".$(basename $(echo $SHELL))rc"
+reconftw_version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --tags)
+generate_resolvers=false
+proxy_url="http://127.0.0.1:8080/"
+#dir_output=/custom/output/path
+
+# Golang Vars (Comment or change on your own)
+export GOROOT=/usr/local/go
+export GOPATH=$HOME/go
+export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH
+
+# Tools config files
+#NOTIFY_CONFIG=~/.config/notify/provider-config.yaml # No need to define
+AMASS_CONFIG=~/.config/amass/config.ini
+GITHUB_TOKENS=${tools}/.github_tokens
+#CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path
+
+# APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line
+#SHODAN_API_KEY="XXXXXXXXXXXXX"
+#WHOISXML_API="XXXXXXXXXX"
+#XSS_SERVER="XXXXXXXXXXXXXXXXX"
+#COLLAB_SERVER="XXXXXXXXXXXXXXXXX"
+#slack_channel="XXXXXXXX"
+#slack_auth="xoXX-XXX-XXX-XXX"
+
+# File descriptors
+DEBUG_STD="&>/dev/null"
+DEBUG_ERROR="2>/dev/null"
+
+# Osint
+OSINT=true
+GOOGLE_DORKS=true
+GITHUB_DORKS=true
+METADATA=true
+EMAILS=true
+DOMAIN_INFO=true
+IP_INFO=true
+METAFINDER_LIMIT=20 # Max 250
+
+# Subdomains
+SUBDOMAINS_GENERAL=true
+SUBPASSIVE=true
+SUBCRT=true
+SUBANALYTICS=true
+SUBBRUTE=true
+SUBSCRAPING=true
+SUBPERMUTE=true
+SUBTAKEOVER=true
+SUBRECURSIVE=true
+SUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queries
+SUB_RECURSIVE_BRUTE=false # Needs big disk space and time to resolve
+ZONETRANSFER=true
+S3BUCKETS=true
+REVERSE_IP=false
+TLS_PORTS="21,22,25,80,110,135,143,261,271,324,443,448,465,563,614,631,636,664,684,695,832,853,854,990,993,989,990,992,993,994,995,1129,1131,1184,2083,2087,2089,2096,2221,2252,2376,2381,2478,2479,2482,2484,2679,2762,3077,3078,3183,3191,3220,3269,3306,3410,3424,3471,3496,3509,3529,3539,3535,3660,36611,3713,3747,3766,3864,3885,3995,3896,4031,4036,4062,4064,4081,4083,4116,4335,4336,4536,4590,4740,4843,4843,4849,5443,5007,5061,5321,5349,5671,5783,5868,5986,5989,5990,6209,6251,6443,6513,6514,6619,6697,6771,6697,7202,7443,7673,7674,7677,7775,8243,8443,8991,8989,9089,9295,9318,9443,9444,9614,9802,10161,10162,11751,12013,12109,14143,15002,16995,41230,16993,20003"
+
+# Web detection
+WEBPROBESIMPLE=true
+WEBPROBEFULL=true
+WEBSCREENSHOT=true
+VIRTUALHOSTS=true
+UNCOMMON_PORTS_WEB="81,300,591,593,832,981,1010,1311,1099,2082,2095,2096,2480,3000,3128,3333,4243,4567,4711,4712,4993,5000,5104,5108,5280,5281,5601,5800,6543,7000,7001,7396,7474,8000,8001,8008,8014,8042,8060,8069,8080,8081,8083,8088,8090,8091,8095,8118,8123,8172,8181,8222,8243,8280,8281,8333,8337,8443,8500,8834,8880,8888,8983,9000,9001,9043,9060,9080,9090,9091,9092,9200,9443,9502,9800,9981,10000,10250,11371,12443,15672,16080,17778,18091,18092,20720,32000,55440,55672"
+# You can change to aquatone if gowitness fails, comment the one you don't want
+AXIOM_SCREENSHOT_MODULE=webscreenshot # Choose between aquatone,gowitness,webscreenshot
+
+# Host
+FAVICON=true
+PORTSCANNER=true
+PORTSCAN_PASSIVE=true
+PORTSCAN_ACTIVE=true
+CDN_IP=true
+
+# Web analysis
+WAF_DETECTION=true
+NUCLEICHECK=true
+NUCLEI_SEVERITY="info,low,medium,high,critical"
+URL_CHECK=true
+URL_GF=true
+URL_EXT=true
+JSCHECKS=true
+FUZZ=true
+CMS_SCANNER=true
+WORDLIST=true
+ROBOTSWORDLIST=true
+PASSWORD_DICT=true
+PASSWORD_MIN_LENGTH=5
+PASSWORD_MAX_LENGTH=14
+
+# Vulns
+VULNS_GENERAL=false
+XSS=true
+CORS=true
+TEST_SSL=true
+OPEN_REDIRECT=true
+SSRF_CHECKS=true
+CRLF_CHECKS=true
+LFI=true
+SSTI=true
+SQLI=true
+BROKENLINKS=true
+SPRAY=true
+COMM_INJ=true
+PROTO_POLLUTION=true
+
+# Extra features
+NOTIFICATION=false # Notification for every function
+SOFT_NOTIFICATION=false # Only for start/end
+DEEP=false
+DEEP_LIMIT=500
+DEEP_LIMIT2=1500
+DIFF=false
+REMOVETMP=false
+REMOVELOG=false
+PROXY=false
+SENDZIPNOTIFY=false
+PRESERVE=true # set to true to avoid deleting the .called_fn files on really large scans
+FFUF_FLAGS="-mc all -fc 404 -ac -sf -s"
+
+# HTTP options
+HEADER="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0"
+
+# Threads
+FFUF_THREADS=40
+HTTPX_THREADS=50
+HTTPX_UNCOMMONPORTS_THREADS=100
+KATANA_THREADS=20
+BRUTESPRAY_THREADS=20
+BRUTESPRAY_CONCURRENCE=10
+GAUPLUS_THREADS=10
+DNSTAKE_THREADS=100
+DALFOX_THREADS=200
+PUREDNS_PUBLIC_LIMIT=0 # Set between 2000 - 10000 if your router blows up, 0 is unlimited
+PUREDNS_TRUSTED_LIMIT=400
+PUREDNS_WILDCARDTEST_LIMIT=30
+PUREDNS_WILDCARDBATCH_LIMIT=1500000
+WEBSCREENSHOT_THREADS=200
+GOWITNESS_THREADS=8
+RESOLVE_DOMAINS_THREADS=150
+PPFUZZ_THREADS=30
+DNSVALIDATOR_THREADS=200
+INTERLACE_THREADS=10
+TLSX_THREADS=1000
+
+# Timeouts
+CMSSCAN_TIMEOUT=3600
+FFUF_MAXTIME=900 # Seconds
+HTTPX_TIMEOUT=10 # Seconds
+HTTPX_UNCOMMONPORTS_TIMEOUT=10 # Seconds
+
+# lists
+fuzz_wordlist=${tools}/fuzz_wordlist.txt
+lfi_wordlist=${tools}/lfi_wordlist.txt
+ssti_wordlist=${tools}/ssti_wordlist.txt
+subs_wordlist=${tools}/subdomains.txt
+subs_wordlist_big=${tools}/subdomains_big.txt
+resolvers=${tools}/resolvers.txt
+resolvers_trusted=${tools}/resolvers_trusted.txt
+
+# Axiom Fleet
+# Will not start a new fleet if one exist w/ same name and size (or larger)
+# AXIOM=false Uncomment only to overwrite command line flags
+AXIOM_FLEET_LAUNCH=false
+AXIOM_FLEET_NAME="reconFTW"
+AXIOM_FLEET_COUNT=5
+AXIOM_FLEET_REGIONS="eu-central"
+AXIOM_FLEET_SHUTDOWN=true
+# This is a script on your reconftw host that might prep things your way...
+#AXIOM_POST_START="~/Tools/axiom_config.sh"
+AXIOM_EXTRA_ARGS="" # Leave empty if you don't want to add extra arguments
+#AXIOM_EXTRA_ARGS="--rm-logs" # Example
+
+# BBRF
+BBRF_CONNECTION=false
+BBRF_SERVER=https://demo.bbrf.me/bbrf
+BBRF_USERNAME="user"
+BBRF_PASSWORD="password"
+
+# TERM COLORS
+bred='\033[1;31m'
+bblue='\033[1;34m'
+bgreen='\033[1;32m'
+byellow='\033[1;33m'
+red='\033[0;31m'
+blue='\033[0;34m'
+green='\033[0;32m'
+yellow='\033[0;33m'
+reset='\033[0m'
diff --git a/install.sh b/install.sh
index f934927..64c42e1 100755
--- a/install.sh
+++ b/install.sh
@@ -60,7 +60,7 @@ gotools["github-endpoints"]="go install -v github.com/gwen001/github-endpoints@l
gotools["dnsx"]="go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest"
gotools["subjs"]="go install -v github.com/lc/subjs@latest"
gotools["Gxss"]="go install -v github.com/KathanP19/Gxss@latest"
-gotools["gospider"]="go install -v github.com/jaeles-project/gospider@latest"
+gotools["katana"]="go install github.com/projectdiscovery/katana/cmd/katana@latest"
gotools["crlfuzz"]="go install -v github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@latest"
gotools["dalfox"]="go install -v github.com/hahwul/dalfox/v2@latest"
gotools["puredns"]="go install -v github.com/d3mondev/puredns/v2@latest"
diff --git a/reconftw.cfg b/reconftw.cfg
index 1813016..4962f2d 100644
--- a/reconftw.cfg
+++ b/reconftw.cfg
@@ -153,7 +153,7 @@ HEADER="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Fire
FFUF_THREADS=40
HTTPX_THREADS=50
HTTPX_UNCOMMONPORTS_THREADS=100
-GOSPIDER_THREADS=20
+KATANA_THREADS=20
BRUTESPRAY_THREADS=20
BRUTESPRAY_CONCURRENCE=10
#GAU_THREADS=10
diff --git a/reconftw.sh b/reconftw.sh
index c01a183..44d9f41 100755
--- a/reconftw.sh
+++ b/reconftw.sh
@@ -67,7 +67,7 @@ function tools_installed(){
which github-endpoints &>/dev/null || { printf "${bred} [*] github-endpoints [NO]${reset}\n"; allinstalled=false;}
which github-subdomains &>/dev/null || { printf "${bred} [*] github-subdomains [NO]${reset}\n"; allinstalled=false;}
which gitlab-subdomains &>/dev/null || { printf "${bred} [*] gitlab-subdomains [NO]${reset}\n"; allinstalled=false;}
- which gospider &>/dev/null || { printf "${bred} [*] gospider [NO]${reset}\n"; allinstalled=false;}
+ which katana &>/dev/null || { printf "${bred} [*] katana [NO]${reset}\n"; allinstalled=false;}
which wafw00f &>/dev/null || { printf "${bred} [*] wafw00f [NO]${reset}\n"; allinstalled=false;}
which dnsvalidator &>/dev/null || { printf "${bred} [*] dnsvalidator [NO]${reset}\n"; allinstalled=false;}
which gowitness &>/dev/null || { printf "${bred} [*] gowitness [NO]${reset}\n"; allinstalled=false;}
@@ -542,18 +542,10 @@ function sub_scraping(){
[ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt
if [ "$DEEP" = true ]; then
- [ -s ".tmp/probed_tmp_scrap.txt" ] && gospider -S .tmp/probed_tmp_scrap.txt --js -t $GOSPIDER_THREADS -d 3 --sitemap --robots -w -r > .tmp/gospider.txt
+ [ -s ".tmp/probed_tmp_scrap.txt" ] && katana -silent -list .tmp/probed_tmp_scrap.txt -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn -o .tmp/katana.txt &>/dev/null
else
- [ -s ".tmp/probed_tmp_scrap.txt" ] && gospider -S .tmp/probed_tmp_scrap.txt --js -t $GOSPIDER_THREADS -d 2 --sitemap --robots -w -r > .tmp/gospider.txt
+ [ -s ".tmp/probed_tmp_scrap.txt" ] && katana -silent -list .tmp/probed_tmp_scrap.txt -jc -kf all -c $KATANA_THREADS -d 2 -fs rdn -o .tmp/katana.txt &>/dev/null
fi
- sed -i '/^.\{2048\}./d' .tmp/gospider.txt
- [ -s ".tmp/gospider.txt" ] && cat .tmp/gospider.txt | grep -aEo 'https?://[^ ]+' | sed 's/]$//' | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | anew -q .tmp/scrap_subs.txt
- [ -s ".tmp/scrap_subs.txt" ] && puredns resolve .tmp/scrap_subs.txt -w .tmp/scrap_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" &>/dev/null
- if [ "$INSCOPE" = true ]; then
- check_inscope .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" &>/dev/null
- fi
- NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l)
- [ -s ".tmp/diff_scrap.txt" ] && cat .tmp/diff_scrap.txt | httpx -follow-host-redirects -H "${HEADER}" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info3.txt 2>>"$LOGFILE" &>/dev/null
else
resolvers_update_quick_axiom
axiom-scan subdomains/subdomains.txt -m httpx -follow-host-redirects -H \"${HEADER}\" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info1.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
@@ -561,20 +553,19 @@ function sub_scraping(){
[ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m httpx -tls-grab -tls-probe -csp-probe -H \"${HEADER}\" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info2.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
[ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt
if [ "$DEEP" = true ]; then
- [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m gospider --js -d 3 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m katana -jc -kf all -d 3 -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
else
- [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m gospider --js -d 2 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m katana -jc -kf all -d 2 -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
fi
- NUMFILES=0
- touch .tmp/gospider.txt
- [[ -d .tmp/gospider/ ]] && NUMFILES=$(find .tmp/gospider/ -type f | wc -l)
- [[ $NUMFILES -gt 0 ]] && find .tmp/gospider/ -type f -exec cat {} + | sed '/^.\{2048\}./d' | anew -q .tmp/gospider.txt
- [ -s ".tmp/gospider.txt" ] && cat .tmp/gospider.txt | grep -aEo 'https?://[^ ]+' | sed 's/]$//' | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | anew -q .tmp/scrap_subs.txt
- [ -s ".tmp/scrap_subs.txt" ] && axiom-scan .tmp/scrap_subs.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/scrap_subs_resolved.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
- [[ "$INSCOPE" = true ]] && check_inscope .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" &>/dev/null
- NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l)
- [ -s ".tmp/diff_scrap.txt" ] && axiom-scan .tmp/diff_scrap.txt -m httpx -follow-host-redirects -H \"${HEADER}\" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info3.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
fi
+ sed -i '/^.\{2048\}./d' .tmp/katana.txt
+ [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | anew -q .tmp/scrap_subs.txt
+ [ -s ".tmp/scrap_subs.txt" ] && puredns resolve .tmp/scrap_subs.txt -w .tmp/scrap_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" &>/dev/null
+ if [ "$INSCOPE" = true ]; then
+ check_inscope .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" &>/dev/null
+ fi
+ NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l)
+ [ -s ".tmp/diff_scrap.txt" ] && cat .tmp/diff_scrap.txt | httpx -follow-host-redirects -H "${HEADER}" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info3.txt 2>>"$LOGFILE" &>/dev/null
[ -s ".tmp/web_full_info3.txt" ] && cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt
cat .tmp/web_full_info1.txt .tmp/web_full_info2.txt .tmp/web_full_info3.txt 2>>"$LOGFILE" | jq -s 'try .' | jq 'try unique_by(.input)' | jq 'try .[]' 2>>"$LOGFILE" > .tmp/web_full_info.txt
end_subfunc "${NUMOFLINES} new subs (code scraping)" ${FUNCNAME[0]}
@@ -1327,17 +1318,15 @@ function urlchecks(){
fi
fi
diff_webs=$(diff <(sort -u .tmp/probed_tmp.txt 2>>"$LOGFILE") <(sort -u .tmp/webs_all.txt 2>>"$LOGFILE") | wc -l)
- if [ $diff_webs != "0" ] || [ ! -s ".tmp/gospider.txt" ]; then
+ if [ $diff_webs != "0" ] || [ ! -s ".tmp/katana.txt" ]; then
if [ "$URL_CHECK_ACTIVE" = true ]; then
if [ "$DEEP" = true ]; then
- gospider -S .tmp/webs_all.txt --js -t $GOSPIDER_THREADS -d 3 --sitemap --robots -w -r > .tmp/gospider.txt
+ katana -silent -list .tmp/webs_all.txt -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn -o .tmp/katana.txt &>/dev/null
else
- gospider -S .tmp/webs_all.txt --js -t $GOSPIDER_THREADS -d 2 --sitemap --robots -w -r > .tmp/gospider.txt
+ katana -silent -list .tmp/webs_all.txt -jc -kf all -c $KATANA_THREADS -d 2 -fs rdn -o .tmp/katana.txt &>/dev/null
fi
fi
fi
- [ -s ".tmp/gospider.txt" ] && sed -i '/^.\{2048\}./d' .tmp/gospider.txt
- [ -s ".tmp/gospider.txt" ] && cat .tmp/gospider.txt | grep -aEo 'https?://[^ ]+' | sed 's/]$//' | grep -E "^(http|https):[\/]{2}([a-zA-Z0-9\-\.]+\.$domain)" | anew -q .tmp/url_extract_tmp.txt
else
if [ "$URL_CHECK_PASSIVE" = true ]; then
axiom-scan .tmp/webs_all.txt -m waymore -o .tmp/url_extract_tmp.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
@@ -1347,19 +1336,18 @@ function urlchecks(){
fi
fi
diff_webs=$(diff <(sort -u .tmp/probed_tmp.txt) <(sort -u .tmp/webs_all.txt) | wc -l)
- if [ $diff_webs != "0" ] || [ ! -s ".tmp/gospider.txt" ]; then
+ if [ $diff_webs != "0" ] || [ ! -s ".tmp/katana.txt" ]; then
if [ "$URL_CHECK_ACTIVE" = true ]; then
if [ "$DEEP" = true ]; then
- axiom-scan .tmp/webs_all.txt -m gospider --js -d 3 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ axiom-scan .tmp/webs_all.txt -m katana -jc -kf all -d 3 -fs rdn -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
else
- axiom-scan .tmp/webs_all.txt -m gospider --js -d 2 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ axiom-scan .tmp/webs_all.txt -m katana -jc -kf all -d 2 -fs rdn -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
fi
- [[ -d .tmp/gospider/ ]] && find .tmp/gospider -type f -exec cat {} + | sed '/^.\{2048\}./d' | anew -q .tmp/gospider.txt
fi
fi
- [[ -d .tmp/gospider/ ]] && NUMFILES=$(find .tmp/gospider/ -type f | wc -l)
- [[ $NUMFILES -gt 0 ]] && cat .tmp/gospider.txt | grep -aEo 'https?://[^ ]+' | sed 's/]$//' | grep -E "^(http|https):[\/]{2}([a-zA-Z0-9\-\.]+\.$domain)" | anew -q .tmp/url_extract_tmp.txt
fi
+ [ -s ".tmp/katana.txt" ] && sed -i '/^.\{2048\}./d' .tmp/katana.txt
+ [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | anew -q .tmp/url_extract_tmp.txt
[ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -aEi "\.(js)" | anew -q .tmp/url_extract_js.txt
if [ "$DEEP" = true ]; then
[ -s ".tmp/url_extract_js.txt" ] && cat .tmp/url_extract_js.txt | python3 $tools/JSA/jsa.py | anew -q .tmp/url_extract_tmp.txt
@@ -1549,25 +1537,25 @@ function brokenLinks(){
start_func ${FUNCNAME[0]} "Broken links checks"
[ ! -s ".tmp/webs_all.txt" ] && cat webs/webs.txt webs/webs_uncommon_ports.txt 2>/dev/null | anew -q .tmp/webs_all.txt
if [ ! "$AXIOM" = true ]; then
- if [ ! -s ".tmp/gospider.txt" ]; then
+ if [ ! -s ".tmp/katana.txt" ]; then
if [ "$DEEP" = true ]; then
- [ -s ".tmp/webs_all.txt" ] && gospider -S .tmp/webs_all.txt --js -t $GOSPIDER_THREADS -d 3 --sitemap --robots -w -r > .tmp/gospider.txt
+ [ -s ".tmp/webs_all.txt" ] && katana -silent -list .tmp/webs_all.txt -jc -kf all -c $KATANA_THREADS -d 3 -o .tmp/katana.txt &>/dev/null
else
- [ -s ".tmp/webs_all.txt" ] && gospider -S .tmp/webs_all.txt --js -t $GOSPIDER_THREADS -d 2 --sitemap --robots -w -r > .tmp/gospider.txt
+ [ -s ".tmp/webs_all.txt" ] && katana -silent -list .tmp/webs_all.txt -jc -kf all -c $KATANA_THREADS -d 2 -o .tmp/katana.txt &>/dev/null
fi
fi
- [ -s ".tmp/gospider.txt" ] && sed -i '/^.\{2048\}./d' .tmp/gospider.txt
+ [ -s ".tmp/katana.txt" ] && sed -i '/^.\{2048\}./d' .tmp/katana.txt
else
- if [ ! -s ".tmp/gospider.txt" ]; then
+ if [ ! -s ".tmp/katana.txt" ]; then
if [ "$DEEP" = true ]; then
- [ -s ".tmp/webs_all.txt" ] && axiom-scan .tmp/webs_all.txt -m gospider --js -d 3 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ [ -s ".tmp/webs_all.txt" ] && axiom-scan .tmp/webs_all.txt -m katana -jc -kf all -d 3 -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
else
- [ -s ".tmp/webs_all.txt" ] && axiom-scan .tmp/webs_all.txt -m gospider --js -d 2 --sitemap --robots -w -r -o .tmp/gospider $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
+ [ -s ".tmp/webs_all.txt" ] && axiom-scan .tmp/webs_all.txt -m katana -jc -kf all -d 2 -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" &>/dev/null
fi
- find .tmp/gospider -type f -exec cat {} + | sed '/^.\{2048\}./d' | anew -q .tmp/gospider.txt
+ [ -s ".tmp/katana.txt" ] && sed -i '/^.\{2048\}./d' .tmp/katana.txt
fi
fi
- [ -s ".tmp/gospider.txt" ] && cat .tmp/gospider.txt | grep -aEo 'https?://[^ ]+' | sed 's/]$//' | sort -u | httpx -follow-redirects -H "${HEADER}" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -no-color | grep "\[4" | cut -d ' ' -f1 | anew -q .tmp/brokenLinks_total.txt
+ [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | sort -u | httpx -follow-redirects -H "${HEADER}" -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -no-color | grep "\[4" | cut -d ' ' -f1 | anew -q .tmp/brokenLinks_total.txt
NUMOFLINES=$(cat .tmp/brokenLinks_total.txt 2>>"$LOGFILE" | anew vulns/brokenLinks.txt | sed '/^$/d' | wc -l)
notification "${NUMOFLINES} new broken links found" info
end_func "Results are saved in vulns/brokenLinks.txt" ${FUNCNAME[0]}