SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_hackertarget: code cleanup
Brendan Coles31fc4ff44638
diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.pyindex 5f102d9f..4c1bd5cd 100644--- a/modules/sfp_hackertarget.py+++ b/modules/sfp_hackertarget.py@@ -14,6 +14,9 @@import jsonimport re+import urllib.error+import urllib.parse+import urllib.requestfrom netaddr import IPNetwork@@ -26,7 +29,7 @@ class sfp_hackertarget(SpiderFootPlugin):'name': "HackerTarget",'summary': "Search HackerTarget.com for hosts sharing the same IP.",'flags': [""],- 'useCases': ["Footprint", "Investigate", "Passive"],+ 'useCases': ["Footprint", "Investigate"],'categories': ["Passive DNS"],'dataSource': {'website': "https://hackertarget.com/",@@ -45,7 +48,6 @@ class sfp_hackertarget(SpiderFootPlugin):}}- # Default optionsopts = {'cohostsamedomain': False,'verify': True,@@ -57,7 +59,6 @@ class sfp_hackertarget(SpiderFootPlugin):'udp_portscan': False}- # Option descriptionsoptdescs = {'cohostsamedomain': "Treat co-hosted sites on the same target domain as co-hosting?",'verify': "Verify co-hosts are valid by checking if they still resolve to the shared IP.",@@ -80,25 +81,51 @@ class sfp_hackertarget(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):- return ["IP_ADDRESS", "NETBLOCK_OWNER", 'DOMAIN_NAME_PARENT']+ return [+ "IP_ADDRESS",+ "NETBLOCK_OWNER",+ 'DOMAIN_NAME_PARENT'+ ]- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):- return ["CO_HOSTED_SITE", "UDP_PORT_OPEN", "TCP_PORT_OPEN", "IP_ADDRESS",- 'WEBSERVER_HTTPHEADERS', 'RAW_DNS_RECORDS',- 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED',- 'DOMAIN_NAME', 'AFFILIATE_DOMAIN_NAME',- 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED']+ return [+ "CO_HOSTED_SITE",+ "UDP_PORT_OPEN",+ "TCP_PORT_OPEN",+ "IP_ADDRESS",+ 'WEBSERVER_HTTPHEADERS',+ 'RAW_DNS_RECORDS',+ 'INTERNET_NAME',+ 'INTERNET_NAME_UNRESOLVED',+ 'DOMAIN_NAME',+ 'AFFILIATE_DOMAIN_NAME',+ 'AFFILIATE_INTERNET_NAME',+ 'AFFILIATE_INTERNET_NAME_UNRESOLVED'+ ]- # Port scan for commonly open UDP portsdef portScanUDP(self, ip):- res = self.sf.fetchUrl("https://hackertarget.com/udp-port-scan/", timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- postData="theinput=" + ip + "&thetest=udpscan&name_of_nonce_field=&_wp_http_referer=%2Fudp-port-scan%2F")+ """Port scan for commonly open UDP ports++ Args:+ ip (str): IPv4 address++ Returns:+ list: open UDP ports+ """+ params = urllib.parse.urlencode({+ "theinput": ip,+ "thetest": "udpscan",+ "name_of_nonce_field": "",+ "_wp_http_referer": "/udp-port-scan/"+ })++ res = self.sf.fetchUrl(+ "https://hackertarget.com/udp-port-scan/",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ postData=params+ )if res['content'] is None:return None@@ -106,24 +133,40 @@ class sfp_hackertarget(SpiderFootPlugin):html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)if not html_data:- self.sf.debug("Found no open UDP ports on " + ip)+ self.sf.debug(f"Found no open UDP ports on {ip}")return Noneopen_ports = re.findall(r'(\d+)/udp\s+open\s+', html_data[0])if not open_ports:- self.sf.debug("Found no open UDP ports on " + ip)+ self.sf.debug(f"Found no open UDP ports on {ip}")return None- self.sf.debug("Found " + str(len(open_ports)) + " open UDP ports on " + ip)+ self.sf.debug(f"Found {len(open_ports)} open UDP ports on {ip}")return open_ports- # Port scan for commonly open TCP portsdef portScanTCP(self, ip):- res = self.sf.fetchUrl("https://hackertarget.com/tcp-port-scan/", timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- postData="theinput=" + ip + "&thetest=tcpscan&name_of_nonce_field=&_wp_http_referer=%2Ftcp-port-scan%2F")+ """Port scan for commonly open TCP ports++ Args:+ ip (str): IPv4 address++ Returns:+ list: open TCP ports+ """+ params = urllib.parse.urlencode({+ "theinput": ip,+ "thetest": "tcpscan",+ "name_of_nonce_field": "",+ "_wp_http_referer": "/tcp-port-scan/"+ })++ res = self.sf.fetchUrl(+ "https://hackertarget.com/tcp-port-scan/", timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ postData=params+ )if res['content'] is None:return None@@ -131,31 +174,44 @@ class sfp_hackertarget(SpiderFootPlugin):html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)if not html_data:- self.sf.debug("Found no open TCP ports on " + ip)+ self.sf.debug(f"Found no open TCP ports on {ip}")return Noneopen_ports = re.findall(r'(\d+)/tcp\s+open\s+', html_data[0])if not open_ports:- self.sf.debug("Found no open TCP ports on " + ip)+ self.sf.debug(f"Found no open TCP ports on {ip}")return None- self.sf.debug("Found " + str(len(open_ports)) + " open TCP ports on " + ip)+ self.sf.debug(f"Found {len(open_ports)} open TCP ports on {ip}")return open_ports- # Retrieve HTTP headersdef httpHeaders(self, ip):- res = self.sf.fetchUrl("https://api.hackertarget.com/httpheaders/?q=" + ip,- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ """Retrieve HTTP headers for IP address++ Args:+ ip (str): IPv4 address++ Returns:+ dict: HTTP headers+ """+ params = urllib.parse.urlencode({+ 'q': ip+ })++ res = self.sf.fetchUrl(+ f"https://api.hackertarget.com/httpheaders/?{params}",+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )if res['content'] is None:- self.sf.error("Unable to fetch HTTP headers for " + ip + " from HackerTarget.com.")+ self.sf.error(f"Unable to fetch HTTP headers for {ip} from HackerTarget.com.")return Noneif not res['content'].startswith('HTTP/'):- self.sf.debug("Found no HTTP headers for " + ip)+ self.sf.debug(f"Found no HTTP headers for {ip}")return Noneheaders = dict()@@ -169,14 +225,27 @@ class sfp_hackertarget(SpiderFootPlugin):return headers- # Retrieve DNS Zonedef zoneTransfer(self, ip):- res = self.sf.fetchUrl("https://api.hackertarget.com/zonetransfer/?q=" + ip,- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ """Retrieve DNS zone transfer++ Args:+ ip (str): IPv4 address++ Returns:+ list: DNS zone+ """+ params = urllib.parse.urlencode({+ 'q': ip+ })++ res = self.sf.fetchUrl(+ f"https://api.hackertarget.com/zonetransfer/?{params}",+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )if res['content'] is None:- self.sf.error("Unable to fetch DNS zone for " + ip + " from HackerTarget.com.")+ self.sf.error(f"Unable to fetch DNS zone for {ip} from HackerTarget.com.")return Nonerecords = list()@@ -190,11 +259,25 @@ class sfp_hackertarget(SpiderFootPlugin):return records- # Reverse lookup hosts on the same IP addressdef reverseIpLookup(self, ip):- res = self.sf.fetchUrl("https://api.hackertarget.com/reverseiplookup/?q=" + ip,- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ """Reverse lookup hosts on the same IP address++ Args:+ ip (str): IPv4 address++ Returns:+ list: (co)hosts on provided IP addresses+ """+ params = urllib.parse.urlencode({+ 'q': ip+ })++ res = self.sf.fetchUrl(+ f"https://api.hackertarget.com/reverseiplookup/?{params}",+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )+if res['content'] is None:self.sf.error("Unable to fetch hackertarget.com content.")return None@@ -204,11 +287,10 @@ class sfp_hackertarget(SpiderFootPlugin):hosts = res['content'].split('\n')- self.sf.debug("Found " + str(len(hosts)) + " on " + ip)+ self.sf.debug(f"Found {len(hosts)} on {ip}")return hosts- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -218,10 +300,9 @@ class sfp_hackertarget(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS":- self.sf.debug("Ignoring " + eventName + ", from self.")+ self.sf.debug(f"Ignoring {eventName}, from self.")return None- # Don't look up stuff twiceif eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None@@ -229,12 +310,12 @@ class sfp_hackertarget(SpiderFootPlugin):if eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: "- + str(IPNetwork(eventData).prefixlen) + " > "- + str(self.opts['maxnetblock']))- return None++ max_netblock = self.opts['maxnetblock']+ net_size = IPNetwork(eventData).prefixlen+ if net_size < max_netblock:+ self.sf.debug(f"Network size bigger than permitted: {net_size} > {max_netblock}")+ return Noneif eventName == 'DOMAIN_NAME_PARENT':records = self.zoneTransfer(eventData)@@ -309,17 +390,18 @@ class sfp_hackertarget(SpiderFootPlugin):if " " in h:continue- self.sf.info("Found something on same IP: " + h)+ self.sf.info(f"Found something on same IP: {h}")if not self.opts['cohostsamedomain']:if self.getTarget().matches(h, includeParents=True):- self.sf.debug("Skipping " + h + " because it is on the same domain.")+ self.sf.debug(f"Skipping {h} because it is on the same domain.")continueif h not in myres and h != ip:if self.opts['verify'] and not self.sf.validateIP(h, ip):- self.sf.debug("Host " + h + " no longer resolves to " + ip)+ self.sf.debug(f"Host {h} no longer resolves to {ip}")continue+if self.cohostcount < self.opts['maxcohost']:# Create an IP Address event stemming from the netblock as the# link to the co-host.@@ -354,14 +436,14 @@ class sfp_hackertarget(SpiderFootPlugin):udp_ports = self.portScanUDP(ip)if udp_ports:for port in udp_ports:- e = SpiderFootEvent("UDP_PORT_OPEN", ip + ":" + port, self.__name__, pevent)+ e = SpiderFootEvent("UDP_PORT_OPEN", f"{ip}:{port}", self.__name__, pevent)self.notifyListeners(e)if self.opts.get('tcp_portscan', True):tcp_ports = self.portScanTCP(ip)if tcp_ports:for port in tcp_ports:- e = SpiderFootEvent("TCP_PORT_OPEN", ip + ":" + port, self.__name__, pevent)+ e = SpiderFootEvent("TCP_PORT_OPEN", f"{ip}:{port}", self.__name__, pevent)self.notifyListeners(e)# End of sfp_hackertarget class