Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_hackertarget: code cleanup

Brendan Coles31fc4ff44638
diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.py
index 5f102d9f..4c1bd5cd 100644
--- a/modules/sfp_hackertarget.py
+++ b/modules/sfp_hackertarget.py
@@ -14,6 +14,9 @@
import json
import re
+import urllib.error
+import urllib.parse
+import urllib.request
from netaddr import IPNetwork
@@ -26,7 +29,7 @@ class sfp_hackertarget(SpiderFootPlugin):
'name': "HackerTarget",
'summary': "Search HackerTarget.com for hosts sharing the same IP.",
'flags': [""],
- 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'useCases': ["Footprint", "Investigate"],
'categories': ["Passive DNS"],
'dataSource': {
'website': "https://hackertarget.com/",
@@ -45,7 +48,6 @@ class sfp_hackertarget(SpiderFootPlugin):
}
}
- # Default options
opts = {
'cohostsamedomain': False,
'verify': True,
@@ -57,7 +59,6 @@ class sfp_hackertarget(SpiderFootPlugin):
'udp_portscan': False
}
- # Option descriptions
optdescs = {
'cohostsamedomain': "Treat co-hosted sites on the same target domain as co-hosting?",
'verify': "Verify co-hosts are valid by checking if they still resolve to the shared IP.",
@@ -80,25 +81,51 @@ class sfp_hackertarget(SpiderFootPlugin):
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
- # What events is this module interested in for input
def watchedEvents(self):
- return ["IP_ADDRESS", "NETBLOCK_OWNER", 'DOMAIN_NAME_PARENT']
+ return [
+ "IP_ADDRESS",
+ "NETBLOCK_OWNER",
+ 'DOMAIN_NAME_PARENT'
+ ]
- # What events this module produces
- # This is to support the end user in selecting modules based on events
- # produced.
def producedEvents(self):
- return ["CO_HOSTED_SITE", "UDP_PORT_OPEN", "TCP_PORT_OPEN", "IP_ADDRESS",
- 'WEBSERVER_HTTPHEADERS', 'RAW_DNS_RECORDS',
- 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED',
- 'DOMAIN_NAME', 'AFFILIATE_DOMAIN_NAME',
- 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED']
+ return [
+ "CO_HOSTED_SITE",
+ "UDP_PORT_OPEN",
+ "TCP_PORT_OPEN",
+ "IP_ADDRESS",
+ 'WEBSERVER_HTTPHEADERS',
+ 'RAW_DNS_RECORDS',
+ 'INTERNET_NAME',
+ 'INTERNET_NAME_UNRESOLVED',
+ 'DOMAIN_NAME',
+ 'AFFILIATE_DOMAIN_NAME',
+ 'AFFILIATE_INTERNET_NAME',
+ 'AFFILIATE_INTERNET_NAME_UNRESOLVED'
+ ]
- # Port scan for commonly open UDP ports
def portScanUDP(self, ip):
- res = self.sf.fetchUrl("https://hackertarget.com/udp-port-scan/", timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- postData="theinput=" + ip + "&thetest=udpscan&name_of_nonce_field=&_wp_http_referer=%2Fudp-port-scan%2F")
+ """Port scan for commonly open UDP ports
+
+ Args:
+ ip (str): IPv4 address
+
+ Returns:
+ list: open UDP ports
+ """
+ params = urllib.parse.urlencode({
+ "theinput": ip,
+ "thetest": "udpscan",
+ "name_of_nonce_field": "",
+ "_wp_http_referer": "/udp-port-scan/"
+ })
+
+ res = self.sf.fetchUrl(
+ "https://hackertarget.com/udp-port-scan/",
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ postData=params
+ )
if res['content'] is None:
return None
@@ -106,24 +133,40 @@ class sfp_hackertarget(SpiderFootPlugin):
html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)
if not html_data:
- self.sf.debug("Found no open UDP ports on " + ip)
+ self.sf.debug(f"Found no open UDP ports on {ip}")
return None
open_ports = re.findall(r'(\d+)/udp\s+open\s+', html_data[0])
if not open_ports:
- self.sf.debug("Found no open UDP ports on " + ip)
+ self.sf.debug(f"Found no open UDP ports on {ip}")
return None
- self.sf.debug("Found " + str(len(open_ports)) + " open UDP ports on " + ip)
+ self.sf.debug(f"Found {len(open_ports)} open UDP ports on {ip}")
return open_ports
- # Port scan for commonly open TCP ports
def portScanTCP(self, ip):
- res = self.sf.fetchUrl("https://hackertarget.com/tcp-port-scan/", timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- postData="theinput=" + ip + "&thetest=tcpscan&name_of_nonce_field=&_wp_http_referer=%2Ftcp-port-scan%2F")
+ """Port scan for commonly open TCP ports
+
+ Args:
+ ip (str): IPv4 address
+
+ Returns:
+ list: open TCP ports
+ """
+ params = urllib.parse.urlencode({
+ "theinput": ip,
+ "thetest": "tcpscan",
+ "name_of_nonce_field": "",
+ "_wp_http_referer": "/tcp-port-scan/"
+ })
+
+ res = self.sf.fetchUrl(
+ "https://hackertarget.com/tcp-port-scan/", timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ postData=params
+ )
if res['content'] is None:
return None
@@ -131,31 +174,44 @@ class sfp_hackertarget(SpiderFootPlugin):
html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)
if not html_data:
- self.sf.debug("Found no open TCP ports on " + ip)
+ self.sf.debug(f"Found no open TCP ports on {ip}")
return None
open_ports = re.findall(r'(\d+)/tcp\s+open\s+', html_data[0])
if not open_ports:
- self.sf.debug("Found no open TCP ports on " + ip)
+ self.sf.debug(f"Found no open TCP ports on {ip}")
return None
- self.sf.debug("Found " + str(len(open_ports)) + " open TCP ports on " + ip)
+ self.sf.debug(f"Found {len(open_ports)} open TCP ports on {ip}")
return open_ports
- # Retrieve HTTP headers
def httpHeaders(self, ip):
- res = self.sf.fetchUrl("https://api.hackertarget.com/httpheaders/?q=" + ip,
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout'])
+ """Retrieve HTTP headers for IP address
+
+ Args:
+ ip (str): IPv4 address
+
+ Returns:
+ dict: HTTP headers
+ """
+ params = urllib.parse.urlencode({
+ 'q': ip
+ })
+
+ res = self.sf.fetchUrl(
+ f"https://api.hackertarget.com/httpheaders/?{params}",
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']
+ )
if res['content'] is None:
- self.sf.error("Unable to fetch HTTP headers for " + ip + " from HackerTarget.com.")
+ self.sf.error(f"Unable to fetch HTTP headers for {ip} from HackerTarget.com.")
return None
if not res['content'].startswith('HTTP/'):
- self.sf.debug("Found no HTTP headers for " + ip)
+ self.sf.debug(f"Found no HTTP headers for {ip}")
return None
headers = dict()
@@ -169,14 +225,27 @@ class sfp_hackertarget(SpiderFootPlugin):
return headers
- # Retrieve DNS Zone
def zoneTransfer(self, ip):
- res = self.sf.fetchUrl("https://api.hackertarget.com/zonetransfer/?q=" + ip,
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout'])
+ """Retrieve DNS zone transfer
+
+ Args:
+ ip (str): IPv4 address
+
+ Returns:
+ list: DNS zone
+ """
+ params = urllib.parse.urlencode({
+ 'q': ip
+ })
+
+ res = self.sf.fetchUrl(
+ f"https://api.hackertarget.com/zonetransfer/?{params}",
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']
+ )
if res['content'] is None:
- self.sf.error("Unable to fetch DNS zone for " + ip + " from HackerTarget.com.")
+ self.sf.error(f"Unable to fetch DNS zone for {ip} from HackerTarget.com.")
return None
records = list()
@@ -190,11 +259,25 @@ class sfp_hackertarget(SpiderFootPlugin):
return records
- # Reverse lookup hosts on the same IP address
def reverseIpLookup(self, ip):
- res = self.sf.fetchUrl("https://api.hackertarget.com/reverseiplookup/?q=" + ip,
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout'])
+ """Reverse lookup hosts on the same IP address
+
+ Args:
+ ip (str): IPv4 address
+
+ Returns:
+ list: (co)hosts on provided IP addresses
+ """
+ params = urllib.parse.urlencode({
+ 'q': ip
+ })
+
+ res = self.sf.fetchUrl(
+ f"https://api.hackertarget.com/reverseiplookup/?{params}",
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']
+ )
+
if res['content'] is None:
self.sf.error("Unable to fetch hackertarget.com content.")
return None
@@ -204,11 +287,10 @@ class sfp_hackertarget(SpiderFootPlugin):
hosts = res['content'].split('\n')
- self.sf.debug("Found " + str(len(hosts)) + " on " + ip)
+ self.sf.debug(f"Found {len(hosts)} on {ip}")
return hosts
- # Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
@@ -218,10 +300,9 @@ class sfp_hackertarget(SpiderFootPlugin):
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS":
- self.sf.debug("Ignoring " + eventName + ", from self.")
+ self.sf.debug(f"Ignoring {eventName}, from self.")
return None
- # Don't look up stuff twice
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
@@ -229,12 +310,12 @@ class sfp_hackertarget(SpiderFootPlugin):
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
- self.sf.debug("Network size bigger than permitted: "
- + str(IPNetwork(eventData).prefixlen) + " > "
- + str(self.opts['maxnetblock']))
- return None
+
+ max_netblock = self.opts['maxnetblock']
+ net_size = IPNetwork(eventData).prefixlen
+ if net_size < max_netblock:
+ self.sf.debug(f"Network size bigger than permitted: {net_size} > {max_netblock}")
+ return None
if eventName == 'DOMAIN_NAME_PARENT':
records = self.zoneTransfer(eventData)
@@ -309,17 +390,18 @@ class sfp_hackertarget(SpiderFootPlugin):
if " " in h:
continue
- self.sf.info("Found something on same IP: " + h)
+ self.sf.info(f"Found something on same IP: {h}")
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(h, includeParents=True):
- self.sf.debug("Skipping " + h + " because it is on the same domain.")
+ self.sf.debug(f"Skipping {h} because it is on the same domain.")
continue
if h not in myres and h != ip:
if self.opts['verify'] and not self.sf.validateIP(h, ip):
- self.sf.debug("Host " + h + " no longer resolves to " + ip)
+ self.sf.debug(f"Host {h} no longer resolves to {ip}")
continue
+
if self.cohostcount < self.opts['maxcohost']:
# Create an IP Address event stemming from the netblock as the
# link to the co-host.
@@ -354,14 +436,14 @@ class sfp_hackertarget(SpiderFootPlugin):
udp_ports = self.portScanUDP(ip)
if udp_ports:
for port in udp_ports:
- e = SpiderFootEvent("UDP_PORT_OPEN", ip + ":" + port, self.__name__, pevent)
+ e = SpiderFootEvent("UDP_PORT_OPEN", f"{ip}:{port}", self.__name__, pevent)
self.notifyListeners(e)
if self.opts.get('tcp_portscan', True):
tcp_ports = self.portScanTCP(ip)
if tcp_ports:
for port in tcp_ports:
- e = SpiderFootEvent("TCP_PORT_OPEN", ip + ":" + port, self.__name__, pevent)
+ e = SpiderFootEvent("TCP_PORT_OPEN", f"{ip}:{port}", self.__name__, pevent)
self.notifyListeners(e)
# End of sfp_hackertarget class