SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Better handling of IPV6 events, usernames as targets, and bug fixes.
smicallefa9b578682c24
diff --git a/dyn/newscan.tmpl b/dyn/newscan.tmplindex 0e35d3d8..12fbb459 100644--- a/dyn/newscan.tmpl+++ b/dyn/newscan.tmpl@@ -52,7 +52,7 @@<div class="control-group"><label class="control-label" for="scantarget">Seed Target</label><div class="controls">- <input type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>"John Smith"</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan.">+ <input type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>"John Smith"</i> (must be in quotes)<br><b>Username</b>: e.g. <i>"jsmith2000"</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan."></div></div>diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 07336c90..5718db61 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -243,46 +243,40 @@ class sfp_accounts(SpiderFootPlugin):if eventName == "HUMAN_NAME":names = [ eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".") ]for name in names:- res = self.batchSites(name)- for site in res:- evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,- self.__name__, event)- self.notifyListeners(evt)- users.append(name)+ users.append(name)if eventName == "DOMAIN_NAME":kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])+ users.append(kw)- res = self.batchSites(kw)- for site in res:- evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,- self.__name__, event)- self.notifyListeners(evt)- users.append(kw)-- if eventName in ["EMAILADDR", "USERNAME"]:+ if eventName == "EMAILADDR":name = eventData.split("@")[0].lower()- adduser = True- if self.opts['generic'] is list() and name in self.opts['generic']:- self.sf.debug(name + " is a generic account name, skipping.")- adduser = False+ users.append(name)- if self.opts['ignorenamedict'] and name in self.commonNames:- self.sf.debug(name + " is found in our name dictionary, skipping.")- adduser = False+ if eventName == "USERNAME":+ users.append(eventData)++ for user in users:+ adduser = True+ if self.opts['generic'] is list() and user in self.opts['generic']:+ self.sf.debug(user + " is a generic account name, skipping.")+ continue- if self.opts['ignoreworddict'] and name in self.words:- self.sf.debug(name + " is found in our word dictionary, skipping.")- adduser = False+ if self.opts['ignorenamedict'] and user in self.commonNames:+ self.sf.debug(user + " is found in our name dictionary, skipping.")+ continue- if eventName == "EMAILADDR" and not self.opts['userfromemail']:- adduser = False+ if self.opts['ignoreworddict'] and user in self.words:+ self.sf.debug(user + " is found in our word dictionary, skipping.")+ continue- if adduser:- users.append(name)+ res = self.batchSites(user)+ for site in res:+ evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,+ self.__name__, event)+ self.notifyListeners(evt)- for user in users:- if user not in self.reportedUsers and user != eventData:+ if user not in self.reportedUsers and eventData != user:evt = SpiderFootEvent("USERNAME", user, self.__name__, event)self.notifyListeners(evt)self.reportedUsers.append(user)diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.pyindex acfe02ce..3dfec71e 100644--- a/modules/sfp_dnsresolve.py+++ b/modules/sfp_dnsresolve.py@@ -13,7 +13,7 @@import re-import urllib.request, urllib.error, urllib.parse+import urllibfrom netaddr import IPNetworkfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent@@ -62,6 +62,8 @@ class sfp_dnsresolve(SpiderFootPlugin):self.sf.debug("Found an alias: " + host)if self.sf.validIP(host):target.setAlias(host, "IP_ADDRESS")+ elif self.sf.validIP6(host):+ target.setAlias(host, "IPV6_ADDRESS")else:target.setAlias(host, "INTERNET_NAME")idnahost = host.encode("idna")@@ -84,7 +86,7 @@ class sfp_dnsresolve(SpiderFootPlugin):return [# Events that need some kind of DNS treatment"CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",- "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",+ "IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",# Events that may contain hostnames in their content"TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS","CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",@@ -148,9 +150,9 @@ class sfp_dnsresolve(SpiderFootPlugin):# Search for IPs/hosts in raw dataif eventName not in [ "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME",- "NETBLOCK_OWNER", "IP_ADDRESS",+ "NETBLOCK_OWNER", "IP_ADDRESS", "IPV6_ADDRESS","INTERNET_NAME", "AFFILIATE_IPADDR"]:- data = urllib.parse.unquote(eventData)+ data = urllib.parse.unquote(eventData).lower()for name in self.getTarget().getNames():if self.checkForStop():return None@@ -162,7 +164,7 @@ class sfp_dnsresolve(SpiderFootPlugin):if offset == 0:offset += len(name)- pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)+ pat = re.compile("[^a-z0-9\-\.\%]([a-z0-9\-\.\%]*\." + name + ")", re.DOTALL|re.MULTILINE)while offset >= 0:offset = data.find(name, offset)#print "found at offset: " + str(offset)@@ -177,10 +179,13 @@ class sfp_dnsresolve(SpiderFootPlugin):if matches:for match in matches:# Wildcard certs will come in as .blah.blah- if match[1].startswith("."):- m = match[1][1:]+ if match.startswith("."):+ m = match[1:]else:- m = match[1]+ m = match+ # Remove URL-encoded stuff+ if '%' in m:+ m = urllib.parse.unquote(m)self.processHost(m, parentEvent, False)except Exception as e:self.sf.error("Error applying regex to data (" + str(e) + ")", False)@@ -231,7 +236,7 @@ class sfp_dnsresolve(SpiderFootPlugin):self.processHost(ipaddr, parentEvent, False)return None- if eventName in ["IP_ADDRESS", "INTERNET_NAME",+ if eventName in ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS","AFFILIATE_IPADDR", "AFFILIATE_INTERNET_NAME"]:if "_NAME" in eventName:@@ -296,6 +301,8 @@ class sfp_dnsresolve(SpiderFootPlugin):else:if self.sf.validIP(host):htype = "IP_ADDRESS"+ elif self.sf.validIP6(host):+ htype = "IPV6_ADDRESS"else:htype = "INTERNET_NAME"diff --git a/sflib.py b/sflib.pyindex 3eff967c..4423e954 100644--- a/sflib.py+++ b/sflib.py@@ -574,7 +574,8 @@ class SpiderFoot:{"^\d+\.\d+\.\d+\.\d+/\d+$": "NETBLOCK_OWNER"},{"^.*@.*$": "EMAILADDR"},{"^\+\d+$": "PHONE_NUMBER"},- {"^\".*\"$": "HUMAN_NAME"},+ {"^\".*\s+.*\"$": "HUMAN_NAME"},+ {"^\".*\"$": "USERNAME"},{"^\d+$": "BGP_AS_OWNER"},{"^[0-9a-f:]+$": "IPV6_ADDRESS"},{"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"}@@ -972,7 +973,7 @@ class SpiderFoot:t = target.getType()v = target.getValue()- if t == "IP_ADDRESS":+ if t in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:r = self.resolveIP(v)if r:ret.extend(r)@@ -1809,7 +1810,7 @@ class SpiderFootPlugin(object):# Class for targetsclass SpiderFootTarget(object):_validTypes = ["IP_ADDRESS", 'IPV6_ADDRESS', "NETBLOCK_OWNER", "INTERNET_NAME",- "EMAILADDR", "HUMAN_NAME", "BGP_AS_OWNER", 'PHONE_NUMBER']+ "EMAILADDR", "HUMAN_NAME", "BGP_AS_OWNER", 'PHONE_NUMBER', "USERNAME"]targetType = NonetargetValue = NonetargetAliases = list()@@ -1872,6 +1873,9 @@ class SpiderFootTarget(object):e = self._getEquivalents("IP_ADDRESS")if self.targetType == "IP_ADDRESS":e.append(self.targetValue)+ e = self._getEquivalents("IPV6_ADDRESS")+ if self.targetType == "IPV6_ADDRESS":+ e.append(self.targetValue)return e# Check whether the supplied value is "tightly" related@@ -1899,8 +1903,9 @@ class SpiderFootTarget(object):if value is None or value == "":return False- # We can't really say anything about names or phone numbers, so everything matches- if self.targetType == "HUMAN_NAME" or self.targetType == "PHONE_NUMBER":+ # We can't really say anything about names, username or phone numbers,+ # so everything matches+ if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME" ]:return Trueif netaddr.valid_ipv4(value):@@ -1911,7 +1916,7 @@ class SpiderFootTarget(object):if self.targetType == "NETBLOCK_OWNER":if netaddr.IPAddress(value) in netaddr.IPNetwork(self.targetValue):return True- if self.targetType == "IP_ADDRESS":+ if self.targetType in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:if netaddr.IPAddress(value) in \netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):return Truediff --git a/sfwebui.py b/sfwebui.pyindex 842370d3..668f3631 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -818,7 +818,7 @@ class SpiderFootWebUi:# Start running a new scanscanId = sf.genScanInstanceGUID(scanname)- if targetType == "HUMAN_NAME":+ if targetType in [ "HUMAN_NAME", "USERNAME" ]:scantarget = scantarget.replace("\"", "")else:scantarget = scantarget.lower()