Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Better handling of IPV6 events, usernames as targets, and bug fixes.

smicallefa9b578682c24
diff --git a/dyn/newscan.tmpl b/dyn/newscan.tmpl
index 0e35d3d8..12fbb459 100644
--- a/dyn/newscan.tmpl
+++ b/dyn/newscan.tmpl
@@ -52,7 +52,7 @@
<div class="control-group">
<label class="control-label" for="scantarget">Seed Target</label>
<div class="controls">
- <input type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>&quot;John Smith&quot;</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan.">
+ <input type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>&quot;John Smith&quot;</i> (must be in quotes)<br><b>Username</b>: e.g. <i>&quot;jsmith2000&quot;</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan.">
</div>
</div>
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 07336c90..5718db61 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -243,46 +243,40 @@ class sfp_accounts(SpiderFootPlugin):
if eventName == "HUMAN_NAME":
names = [ eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".") ]
for name in names:
- res = self.batchSites(name)
- for site in res:
- evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,
- self.__name__, event)
- self.notifyListeners(evt)
- users.append(name)
+ users.append(name)
if eventName == "DOMAIN_NAME":
kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
+ users.append(kw)
- res = self.batchSites(kw)
- for site in res:
- evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,
- self.__name__, event)
- self.notifyListeners(evt)
- users.append(kw)
-
- if eventName in ["EMAILADDR", "USERNAME"]:
+ if eventName == "EMAILADDR":
name = eventData.split("@")[0].lower()
- adduser = True
- if self.opts['generic'] is list() and name in self.opts['generic']:
- self.sf.debug(name + " is a generic account name, skipping.")
- adduser = False
+ users.append(name)
- if self.opts['ignorenamedict'] and name in self.commonNames:
- self.sf.debug(name + " is found in our name dictionary, skipping.")
- adduser = False
+ if eventName == "USERNAME":
+ users.append(eventData)
+
+ for user in users:
+ adduser = True
+ if self.opts['generic'] is list() and user in self.opts['generic']:
+ self.sf.debug(user + " is a generic account name, skipping.")
+ continue
- if self.opts['ignoreworddict'] and name in self.words:
- self.sf.debug(name + " is found in our word dictionary, skipping.")
- adduser = False
+ if self.opts['ignorenamedict'] and user in self.commonNames:
+ self.sf.debug(user + " is found in our name dictionary, skipping.")
+ continue
- if eventName == "EMAILADDR" and not self.opts['userfromemail']:
- adduser = False
+ if self.opts['ignoreworddict'] and user in self.words:
+ self.sf.debug(user + " is found in our word dictionary, skipping.")
+ continue
- if adduser:
- users.append(name)
+ res = self.batchSites(user)
+ for site in res:
+ evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,
+ self.__name__, event)
+ self.notifyListeners(evt)
- for user in users:
- if user not in self.reportedUsers and user != eventData:
+ if user not in self.reportedUsers and eventData != user:
evt = SpiderFootEvent("USERNAME", user, self.__name__, event)
self.notifyListeners(evt)
self.reportedUsers.append(user)
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index acfe02ce..3dfec71e 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -13,7 +13,7 @@
import re
-import urllib.request, urllib.error, urllib.parse
+import urllib
from netaddr import IPNetwork
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
@@ -62,6 +62,8 @@ class sfp_dnsresolve(SpiderFootPlugin):
self.sf.debug("Found an alias: " + host)
if self.sf.validIP(host):
target.setAlias(host, "IP_ADDRESS")
+ elif self.sf.validIP6(host):
+ target.setAlias(host, "IPV6_ADDRESS")
else:
target.setAlias(host, "INTERNET_NAME")
idnahost = host.encode("idna")
@@ -84,7 +86,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
return [
# Events that need some kind of DNS treatment
"CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",
- "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
+ "IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
# Events that may contain hostnames in their content
"TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS",
"CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",
@@ -148,9 +150,9 @@ class sfp_dnsresolve(SpiderFootPlugin):
# Search for IPs/hosts in raw data
if eventName not in [ "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME",
- "NETBLOCK_OWNER", "IP_ADDRESS",
+ "NETBLOCK_OWNER", "IP_ADDRESS", "IPV6_ADDRESS",
"INTERNET_NAME", "AFFILIATE_IPADDR"]:
- data = urllib.parse.unquote(eventData)
+ data = urllib.parse.unquote(eventData).lower()
for name in self.getTarget().getNames():
if self.checkForStop():
return None
@@ -162,7 +164,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
if offset == 0:
offset += len(name)
- pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)
+ pat = re.compile("[^a-z0-9\-\.\%]([a-z0-9\-\.\%]*\." + name + ")", re.DOTALL|re.MULTILINE)
while offset >= 0:
offset = data.find(name, offset)
#print "found at offset: " + str(offset)
@@ -177,10 +179,13 @@ class sfp_dnsresolve(SpiderFootPlugin):
if matches:
for match in matches:
# Wildcard certs will come in as .blah.blah
- if match[1].startswith("."):
- m = match[1][1:]
+ if match.startswith("."):
+ m = match[1:]
else:
- m = match[1]
+ m = match
+ # Remove URL-encoded stuff
+ if '%' in m:
+ m = urllib.parse.unquote(m)
self.processHost(m, parentEvent, False)
except Exception as e:
self.sf.error("Error applying regex to data (" + str(e) + ")", False)
@@ -231,7 +236,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
self.processHost(ipaddr, parentEvent, False)
return None
- if eventName in ["IP_ADDRESS", "INTERNET_NAME",
+ if eventName in ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS",
"AFFILIATE_IPADDR", "AFFILIATE_INTERNET_NAME"]:
if "_NAME" in eventName:
@@ -296,6 +301,8 @@ class sfp_dnsresolve(SpiderFootPlugin):
else:
if self.sf.validIP(host):
htype = "IP_ADDRESS"
+ elif self.sf.validIP6(host):
+ htype = "IPV6_ADDRESS"
else:
htype = "INTERNET_NAME"
diff --git a/sflib.py b/sflib.py
index 3eff967c..4423e954 100644
--- a/sflib.py
+++ b/sflib.py
@@ -574,7 +574,8 @@ class SpiderFoot:
{"^\d+\.\d+\.\d+\.\d+/\d+$": "NETBLOCK_OWNER"},
{"^.*@.*$": "EMAILADDR"},
{"^\+\d+$": "PHONE_NUMBER"},
- {"^\".*\"$": "HUMAN_NAME"},
+ {"^\".*\s+.*\"$": "HUMAN_NAME"},
+ {"^\".*\"$": "USERNAME"},
{"^\d+$": "BGP_AS_OWNER"},
{"^[0-9a-f:]+$": "IPV6_ADDRESS"},
{"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"}
@@ -972,7 +973,7 @@ class SpiderFoot:
t = target.getType()
v = target.getValue()
- if t == "IP_ADDRESS":
+ if t in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:
r = self.resolveIP(v)
if r:
ret.extend(r)
@@ -1809,7 +1810,7 @@ class SpiderFootPlugin(object):
# Class for targets
class SpiderFootTarget(object):
_validTypes = ["IP_ADDRESS", 'IPV6_ADDRESS', "NETBLOCK_OWNER", "INTERNET_NAME",
- "EMAILADDR", "HUMAN_NAME", "BGP_AS_OWNER", 'PHONE_NUMBER']
+ "EMAILADDR", "HUMAN_NAME", "BGP_AS_OWNER", 'PHONE_NUMBER', "USERNAME"]
targetType = None
targetValue = None
targetAliases = list()
@@ -1872,6 +1873,9 @@ class SpiderFootTarget(object):
e = self._getEquivalents("IP_ADDRESS")
if self.targetType == "IP_ADDRESS":
e.append(self.targetValue)
+ e = self._getEquivalents("IPV6_ADDRESS")
+ if self.targetType == "IPV6_ADDRESS":
+ e.append(self.targetValue)
return e
# Check whether the supplied value is "tightly" related
@@ -1899,8 +1903,9 @@ class SpiderFootTarget(object):
if value is None or value == "":
return False
- # We can't really say anything about names or phone numbers, so everything matches
- if self.targetType == "HUMAN_NAME" or self.targetType == "PHONE_NUMBER":
+ # We can't really say anything about names, username or phone numbers,
+ # so everything matches
+ if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME" ]:
return True
if netaddr.valid_ipv4(value):
@@ -1911,7 +1916,7 @@ class SpiderFootTarget(object):
if self.targetType == "NETBLOCK_OWNER":
if netaddr.IPAddress(value) in netaddr.IPNetwork(self.targetValue):
return True
- if self.targetType == "IP_ADDRESS":
+ if self.targetType in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:
if netaddr.IPAddress(value) in \
netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):
return True
diff --git a/sfwebui.py b/sfwebui.py
index 842370d3..668f3631 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -818,7 +818,7 @@ class SpiderFootWebUi:
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
- if targetType == "HUMAN_NAME":
+ if targetType in [ "HUMAN_NAME", "USERNAME" ]:
scantarget = scantarget.replace("\"", "")
else:
scantarget = scantarget.lower()